From 92697139b01339b6c0767fa1305a4df9a7c1f37f Mon Sep 17 00:00:00 2001 From: Guanjun Date: Mon, 27 Nov 2023 16:31:27 +0800 Subject: lib/find_bit: Fix the code comments about find_next_bit_wrap The function find_next_bit_wrap only has one memory region to search on. Adjust the comments. Signed-off-by: Guanjun Signed-off-by: Yury Norov --- include/linux/find.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/find.h b/include/linux/find.h index 5e4f39ef2e72..af63ae5b9013 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -413,8 +413,8 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1, } /** - * find_next_bit_wrap - find the next set bit in both memory regions - * @addr: The first address to base the search on + * find_next_bit_wrap - find the next set bit in a memory region + * @addr: The address to base the search on * @size: The bitmap size in bits * @offset: The bitnumber to start searching at * -- cgit v1.2.3 From 27c82f14e6d2bcb9f085bad37fe339227571de60 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 28 Oct 2023 12:05:29 -0700 Subject: lib/find: optimize find_*_bit_wrap When an offset is 0, there's no need to search a bitmap from the beginning after the 1st search failed, because each bit has already been tested. Signed-off-by: Yury Norov --- include/linux/find.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/find.h b/include/linux/find.h index af63ae5b9013..c69598e383c1 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -405,7 +405,7 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1, { unsigned long bit = find_next_and_bit(addr1, addr2, size, offset); - if (bit < size) + if (bit < size || offset == 0) return bit; bit = find_first_and_bit(addr1, addr2, offset); @@ -427,7 +427,7 @@ unsigned long find_next_bit_wrap(const unsigned long *addr, { unsigned long bit = find_next_bit(addr, size, offset); - if (bit < size) + if (bit < size || offset == 0) return bit; bit = find_first_bit(addr, offset); -- cgit v1.2.3 From f0eb58dd08770a2e24bfc41db5ee3ff7c3a684ee Mon Sep 17 00:00:00 2001 From: Duje Mihanović Date: Tue, 16 Jan 2024 13:54:31 -0800 Subject: Input: navpoint - remove driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This driver does not use the SPI core as it should, instead tampering with the SSP registers manually. Refactoring the driver is almost certainly not worth it as the hardware seems to have been designed for and used only in the HP iPAQ hx4700 removed more than a year ago in d6df7df7ae5a ("ARM: pxa: remove unused board files"), so let's remove it. Signed-off-by: Duje Mihanović Link: https://lore.kernel.org/r/20240116-navpoint-removal-v2-2-e566806f1009@skole.hr Signed-off-by: Dmitry Torokhov --- drivers/input/mouse/Kconfig | 12 -- drivers/input/mouse/Makefile | 1 - drivers/input/mouse/navpoint.c | 350 ----------------------------------------- include/linux/input/navpoint.h | 8 - 4 files changed, 371 deletions(-) delete mode 100644 drivers/input/mouse/navpoint.c delete mode 100644 include/linux/input/navpoint.h (limited to 'include/linux') diff --git a/drivers/input/mouse/Kconfig b/drivers/input/mouse/Kconfig index 32cc4c62a716..833b643f0616 100644 --- a/drivers/input/mouse/Kconfig +++ b/drivers/input/mouse/Kconfig @@ -439,16 +439,4 @@ config MOUSE_SYNAPTICS_USB To compile this driver as a module, choose M here: the module will be called synaptics_usb. -config MOUSE_NAVPOINT_PXA27x - tristate "Synaptics NavPoint (PXA27x SSP/SPI)" - depends on PXA27x && PXA_SSP - help - This driver adds support for the Synaptics NavPoint touchpad connected - to a PXA27x SSP port in SPI slave mode. The device emulates a mouse; - a tap or tap-and-a-half drag gesture emulates the left mouse button. - For example, use the xf86-input-evdev driver for an X pointing device. - - To compile this driver as a module, choose M here: the - module will be called navpoint. - endif diff --git a/drivers/input/mouse/Makefile b/drivers/input/mouse/Makefile index 92b3204ce84e..a1336d5bee6f 100644 --- a/drivers/input/mouse/Makefile +++ b/drivers/input/mouse/Makefile @@ -15,7 +15,6 @@ obj-$(CONFIG_MOUSE_GPIO) += gpio_mouse.o obj-$(CONFIG_MOUSE_INPORT) += inport.o obj-$(CONFIG_MOUSE_LOGIBM) += logibm.o obj-$(CONFIG_MOUSE_MAPLE) += maplemouse.o -obj-$(CONFIG_MOUSE_NAVPOINT_PXA27x) += navpoint.o obj-$(CONFIG_MOUSE_PC110PAD) += pc110pad.o obj-$(CONFIG_MOUSE_PS2) += psmouse.o obj-$(CONFIG_MOUSE_RISCPC) += rpcmouse.o diff --git a/drivers/input/mouse/navpoint.c b/drivers/input/mouse/navpoint.c deleted file mode 100644 index ba757783c258..000000000000 --- a/drivers/input/mouse/navpoint.c +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Synaptics NavPoint (PXA27x SSP/SPI) driver. - * - * Copyright (C) 2012 Paul Parsons - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Synaptics Modular Embedded Protocol: Module Packet Format. - * Module header byte 2:0 = Length (# bytes that follow) - * Module header byte 4:3 = Control - * Module header byte 7:5 = Module Address - */ -#define HEADER_LENGTH(byte) ((byte) & 0x07) -#define HEADER_CONTROL(byte) (((byte) >> 3) & 0x03) -#define HEADER_ADDRESS(byte) ((byte) >> 5) - -struct navpoint { - struct ssp_device *ssp; - struct input_dev *input; - struct device *dev; - struct gpio_desc *gpiod; - int index; - u8 data[1 + HEADER_LENGTH(0xff)]; -}; - -/* - * Initialization values for SSCR0_x, SSCR1_x, SSSR_x. - */ -static const u32 sscr0 = 0 - | SSCR0_TUM /* TIM = 1; No TUR interrupts */ - | SSCR0_RIM /* RIM = 1; No ROR interrupts */ - | SSCR0_SSE /* SSE = 1; SSP enabled */ - | SSCR0_Motorola /* FRF = 0; Motorola SPI */ - | SSCR0_DataSize(16) /* DSS = 15; Data size = 16-bit */ - ; -static const u32 sscr1 = 0 - | SSCR1_SCFR /* SCFR = 1; SSPSCLK only during transfers */ - | SSCR1_SCLKDIR /* SCLKDIR = 1; Slave mode */ - | SSCR1_SFRMDIR /* SFRMDIR = 1; Slave mode */ - | SSCR1_RWOT /* RWOT = 1; Receive without transmit mode */ - | SSCR1_RxTresh(1) /* RFT = 0; Receive FIFO threshold = 1 */ - | SSCR1_SPH /* SPH = 1; SSPSCLK inactive 0.5 + 1 cycles */ - | SSCR1_RIE /* RIE = 1; Receive FIFO interrupt enabled */ - ; -static const u32 sssr = 0 - | SSSR_BCE /* BCE = 1; Clear BCE */ - | SSSR_TUR /* TUR = 1; Clear TUR */ - | SSSR_EOC /* EOC = 1; Clear EOC */ - | SSSR_TINT /* TINT = 1; Clear TINT */ - | SSSR_PINT /* PINT = 1; Clear PINT */ - | SSSR_ROR /* ROR = 1; Clear ROR */ - ; - -/* - * MEP Query $22: Touchpad Coordinate Range Query is not supported by - * the NavPoint module, so sampled values provide the default limits. - */ -#define NAVPOINT_X_MIN 1278 -#define NAVPOINT_X_MAX 5340 -#define NAVPOINT_Y_MIN 1572 -#define NAVPOINT_Y_MAX 4396 -#define NAVPOINT_PRESSURE_MIN 0 -#define NAVPOINT_PRESSURE_MAX 255 - -static void navpoint_packet(struct navpoint *navpoint) -{ - int finger; - int gesture; - int x, y, z; - - switch (navpoint->data[0]) { - case 0xff: /* Garbage (packet?) between reset and Hello packet */ - case 0x00: /* Module 0, NULL packet */ - break; - - case 0x0e: /* Module 0, Absolute packet */ - finger = (navpoint->data[1] & 0x01); - gesture = (navpoint->data[1] & 0x02); - x = ((navpoint->data[2] & 0x1f) << 8) | navpoint->data[3]; - y = ((navpoint->data[4] & 0x1f) << 8) | navpoint->data[5]; - z = navpoint->data[6]; - input_report_key(navpoint->input, BTN_TOUCH, finger); - input_report_abs(navpoint->input, ABS_X, x); - input_report_abs(navpoint->input, ABS_Y, y); - input_report_abs(navpoint->input, ABS_PRESSURE, z); - input_report_key(navpoint->input, BTN_TOOL_FINGER, finger); - input_report_key(navpoint->input, BTN_LEFT, gesture); - input_sync(navpoint->input); - break; - - case 0x19: /* Module 0, Hello packet */ - if ((navpoint->data[1] & 0xf0) == 0x10) - break; - fallthrough; - default: - dev_warn(navpoint->dev, - "spurious packet: data=0x%02x,0x%02x,...\n", - navpoint->data[0], navpoint->data[1]); - break; - } -} - -static irqreturn_t navpoint_irq(int irq, void *dev_id) -{ - struct navpoint *navpoint = dev_id; - struct ssp_device *ssp = navpoint->ssp; - irqreturn_t ret = IRQ_NONE; - u32 status; - - status = pxa_ssp_read_reg(ssp, SSSR); - if (status & sssr) { - dev_warn(navpoint->dev, - "unexpected interrupt: status=0x%08x\n", status); - pxa_ssp_write_reg(ssp, SSSR, (status & sssr)); - ret = IRQ_HANDLED; - } - - while (status & SSSR_RNE) { - u32 data; - - data = pxa_ssp_read_reg(ssp, SSDR); - navpoint->data[navpoint->index + 0] = (data >> 8); - navpoint->data[navpoint->index + 1] = data; - navpoint->index += 2; - if (HEADER_LENGTH(navpoint->data[0]) < navpoint->index) { - navpoint_packet(navpoint); - navpoint->index = 0; - } - status = pxa_ssp_read_reg(ssp, SSSR); - ret = IRQ_HANDLED; - } - - return ret; -} - -static void navpoint_up(struct navpoint *navpoint) -{ - struct ssp_device *ssp = navpoint->ssp; - int timeout; - - clk_prepare_enable(ssp->clk); - - pxa_ssp_write_reg(ssp, SSCR1, sscr1); - pxa_ssp_write_reg(ssp, SSSR, sssr); - pxa_ssp_write_reg(ssp, SSTO, 0); - pxa_ssp_write_reg(ssp, SSCR0, sscr0); /* SSCR0_SSE written last */ - - /* Wait until SSP port is ready for slave clock operations */ - for (timeout = 100; timeout != 0; --timeout) { - if (!(pxa_ssp_read_reg(ssp, SSSR) & SSSR_CSS)) - break; - msleep(1); - } - - if (timeout == 0) - dev_err(navpoint->dev, - "timeout waiting for SSSR[CSS] to clear\n"); - - gpiod_set_value(navpoint->gpiod, 1); -} - -static void navpoint_down(struct navpoint *navpoint) -{ - struct ssp_device *ssp = navpoint->ssp; - - gpiod_set_value(navpoint->gpiod, 0); - - pxa_ssp_write_reg(ssp, SSCR0, 0); - - clk_disable_unprepare(ssp->clk); -} - -static int navpoint_open(struct input_dev *input) -{ - struct navpoint *navpoint = input_get_drvdata(input); - - navpoint_up(navpoint); - - return 0; -} - -static void navpoint_close(struct input_dev *input) -{ - struct navpoint *navpoint = input_get_drvdata(input); - - navpoint_down(navpoint); -} - -static int navpoint_probe(struct platform_device *pdev) -{ - const struct navpoint_platform_data *pdata = - dev_get_platdata(&pdev->dev); - struct ssp_device *ssp; - struct input_dev *input; - struct navpoint *navpoint; - int error; - - if (!pdata) { - dev_err(&pdev->dev, "no platform data\n"); - return -EINVAL; - } - - ssp = pxa_ssp_request(pdata->port, pdev->name); - if (!ssp) - return -ENODEV; - - /* HaRET does not disable devices before jumping into Linux */ - if (pxa_ssp_read_reg(ssp, SSCR0) & SSCR0_SSE) { - pxa_ssp_write_reg(ssp, SSCR0, 0); - dev_warn(&pdev->dev, "ssp%d already enabled\n", pdata->port); - } - - navpoint = kzalloc(sizeof(*navpoint), GFP_KERNEL); - input = input_allocate_device(); - if (!navpoint || !input) { - error = -ENOMEM; - goto err_free_mem; - } - - navpoint->gpiod = gpiod_get_optional(&pdev->dev, - NULL, GPIOD_OUT_LOW); - if (IS_ERR(navpoint->gpiod)) { - error = PTR_ERR(navpoint->gpiod); - dev_err(&pdev->dev, "error getting GPIO\n"); - goto err_free_mem; - } - gpiod_set_consumer_name(navpoint->gpiod, "SYNAPTICS_ON"); - - navpoint->ssp = ssp; - navpoint->input = input; - navpoint->dev = &pdev->dev; - - input->name = pdev->name; - input->dev.parent = &pdev->dev; - - __set_bit(EV_KEY, input->evbit); - __set_bit(EV_ABS, input->evbit); - __set_bit(BTN_LEFT, input->keybit); - __set_bit(BTN_TOUCH, input->keybit); - __set_bit(BTN_TOOL_FINGER, input->keybit); - - input_set_abs_params(input, ABS_X, - NAVPOINT_X_MIN, NAVPOINT_X_MAX, 0, 0); - input_set_abs_params(input, ABS_Y, - NAVPOINT_Y_MIN, NAVPOINT_Y_MAX, 0, 0); - input_set_abs_params(input, ABS_PRESSURE, - NAVPOINT_PRESSURE_MIN, NAVPOINT_PRESSURE_MAX, - 0, 0); - - input->open = navpoint_open; - input->close = navpoint_close; - - input_set_drvdata(input, navpoint); - - error = request_irq(ssp->irq, navpoint_irq, 0, pdev->name, navpoint); - if (error) - goto err_free_mem; - - error = input_register_device(input); - if (error) - goto err_free_irq; - - platform_set_drvdata(pdev, navpoint); - dev_dbg(&pdev->dev, "ssp%d, irq %d\n", pdata->port, ssp->irq); - - return 0; - -err_free_irq: - free_irq(ssp->irq, navpoint); -err_free_mem: - input_free_device(input); - kfree(navpoint); - pxa_ssp_free(ssp); - - return error; -} - -static void navpoint_remove(struct platform_device *pdev) -{ - struct navpoint *navpoint = platform_get_drvdata(pdev); - struct ssp_device *ssp = navpoint->ssp; - - free_irq(ssp->irq, navpoint); - - input_unregister_device(navpoint->input); - kfree(navpoint); - - pxa_ssp_free(ssp); -} - -static int navpoint_suspend(struct device *dev) -{ - struct platform_device *pdev = to_platform_device(dev); - struct navpoint *navpoint = platform_get_drvdata(pdev); - struct input_dev *input = navpoint->input; - - mutex_lock(&input->mutex); - if (input_device_enabled(input)) - navpoint_down(navpoint); - mutex_unlock(&input->mutex); - - return 0; -} - -static int navpoint_resume(struct device *dev) -{ - struct platform_device *pdev = to_platform_device(dev); - struct navpoint *navpoint = platform_get_drvdata(pdev); - struct input_dev *input = navpoint->input; - - mutex_lock(&input->mutex); - if (input_device_enabled(input)) - navpoint_up(navpoint); - mutex_unlock(&input->mutex); - - return 0; -} - -static DEFINE_SIMPLE_DEV_PM_OPS(navpoint_pm_ops, - navpoint_suspend, navpoint_resume); - -static struct platform_driver navpoint_driver = { - .probe = navpoint_probe, - .remove_new = navpoint_remove, - .driver = { - .name = "navpoint", - .pm = pm_sleep_ptr(&navpoint_pm_ops), - }, -}; - -module_platform_driver(navpoint_driver); - -MODULE_AUTHOR("Paul Parsons "); -MODULE_DESCRIPTION("Synaptics NavPoint (PXA27x SSP/SPI) driver"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:navpoint"); diff --git a/include/linux/input/navpoint.h b/include/linux/input/navpoint.h deleted file mode 100644 index 5192ae3f5ec1..000000000000 --- a/include/linux/input/navpoint.h +++ /dev/null @@ -1,8 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 Paul Parsons - */ - -struct navpoint_platform_data { - int port; /* PXA SSP port for pxa_ssp_request() */ -}; -- cgit v1.2.3 From 6df534cc7136fc9e023cbd4e0011a04e3659e74d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 5 Jan 2024 11:32:50 +0100 Subject: spi: make spi_bus_type const Now that the driver core can properly handle constant struct bus_type, move the spi_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Mark Brown Cc: Signed-off-by: Greg Kroah-Hartman Link: https://msgid.link/r/2024010549-erasure-swoop-1cc6@gregkh Signed-off-by: Mark Brown --- drivers/spi/spi.c | 2 +- include/linux/spi/spi.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 7477a11e12be..bc3d7c0ce6b4 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -459,7 +459,7 @@ static void spi_shutdown(struct device *dev) } } -struct bus_type spi_bus_type = { +const struct bus_type spi_bus_type = { .name = "spi", .dev_groups = spi_dev_groups, .match = spi_match_device, diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 471fe2ff9066..f306aececeaf 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -36,7 +36,7 @@ struct spi_message; * INTERFACES between SPI master-side drivers and SPI slave protocol handlers, * and SPI infrastructure. */ -extern struct bus_type spi_bus_type; +extern const struct bus_type spi_bus_type; /** * struct spi_statistics - statistics for spi transfers -- cgit v1.2.3 From 28c5d4e40752fc39507a647b20649c5ca1cf33b7 Mon Sep 17 00:00:00 2001 From: Kuninori Morimoto Date: Wed, 10 Jan 2024 01:14:50 +0000 Subject: of: Add for_each_reserved_child_of_node() We would like to use for_each loop for status = "reserved" nodes. Add for_each_reserved_child_of_node() for it. Signed-off-by: Kuninori Morimoto Tested-by: Yusuke Goda Reviewed-by: Geert Uytterhoeven Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/87a5pegfau.wl-kuninori.morimoto.gx@renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/of/base.c | 29 +++++++++++++++++++++++++++++ include/linux/of.h | 11 +++++++++++ 2 files changed, 40 insertions(+) (limited to 'include/linux') diff --git a/drivers/of/base.c b/drivers/of/base.c index ae26a15390f7..49a9ad8134db 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -465,6 +465,20 @@ static bool __of_device_is_available(const struct device_node *device) __of_device_is_status(device, ok); } +/** + * __of_device_is_reserved - check if a device is reserved + * + * @device: Node to check for availability, with locks already held + * + * Return: True if the status property is set to "reserved", false otherwise + */ +static bool __of_device_is_reserved(const struct device_node *device) +{ + static const char * const reserved[] = {"reserved", NULL}; + + return __of_device_is_status(device, reserved); +} + /** * of_device_is_available - check if a device is available for use * @@ -650,6 +664,21 @@ struct device_node *of_get_next_available_child(const struct device_node *node, } EXPORT_SYMBOL(of_get_next_available_child); +/** + * of_get_next_reserved_child - Find the next reserved child node + * @node: parent node + * @prev: previous child of the parent node, or NULL to get first + * + * This function is like of_get_next_child(), except that it + * automatically skips any disabled nodes (i.e. status = "disabled"). + */ +struct device_node *of_get_next_reserved_child(const struct device_node *node, + struct device_node *prev) +{ + return of_get_next_status_child(node, prev, __of_device_is_reserved); +} +EXPORT_SYMBOL(of_get_next_reserved_child); + /** * of_get_next_cpu_node - Iterate on cpu nodes * @prev: previous child of the /cpus node, or NULL to get first diff --git a/include/linux/of.h b/include/linux/of.h index 6a9ddf20e79a..331e05918f11 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -294,6 +294,8 @@ extern struct device_node *of_get_next_child(const struct device_node *node, struct device_node *prev); extern struct device_node *of_get_next_available_child( const struct device_node *node, struct device_node *prev); +extern struct device_node *of_get_next_reserved_child( + const struct device_node *node, struct device_node *prev); extern struct device_node *of_get_compatible_child(const struct device_node *parent, const char *compatible); @@ -541,6 +543,12 @@ static inline struct device_node *of_get_next_available_child( return NULL; } +static inline struct device_node *of_get_next_reserved_child( + const struct device_node *node, struct device_node *prev) +{ + return NULL; +} + static inline struct device_node *of_find_node_with_property( struct device_node *from, const char *prop_name) { @@ -1431,6 +1439,9 @@ static inline int of_property_read_s32(const struct device_node *np, #define for_each_available_child_of_node(parent, child) \ for (child = of_get_next_available_child(parent, NULL); child != NULL; \ child = of_get_next_available_child(parent, child)) +#define for_each_reserved_child_of_node(parent, child) \ + for (child = of_get_next_reserved_child(parent, NULL); child != NULL; \ + child = of_get_next_reserved_child(parent, child)) #define for_each_of_cpu_node(cpu) \ for (cpu = of_get_next_cpu_node(NULL); cpu != NULL; \ -- cgit v1.2.3 From 71fc3249f50ac22f495185872e71393cfa9d6f07 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Thu, 18 Jan 2024 10:05:28 +0100 Subject: video/nomodeset: Select nomodeset= parameter with CONFIG_VIDEO Enable support for nomodeset= parameter via CONFIG_VIDEO. Both, DRM and fbdev, already select this option. Remove the existing option CONFIG_VIDEO_NOMODESET. Simplifies the Kconfig rules. Signed-off-by: Thomas Zimmermann Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20240118090721.7995-4-tzimmermann@suse.de --- drivers/gpu/drm/Kconfig | 1 - drivers/staging/sm750fb/Kconfig | 1 - drivers/video/Kconfig | 4 ---- drivers/video/Makefile | 3 +-- drivers/video/fbdev/Kconfig | 37 ------------------------------------- drivers/video/fbdev/core/fbmem.c | 2 -- drivers/video/fbdev/geode/Kconfig | 3 --- include/linux/fb.h | 7 ------- 8 files changed, 1 insertion(+), 57 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index da8750e1c867..23764316efcf 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -20,7 +20,6 @@ menuconfig DRM # device and dmabuf fd. Let's make sure that is available for our userspace. select KCMP select VIDEO - select VIDEO_NOMODESET help Kernel-level support for the Direct Rendering Infrastructure (DRI) introduced in XFree86 4.0. If you say Y here, you need to select diff --git a/drivers/staging/sm750fb/Kconfig b/drivers/staging/sm750fb/Kconfig index ab3d9b057d56..08bcccdd0f1c 100644 --- a/drivers/staging/sm750fb/Kconfig +++ b/drivers/staging/sm750fb/Kconfig @@ -6,7 +6,6 @@ config FB_SM750 select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT - select VIDEO_NOMODESET help Frame buffer driver for the Silicon Motion SM750 chip with 2D acceleration and dual head support. diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 253b129a82dc..130ebccb8338 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -22,10 +22,6 @@ config VIDEO bool default n -config VIDEO_NOMODESET - bool - default n - source "drivers/auxdisplay/Kconfig" if HAS_IOMEM diff --git a/drivers/video/Makefile b/drivers/video/Makefile index 287c198f0c82..9eb5557911de 100644 --- a/drivers/video/Makefile +++ b/drivers/video/Makefile @@ -3,8 +3,7 @@ obj-$(CONFIG_APERTURE_HELPERS) += aperture.o obj-$(CONFIG_STI_CORE) += sticore.o obj-$(CONFIG_VGASTATE) += vgastate.o -obj-$(CONFIG_VIDEO) += cmdline.o -obj-$(CONFIG_VIDEO_NOMODESET) += nomodeset.o +obj-$(CONFIG_VIDEO) += cmdline.o nomodeset.o obj-$(CONFIG_HDMI) += hdmi.o obj-$(CONFIG_VT) += console/ diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig index d5909a9206ff..f15ba4b2f306 100644 --- a/drivers/video/fbdev/Kconfig +++ b/drivers/video/fbdev/Kconfig @@ -75,7 +75,6 @@ config FB_CIRRUS select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS - select VIDEO_NOMODESET help This enables support for Cirrus Logic GD542x/543x based boards on Amiga: SD64, Piccolo, Picasso II/II+, Picasso IV, or EGS Spectrum. @@ -95,7 +94,6 @@ config FB_PM2 select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS - select VIDEO_NOMODESET help This is the frame buffer device driver for cards based on the 3D Labs Permedia, Permedia 2 and Permedia 2V chips. @@ -179,7 +177,6 @@ config FB_CYBER2000 tristate "CyberPro 2000/2010/5000 support" depends on FB && PCI && (BROKEN || !SPARC64) select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help This enables support for the Integraphics CyberPro 20x0 and 5000 VGA chips used in the Rebel.com Netwinder and other machines. @@ -330,7 +327,6 @@ config FB_CT65550 bool "Chips 65550 display support" depends on (FB = y) && PPC32 && PCI select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help This is the frame buffer device driver for the Chips & Technologies 65550 graphics chip in PowerBooks. @@ -339,7 +335,6 @@ config FB_ASILIANT bool "Asiliant (Chips) 69000 display support" depends on (FB = y) && PCI select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help This is the frame buffer device driver for the Asiliant 69030 chipset @@ -349,7 +344,6 @@ config FB_IMSTT select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS select FB_MACMODES if PPC_PMAC - select VIDEO_NOMODESET help The IMS Twin Turbo is a PCI-based frame buffer card bundled with many Macintosh and compatible computers. @@ -414,7 +408,6 @@ config FB_TGA select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS - select VIDEO_NOMODESET help This is the frame buffer device driver for generic TGA and SFB+ graphic cards. These include DEC ZLXp-E1, -E2 and -E3 PCI cards, @@ -591,7 +584,6 @@ config FB_XVR500 select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS - select VIDEO_NOMODESET help This is the framebuffer device for the Sun XVR-500 and similar graphics cards based upon the 3DLABS Wildcat chipset. The driver @@ -603,7 +595,6 @@ config FB_XVR2500 bool "Sun XVR-2500 3DLABS Wildcat support" depends on (FB = y) && PCI && SPARC64 select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help This is the framebuffer device for the Sun XVR-2500 and similar graphics cards based upon the 3DLABS Wildcat chipset. The driver @@ -629,7 +620,6 @@ config FB_PVR2 select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS - select VIDEO_NOMODESET help Say Y here if you have a PowerVR 2 card in your box. If you plan to run linux on your Dreamcast, you will have to say Y here. @@ -692,7 +682,6 @@ config FB_NVIDIA select FB_IOMEM_FOPS select BITREVERSE select VGASTATE - select VIDEO_NOMODESET help This driver supports graphics boards with the nVidia chips, TNT and newer. For very old chipsets, such as the RIVA128, then use @@ -741,7 +730,6 @@ config FB_RIVA select FB_MODE_HELPERS select BITREVERSE select VGASTATE - select VIDEO_NOMODESET help This driver supports graphics boards with the nVidia Riva/Geforce chips. @@ -784,7 +772,6 @@ config FB_I740 select FB_IOMEM_HELPERS select FB_MODE_HELPERS select VGASTATE - select VIDEO_NOMODESET select FB_DDC help This driver supports graphics cards based on Intel740 chip. @@ -795,7 +782,6 @@ config FB_I810 select FB_IOMEM_FOPS select FB_MODE_HELPERS select VGASTATE - select VIDEO_NOMODESET help This driver supports the on-board graphics built in to the Intel 810 and 815 chipsets. Say Y if you have and plan to use such a board. @@ -844,7 +830,6 @@ config FB_LE80578 depends on FB && PCI && X86 select FB_IOMEM_HELPERS select FB_MODE_HELPERS - select VIDEO_NOMODESET help This driver supports the LE80578 (Vermilion Range) chipset @@ -863,7 +848,6 @@ config FB_INTEL select FB_IOMEM_FOPS select FB_MODE_HELPERS select BOOT_VESA_SUPPORT if FB_INTEL = y - select VIDEO_NOMODESET depends on !DRM_I915 help This driver supports the on-board graphics built in to the Intel @@ -902,7 +886,6 @@ config FB_MATROX select FB_IOMEM_FOPS select FB_TILEBLITTING select FB_MACMODES if PPC_PMAC - select VIDEO_NOMODESET help Say Y here if you have a Matrox Millennium, Matrox Millennium II, Matrox Mystique, Matrox Mystique 220, Matrox Productiva G100, Matrox @@ -1025,7 +1008,6 @@ config FB_RADEON select FB_IOMEM_FOPS select FB_MACMODES if PPC select FB_MODE_HELPERS - select VIDEO_NOMODESET help Choose this option if you want to use an ATI Radeon graphics card as a framebuffer device. There are both PCI and AGP versions. You @@ -1063,7 +1045,6 @@ config FB_ATY128 select FB_BACKLIGHT if FB_ATY128_BACKLIGHT select FB_IOMEM_HELPERS select FB_MACMODES if PPC_PMAC - select VIDEO_NOMODESET help This driver supports graphics boards with the ATI Rage128 chips. Say Y if you have such a graphics board and read @@ -1089,7 +1070,6 @@ config FB_ATY select FB_IOMEM_FOPS select FB_MACMODES if PPC select FB_ATY_CT if SPARC64 && PCI - select VIDEO_NOMODESET help This driver supports graphics boards with the ATI Mach64 chips. Say Y if you have such a graphics board. @@ -1141,7 +1121,6 @@ config FB_S3 select FB_TILEBLITTING select FB_SVGALIB select VGASTATE - select VIDEO_NOMODESET select FONT_8x16 if FRAMEBUFFER_CONSOLE help Driver for graphics boards with S3 Trio / S3 Virge chip. @@ -1163,7 +1142,6 @@ config FB_SAVAGE select FB_IOMEM_FOPS select FB_MODE_HELPERS select VGASTATE - select VIDEO_NOMODESET help This driver supports notebooks and computers with S3 Savage PCI/AGP chips. @@ -1203,7 +1181,6 @@ config FB_SIS select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS select FB_SIS_300 if !FB_SIS_315 - select VIDEO_NOMODESET help This is the frame buffer device driver for the SiS 300, 315, 330 and 340 series as well as XGI V3XT, V5, V8, Z7 graphics chipsets. @@ -1234,7 +1211,6 @@ config FB_VIA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS select I2C_ALGOBIT - select VIDEO_NOMODESET help This is the frame buffer device driver for Graphics chips of VIA UniChrome (Pro) Family (CLE266,PM800/CN400,P4M800CE/P4M800Pro/ @@ -1275,7 +1251,6 @@ config FB_NEOMAGIC select FB_IOMEM_FOPS select FB_MODE_HELPERS select VGASTATE - select VIDEO_NOMODESET help This driver supports notebooks with NeoMagic PCI chips. Say Y if you have such a graphics card. @@ -1287,7 +1262,6 @@ config FB_KYRO tristate "IMG Kyro support" depends on FB && PCI select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Say Y here if you have a STG4000 / Kyro / PowerVR 3 based graphics board. @@ -1303,7 +1277,6 @@ config FB_3DFX select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS select FB_MODE_HELPERS - select VIDEO_NOMODESET help This driver supports graphics boards with the 3Dfx Banshee, Voodoo3 or VSA-100 (aka Voodoo4/5) chips. Say Y if you have @@ -1332,7 +1305,6 @@ config FB_VOODOO1 depends on FB && PCI depends on FB_DEVICE select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Say Y here if you have a 3Dfx Voodoo Graphics (Voodoo1/sst1) or Voodoo2 (cvg) based graphics card. @@ -1355,7 +1327,6 @@ config FB_VT8623 select FB_TILEBLITTING select FB_SVGALIB select VGASTATE - select VIDEO_NOMODESET select FONT_8x16 if FRAMEBUFFER_CONSOLE help Driver for CastleRock integrated graphics core in the @@ -1370,7 +1341,6 @@ config FB_TRIDENT select FB_DDC select FB_IOMEM_FOPS select FB_MODE_HELPERS - select VIDEO_NOMODESET help This is the frame buffer device driver for Trident PCI/AGP chipsets. Supported chipset families are TGUI 9440/96XX, 3DImage, Blade3D @@ -1395,7 +1365,6 @@ config FB_ARK select FB_TILEBLITTING select FB_SVGALIB select VGASTATE - select VIDEO_NOMODESET select FONT_8x16 if FRAMEBUFFER_CONSOLE help Driver for PCI graphics boards with ARK 2000PV chip @@ -1408,7 +1377,6 @@ config FB_PM3 select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_IOMEM_FOPS - select VIDEO_NOMODESET help This is the frame buffer device driver for the 3DLabs Permedia3 chipset, used in Formac ProFormance III, 3DLabs Oxygen VX1 & @@ -1419,7 +1387,6 @@ config FB_CARMINE tristate "Fujitsu carmine frame buffer support" depends on FB && PCI select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help This is the frame buffer device driver for the Fujitsu Carmine chip. The driver provides two independent frame buffer devices. @@ -1701,7 +1668,6 @@ config FB_IBM_GXT4500 tristate "Framebuffer support for IBM GXT4000P/4500P/6000P/6500P adaptors" depends on FB select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Say Y here to enable support for the IBM GXT4000P/6000P and GXT4500P/6500P display adaptor based on Raster Engine RC1000, @@ -1819,7 +1785,6 @@ config FB_MB862XX depends on FB depends on PCI || (OF && PPC) select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Frame buffer driver for Fujitsu Carmine/Coral-P(A)/Lime controllers. @@ -1885,7 +1850,6 @@ config FB_HYPERV depends on FB && HYPERV select DMA_CMA if HAVE_DMA_CONTIGUOUS && CMA select FB_IOMEM_HELPERS_DEFERRED - select VIDEO_NOMODESET help This framebuffer driver supports Microsoft Hyper-V Synthetic Video. @@ -1919,7 +1883,6 @@ config FB_SM712 tristate "Silicon Motion SM712 framebuffer support" depends on FB && PCI select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Frame buffer driver for the Silicon Motion SM710, SM712, SM721 and SM722 chips. diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c index fc206755f5f6..48287366e0d4 100644 --- a/drivers/video/fbdev/core/fbmem.c +++ b/drivers/video/fbdev/core/fbmem.c @@ -645,7 +645,6 @@ int fb_new_modelist(struct fb_info *info) return 0; } -#if defined(CONFIG_VIDEO_NOMODESET) bool fb_modesetting_disabled(const char *drvname) { bool fwonly = video_firmware_drivers_only(); @@ -657,6 +656,5 @@ bool fb_modesetting_disabled(const char *drvname) return fwonly; } EXPORT_SYMBOL(fb_modesetting_disabled); -#endif MODULE_LICENSE("GPL"); diff --git a/drivers/video/fbdev/geode/Kconfig b/drivers/video/fbdev/geode/Kconfig index 9a49916e0492..3b20420cc94d 100644 --- a/drivers/video/fbdev/geode/Kconfig +++ b/drivers/video/fbdev/geode/Kconfig @@ -14,7 +14,6 @@ config FB_GEODE_LX tristate "AMD Geode LX framebuffer support" depends on FB && FB_GEODE select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Framebuffer driver for the display controller integrated into the AMD Geode LX processors. @@ -28,7 +27,6 @@ config FB_GEODE_GX tristate "AMD Geode GX framebuffer support" depends on FB && FB_GEODE select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Framebuffer driver for the display controller integrated into the AMD Geode GX processors. @@ -42,7 +40,6 @@ config FB_GEODE_GX1 tristate "AMD Geode GX1 framebuffer support" depends on FB && FB_GEODE select FB_IOMEM_HELPERS - select VIDEO_NOMODESET help Framebuffer driver for the display controller integrated into the AMD Geode GX1 processor. diff --git a/include/linux/fb.h b/include/linux/fb.h index 05dc9624897d..2ce2f5c2fca9 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -840,14 +840,7 @@ extern int fb_find_mode(struct fb_var_screeninfo *var, const struct fb_videomode *default_mode, unsigned int default_bpp); -#if defined(CONFIG_VIDEO_NOMODESET) bool fb_modesetting_disabled(const char *drvname); -#else -static inline bool fb_modesetting_disabled(const char *drvname) -{ - return false; -} -#endif /* * Convenience logging macros -- cgit v1.2.3 From db5914695a84a7b128ec2e4e9272e6e8091753e1 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 22 Jan 2024 11:25:57 +0000 Subject: inet_diag: add module pointer to "struct inet_diag_handler" Following patch is going to use RCU instead of inet_diag_table_mutex acquisition. This patch is a preparation, no change of behavior yet. Signed-off-by: Eric Dumazet Reviewed-by: Guillaume Nault Reviewed-by: Kuniyuki Iwashima Reviewed-by: Willem de Bruijn Signed-off-by: Paolo Abeni --- include/linux/inet_diag.h | 1 + net/dccp/diag.c | 1 + net/ipv4/raw_diag.c | 1 + net/ipv4/tcp_diag.c | 1 + net/ipv4/udp_diag.c | 2 ++ net/mptcp/mptcp_diag.c | 1 + net/sctp/diag.c | 1 + 7 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/inet_diag.h b/include/linux/inet_diag.h index 84abb30a3fbb..a9033696b0aa 100644 --- a/include/linux/inet_diag.h +++ b/include/linux/inet_diag.h @@ -8,6 +8,7 @@ struct inet_hashinfo; struct inet_diag_handler { + struct module *owner; void (*dump)(struct sk_buff *skb, struct netlink_callback *cb, const struct inet_diag_req_v2 *r); diff --git a/net/dccp/diag.c b/net/dccp/diag.c index 8a82c5a2c5a8..f5019d95c3ae 100644 --- a/net/dccp/diag.c +++ b/net/dccp/diag.c @@ -58,6 +58,7 @@ static int dccp_diag_dump_one(struct netlink_callback *cb, } static const struct inet_diag_handler dccp_diag_handler = { + .owner = THIS_MODULE, .dump = dccp_diag_dump, .dump_one = dccp_diag_dump_one, .idiag_get_info = dccp_diag_get_info, diff --git a/net/ipv4/raw_diag.c b/net/ipv4/raw_diag.c index fe2140c8375c..cc793bd8de25 100644 --- a/net/ipv4/raw_diag.c +++ b/net/ipv4/raw_diag.c @@ -213,6 +213,7 @@ static int raw_diag_destroy(struct sk_buff *in_skb, #endif static const struct inet_diag_handler raw_diag_handler = { + .owner = THIS_MODULE, .dump = raw_diag_dump, .dump_one = raw_diag_dump_one, .idiag_get_info = raw_diag_get_info, diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c index 4cbe4b44425a..f428ecf9120f 100644 --- a/net/ipv4/tcp_diag.c +++ b/net/ipv4/tcp_diag.c @@ -222,6 +222,7 @@ static int tcp_diag_destroy(struct sk_buff *in_skb, #endif static const struct inet_diag_handler tcp_diag_handler = { + .owner = THIS_MODULE, .dump = tcp_diag_dump, .dump_one = tcp_diag_dump_one, .idiag_get_info = tcp_diag_get_info, diff --git a/net/ipv4/udp_diag.c b/net/ipv4/udp_diag.c index dc41a22ee80e..38cb3a28e4ed 100644 --- a/net/ipv4/udp_diag.c +++ b/net/ipv4/udp_diag.c @@ -237,6 +237,7 @@ static int udplite_diag_destroy(struct sk_buff *in_skb, #endif static const struct inet_diag_handler udp_diag_handler = { + .owner = THIS_MODULE, .dump = udp_diag_dump, .dump_one = udp_diag_dump_one, .idiag_get_info = udp_diag_get_info, @@ -260,6 +261,7 @@ static int udplite_diag_dump_one(struct netlink_callback *cb, } static const struct inet_diag_handler udplite_diag_handler = { + .owner = THIS_MODULE, .dump = udplite_diag_dump, .dump_one = udplite_diag_dump_one, .idiag_get_info = udp_diag_get_info, diff --git a/net/mptcp/mptcp_diag.c b/net/mptcp/mptcp_diag.c index 5409c2ea3f57..bd8ff5950c8d 100644 --- a/net/mptcp/mptcp_diag.c +++ b/net/mptcp/mptcp_diag.c @@ -225,6 +225,7 @@ static void mptcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r, } static const struct inet_diag_handler mptcp_diag_handler = { + .owner = THIS_MODULE, .dump = mptcp_diag_dump, .dump_one = mptcp_diag_dump_one, .idiag_get_info = mptcp_diag_get_info, diff --git a/net/sctp/diag.c b/net/sctp/diag.c index eb05131ff1dd..23359e522273 100644 --- a/net/sctp/diag.c +++ b/net/sctp/diag.c @@ -507,6 +507,7 @@ done: } static const struct inet_diag_handler sctp_diag_handler = { + .owner = THIS_MODULE, .dump = sctp_diag_dump, .dump_one = sctp_diag_dump_one, .idiag_get_info = sctp_diag_get_info, -- cgit v1.2.3 From 114b4bb1cc19239b272d52ebbe156053483fe2f8 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 22 Jan 2024 11:25:59 +0000 Subject: sock_diag: add module pointer to "struct sock_diag_handler" Following patch is going to use RCU instead of sock_diag_table_mutex acquisition. This patch is a preparation, no change of behavior yet. Signed-off-by: Eric Dumazet Reviewed-by: Guillaume Nault Reviewed-by: Kuniyuki Iwashima Reviewed-by: Willem de Bruijn Signed-off-by: Paolo Abeni --- include/linux/sock_diag.h | 1 + net/ipv4/inet_diag.c | 2 ++ net/netlink/diag.c | 1 + net/packet/diag.c | 1 + net/smc/smc_diag.c | 1 + net/tipc/diag.c | 1 + net/unix/diag.c | 1 + net/vmw_vsock/diag.c | 1 + net/xdp/xsk_diag.c | 1 + 9 files changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sock_diag.h b/include/linux/sock_diag.h index 0b9ecd8cf979..7c07754d711b 100644 --- a/include/linux/sock_diag.h +++ b/include/linux/sock_diag.h @@ -13,6 +13,7 @@ struct nlmsghdr; struct sock; struct sock_diag_handler { + struct module *owner; __u8 family; int (*dump)(struct sk_buff *skb, struct nlmsghdr *nlh); int (*get_info)(struct sk_buff *skb, struct sock *sk); diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c index abf7dc982796..52ce20691e4e 100644 --- a/net/ipv4/inet_diag.c +++ b/net/ipv4/inet_diag.c @@ -1488,6 +1488,7 @@ int inet_diag_handler_get_info(struct sk_buff *skb, struct sock *sk) } static const struct sock_diag_handler inet_diag_handler = { + .owner = THIS_MODULE, .family = AF_INET, .dump = inet_diag_handler_cmd, .get_info = inet_diag_handler_get_info, @@ -1495,6 +1496,7 @@ static const struct sock_diag_handler inet_diag_handler = { }; static const struct sock_diag_handler inet6_diag_handler = { + .owner = THIS_MODULE, .family = AF_INET6, .dump = inet_diag_handler_cmd, .get_info = inet_diag_handler_get_info, diff --git a/net/netlink/diag.c b/net/netlink/diag.c index 1eeff9422856..e12c90d5f6ad 100644 --- a/net/netlink/diag.c +++ b/net/netlink/diag.c @@ -241,6 +241,7 @@ static int netlink_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h) } static const struct sock_diag_handler netlink_diag_handler = { + .owner = THIS_MODULE, .family = AF_NETLINK, .dump = netlink_diag_handler_dump, }; diff --git a/net/packet/diag.c b/net/packet/diag.c index 9a7980e3309d..b3bd2f6c2bf7 100644 --- a/net/packet/diag.c +++ b/net/packet/diag.c @@ -245,6 +245,7 @@ static int packet_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h) } static const struct sock_diag_handler packet_diag_handler = { + .owner = THIS_MODULE, .family = AF_PACKET, .dump = packet_diag_handler_dump, }; diff --git a/net/smc/smc_diag.c b/net/smc/smc_diag.c index 52f7c4f1e767..32bad267fa3e 100644 --- a/net/smc/smc_diag.c +++ b/net/smc/smc_diag.c @@ -255,6 +255,7 @@ static int smc_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h) } static const struct sock_diag_handler smc_diag_handler = { + .owner = THIS_MODULE, .family = AF_SMC, .dump = smc_diag_handler_dump, }; diff --git a/net/tipc/diag.c b/net/tipc/diag.c index 18733451c9e0..54dde8c4e4d4 100644 --- a/net/tipc/diag.c +++ b/net/tipc/diag.c @@ -95,6 +95,7 @@ static int tipc_sock_diag_handler_dump(struct sk_buff *skb, } static const struct sock_diag_handler tipc_sock_diag_handler = { + .owner = THIS_MODULE, .family = AF_TIPC, .dump = tipc_sock_diag_handler_dump, }; diff --git a/net/unix/diag.c b/net/unix/diag.c index bec09a3a1d44..c3648b706509 100644 --- a/net/unix/diag.c +++ b/net/unix/diag.c @@ -322,6 +322,7 @@ static int unix_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h) } static const struct sock_diag_handler unix_diag_handler = { + .owner = THIS_MODULE, .family = AF_UNIX, .dump = unix_diag_handler_dump, }; diff --git a/net/vmw_vsock/diag.c b/net/vmw_vsock/diag.c index 2e29994f92ff..ab87ef66c1e8 100644 --- a/net/vmw_vsock/diag.c +++ b/net/vmw_vsock/diag.c @@ -157,6 +157,7 @@ static int vsock_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h) } static const struct sock_diag_handler vsock_diag_handler = { + .owner = THIS_MODULE, .family = AF_VSOCK, .dump = vsock_diag_handler_dump, }; diff --git a/net/xdp/xsk_diag.c b/net/xdp/xsk_diag.c index 9f8955367275..09dcea0cbbed 100644 --- a/net/xdp/xsk_diag.c +++ b/net/xdp/xsk_diag.c @@ -194,6 +194,7 @@ static int xsk_diag_handler_dump(struct sk_buff *nlskb, struct nlmsghdr *hdr) } static const struct sock_diag_handler xsk_diag_handler = { + .owner = THIS_MODULE, .family = AF_XDP, .dump = xsk_diag_handler_dump, }; -- cgit v1.2.3 From 86e8921df05c6e9423ab74ab8d41022775d8b83a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 22 Jan 2024 11:26:01 +0000 Subject: sock_diag: allow concurrent operation in sock_diag_rcv_msg() TCPDIAG_GETSOCK and DCCPDIAG_GETSOCK diag are serialized on sock_diag_table_mutex. This is to make sure inet_diag module is not unloaded while diag was ongoing. It is time to get rid of this mutex and use RCU protection, allowing full parallelism. Signed-off-by: Eric Dumazet Reviewed-by: Guillaume Nault Reviewed-by: Kuniyuki Iwashima Reviewed-by: Willem de Bruijn Signed-off-by: Paolo Abeni --- include/linux/sock_diag.h | 9 +++++++-- net/core/sock_diag.c | 43 ++++++++++++++++++++++++++----------------- net/ipv4/inet_diag.c | 9 +++++++-- 3 files changed, 40 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sock_diag.h b/include/linux/sock_diag.h index 7c07754d711b..110978dc9af1 100644 --- a/include/linux/sock_diag.h +++ b/include/linux/sock_diag.h @@ -23,8 +23,13 @@ struct sock_diag_handler { int sock_diag_register(const struct sock_diag_handler *h); void sock_diag_unregister(const struct sock_diag_handler *h); -void sock_diag_register_inet_compat(int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh)); -void sock_diag_unregister_inet_compat(int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh)); +struct sock_diag_inet_compat { + struct module *owner; + int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh); +}; + +void sock_diag_register_inet_compat(const struct sock_diag_inet_compat *ptr); +void sock_diag_unregister_inet_compat(const struct sock_diag_inet_compat *ptr); u64 __sock_gen_cookie(struct sock *sk); diff --git a/net/core/sock_diag.c b/net/core/sock_diag.c index 72009e1f4380..5c3666431df4 100644 --- a/net/core/sock_diag.c +++ b/net/core/sock_diag.c @@ -17,8 +17,9 @@ #include static const struct sock_diag_handler __rcu *sock_diag_handlers[AF_MAX]; -static int (*inet_rcv_compat)(struct sk_buff *skb, struct nlmsghdr *nlh); -static DEFINE_MUTEX(sock_diag_table_mutex); + +static struct sock_diag_inet_compat __rcu *inet_rcv_compat; + static struct workqueue_struct *broadcast_wq; DEFINE_COOKIE(sock_cookie); @@ -184,19 +185,20 @@ void sock_diag_broadcast_destroy(struct sock *sk) queue_work(broadcast_wq, &bsk->work); } -void sock_diag_register_inet_compat(int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh)) +void sock_diag_register_inet_compat(const struct sock_diag_inet_compat *ptr) { - mutex_lock(&sock_diag_table_mutex); - inet_rcv_compat = fn; - mutex_unlock(&sock_diag_table_mutex); + xchg((__force const struct sock_diag_inet_compat **)&inet_rcv_compat, + ptr); } EXPORT_SYMBOL_GPL(sock_diag_register_inet_compat); -void sock_diag_unregister_inet_compat(int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh)) +void sock_diag_unregister_inet_compat(const struct sock_diag_inet_compat *ptr) { - mutex_lock(&sock_diag_table_mutex); - inet_rcv_compat = NULL; - mutex_unlock(&sock_diag_table_mutex); + const struct sock_diag_inet_compat *old; + + old = xchg((__force const struct sock_diag_inet_compat **)&inet_rcv_compat, + NULL); + WARN_ON_ONCE(old != ptr); } EXPORT_SYMBOL_GPL(sock_diag_unregister_inet_compat); @@ -259,20 +261,27 @@ static int __sock_diag_cmd(struct sk_buff *skb, struct nlmsghdr *nlh) static int sock_diag_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { + const struct sock_diag_inet_compat *ptr; int ret; switch (nlh->nlmsg_type) { case TCPDIAG_GETSOCK: case DCCPDIAG_GETSOCK: - if (inet_rcv_compat == NULL) + + if (!rcu_access_pointer(inet_rcv_compat)) sock_load_diag_module(AF_INET, 0); - mutex_lock(&sock_diag_table_mutex); - if (inet_rcv_compat != NULL) - ret = inet_rcv_compat(skb, nlh); - else - ret = -EOPNOTSUPP; - mutex_unlock(&sock_diag_table_mutex); + rcu_read_lock(); + ptr = rcu_dereference(inet_rcv_compat); + if (ptr && !try_module_get(ptr->owner)) + ptr = NULL; + rcu_read_unlock(); + + ret = -EOPNOTSUPP; + if (ptr) { + ret = ptr->fn(skb, nlh); + module_put(ptr->owner); + } return ret; case SOCK_DIAG_BY_FAMILY: diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c index 52ce20691e4e..2c2d8b9dd8e9 100644 --- a/net/ipv4/inet_diag.c +++ b/net/ipv4/inet_diag.c @@ -1527,6 +1527,11 @@ void inet_diag_unregister(const struct inet_diag_handler *h) } EXPORT_SYMBOL_GPL(inet_diag_unregister); +static const struct sock_diag_inet_compat inet_diag_compat = { + .owner = THIS_MODULE, + .fn = inet_diag_rcv_msg_compat, +}; + static int __init inet_diag_init(void) { const int inet_diag_table_size = (IPPROTO_MAX * @@ -1545,7 +1550,7 @@ static int __init inet_diag_init(void) if (err) goto out_free_inet; - sock_diag_register_inet_compat(inet_diag_rcv_msg_compat); + sock_diag_register_inet_compat(&inet_diag_compat); out: return err; @@ -1560,7 +1565,7 @@ static void __exit inet_diag_exit(void) { sock_diag_unregister(&inet6_diag_handler); sock_diag_unregister(&inet_diag_handler); - sock_diag_unregister_inet_compat(inet_diag_rcv_msg_compat); + sock_diag_unregister_inet_compat(&inet_diag_compat); kfree(inet_diag_table); } -- cgit v1.2.3 From 7c05e7f3e74e7e550534d524e04d7e6f78d6fa24 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Fri, 5 Jan 2024 18:48:17 +0800 Subject: bpf: Support inlining bpf_kptr_xchg() helper The motivation of inlining bpf_kptr_xchg() comes from the performance profiling of bpf memory allocator benchmark. The benchmark uses bpf_kptr_xchg() to stash the allocated objects and to pop the stashed objects for free. After inling bpf_kptr_xchg(), the performance for object free on 8-CPUs VM increases about 2%~10%. The inline also has downside: both the kasan and kcsan checks on the pointer will be unavailable. bpf_kptr_xchg() can be inlined by converting the calling of bpf_kptr_xchg() into an atomic_xchg() instruction. But the conversion depends on two conditions: 1) JIT backend supports atomic_xchg() on pointer-sized word 2) For the specific arch, the implementation of xchg is the same as atomic_xchg() on pointer-sized words. It seems most 64-bit JIT backends satisfies these two conditions. But as a precaution, defining a weak function bpf_jit_supports_ptr_xchg() to state whether such conversion is safe and only supporting inline for 64-bit host. For x86-64, it supports BPF_XCHG atomic operation and both xchg() and atomic_xchg() use arch_xchg() to implement the exchange, so enabling the inline of bpf_kptr_xchg() on x86-64 first. Reviewed-by: Eduard Zingerman Signed-off-by: Hou Tao Link: https://lore.kernel.org/r/20240105104819.3916743-2-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 5 +++++ include/linux/filter.h | 1 + kernel/bpf/core.c | 10 ++++++++++ kernel/bpf/helpers.c | 1 + kernel/bpf/verifier.c | 17 +++++++++++++++++ 5 files changed, 34 insertions(+) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 919f647c740f..e1390d1e331b 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3242,3 +3242,8 @@ void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke, BUG_ON(ret < 0); } } + +bool bpf_jit_supports_ptr_xchg(void) +{ + return true; +} diff --git a/include/linux/filter.h b/include/linux/filter.h index 68fb6c8142fe..35f067fd3840 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -955,6 +955,7 @@ bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_kfunc_call(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); +bool bpf_jit_supports_ptr_xchg(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); bool bpf_helper_changes_pkt_data(void *func); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index ea6843be2616..fbb1d95a9b44 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2925,6 +2925,16 @@ bool __weak bpf_jit_supports_far_kfunc_call(void) return false; } +/* Return TRUE if the JIT backend satisfies the following two conditions: + * 1) JIT backend supports atomic_xchg() on pointer-sized words. + * 2) Under the specific arch, the implementation of xchg() is the same + * as atomic_xchg() on pointer-sized words. + */ +bool __weak bpf_jit_supports_ptr_xchg(void) +{ + return false; +} + /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call * skb_copy_bits(), so provide a weak definition of it for NET-less config. */ diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index be72824f32b2..e04ca1af8927 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1414,6 +1414,7 @@ BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr) { unsigned long *kptr = map_value; + /* This helper may be inlined by verifier. */ return xchg(kptr, (unsigned long)ptr); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 65f598694d55..5b33d65eef7b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19809,6 +19809,23 @@ patch_map_ops_generic: continue; } + /* Implement bpf_kptr_xchg inline */ + if (prog->jit_requested && BITS_PER_LONG == 64 && + insn->imm == BPF_FUNC_kptr_xchg && + bpf_jit_supports_ptr_xchg()) { + insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); + insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); + cnt = 2; + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; + } patch_call_imm: fn = env->ops->get_func_proto(insn->imm, env->prog); /* all functions that have prototype and verifier allowed -- cgit v1.2.3 From 522bb2c1f82b12eb7befaae815d1d959b8e6bba2 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 4 Jan 2024 16:09:05 -0800 Subject: bpf: support multiple tags per argument Add ability to iterate multiple decl_tag types pointed to the same function argument. Use this to support multiple __arg_xxx tags per global subprog argument. We leave btf_find_decl_tag_value() intact, but change its implementation to use a new btf_find_next_decl_tag() which can be straightforwardly used to find next BTF type ID of a matching btf_decl_tag type. btf_prepare_func_args() is switched from btf_find_decl_tag_value() to btf_find_next_decl_tag() to gain multiple tags per argument support. Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20240105000909.2818934-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 ++ kernel/bpf/btf.c | 64 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e30100597d0a..377857b232c6 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2472,6 +2472,8 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr struct btf *btf, const struct btf_type *t); const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, int comp_idx, const char *tag_key); +int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt, + int comp_idx, const char *tag_key, int last_id); struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 1636b574a63d..4f532b303a27 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3310,30 +3310,48 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, return BTF_FIELD_FOUND; } -const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, - int comp_idx, const char *tag_key) +int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt, + int comp_idx, const char *tag_key, int last_id) { - const char *value = NULL; - int i; + int len = strlen(tag_key); + int i, n; - for (i = 1; i < btf_nr_types(btf); i++) { + for (i = last_id + 1, n = btf_nr_types(btf); i < n; i++) { const struct btf_type *t = btf_type_by_id(btf, i); - int len = strlen(tag_key); if (!btf_type_is_decl_tag(t)) continue; - if (pt != btf_type_by_id(btf, t->type) || - btf_type_decl_tag(t)->component_idx != comp_idx) + if (pt != btf_type_by_id(btf, t->type)) + continue; + if (btf_type_decl_tag(t)->component_idx != comp_idx) continue; if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) continue; - /* Prevent duplicate entries for same type */ - if (value) - return ERR_PTR(-EEXIST); - value = __btf_name_by_offset(btf, t->name_off) + len; + return i; } - if (!value) - return ERR_PTR(-ENOENT); + return -ENOENT; +} + +const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, + int comp_idx, const char *tag_key) +{ + const char *value = NULL; + const struct btf_type *t; + int len, id; + + id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, 0); + if (id < 0) + return ERR_PTR(id); + + t = btf_type_by_id(btf, id); + len = strlen(tag_key); + value = __btf_name_by_offset(btf, t->name_off) + len; + + /* Prevent duplicate entries for same type */ + id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, id); + if (id >= 0) + return ERR_PTR(-EEXIST); + return value; } @@ -7032,20 +7050,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) * Only PTR_TO_CTX and SCALAR are supported atm. */ for (i = 0; i < nargs; i++) { - const char *tag; u32 tags = 0; + int id = 0; - tag = btf_find_decl_tag_value(btf, fn_t, i, "arg:"); - if (IS_ERR(tag) && PTR_ERR(tag) == -ENOENT) { - tag = NULL; - } else if (IS_ERR(tag)) { - bpf_log(log, "arg#%d type's tag fetching failure: %ld\n", i, PTR_ERR(tag)); - return PTR_ERR(tag); - } /* 'arg:' decl_tag takes precedence over derivation of * register type from BTF type itself */ - if (tag) { + while ((id = btf_find_next_decl_tag(btf, fn_t, i, "arg:", id)) > 0) { + const struct btf_type *tag_t = btf_type_by_id(btf, id); + const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; + /* disallow arg tags in static subprogs */ if (!is_global) { bpf_log(log, "arg#%d type tag is not supported in static functions\n", i); @@ -7061,6 +7075,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EOPNOTSUPP; } } + if (id != -ENOENT) { + bpf_log(log, "arg#%d type tag fetching failure: %d\n", i, id); + return id; + } t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) -- cgit v1.2.3 From 32f55dd4add4df1a5bc8febc1fafd3086290dbf6 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Mon, 8 Jan 2024 22:51:58 +0200 Subject: bpf: Make bpf_for_each_spilled_reg consider narrow spills Adjust the check in bpf_get_spilled_reg to take into account spilled registers narrower than 64 bits. That allows find_equal_scalars to properly adjust the range of all spilled registers that have the same ID. Before this change, it was possible for a register and a spilled register to have the same IDs but different ranges if the spill was narrower than 64 bits and a range check was performed on the register. Signed-off-by: Maxim Mikityanskiy Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20240108205209.838365-5-maxtram95@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d07d857ca67f..e11baecbde68 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -453,7 +453,7 @@ struct bpf_verifier_state { #define bpf_get_spilled_reg(slot, frame, mask) \ (((slot < frame->allocated_stack / BPF_REG_SIZE) && \ - ((1 << frame->stack[slot].slot_type[0]) & (mask))) \ + ((1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & (mask))) \ ? &frame->stack[slot].spilled_ptr : NULL) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ -- cgit v1.2.3 From 3b1f89e747cd4b24244f2798a35d28815b744303 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:49:52 -0800 Subject: bpf: refactory struct_ops type initialization to a function. Move the majority of the code to bpf_struct_ops_init_one(), which can then be utilized for the initialization of newly registered dynamically allocated struct_ops types in the following patches. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-2-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/btf.h | 1 + kernel/bpf/bpf_struct_ops.c | 157 +++++++++++++++++++++++--------------------- kernel/bpf/btf.c | 5 ++ 3 files changed, 89 insertions(+), 74 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index cf5c6ff48981..932af1680bb5 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -137,6 +137,7 @@ struct btf_struct_metas { extern const struct file_operations btf_fops; +const char *btf_get_name(const struct btf *btf); void btf_get(struct btf *btf); void btf_put(struct btf *btf); int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz); diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 02068bd0e4d9..96cba76f4ac3 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -110,102 +110,111 @@ const struct bpf_prog_ops bpf_struct_ops_prog_ops = { static const struct btf_type *module_type; -void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) +static void bpf_struct_ops_init_one(struct bpf_struct_ops *st_ops, + struct btf *btf, + struct bpf_verifier_log *log) { - s32 type_id, value_id, module_id; const struct btf_member *member; - struct bpf_struct_ops *st_ops; const struct btf_type *t; + s32 type_id, value_id; char value_name[128]; const char *mname; - u32 i, j; + int i; - /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */ -#define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name); -#include "bpf_struct_ops_types.h" -#undef BPF_STRUCT_OPS_TYPE + if (strlen(st_ops->name) + VALUE_PREFIX_LEN >= + sizeof(value_name)) { + pr_warn("struct_ops name %s is too long\n", + st_ops->name); + return; + } + sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name); - module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT); - if (module_id < 0) { - pr_warn("Cannot find struct module in btf_vmlinux\n"); + value_id = btf_find_by_name_kind(btf, value_name, + BTF_KIND_STRUCT); + if (value_id < 0) { + pr_warn("Cannot find struct %s in %s\n", + value_name, btf_get_name(btf)); return; } - module_type = btf_type_by_id(btf, module_id); - for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - st_ops = bpf_struct_ops[i]; + type_id = btf_find_by_name_kind(btf, st_ops->name, + BTF_KIND_STRUCT); + if (type_id < 0) { + pr_warn("Cannot find struct %s in %s\n", + st_ops->name, btf_get_name(btf)); + return; + } + t = btf_type_by_id(btf, type_id); + if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) { + pr_warn("Cannot support #%u members in struct %s\n", + btf_type_vlen(t), st_ops->name); + return; + } - if (strlen(st_ops->name) + VALUE_PREFIX_LEN >= - sizeof(value_name)) { - pr_warn("struct_ops name %s is too long\n", + for_each_member(i, t, member) { + const struct btf_type *func_proto; + + mname = btf_name_by_offset(btf, member->name_off); + if (!*mname) { + pr_warn("anon member in struct %s is not supported\n", st_ops->name); - continue; + break; } - sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name); - value_id = btf_find_by_name_kind(btf, value_name, - BTF_KIND_STRUCT); - if (value_id < 0) { - pr_warn("Cannot find struct %s in btf_vmlinux\n", - value_name); - continue; + if (__btf_member_bitfield_size(t, member)) { + pr_warn("bit field member %s in struct %s is not supported\n", + mname, st_ops->name); + break; } - type_id = btf_find_by_name_kind(btf, st_ops->name, - BTF_KIND_STRUCT); - if (type_id < 0) { - pr_warn("Cannot find struct %s in btf_vmlinux\n", - st_ops->name); - continue; - } - t = btf_type_by_id(btf, type_id); - if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) { - pr_warn("Cannot support #%u members in struct %s\n", - btf_type_vlen(t), st_ops->name); - continue; + func_proto = btf_type_resolve_func_ptr(btf, + member->type, + NULL); + if (func_proto && + btf_distill_func_proto(log, btf, + func_proto, mname, + &st_ops->func_models[i])) { + pr_warn("Error in parsing func ptr %s in struct %s\n", + mname, st_ops->name); + break; } + } - for_each_member(j, t, member) { - const struct btf_type *func_proto; + if (i == btf_type_vlen(t)) { + if (st_ops->init(btf)) { + pr_warn("Error in init bpf_struct_ops %s\n", + st_ops->name); + } else { + st_ops->type_id = type_id; + st_ops->type = t; + st_ops->value_id = value_id; + st_ops->value_type = btf_type_by_id(btf, + value_id); + } + } +} - mname = btf_name_by_offset(btf, member->name_off); - if (!*mname) { - pr_warn("anon member in struct %s is not supported\n", - st_ops->name); - break; - } +void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) +{ + struct bpf_struct_ops *st_ops; + s32 module_id; + u32 i; - if (__btf_member_bitfield_size(t, member)) { - pr_warn("bit field member %s in struct %s is not supported\n", - mname, st_ops->name); - break; - } + /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */ +#define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name); +#include "bpf_struct_ops_types.h" +#undef BPF_STRUCT_OPS_TYPE - func_proto = btf_type_resolve_func_ptr(btf, - member->type, - NULL); - if (func_proto && - btf_distill_func_proto(log, btf, - func_proto, mname, - &st_ops->func_models[j])) { - pr_warn("Error in parsing func ptr %s in struct %s\n", - mname, st_ops->name); - break; - } - } + module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT); + if (module_id < 0) { + pr_warn("Cannot find struct module in %s\n", btf_get_name(btf)); + return; + } + module_type = btf_type_by_id(btf, module_id); - if (j == btf_type_vlen(t)) { - if (st_ops->init(btf)) { - pr_warn("Error in init bpf_struct_ops %s\n", - st_ops->name); - } else { - st_ops->type_id = type_id; - st_ops->type = t; - st_ops->value_id = value_id; - st_ops->value_type = btf_type_by_id(btf, - value_id); - } - } + for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { + st_ops = bpf_struct_ops[i]; + bpf_struct_ops_init_one(st_ops, btf, log); } } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4f532b303a27..f8735a2ec5d4 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1707,6 +1707,11 @@ static void btf_free_rcu(struct rcu_head *rcu) btf_free(btf); } +const char *btf_get_name(const struct btf *btf) +{ + return btf->name; +} + void btf_get(struct btf *btf) { refcount_inc(&btf->refcnt); -- cgit v1.2.3 From 4c5763ed996a61b51d721d0968d0df957826ea49 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:49:54 -0800 Subject: bpf, net: introduce bpf_struct_ops_desc. Move some of members of bpf_struct_ops to bpf_struct_ops_desc. type_id is unavailabe in bpf_struct_ops anymore. Modules should get it from the btf received by kmod's init function. Cc: netdev@vger.kernel.org Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-4-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 15 +++++--- kernel/bpf/bpf_struct_ops.c | 80 ++++++++++++++++++++++-------------------- kernel/bpf/verifier.c | 8 +++-- net/bpf/bpf_dummy_struct_ops.c | 11 +++++- net/ipv4/bpf_tcp_ca.c | 8 ++++- 5 files changed, 73 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 377857b232c6..7fc95e7babab 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1673,18 +1673,23 @@ struct bpf_struct_ops { void (*unreg)(void *kdata); int (*update)(void *kdata, void *old_kdata); int (*validate)(void *kdata); - const struct btf_type *type; - const struct btf_type *value_type; + void *cfi_stubs; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; +}; + +struct bpf_struct_ops_desc { + struct bpf_struct_ops *st_ops; + + const struct btf_type *type; + const struct btf_type *value_type; u32 type_id; u32 value_id; - void *cfi_stubs; }; #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) -const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); +const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id); void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); bool bpf_struct_ops_get(const void *kdata); void bpf_struct_ops_put(const void *kdata); @@ -1728,7 +1733,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif #else -static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) +static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id) { return NULL; } diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 5b3ebcb435d0..9774f7824e8b 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -32,7 +32,7 @@ struct bpf_struct_ops_value { struct bpf_struct_ops_map { struct bpf_map map; struct rcu_head rcu; - const struct bpf_struct_ops *st_ops; + const struct bpf_struct_ops_desc *st_ops_desc; /* protect map_update */ struct mutex lock; /* link has all the bpf_links that is populated @@ -92,9 +92,9 @@ enum { __NR_BPF_STRUCT_OPS_TYPE, }; -static struct bpf_struct_ops * const bpf_struct_ops[] = { +static struct bpf_struct_ops_desc bpf_struct_ops[] = { #define BPF_STRUCT_OPS_TYPE(_name) \ - [BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name, + [BPF_STRUCT_OPS_TYPE_##_name] = { .st_ops = &bpf_##_name }, #include "bpf_struct_ops_types.h" #undef BPF_STRUCT_OPS_TYPE }; @@ -115,10 +115,11 @@ enum { IDX_MODULE_ID, }; -static void bpf_struct_ops_init_one(struct bpf_struct_ops *st_ops, - struct btf *btf, - struct bpf_verifier_log *log) +static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, + struct btf *btf, + struct bpf_verifier_log *log) { + struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; const struct btf_member *member; const struct btf_type *t; s32 type_id, value_id; @@ -190,18 +191,18 @@ static void bpf_struct_ops_init_one(struct bpf_struct_ops *st_ops, pr_warn("Error in init bpf_struct_ops %s\n", st_ops->name); } else { - st_ops->type_id = type_id; - st_ops->type = t; - st_ops->value_id = value_id; - st_ops->value_type = btf_type_by_id(btf, - value_id); + st_ops_desc->type_id = type_id; + st_ops_desc->type = t; + st_ops_desc->value_id = value_id; + st_ops_desc->value_type = btf_type_by_id(btf, + value_id); } } } void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) { - struct bpf_struct_ops *st_ops; + struct bpf_struct_ops_desc *st_ops_desc; u32 i; /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */ @@ -210,14 +211,14 @@ void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) #undef BPF_STRUCT_OPS_TYPE for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - st_ops = bpf_struct_ops[i]; - bpf_struct_ops_init_one(st_ops, btf, log); + st_ops_desc = &bpf_struct_ops[i]; + bpf_struct_ops_desc_init(st_ops_desc, btf, log); } } extern struct btf *btf_vmlinux; -static const struct bpf_struct_ops * +static const struct bpf_struct_ops_desc * bpf_struct_ops_find_value(u32 value_id) { unsigned int i; @@ -226,14 +227,14 @@ bpf_struct_ops_find_value(u32 value_id) return NULL; for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - if (bpf_struct_ops[i]->value_id == value_id) - return bpf_struct_ops[i]; + if (bpf_struct_ops[i].value_id == value_id) + return &bpf_struct_ops[i]; } return NULL; } -const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) +const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id) { unsigned int i; @@ -241,8 +242,8 @@ const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) return NULL; for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - if (bpf_struct_ops[i]->type_id == type_id) - return bpf_struct_ops[i]; + if (bpf_struct_ops[i].type_id == type_id) + return &bpf_struct_ops[i]; } return NULL; @@ -302,7 +303,7 @@ static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key) static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) { - const struct btf_type *t = st_map->st_ops->type; + const struct btf_type *t = st_map->st_ops_desc->type; u32 i; for (i = 0; i < btf_type_vlen(t); i++) { @@ -382,11 +383,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, void *value, u64 flags) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; - const struct bpf_struct_ops *st_ops = st_map->st_ops; + const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc; + const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; struct bpf_struct_ops_value *uvalue, *kvalue; const struct btf_type *module_type; const struct btf_member *member; - const struct btf_type *t = st_ops->type; + const struct btf_type *t = st_ops_desc->type; struct bpf_tramp_links *tlinks; void *udata, *kdata; int prog_fd, err; @@ -399,7 +401,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (*(u32 *)key != 0) return -E2BIG; - err = check_zero_holes(st_ops->value_type, value); + err = check_zero_holes(st_ops_desc->value_type, value); if (err) return err; @@ -492,7 +494,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, } if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || - prog->aux->attach_btf_id != st_ops->type_id || + prog->aux->attach_btf_id != st_ops_desc->type_id || prog->expected_attach_type != i) { bpf_prog_put(prog); err = -EINVAL; @@ -588,7 +590,7 @@ static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key) BPF_STRUCT_OPS_STATE_TOBEFREE); switch (prev_state) { case BPF_STRUCT_OPS_STATE_INUSE: - st_map->st_ops->unreg(&st_map->kvalue.data); + st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data); bpf_map_put(map); return 0; case BPF_STRUCT_OPS_STATE_TOBEFREE: @@ -669,22 +671,22 @@ static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) { - const struct bpf_struct_ops *st_ops; + const struct bpf_struct_ops_desc *st_ops_desc; size_t st_map_size; struct bpf_struct_ops_map *st_map; const struct btf_type *t, *vt; struct bpf_map *map; int ret; - st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id); - if (!st_ops) + st_ops_desc = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id); + if (!st_ops_desc) return ERR_PTR(-ENOTSUPP); - vt = st_ops->value_type; + vt = st_ops_desc->value_type; if (attr->value_size != vt->size) return ERR_PTR(-EINVAL); - t = st_ops->type; + t = st_ops_desc->type; st_map_size = sizeof(*st_map) + /* kvalue stores the @@ -696,7 +698,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) if (!st_map) return ERR_PTR(-ENOMEM); - st_map->st_ops = st_ops; + st_map->st_ops_desc = st_ops_desc; map = &st_map->map; ret = bpf_jit_charge_modmem(PAGE_SIZE); @@ -733,8 +735,8 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; - const struct bpf_struct_ops *st_ops = st_map->st_ops; - const struct btf_type *vt = st_ops->value_type; + const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc; + const struct btf_type *vt = st_ops_desc->value_type; u64 usage; usage = sizeof(*st_map) + @@ -808,7 +810,7 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) /* st_link->map can be NULL if * bpf_struct_ops_link_create() fails to register. */ - st_map->st_ops->unreg(&st_map->kvalue.data); + st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data); bpf_map_put(&st_map->map); } kfree(st_link); @@ -855,7 +857,7 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map if (!bpf_struct_ops_valid_to_reg(new_map)) return -EINVAL; - if (!st_map->st_ops->update) + if (!st_map->st_ops_desc->st_ops->update) return -EOPNOTSUPP; mutex_lock(&update_mutex); @@ -868,12 +870,12 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); /* The new and old struct_ops must be the same type. */ - if (st_map->st_ops != old_st_map->st_ops) { + if (st_map->st_ops_desc != old_st_map->st_ops_desc) { err = -EINVAL; goto err_out; } - err = st_map->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data); + err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data); if (err) goto err_out; @@ -924,7 +926,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) if (err) goto err_out; - err = st_map->st_ops->reg(st_map->kvalue.data); + err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data); if (err) { bpf_link_cleanup(&link_primer); link = NULL; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9e9cc132dd02..e279491118b7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20285,6 +20285,7 @@ static void print_verification_stats(struct bpf_verifier_env *env) static int check_struct_ops_btf_id(struct bpf_verifier_env *env) { const struct btf_type *t, *func_proto; + const struct bpf_struct_ops_desc *st_ops_desc; const struct bpf_struct_ops *st_ops; const struct btf_member *member; struct bpf_prog *prog = env->prog; @@ -20297,14 +20298,15 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) } btf_id = prog->aux->attach_btf_id; - st_ops = bpf_struct_ops_find(btf_id); - if (!st_ops) { + st_ops_desc = bpf_struct_ops_find(btf_id); + if (!st_ops_desc) { verbose(env, "attach_btf_id %u is not a supported struct\n", btf_id); return -ENOTSUPP; } + st_ops = st_ops_desc->st_ops; - t = st_ops->type; + t = st_ops_desc->type; member_idx = prog->expected_attach_type; if (member_idx >= btf_type_vlen(t)) { verbose(env, "attach to invalid member idx %u of struct %s\n", diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 8906f7bdf4a9..ba2c58dba2da 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -22,6 +22,8 @@ struct bpf_dummy_ops_test_args { struct bpf_dummy_ops_state state; }; +static struct btf *bpf_dummy_ops_btf; + static struct bpf_dummy_ops_test_args * dummy_ops_init_args(const union bpf_attr *kattr, unsigned int nr) { @@ -90,9 +92,15 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, void *image = NULL; unsigned int op_idx; int prog_ret; + s32 type_id; int err; - if (prog->aux->attach_btf_id != st_ops->type_id) + type_id = btf_find_by_name_kind(bpf_dummy_ops_btf, + bpf_bpf_dummy_ops.name, + BTF_KIND_STRUCT); + if (type_id < 0) + return -EINVAL; + if (prog->aux->attach_btf_id != type_id) return -EOPNOTSUPP; func_proto = prog->aux->attach_func_proto; @@ -148,6 +156,7 @@ out: static int bpf_dummy_init(struct btf *btf) { + bpf_dummy_ops_btf = btf; return 0; } diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index ae8b15e6896f..dffd8828079b 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -20,6 +20,7 @@ static u32 unsupported_ops[] = { static const struct btf_type *tcp_sock_type; static u32 tcp_sock_id, sock_id; +static const struct btf_type *tcp_congestion_ops_type; static int bpf_tcp_ca_init(struct btf *btf) { @@ -36,6 +37,11 @@ static int bpf_tcp_ca_init(struct btf *btf) tcp_sock_id = type_id; tcp_sock_type = btf_type_by_id(btf, tcp_sock_id); + type_id = btf_find_by_name_kind(btf, "tcp_congestion_ops", BTF_KIND_STRUCT); + if (type_id < 0) + return -EINVAL; + tcp_congestion_ops_type = btf_type_by_id(btf, type_id); + return 0; } @@ -149,7 +155,7 @@ static u32 prog_ops_moff(const struct bpf_prog *prog) u32 midx; midx = prog->expected_attach_type; - t = bpf_tcp_congestion_ops.type; + t = tcp_congestion_ops_type; m = &btf_type_member(t)[midx]; return __btf_member_bit_offset(t, m) / 8; -- cgit v1.2.3 From 1338b93346587a2a6ac79bbcf55ef5b357745573 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:49:57 -0800 Subject: bpf: pass btf object id in bpf_map_info. Include btf object id (btf_obj_id) in bpf_map_info so that tools (ex: bpftools struct_ops dump) know the correct btf from the kernel to look up type information of struct_ops types. Since struct_ops types can be defined and registered in a module. The type information of a struct_ops type are defined in the btf of the module defining it. The userspace tools need to know which btf is for the module defining a struct_ops type. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-7-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 4 ++++ include/uapi/linux/bpf.h | 2 +- kernel/bpf/bpf_struct_ops.c | 7 +++++++ kernel/bpf/syscall.c | 2 ++ tools/include/uapi/linux/bpf.h | 2 +- 5 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7fc95e7babab..29fcae9fa8ed 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1732,6 +1732,7 @@ struct bpf_dummy_ops { int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif +void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map); #else static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id) { @@ -1759,6 +1760,9 @@ static inline int bpf_struct_ops_link_create(union bpf_attr *attr) { return -EOPNOTSUPP; } +static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map) +{ +} #endif diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 287d05732668..a380047c86af 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6487,7 +6487,7 @@ struct bpf_map_info { __u32 btf_id; __u32 btf_key_type_id; __u32 btf_value_type_id; - __u32 :32; /* alignment pad */ + __u32 btf_vmlinux_id; __u64 map_extra; } __attribute__((aligned(8))); diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 5ddcca4c4fba..5e98af4fc2e2 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -947,3 +947,10 @@ err_out: kfree(link); return err; } + +void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + + info->btf_vmlinux_id = btf_obj_id(st_map->btf); +} diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 13193aaafb64..55b458429705 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4691,6 +4691,8 @@ static int bpf_map_get_info_by_fd(struct file *file, info.btf_value_type_id = map->btf_value_type_id; } info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; + if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) + bpf_map_struct_ops_info_fill(&info, map); if (bpf_map_is_offloaded(map)) { err = bpf_map_offload_info_fill(&info, map); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 287d05732668..a380047c86af 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -6487,7 +6487,7 @@ struct bpf_map_info { __u32 btf_id; __u32 btf_key_type_id; __u32 btf_value_type_id; - __u32 :32; /* alignment pad */ + __u32 btf_vmlinux_id; __u64 map_extra; } __attribute__((aligned(8))); -- cgit v1.2.3 From 689423db3bda2244c24db8a64de4cdb37be1de41 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:49:58 -0800 Subject: bpf: lookup struct_ops types from a given module BTF. This is a preparation for searching for struct_ops types from a specified module. BTF is always btf_vmlinux now. This patch passes a pointer of BTF to bpf_struct_ops_find_value() and bpf_struct_ops_find(). Once the new registration API of struct_ops types is used, other BTFs besides btf_vmlinux can also be passed to them. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-8-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 4 ++-- kernel/bpf/bpf_struct_ops.c | 11 ++++++----- kernel/bpf/verifier.c | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 29fcae9fa8ed..86ff8911d7ee 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1689,7 +1689,7 @@ struct bpf_struct_ops_desc { #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) -const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id); +const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id); void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); bool bpf_struct_ops_get(const void *kdata); void bpf_struct_ops_put(const void *kdata); @@ -1734,7 +1734,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, #endif void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map); #else -static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id) +static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id) { return NULL; } diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 5e98af4fc2e2..7505f515aac3 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -221,11 +221,11 @@ void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) extern struct btf *btf_vmlinux; static const struct bpf_struct_ops_desc * -bpf_struct_ops_find_value(u32 value_id) +bpf_struct_ops_find_value(struct btf *btf, u32 value_id) { unsigned int i; - if (!value_id || !btf_vmlinux) + if (!value_id || !btf) return NULL; for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { @@ -236,11 +236,12 @@ bpf_struct_ops_find_value(u32 value_id) return NULL; } -const struct bpf_struct_ops_desc *bpf_struct_ops_find(u32 type_id) +const struct bpf_struct_ops_desc * +bpf_struct_ops_find(struct btf *btf, u32 type_id) { unsigned int i; - if (!type_id || !btf_vmlinux) + if (!type_id || !btf) return NULL; for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { @@ -682,7 +683,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) struct bpf_map *map; int ret; - st_ops_desc = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id); + st_ops_desc = bpf_struct_ops_find_value(btf_vmlinux, attr->btf_vmlinux_value_type_id); if (!st_ops_desc) return ERR_PTR(-ENOTSUPP); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e279491118b7..2a0fd2ccdb11 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20298,7 +20298,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) } btf_id = prog->aux->attach_btf_id; - st_ops_desc = bpf_struct_ops_find(btf_id); + st_ops_desc = bpf_struct_ops_find(btf_vmlinux, btf_id); if (!st_ops_desc) { verbose(env, "attach_btf_id %u is not a supported struct\n", btf_id); -- cgit v1.2.3 From e3f87fdfed7b770dd7066b02262b12747881e76d Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:50:00 -0800 Subject: bpf: hold module refcnt in bpf_struct_ops map creation and prog verification. To ensure that a module remains accessible whenever a struct_ops object of a struct_ops type provided by the module is still in use. struct bpf_struct_ops_map doesn't hold a refcnt to btf anymore since a module will hold a refcnt to it's btf already. But, struct_ops programs are different. They hold their associated btf, not the module since they need only btf to assure their types (signatures). However, verifier holds the refcnt of the associated module of a struct_ops type temporarily when verify a struct_ops prog. Verifier needs the help from the verifier operators (struct bpf_verifier_ops) provided by the owner module to verify data access of a prog, provide information, and generate code. This patch also add a count of links (links_cnt) to bpf_struct_ops_map. It avoids bpf_struct_ops_map_put_progs() from accessing btf after calling module_put() in bpf_struct_ops_map_free(). Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-10-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 1 + include/linux/bpf_verifier.h | 1 + kernel/bpf/bpf_struct_ops.c | 29 +++++++++++++++++++++++------ kernel/bpf/verifier.c | 11 +++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 86ff8911d7ee..a5b425893d38 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1674,6 +1674,7 @@ struct bpf_struct_ops { int (*update)(void *kdata, void *old_kdata); int (*validate)(void *kdata); void *cfi_stubs; + struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; }; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e11baecbde68..7f5816482a10 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -662,6 +662,7 @@ struct bpf_verifier_env { u32 prev_insn_idx; struct bpf_prog *prog; /* eBPF program being verified */ const struct bpf_verifier_ops *ops; + struct module *attach_btf_mod; /* The owner module of prog->aux->attach_btf */ struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */ int stack_size; /* number of states to be processed */ bool strict_alignment; /* perform strict pointer alignment checks */ diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 3b8d689ece5d..02216a8d9265 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -40,6 +40,7 @@ struct bpf_struct_ops_map { * (in kvalue.data). */ struct bpf_link **links; + u32 links_cnt; /* image is a page that has all the trampolines * that stores the func args before calling the bpf_prog. * A PAGE_SIZE "image" is enough to store all trampoline for @@ -306,10 +307,9 @@ static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key) static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) { - const struct btf_type *t = st_map->st_ops_desc->type; u32 i; - for (i = 0; i < btf_type_vlen(t); i++) { + for (i = 0; i < st_map->links_cnt; i++) { if (st_map->links[i]) { bpf_link_put(st_map->links[i]); st_map->links[i] = NULL; @@ -641,12 +641,20 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_jit_uncharge_modmem(PAGE_SIZE); } bpf_map_area_free(st_map->uvalue); - btf_put(st_map->btf); bpf_map_area_free(st_map); } static void bpf_struct_ops_map_free(struct bpf_map *map) { + struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + + /* st_ops->owner was acquired during map_alloc to implicitly holds + * the btf's refcnt. The acquire was only done when btf_is_module() + * st_map->btf cannot be NULL here. + */ + if (btf_is_module(st_map->btf)) + module_put(st_map->st_ops_desc->st_ops->owner); + /* The struct_ops's function may switch to another struct_ops. * * For example, bpf_tcp_cc_x->init() may switch to @@ -682,6 +690,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) size_t st_map_size; struct bpf_struct_ops_map *st_map; const struct btf_type *t, *vt; + struct module *mod = NULL; struct bpf_map *map; struct btf *btf; int ret; @@ -695,11 +704,18 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) btf_put(btf); return ERR_PTR(-EINVAL); } + + mod = btf_try_get_module(btf); + /* mod holds a refcnt to btf. We don't need an extra refcnt + * here. + */ + btf_put(btf); + if (!mod) + return ERR_PTR(-EINVAL); } else { btf = bpf_get_btf_vmlinux(); if (IS_ERR(btf)) return ERR_CAST(btf); - btf_get(btf); } st_ops_desc = bpf_struct_ops_find_value(btf, attr->btf_vmlinux_value_type_id); @@ -746,8 +762,9 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) goto errout_free; } st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); + st_map->links_cnt = btf_type_vlen(t); st_map->links = - bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *), + bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *), NUMA_NO_NODE); if (!st_map->uvalue || !st_map->links) { ret = -ENOMEM; @@ -763,7 +780,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) errout_free: __bpf_struct_ops_map_free(map); errout: - btf_put(btf); + module_put(mod); return ERR_PTR(ret); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6081512deb79..f31868ba0c2d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20299,6 +20299,15 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) } btf = prog->aux->attach_btf ?: bpf_get_btf_vmlinux(); + if (btf_is_module(btf)) { + /* Make sure st_ops is valid through the lifetime of env */ + env->attach_btf_mod = btf_try_get_module(btf); + if (!env->attach_btf_mod) { + verbose(env, "struct_ops module %s is not found\n", + btf_get_name(btf)); + return -ENOTSUPP; + } + } btf_id = prog->aux->attach_btf_id; st_ops_desc = bpf_struct_ops_find(btf, btf_id); @@ -21024,6 +21033,8 @@ err_release_maps: env->prog->expected_attach_type = 0; *prog = env->prog; + + module_put(env->attach_btf_mod); err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); -- cgit v1.2.3 From 612d087d4ba54cef47946e22e5dabad762dd7ed5 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:50:01 -0800 Subject: bpf: validate value_type A value_type should consist of three components: refcnt, state, and data. refcnt and state has been move to struct bpf_struct_ops_common_value to make it easier to check the value type. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-11-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 12 ++++++ kernel/bpf/bpf_struct_ops.c | 93 +++++++++++++++++++++++++++++---------------- 2 files changed, 72 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a5b425893d38..7c178170f93f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1688,6 +1688,18 @@ struct bpf_struct_ops_desc { u32 value_id; }; +enum bpf_struct_ops_state { + BPF_STRUCT_OPS_STATE_INIT, + BPF_STRUCT_OPS_STATE_INUSE, + BPF_STRUCT_OPS_STATE_TOBEFREE, + BPF_STRUCT_OPS_STATE_READY, +}; + +struct bpf_struct_ops_common_value { + refcount_t refcnt; + enum bpf_struct_ops_state state; +}; + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id); diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 02216a8d9265..30ab34fab0f8 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -13,19 +13,8 @@ #include #include -enum bpf_struct_ops_state { - BPF_STRUCT_OPS_STATE_INIT, - BPF_STRUCT_OPS_STATE_INUSE, - BPF_STRUCT_OPS_STATE_TOBEFREE, - BPF_STRUCT_OPS_STATE_READY, -}; - -#define BPF_STRUCT_OPS_COMMON_VALUE \ - refcount_t refcnt; \ - enum bpf_struct_ops_state state - struct bpf_struct_ops_value { - BPF_STRUCT_OPS_COMMON_VALUE; + struct bpf_struct_ops_common_value common; char data[] ____cacheline_aligned_in_smp; }; @@ -81,8 +70,8 @@ static DEFINE_MUTEX(update_mutex); #define BPF_STRUCT_OPS_TYPE(_name) \ extern struct bpf_struct_ops bpf_##_name; \ \ -struct bpf_struct_ops_##_name { \ - BPF_STRUCT_OPS_COMMON_VALUE; \ +struct bpf_struct_ops_##_name { \ + struct bpf_struct_ops_common_value common; \ struct _name data ____cacheline_aligned_in_smp; \ }; #include "bpf_struct_ops_types.h" @@ -113,11 +102,49 @@ const struct bpf_prog_ops bpf_struct_ops_prog_ops = { BTF_ID_LIST(st_ops_ids) BTF_ID(struct, module) +BTF_ID(struct, bpf_struct_ops_common_value) enum { IDX_MODULE_ID, + IDX_ST_OPS_COMMON_VALUE_ID, }; +extern struct btf *btf_vmlinux; + +static bool is_valid_value_type(struct btf *btf, s32 value_id, + const struct btf_type *type, + const char *value_name) +{ + const struct btf_type *common_value_type; + const struct btf_member *member; + const struct btf_type *vt, *mt; + + vt = btf_type_by_id(btf, value_id); + if (btf_vlen(vt) != 2) { + pr_warn("The number of %s's members should be 2, but we get %d\n", + value_name, btf_vlen(vt)); + return false; + } + member = btf_type_member(vt); + mt = btf_type_by_id(btf, member->type); + common_value_type = btf_type_by_id(btf_vmlinux, + st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]); + if (mt != common_value_type) { + pr_warn("The first member of %s should be bpf_struct_ops_common_value\n", + value_name); + return false; + } + member++; + mt = btf_type_by_id(btf, member->type); + if (mt != type) { + pr_warn("The second member of %s should be %s\n", + value_name, btf_name_by_offset(btf, type->name_off)); + return false; + } + + return true; +} + static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, struct btf *btf, struct bpf_verifier_log *log) @@ -138,14 +165,6 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, } sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name); - value_id = btf_find_by_name_kind(btf, value_name, - BTF_KIND_STRUCT); - if (value_id < 0) { - pr_warn("Cannot find struct %s in %s\n", - value_name, btf_get_name(btf)); - return; - } - type_id = btf_find_by_name_kind(btf, st_ops->name, BTF_KIND_STRUCT); if (type_id < 0) { @@ -160,6 +179,16 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, return; } + value_id = btf_find_by_name_kind(btf, value_name, + BTF_KIND_STRUCT); + if (value_id < 0) { + pr_warn("Cannot find struct %s in %s\n", + value_name, btf_get_name(btf)); + return; + } + if (!is_valid_value_type(btf, value_id, t, value_name)) + return; + for_each_member(i, t, member) { const struct btf_type *func_proto; @@ -219,8 +248,6 @@ void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) } } -extern struct btf *btf_vmlinux; - static const struct bpf_struct_ops_desc * bpf_struct_ops_find_value(struct btf *btf, u32 value_id) { @@ -276,7 +303,7 @@ int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, kvalue = &st_map->kvalue; /* Pair with smp_store_release() during map_update */ - state = smp_load_acquire(&kvalue->state); + state = smp_load_acquire(&kvalue->common.state); if (state == BPF_STRUCT_OPS_STATE_INIT) { memset(value, 0, map->value_size); return 0; @@ -287,7 +314,7 @@ int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, */ uvalue = value; memcpy(uvalue, st_map->uvalue, map->value_size); - uvalue->state = state; + uvalue->common.state = state; /* This value offers the user space a general estimate of how * many sockets are still utilizing this struct_ops for TCP @@ -295,7 +322,7 @@ int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, * should sufficiently meet our present goals. */ refcnt = atomic64_read(&map->refcnt) - atomic64_read(&map->usercnt); - refcount_set(&uvalue->refcnt, max_t(s64, refcnt, 0)); + refcount_set(&uvalue->common.refcnt, max_t(s64, refcnt, 0)); return 0; } @@ -413,7 +440,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (err) return err; - if (uvalue->state || refcount_read(&uvalue->refcnt)) + if (uvalue->common.state || refcount_read(&uvalue->common.refcnt)) return -EINVAL; tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL); @@ -425,7 +452,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, mutex_lock(&st_map->lock); - if (kvalue->state != BPF_STRUCT_OPS_STATE_INIT) { + if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) { err = -EBUSY; goto unlock; } @@ -540,7 +567,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, * * Pair with smp_load_acquire() during lookup_elem(). */ - smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_READY); + smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY); goto unlock; } @@ -558,7 +585,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, * It ensures the above udata updates (e.g. prog->aux->id) * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set. */ - smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_INUSE); + smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE); goto unlock; } @@ -588,7 +615,7 @@ static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key) if (st_map->map.map_flags & BPF_F_LINK) return -EOPNOTSUPP; - prev_state = cmpxchg(&st_map->kvalue.state, + prev_state = cmpxchg(&st_map->kvalue.common.state, BPF_STRUCT_OPS_STATE_INUSE, BPF_STRUCT_OPS_STATE_TOBEFREE); switch (prev_state) { @@ -848,7 +875,7 @@ static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) return map->map_type == BPF_MAP_TYPE_STRUCT_OPS && map->map_flags & BPF_F_LINK && /* Pair with smp_store_release() during map_update */ - smp_load_acquire(&st_map->kvalue.state) == BPF_STRUCT_OPS_STATE_READY; + smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY; } static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) -- cgit v1.2.3 From f6be98d19985411ca1f3d53413d94d5b7f41c200 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Fri, 19 Jan 2024 14:50:02 -0800 Subject: bpf, net: switch to dynamic registration Replace the static list of struct_ops types with per-btf struct_ops_tab to enable dynamic registration. Both bpf_dummy_ops and bpf_tcp_ca now utilize the registration function instead of being listed in bpf_struct_ops_types.h. Cc: netdev@vger.kernel.org Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240119225005.668602-12-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 27 ++++++---- include/linux/btf.h | 12 +++++ kernel/bpf/bpf_struct_ops.c | 100 +++++--------------------------------- kernel/bpf/bpf_struct_ops_types.h | 12 ----- kernel/bpf/btf.c | 86 ++++++++++++++++++++++++++++++-- net/bpf/bpf_dummy_struct_ops.c | 11 ++++- net/ipv4/bpf_tcp_ca.c | 12 +++-- 7 files changed, 142 insertions(+), 118 deletions(-) delete mode 100644 kernel/bpf/bpf_struct_ops_types.h (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7c178170f93f..75b7f9b19c6a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1701,9 +1701,20 @@ struct bpf_struct_ops_common_value { }; #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) +/* This macro helps developer to register a struct_ops type and generate + * type information correctly. Developers should use this macro to register + * a struct_ops type instead of calling __register_bpf_struct_ops() directly. + */ +#define register_bpf_struct_ops(st_ops, type) \ + ({ \ + struct bpf_struct_ops_##type { \ + struct bpf_struct_ops_common_value common; \ + struct type data ____cacheline_aligned_in_smp; \ + }; \ + BTF_TYPE_EMIT(struct bpf_struct_ops_##type); \ + __register_bpf_struct_ops(st_ops); \ + }) #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) -const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id); -void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); bool bpf_struct_ops_get(const void *kdata); void bpf_struct_ops_put(const void *kdata); int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, @@ -1745,16 +1756,12 @@ struct bpf_dummy_ops { int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif +int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, + struct btf *btf, + struct bpf_verifier_log *log); void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map); #else -static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id) -{ - return NULL; -} -static inline void bpf_struct_ops_init(struct btf *btf, - struct bpf_verifier_log *log) -{ -} +#define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; }) static inline bool bpf_try_module_get(const void *data, struct module *owner) { return try_module_get(owner); diff --git a/include/linux/btf.h b/include/linux/btf.h index 932af1680bb5..1ee8977b8c95 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -497,6 +497,18 @@ static inline void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id) struct bpf_verifier_log; +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) +struct bpf_struct_ops; +int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops); +const struct bpf_struct_ops_desc *bpf_struct_ops_find_value(struct btf *btf, u32 value_id); +const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id); +#else +static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf *btf, u32 type_id) +{ + return NULL; +} +#endif + #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); const char *btf_name_by_offset(const struct btf *btf, u32 offset); diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 30ab34fab0f8..defc052e4622 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -62,35 +62,6 @@ static DEFINE_MUTEX(update_mutex); #define VALUE_PREFIX "bpf_struct_ops_" #define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1) -/* bpf_struct_ops_##_name (e.g. bpf_struct_ops_tcp_congestion_ops) is - * the map's value exposed to the userspace and its btf-type-id is - * stored at the map->btf_vmlinux_value_type_id. - * - */ -#define BPF_STRUCT_OPS_TYPE(_name) \ -extern struct bpf_struct_ops bpf_##_name; \ - \ -struct bpf_struct_ops_##_name { \ - struct bpf_struct_ops_common_value common; \ - struct _name data ____cacheline_aligned_in_smp; \ -}; -#include "bpf_struct_ops_types.h" -#undef BPF_STRUCT_OPS_TYPE - -enum { -#define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name, -#include "bpf_struct_ops_types.h" -#undef BPF_STRUCT_OPS_TYPE - __NR_BPF_STRUCT_OPS_TYPE, -}; - -static struct bpf_struct_ops_desc bpf_struct_ops[] = { -#define BPF_STRUCT_OPS_TYPE(_name) \ - [BPF_STRUCT_OPS_TYPE_##_name] = { .st_ops = &bpf_##_name }, -#include "bpf_struct_ops_types.h" -#undef BPF_STRUCT_OPS_TYPE -}; - const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = { }; @@ -145,9 +116,9 @@ static bool is_valid_value_type(struct btf *btf, s32 value_id, return true; } -static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, - struct btf *btf, - struct bpf_verifier_log *log) +int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, + struct btf *btf, + struct bpf_verifier_log *log) { struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; const struct btf_member *member; @@ -161,7 +132,7 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, sizeof(value_name)) { pr_warn("struct_ops name %s is too long\n", st_ops->name); - return; + return -EINVAL; } sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name); @@ -170,13 +141,13 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, if (type_id < 0) { pr_warn("Cannot find struct %s in %s\n", st_ops->name, btf_get_name(btf)); - return; + return -EINVAL; } t = btf_type_by_id(btf, type_id); if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) { pr_warn("Cannot support #%u members in struct %s\n", btf_type_vlen(t), st_ops->name); - return; + return -EINVAL; } value_id = btf_find_by_name_kind(btf, value_name, @@ -184,10 +155,10 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, if (value_id < 0) { pr_warn("Cannot find struct %s in %s\n", value_name, btf_get_name(btf)); - return; + return -EINVAL; } if (!is_valid_value_type(btf, value_id, t, value_name)) - return; + return -EINVAL; for_each_member(i, t, member) { const struct btf_type *func_proto; @@ -196,13 +167,13 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, if (!*mname) { pr_warn("anon member in struct %s is not supported\n", st_ops->name); - break; + return -EOPNOTSUPP; } if (__btf_member_bitfield_size(t, member)) { pr_warn("bit field member %s in struct %s is not supported\n", mname, st_ops->name); - break; + return -EOPNOTSUPP; } func_proto = btf_type_resolve_func_ptr(btf, @@ -214,7 +185,7 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, &st_ops->func_models[i])) { pr_warn("Error in parsing func ptr %s in struct %s\n", mname, st_ops->name); - break; + return -EINVAL; } } @@ -222,6 +193,7 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, if (st_ops->init(btf)) { pr_warn("Error in init bpf_struct_ops %s\n", st_ops->name); + return -EINVAL; } else { st_ops_desc->type_id = type_id; st_ops_desc->type = t; @@ -230,54 +202,8 @@ static void bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, value_id); } } -} -void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) -{ - struct bpf_struct_ops_desc *st_ops_desc; - u32 i; - - /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */ -#define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name); -#include "bpf_struct_ops_types.h" -#undef BPF_STRUCT_OPS_TYPE - - for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - st_ops_desc = &bpf_struct_ops[i]; - bpf_struct_ops_desc_init(st_ops_desc, btf, log); - } -} - -static const struct bpf_struct_ops_desc * -bpf_struct_ops_find_value(struct btf *btf, u32 value_id) -{ - unsigned int i; - - if (!value_id || !btf) - return NULL; - - for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - if (bpf_struct_ops[i].value_id == value_id) - return &bpf_struct_ops[i]; - } - - return NULL; -} - -const struct bpf_struct_ops_desc * -bpf_struct_ops_find(struct btf *btf, u32 type_id) -{ - unsigned int i; - - if (!type_id || !btf) - return NULL; - - for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) { - if (bpf_struct_ops[i].type_id == type_id) - return &bpf_struct_ops[i]; - } - - return NULL; + return 0; } static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key, diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h deleted file mode 100644 index 5678a9ddf817..000000000000 --- a/kernel/bpf/bpf_struct_ops_types.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* internal file - do not include directly */ - -#ifdef CONFIG_BPF_JIT -#ifdef CONFIG_NET -BPF_STRUCT_OPS_TYPE(bpf_dummy_ops) -#endif -#ifdef CONFIG_INET -#include -BPF_STRUCT_OPS_TYPE(tcp_congestion_ops) -#endif -#endif diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 2e5e6ec5d965..5c3e526a2dec 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -5972,8 +5973,6 @@ struct btf *btf_parse_vmlinux(void) /* btf_parse_vmlinux() runs under bpf_verifier_lock */ bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); - bpf_struct_ops_init(btf, log); - refcount_set(&btf->refcnt, 1); err = btf_alloc_id(btf); @@ -8706,11 +8705,13 @@ bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log, return !strncmp(reg_name, arg_name, cmp_len); } +#ifdef CONFIG_BPF_JIT static int -btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops) +btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, + struct bpf_verifier_log *log) { struct btf_struct_ops_tab *tab, *new_tab; - int i; + int i, err; tab = btf->struct_ops_tab; if (!tab) { @@ -8740,7 +8741,84 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops) tab->ops[btf->struct_ops_tab->cnt].st_ops = st_ops; + err = bpf_struct_ops_desc_init(&tab->ops[btf->struct_ops_tab->cnt], btf, log); + if (err) + return err; + btf->struct_ops_tab->cnt++; return 0; } + +const struct bpf_struct_ops_desc * +bpf_struct_ops_find_value(struct btf *btf, u32 value_id) +{ + const struct bpf_struct_ops_desc *st_ops_list; + unsigned int i; + u32 cnt; + + if (!value_id) + return NULL; + if (!btf->struct_ops_tab) + return NULL; + + cnt = btf->struct_ops_tab->cnt; + st_ops_list = btf->struct_ops_tab->ops; + for (i = 0; i < cnt; i++) { + if (st_ops_list[i].value_id == value_id) + return &st_ops_list[i]; + } + + return NULL; +} + +const struct bpf_struct_ops_desc * +bpf_struct_ops_find(struct btf *btf, u32 type_id) +{ + const struct bpf_struct_ops_desc *st_ops_list; + unsigned int i; + u32 cnt; + + if (!type_id) + return NULL; + if (!btf->struct_ops_tab) + return NULL; + + cnt = btf->struct_ops_tab->cnt; + st_ops_list = btf->struct_ops_tab->ops; + for (i = 0; i < cnt; i++) { + if (st_ops_list[i].type_id == type_id) + return &st_ops_list[i]; + } + + return NULL; +} + +int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops) +{ + struct bpf_verifier_log *log; + struct btf *btf; + int err = 0; + + btf = btf_get_module_btf(st_ops->owner); + if (!btf) + return -EINVAL; + + log = kzalloc(sizeof(*log), GFP_KERNEL | __GFP_NOWARN); + if (!log) { + err = -ENOMEM; + goto errout; + } + + log->level = BPF_LOG_KERNEL; + + err = btf_add_struct_ops(btf, st_ops, log); + +errout: + kfree(log); + btf_put(btf); + + return err; +} +EXPORT_SYMBOL_GPL(__register_bpf_struct_ops); +#endif diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index ba2c58dba2da..02de71719aed 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -7,7 +7,7 @@ #include #include -extern struct bpf_struct_ops bpf_bpf_dummy_ops; +static struct bpf_struct_ops bpf_bpf_dummy_ops; /* A common type for test_N with return value in bpf_dummy_ops */ typedef int (*dummy_ops_test_ret_fn)(struct bpf_dummy_ops_state *state, ...); @@ -256,7 +256,7 @@ static struct bpf_dummy_ops __bpf_bpf_dummy_ops = { .test_sleepable = bpf_dummy_test_sleepable, }; -struct bpf_struct_ops bpf_bpf_dummy_ops = { +static struct bpf_struct_ops bpf_bpf_dummy_ops = { .verifier_ops = &bpf_dummy_verifier_ops, .init = bpf_dummy_init, .check_member = bpf_dummy_ops_check_member, @@ -265,4 +265,11 @@ struct bpf_struct_ops bpf_bpf_dummy_ops = { .unreg = bpf_dummy_unreg, .name = "bpf_dummy_ops", .cfi_stubs = &__bpf_bpf_dummy_ops, + .owner = THIS_MODULE, }; + +static int __init bpf_dummy_struct_ops_init(void) +{ + return register_bpf_struct_ops(&bpf_bpf_dummy_ops, bpf_dummy_ops); +} +late_initcall(bpf_dummy_struct_ops_init); diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index dffd8828079b..8e7716256d3c 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -12,7 +12,7 @@ #include /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ -extern struct bpf_struct_ops bpf_tcp_congestion_ops; +static struct bpf_struct_ops bpf_tcp_congestion_ops; static u32 unsupported_ops[] = { offsetof(struct tcp_congestion_ops, get_info), @@ -345,7 +345,7 @@ static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = { .release = __bpf_tcp_ca_release, }; -struct bpf_struct_ops bpf_tcp_congestion_ops = { +static struct bpf_struct_ops bpf_tcp_congestion_ops = { .verifier_ops = &bpf_tcp_ca_verifier_ops, .reg = bpf_tcp_ca_reg, .unreg = bpf_tcp_ca_unreg, @@ -356,10 +356,16 @@ struct bpf_struct_ops bpf_tcp_congestion_ops = { .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", .cfi_stubs = &__bpf_ops_tcp_congestion_ops, + .owner = THIS_MODULE, }; static int __init bpf_tcp_ca_kfunc_init(void) { - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_tcp_ca_kfunc_set); + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_tcp_ca_kfunc_set); + ret = ret ?: register_bpf_struct_ops(&bpf_tcp_congestion_ops, tcp_congestion_ops); + + return ret; } late_initcall(bpf_tcp_ca_kfunc_init); -- cgit v1.2.3 From 71a5849aedaa9ea028fc51ee74576cad61954743 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Fri, 29 Sep 2023 21:11:57 +0000 Subject: mm: Change mmap_rnd_bits_max to __ro_after_init Allow mmap_rnd_bits_max to be updated on architectures that determine virtual address space size at runtime instead of relying on Kconfig options by changing it from const to __ro_after_init. Signed-off-by: Sami Tolvanen Reviewed-by: Kees Cook Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20230929211155.3910949-5-samitolvanen@google.com Signed-off-by: Palmer Dabbelt --- include/linux/mm.h | 2 +- mm/mmap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index f5a97dec5169..2488c0c5a288 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -86,7 +86,7 @@ extern int sysctl_legacy_va_layout; #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS extern const int mmap_rnd_bits_min; -extern const int mmap_rnd_bits_max; +extern int mmap_rnd_bits_max __ro_after_init; extern int mmap_rnd_bits __read_mostly; #endif #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS diff --git a/mm/mmap.c b/mm/mmap.c index b78e83d351d2..8f47011de22e 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -64,7 +64,7 @@ #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; -const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX; +int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX; int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; #endif #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS -- cgit v1.2.3 From b7dbaace39713025f1fd33407c89651a0c09f667 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 24 Jan 2024 16:29:33 +0100 Subject: fsnotify: Add fsnotify_sb_has_watchers() helper Instead of opencoded checks for number of fsnotify connectors add a helper fsnotify_sb_has_watchers(). Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- include/linux/fsnotify.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index 8300a5286988..1a9de119a0f7 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -17,6 +17,12 @@ #include #include +/* Are there any inode/mount/sb objects that are being watched at all? */ +static inline bool fsnotify_sb_has_watchers(struct super_block *sb) +{ + return atomic_long_read(&sb->s_fsnotify_connectors); +} + /* * Notify this @dir inode about a change in a child directory entry. * The directory entry may have turned positive or negative or its inode may @@ -30,7 +36,7 @@ static inline int fsnotify_name(__u32 mask, const void *data, int data_type, struct inode *dir, const struct qstr *name, u32 cookie) { - if (atomic_long_read(&dir->i_sb->s_fsnotify_connectors) == 0) + if (!fsnotify_sb_has_watchers(dir->i_sb)) return 0; return fsnotify(mask, data, data_type, dir, name, NULL, cookie); @@ -44,7 +50,7 @@ static inline void fsnotify_dirent(struct inode *dir, struct dentry *dentry, static inline void fsnotify_inode(struct inode *inode, __u32 mask) { - if (atomic_long_read(&inode->i_sb->s_fsnotify_connectors) == 0) + if (!fsnotify_sb_has_watchers(inode->i_sb)) return; if (S_ISDIR(inode->i_mode)) @@ -59,7 +65,7 @@ static inline int fsnotify_parent(struct dentry *dentry, __u32 mask, { struct inode *inode = d_inode(dentry); - if (atomic_long_read(&inode->i_sb->s_fsnotify_connectors) == 0) + if (!fsnotify_sb_has_watchers(inode->i_sb)) return 0; if (S_ISDIR(inode->i_mode)) { -- cgit v1.2.3 From b017500ab53c06441ff7d3a681484e37039b4f57 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 22 Jan 2024 17:11:26 +0100 Subject: PM: sleep: Use bool for all 1-bit fields in struct dev_pm_info For some 1-bit fields in struct dev_pm_info the data type is bool, while for some other 1-bit fields in there it is unsigned int, and these differences are somewhat arbitrary. For consistency, change the data type of the latter to bool, so that all of the 1-bit fields in struct dev_pm_info fields are bool. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Greg Kroah-Hartman --- include/linux/pm.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm.h b/include/linux/pm.h index a2f3e53a8196..97b0e23363c8 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -662,8 +662,8 @@ struct pm_subsys_data { struct dev_pm_info { pm_message_t power_state; - unsigned int can_wakeup:1; - unsigned int async_suspend:1; + bool can_wakeup:1; + bool async_suspend:1; bool in_dpm_list:1; /* Owned by the PM core */ bool is_prepared:1; /* Owned by the PM core */ bool is_suspended:1; /* Ditto */ @@ -682,10 +682,10 @@ struct dev_pm_info { bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ bool async_in_progress:1; /* Owned by the PM core */ - unsigned int must_resume:1; /* Owned by the PM core */ - unsigned int may_skip_resume:1; /* Set by subsystems */ + bool must_resume:1; /* Owned by the PM core */ + bool may_skip_resume:1; /* Set by subsystems */ #else - unsigned int should_wakeup:1; + bool should_wakeup:1; #endif #ifdef CONFIG_PM struct hrtimer suspend_timer; @@ -696,17 +696,17 @@ struct dev_pm_info { atomic_t usage_count; atomic_t child_count; unsigned int disable_depth:3; - unsigned int idle_notification:1; - unsigned int request_pending:1; - unsigned int deferred_resume:1; - unsigned int needs_force_resume:1; - unsigned int runtime_auto:1; + bool idle_notification:1; + bool request_pending:1; + bool deferred_resume:1; + bool needs_force_resume:1; + bool runtime_auto:1; bool ignore_children:1; - unsigned int no_callbacks:1; - unsigned int irq_safe:1; - unsigned int use_autosuspend:1; - unsigned int timer_autosuspends:1; - unsigned int memalloc_noio:1; + bool no_callbacks:1; + bool irq_safe:1; + bool use_autosuspend:1; + bool timer_autosuspends:1; + bool memalloc_noio:1; unsigned int links_count; enum rpm_request request; enum rpm_status runtime_status; -- cgit v1.2.3 From 6fe01d3cbb924a72493eb3f4722dfcfd1c194234 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:20:59 -0800 Subject: bpf: Add BPF token delegation mount options to BPF FS Add few new mount options to BPF FS that allow to specify that a given BPF FS instance allows creation of BPF token (added in the next patch), and what sort of operations are allowed under BPF token. As such, we get 4 new mount options, each is a bit mask - `delegate_cmds` allow to specify which bpf() syscall commands are allowed with BPF token derived from this BPF FS instance; - if BPF_MAP_CREATE command is allowed, `delegate_maps` specifies a set of allowable BPF map types that could be created with BPF token; - if BPF_PROG_LOAD command is allowed, `delegate_progs` specifies a set of allowable BPF program types that could be loaded with BPF token; - if BPF_PROG_LOAD command is allowed, `delegate_attachs` specifies a set of allowable BPF program attach types that could be loaded with BPF token; delegate_progs and delegate_attachs are meant to be used together, as full BPF program type is, in general, determined through both program type and program attach type. Currently, these mount options accept the following forms of values: - a special value "any", that enables all possible values of a given bit set; - numeric value (decimal or hexadecimal, determined by kernel automatically) that specifies a bit mask value directly; - all the values for a given mount option are combined, if specified multiple times. E.g., `mount -t bpf nodev /path/to/mount -o delegate_maps=0x1 -o delegate_maps=0x2` will result in a combined 0x3 mask. Ideally, more convenient (for humans) symbolic form derived from corresponding UAPI enums would be accepted (e.g., `-o delegate_progs=kprobe|tracepoint`) and I intend to implement this, but it requires a bunch of UAPI header churn, so I postponed it until this feature lands upstream or at least there is a definite consensus that this feature is acceptable and is going to make it, just to minimize amount of wasted effort and not increase amount of non-essential code to be reviewed. Attentive reader will notice that BPF FS is now marked as FS_USERNS_MOUNT, which theoretically makes it mountable inside non-init user namespace as long as the process has sufficient *namespaced* capabilities within that user namespace. But in reality we still restrict BPF FS to be mountable only by processes with CAP_SYS_ADMIN *in init userns* (extra check in bpf_fill_super()). FS_USERNS_MOUNT is added to allow creating BPF FS context object (i.e., fsopen("bpf")) from inside unprivileged process inside non-init userns, to capture that userns as the owning userns. It will still be required to pass this context object back to privileged process to instantiate and mount it. This manipulation is important, because capturing non-init userns as the owning userns of BPF FS instance (super block) allows to use that userns to constraint BPF token to that userns later on (see next patch). So creating BPF FS with delegation inside unprivileged userns will restrict derived BPF token objects to only "work" inside that intended userns, making it scoped to a intended "container". Also, setting these delegation options requires capable(CAP_SYS_ADMIN), so unprivileged process cannot set this up without involvement of a privileged process. There is a set of selftests at the end of the patch set that simulates this sequence of steps and validates that everything works as intended. But careful review is requested to make sure there are no missed gaps in the implementation and testing. This somewhat subtle set of aspects is the result of previous discussions ([0]) about various user namespace implications and interactions with BPF token functionality and is necessary to contain BPF token inside intended user namespace. [0] https://lore.kernel.org/bpf/20230704-hochverdient-lehne-eeb9eeef785e@brauner/ Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Christian Brauner Link: https://lore.kernel.org/bpf/20240124022127.2379740-3-andrii@kernel.org --- include/linux/bpf.h | 12 +++++++ kernel/bpf/inode.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 90 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 75b7f9b19c6a..28374cec49df 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1609,6 +1609,18 @@ struct bpf_link_primer { u32 id; }; +struct bpf_mount_opts { + kuid_t uid; + kgid_t gid; + umode_t mode; + + /* BPF token-related delegation options */ + u64 delegate_cmds; + u64 delegate_maps; + u64 delegate_progs; + u64 delegate_attachs; +}; + struct bpf_struct_ops_value; struct btf_member; diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 41e0a55c35f5..70b748f6228c 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "preload/bpf_preload.h" enum bpf_type { @@ -601,6 +602,7 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) { struct inode *inode = d_inode(root); umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX; + struct bpf_mount_opts *opts = root->d_sb->s_fs_info; if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) seq_printf(m, ",uid=%u", @@ -610,6 +612,26 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) from_kgid_munged(&init_user_ns, inode->i_gid)); if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); + + if (opts->delegate_cmds == ~0ULL) + seq_printf(m, ",delegate_cmds=any"); + else if (opts->delegate_cmds) + seq_printf(m, ",delegate_cmds=0x%llx", opts->delegate_cmds); + + if (opts->delegate_maps == ~0ULL) + seq_printf(m, ",delegate_maps=any"); + else if (opts->delegate_maps) + seq_printf(m, ",delegate_maps=0x%llx", opts->delegate_maps); + + if (opts->delegate_progs == ~0ULL) + seq_printf(m, ",delegate_progs=any"); + else if (opts->delegate_progs) + seq_printf(m, ",delegate_progs=0x%llx", opts->delegate_progs); + + if (opts->delegate_attachs == ~0ULL) + seq_printf(m, ",delegate_attachs=any"); + else if (opts->delegate_attachs) + seq_printf(m, ",delegate_attachs=0x%llx", opts->delegate_attachs); return 0; } @@ -635,28 +657,31 @@ enum { OPT_UID, OPT_GID, OPT_MODE, + OPT_DELEGATE_CMDS, + OPT_DELEGATE_MAPS, + OPT_DELEGATE_PROGS, + OPT_DELEGATE_ATTACHS, }; static const struct fs_parameter_spec bpf_fs_parameters[] = { fsparam_u32 ("uid", OPT_UID), fsparam_u32 ("gid", OPT_GID), fsparam_u32oct ("mode", OPT_MODE), + fsparam_string ("delegate_cmds", OPT_DELEGATE_CMDS), + fsparam_string ("delegate_maps", OPT_DELEGATE_MAPS), + fsparam_string ("delegate_progs", OPT_DELEGATE_PROGS), + fsparam_string ("delegate_attachs", OPT_DELEGATE_ATTACHS), {} }; -struct bpf_mount_opts { - kuid_t uid; - kgid_t gid; - umode_t mode; -}; - static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) { - struct bpf_mount_opts *opts = fc->fs_private; + struct bpf_mount_opts *opts = fc->s_fs_info; struct fs_parse_result result; kuid_t uid; kgid_t gid; - int opt; + int opt, err; + u64 msk; opt = fs_parse(fc, bpf_fs_parameters, param, &result); if (opt < 0) { @@ -708,6 +733,28 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) case OPT_MODE: opts->mode = result.uint_32 & S_IALLUGO; break; + case OPT_DELEGATE_CMDS: + case OPT_DELEGATE_MAPS: + case OPT_DELEGATE_PROGS: + case OPT_DELEGATE_ATTACHS: + if (strcmp(param->string, "any") == 0) { + msk = ~0ULL; + } else { + err = kstrtou64(param->string, 0, &msk); + if (err) + return err; + } + /* Setting delegation mount options requires privileges */ + if (msk && !capable(CAP_SYS_ADMIN)) + return -EPERM; + switch (opt) { + case OPT_DELEGATE_CMDS: opts->delegate_cmds |= msk; break; + case OPT_DELEGATE_MAPS: opts->delegate_maps |= msk; break; + case OPT_DELEGATE_PROGS: opts->delegate_progs |= msk; break; + case OPT_DELEGATE_ATTACHS: opts->delegate_attachs |= msk; break; + default: return -EINVAL; + } + break; } return 0; @@ -784,10 +831,14 @@ out: static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) { static const struct tree_descr bpf_rfiles[] = { { "" } }; - struct bpf_mount_opts *opts = fc->fs_private; + struct bpf_mount_opts *opts = sb->s_fs_info; struct inode *inode; int ret; + /* Mounting an instance of BPF FS requires privileges */ + if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN)) + return -EPERM; + ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); if (ret) return ret; @@ -811,7 +862,7 @@ static int bpf_get_tree(struct fs_context *fc) static void bpf_free_fc(struct fs_context *fc) { - kfree(fc->fs_private); + kfree(fc->s_fs_info); } static const struct fs_context_operations bpf_context_ops = { @@ -835,17 +886,32 @@ static int bpf_init_fs_context(struct fs_context *fc) opts->uid = current_fsuid(); opts->gid = current_fsgid(); - fc->fs_private = opts; + /* start out with no BPF token delegation enabled */ + opts->delegate_cmds = 0; + opts->delegate_maps = 0; + opts->delegate_progs = 0; + opts->delegate_attachs = 0; + + fc->s_fs_info = opts; fc->ops = &bpf_context_ops; return 0; } +static void bpf_kill_super(struct super_block *sb) +{ + struct bpf_mount_opts *opts = sb->s_fs_info; + + kill_litter_super(sb); + kfree(opts); +} + static struct file_system_type bpf_fs_type = { .owner = THIS_MODULE, .name = "bpf", .init_fs_context = bpf_init_fs_context, .parameters = bpf_fs_parameters, - .kill_sb = kill_litter_super, + .kill_sb = bpf_kill_super, + .fs_flags = FS_USERNS_MOUNT, }; static int __init bpf_init(void) -- cgit v1.2.3 From 35f96de04127d332a5c5e8a155d31f452f88c76d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:00 -0800 Subject: bpf: Introduce BPF token object Add new kind of BPF kernel object, BPF token. BPF token is meant to allow delegating privileged BPF functionality, like loading a BPF program or creating a BPF map, from privileged process to a *trusted* unprivileged process, all while having a good amount of control over which privileged operations could be performed using provided BPF token. This is achieved through mounting BPF FS instance with extra delegation mount options, which determine what operations are delegatable, and also constraining it to the owning user namespace (as mentioned in the previous patch). BPF token itself is just a derivative from BPF FS and can be created through a new bpf() syscall command, BPF_TOKEN_CREATE, which accepts BPF FS FD, which can be attained through open() API by opening BPF FS mount point. Currently, BPF token "inherits" delegated command, map types, prog type, and attach type bit sets from BPF FS as is. In the future, having an BPF token as a separate object with its own FD, we can allow to further restrict BPF token's allowable set of things either at the creation time or after the fact, allowing the process to guard itself further from unintentionally trying to load undesired kind of BPF programs. But for now we keep things simple and just copy bit sets as is. When BPF token is created from BPF FS mount, we take reference to the BPF super block's owning user namespace, and then use that namespace for checking all the {CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN} capabilities that are normally only checked against init userns (using capable()), but now we check them using ns_capable() instead (if BPF token is provided). See bpf_token_capable() for details. Such setup means that BPF token in itself is not sufficient to grant BPF functionality. User namespaced process has to *also* have necessary combination of capabilities inside that user namespace. So while previously CAP_BPF was useless when granted within user namespace, now it gains a meaning and allows container managers and sys admins to have a flexible control over which processes can and need to use BPF functionality within the user namespace (i.e., container in practice). And BPF FS delegation mount options and derived BPF tokens serve as a per-container "flag" to grant overall ability to use bpf() (plus further restrict on which parts of bpf() syscalls are treated as namespaced). Note also, BPF_TOKEN_CREATE command itself requires ns_capable(CAP_BPF) within the BPF FS owning user namespace, rounding up the ns_capable() story of BPF token. Also creating BPF token in init user namespace is currently not supported, given BPF token doesn't have any effect in init user namespace anyways. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Christian Brauner Link: https://lore.kernel.org/bpf/20240124022127.2379740-4-andrii@kernel.org --- include/linux/bpf.h | 41 ++++++++ include/uapi/linux/bpf.h | 37 +++++++ kernel/bpf/Makefile | 2 +- kernel/bpf/inode.c | 12 ++- kernel/bpf/syscall.c | 17 ++++ kernel/bpf/token.c | 217 +++++++++++++++++++++++++++++++++++++++++ tools/include/uapi/linux/bpf.h | 37 +++++++ 7 files changed, 357 insertions(+), 6 deletions(-) create mode 100644 kernel/bpf/token.c (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 28374cec49df..d9ff7ce547b4 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -52,6 +52,10 @@ struct module; struct bpf_func_state; struct ftrace_ops; struct cgroup; +struct bpf_token; +struct user_namespace; +struct super_block; +struct inode; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -1621,6 +1625,13 @@ struct bpf_mount_opts { u64 delegate_attachs; }; +struct bpf_token { + struct work_struct work; + atomic64_t refcnt; + struct user_namespace *userns; + u64 allowed_cmds; +}; + struct bpf_struct_ops_value; struct btf_member; @@ -2109,6 +2120,7 @@ static inline void bpf_enable_instrumentation(void) migrate_enable(); } +extern const struct super_operations bpf_super_ops; extern const struct file_operations bpf_map_fops; extern const struct file_operations bpf_prog_fops; extern const struct file_operations bpf_iter_fops; @@ -2243,6 +2255,8 @@ static inline void bpf_map_dec_elem_count(struct bpf_map *map) extern int sysctl_unprivileged_bpf_disabled; +bool bpf_token_capable(const struct bpf_token *token, int cap); + static inline bool bpf_allow_ptr_leaks(void) { return perfmon_capable(); @@ -2277,8 +2291,17 @@ int bpf_link_new_fd(struct bpf_link *link); struct bpf_link *bpf_link_get_from_fd(u32 ufd); struct bpf_link *bpf_link_get_curr_or_next(u32 *id); +void bpf_token_inc(struct bpf_token *token); +void bpf_token_put(struct bpf_token *token); +int bpf_token_create(union bpf_attr *attr); +struct bpf_token *bpf_token_get_from_fd(u32 ufd); + +bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); + int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); +struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, + umode_t mode); #define BPF_ITER_FUNC_PREFIX "bpf_iter_" #define DEFINE_BPF_ITER_FUNC(target, args...) \ @@ -2638,6 +2661,24 @@ static inline int bpf_obj_get_user(const char __user *pathname, int flags) return -EOPNOTSUPP; } +static inline bool bpf_token_capable(const struct bpf_token *token, int cap) +{ + return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); +} + +static inline void bpf_token_inc(struct bpf_token *token) +{ +} + +static inline void bpf_token_put(struct bpf_token *token) +{ +} + +static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd) +{ + return ERR_PTR(-EOPNOTSUPP); +} + static inline void __dev_flush(void) { } diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 1fef6d5a1330..b9dc0cca172c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -847,6 +847,36 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * + * BPF_TOKEN_CREATE + * Description + * Create BPF token with embedded information about what + * BPF-related functionality it allows: + * - a set of allowed bpf() syscall commands; + * - a set of allowed BPF map types to be created with + * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; + * - a set of allowed BPF program types and BPF program attach + * types to be loaded with BPF_PROG_LOAD command, if + * BPF_PROG_LOAD itself is allowed. + * + * BPF token is created (derived) from an instance of BPF FS, + * assuming it has necessary delegation mount options specified. + * This BPF token can be passed as an extra parameter to various + * bpf() syscall commands to grant BPF subsystem functionality to + * unprivileged processes. + * + * When created, BPF token is "associated" with the owning + * user namespace of BPF FS instance (super block) that it was + * derived from, and subsequent BPF operations performed with + * BPF token would be performing capabilities checks (i.e., + * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within + * that user namespace. Without BPF token, such capabilities + * have to be granted in init user namespace, making bpf() + * syscall incompatible with user namespace, for the most part. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -901,6 +931,8 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_TOKEN_CREATE, + __MAX_BPF_CMD, }; enum bpf_map_type { @@ -1722,6 +1754,11 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; + struct { /* struct used by BPF_TOKEN_CREATE command */ + __u32 flags; + __u32 bpffs_fd; + } token_create; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index f526b7573e97..4ce95acfcaa7 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 70b748f6228c..565be1f3f1ea 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -99,9 +99,9 @@ static const struct inode_operations bpf_prog_iops = { }; static const struct inode_operations bpf_map_iops = { }; static const struct inode_operations bpf_link_iops = { }; -static struct inode *bpf_get_inode(struct super_block *sb, - const struct inode *dir, - umode_t mode) +struct inode *bpf_get_inode(struct super_block *sb, + const struct inode *dir, + umode_t mode) { struct inode *inode; @@ -603,6 +603,7 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) struct inode *inode = d_inode(root); umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX; struct bpf_mount_opts *opts = root->d_sb->s_fs_info; + u64 mask; if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) seq_printf(m, ",uid=%u", @@ -613,7 +614,8 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); - if (opts->delegate_cmds == ~0ULL) + mask = (1ULL << __MAX_BPF_CMD) - 1; + if ((opts->delegate_cmds & mask) == mask) seq_printf(m, ",delegate_cmds=any"); else if (opts->delegate_cmds) seq_printf(m, ",delegate_cmds=0x%llx", opts->delegate_cmds); @@ -646,7 +648,7 @@ static void bpf_free_inode(struct inode *inode) free_inode_nonrcu(inode); } -static const struct super_operations bpf_super_ops = { +const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = generic_delete_inode, .show_options = bpf_show_options, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d5f1edee2d50..3ecba592b8ed 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5426,6 +5426,20 @@ out_prog_put: return ret; } +#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd + +static int token_create(union bpf_attr *attr) +{ + if (CHECK_ATTR(BPF_TOKEN_CREATE)) + return -EINVAL; + + /* no flags are supported yet */ + if (attr->token_create.flags) + return -EINVAL; + + return bpf_token_create(attr); +} + static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) { union bpf_attr attr; @@ -5559,6 +5573,9 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) case BPF_PROG_BIND_MAP: err = bpf_prog_bind_map(&attr); break; + case BPF_TOKEN_CREATE: + err = token_create(&attr); + break; default: err = -EINVAL; break; diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c new file mode 100644 index 000000000000..bdb6fe697568 --- /dev/null +++ b/kernel/bpf/token.c @@ -0,0 +1,217 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool bpf_ns_capable(struct user_namespace *ns, int cap) +{ + return ns_capable(ns, cap) || (cap != CAP_SYS_ADMIN && ns_capable(ns, CAP_SYS_ADMIN)); +} + +bool bpf_token_capable(const struct bpf_token *token, int cap) +{ + struct user_namespace *userns; + + /* BPF token allows ns_capable() level of capabilities */ + userns = token ? token->userns : &init_user_ns; + if (!bpf_ns_capable(userns, cap)) + return false; + return true; +} + +void bpf_token_inc(struct bpf_token *token) +{ + atomic64_inc(&token->refcnt); +} + +static void bpf_token_free(struct bpf_token *token) +{ + put_user_ns(token->userns); + kfree(token); +} + +static void bpf_token_put_deferred(struct work_struct *work) +{ + struct bpf_token *token = container_of(work, struct bpf_token, work); + + bpf_token_free(token); +} + +void bpf_token_put(struct bpf_token *token) +{ + if (!token) + return; + + if (!atomic64_dec_and_test(&token->refcnt)) + return; + + INIT_WORK(&token->work, bpf_token_put_deferred); + schedule_work(&token->work); +} + +static int bpf_token_release(struct inode *inode, struct file *filp) +{ + struct bpf_token *token = filp->private_data; + + bpf_token_put(token); + return 0; +} + +static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) +{ + struct bpf_token *token = filp->private_data; + u64 mask; + + BUILD_BUG_ON(__MAX_BPF_CMD >= 64); + mask = (1ULL << __MAX_BPF_CMD) - 1; + if ((token->allowed_cmds & mask) == mask) + seq_printf(m, "allowed_cmds:\tany\n"); + else + seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds); +} + +#define BPF_TOKEN_INODE_NAME "bpf-token" + +static const struct inode_operations bpf_token_iops = { }; + +static const struct file_operations bpf_token_fops = { + .release = bpf_token_release, + .show_fdinfo = bpf_token_show_fdinfo, +}; + +int bpf_token_create(union bpf_attr *attr) +{ + struct bpf_mount_opts *mnt_opts; + struct bpf_token *token = NULL; + struct user_namespace *userns; + struct inode *inode; + struct file *file; + struct path path; + struct fd f; + umode_t mode; + int err, fd; + + f = fdget(attr->token_create.bpffs_fd); + if (!f.file) + return -EBADF; + + path = f.file->f_path; + path_get(&path); + fdput(f); + + if (path.dentry != path.mnt->mnt_sb->s_root) { + err = -EINVAL; + goto out_path; + } + if (path.mnt->mnt_sb->s_op != &bpf_super_ops) { + err = -EINVAL; + goto out_path; + } + err = path_permission(&path, MAY_ACCESS); + if (err) + goto out_path; + + userns = path.dentry->d_sb->s_user_ns; + /* + * Enforce that creators of BPF tokens are in the same user + * namespace as the BPF FS instance. This makes reasoning about + * permissions a lot easier and we can always relax this later. + */ + if (current_user_ns() != userns) { + err = -EPERM; + goto out_path; + } + if (!ns_capable(userns, CAP_BPF)) { + err = -EPERM; + goto out_path; + } + + /* Creating BPF token in init_user_ns doesn't make much sense. */ + if (current_user_ns() == &init_user_ns) { + err = -EOPNOTSUPP; + goto out_path; + } + + mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask()); + inode = bpf_get_inode(path.mnt->mnt_sb, NULL, mode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_path; + } + + inode->i_op = &bpf_token_iops; + inode->i_fop = &bpf_token_fops; + clear_nlink(inode); /* make sure it is unlinked */ + + file = alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops); + if (IS_ERR(file)) { + iput(inode); + err = PTR_ERR(file); + goto out_path; + } + + token = kzalloc(sizeof(*token), GFP_USER); + if (!token) { + err = -ENOMEM; + goto out_file; + } + + atomic64_set(&token->refcnt, 1); + + /* remember bpffs owning userns for future ns_capable() checks */ + token->userns = get_user_ns(userns); + + mnt_opts = path.dentry->d_sb->s_fs_info; + token->allowed_cmds = mnt_opts->delegate_cmds; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + err = fd; + goto out_token; + } + + file->private_data = token; + fd_install(fd, file); + + path_put(&path); + return fd; + +out_token: + bpf_token_free(token); +out_file: + fput(file); +out_path: + path_put(&path); + return err; +} + +struct bpf_token *bpf_token_get_from_fd(u32 ufd) +{ + struct fd f = fdget(ufd); + struct bpf_token *token; + + if (!f.file) + return ERR_PTR(-EBADF); + if (f.file->f_op != &bpf_token_fops) { + fdput(f); + return ERR_PTR(-EINVAL); + } + + token = f.file->private_data; + bpf_token_inc(token); + fdput(f); + + return token; +} + +bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + if (!token) + return false; + return token->allowed_cmds & (1ULL << cmd); +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 1fef6d5a1330..b9dc0cca172c 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -847,6 +847,36 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * + * BPF_TOKEN_CREATE + * Description + * Create BPF token with embedded information about what + * BPF-related functionality it allows: + * - a set of allowed bpf() syscall commands; + * - a set of allowed BPF map types to be created with + * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; + * - a set of allowed BPF program types and BPF program attach + * types to be loaded with BPF_PROG_LOAD command, if + * BPF_PROG_LOAD itself is allowed. + * + * BPF token is created (derived) from an instance of BPF FS, + * assuming it has necessary delegation mount options specified. + * This BPF token can be passed as an extra parameter to various + * bpf() syscall commands to grant BPF subsystem functionality to + * unprivileged processes. + * + * When created, BPF token is "associated" with the owning + * user namespace of BPF FS instance (super block) that it was + * derived from, and subsequent BPF operations performed with + * BPF token would be performing capabilities checks (i.e., + * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within + * that user namespace. Without BPF token, such capabilities + * have to be granted in init user namespace, making bpf() + * syscall incompatible with user namespace, for the most part. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -901,6 +931,8 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_TOKEN_CREATE, + __MAX_BPF_CMD, }; enum bpf_map_type { @@ -1722,6 +1754,11 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; + struct { /* struct used by BPF_TOKEN_CREATE command */ + __u32 flags; + __u32 bpffs_fd; + } token_create; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF -- cgit v1.2.3 From a177fc2bf6fd83704854feaf7aae926b1df4f0b9 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:01 -0800 Subject: bpf: Add BPF token support to BPF_MAP_CREATE command Allow providing token_fd for BPF_MAP_CREATE command to allow controlled BPF map creation from unprivileged process through delegated BPF token. New BPF_F_TOKEN_FD flag is added to specify together with BPF token FD for BPF_MAP_CREATE command. Wire through a set of allowed BPF map types to BPF token, derived from BPF FS at BPF token creation time. This, in combination with allowed_cmds allows to create a narrowly-focused BPF token (controlled by privileged agent) with a restrictive set of BPF maps that application can attempt to create. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20240124022127.2379740-5-andrii@kernel.org --- include/linux/bpf.h | 2 + include/uapi/linux/bpf.h | 8 +++ kernel/bpf/inode.c | 3 +- kernel/bpf/syscall.c | 59 +++++++++++++++++----- kernel/bpf/token.c | 16 ++++++ tools/include/uapi/linux/bpf.h | 8 +++ .../selftests/bpf/prog_tests/libbpf_probes.c | 2 + .../testing/selftests/bpf/prog_tests/libbpf_str.c | 3 ++ 8 files changed, 86 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d9ff7ce547b4..8252452d0c4d 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1630,6 +1630,7 @@ struct bpf_token { atomic64_t refcnt; struct user_namespace *userns; u64 allowed_cmds; + u64 allowed_maps; }; struct bpf_struct_ops_value; @@ -2297,6 +2298,7 @@ int bpf_token_create(union bpf_attr *attr); struct bpf_token *bpf_token_get_from_fd(u32 ufd); bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); +bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index b9dc0cca172c..c78cab8b462d 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -983,6 +983,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -1365,6 +1366,9 @@ enum { /* Flag for value_type_btf_obj_fd, the fd is available */ BPF_F_VTYPE_BTF_OBJ_FD = (1U << 15), + +/* BPF token FD is passed in a corresponding command's token_fd field */ + BPF_F_TOKEN_FD = (1U << 16), }; /* Flags for BPF_PROG_QUERY. */ @@ -1443,6 +1447,10 @@ union bpf_attr { * type data for * btf_vmlinux_value_type_id. */ + /* BPF token FD to use with BPF_MAP_CREATE operation. + * If provided, map_flags should have BPF_F_TOKEN_FD flag set. + */ + __s32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 565be1f3f1ea..034b7e4d8f19 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -620,7 +620,8 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) else if (opts->delegate_cmds) seq_printf(m, ",delegate_cmds=0x%llx", opts->delegate_cmds); - if (opts->delegate_maps == ~0ULL) + mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; + if ((opts->delegate_maps & mask) == mask) seq_printf(m, ",delegate_maps=any"); else if (opts->delegate_maps) seq_printf(m, ",delegate_maps=0x%llx", opts->delegate_maps); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 3ecba592b8ed..b13a4bdcd3a0 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1011,8 +1011,8 @@ int map_check_no_btf(const struct bpf_map *map, return -ENOTSUPP; } -static int map_check_btf(struct bpf_map *map, const struct btf *btf, - u32 btf_key_id, u32 btf_value_id) +static int map_check_btf(struct bpf_map *map, struct bpf_token *token, + const struct btf *btf, u32 btf_key_id, u32 btf_value_id) { const struct btf_type *key_type, *value_type; u32 key_size, value_size; @@ -1040,7 +1040,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, if (!IS_ERR_OR_NULL(map->record)) { int i; - if (!bpf_capable()) { + if (!bpf_token_capable(token, CAP_BPF)) { ret = -EPERM; goto free_map_tab; } @@ -1128,14 +1128,16 @@ static bool bpf_net_capable(void) return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); } -#define BPF_MAP_CREATE_LAST_FIELD value_type_btf_obj_fd +#define BPF_MAP_CREATE_LAST_FIELD map_token_fd /* called via syscall */ static int map_create(union bpf_attr *attr) { const struct bpf_map_ops *ops; + struct bpf_token *token = NULL; int numa_node = bpf_map_attr_numa_node(attr); u32 map_type = attr->map_type; struct bpf_map *map; + bool token_flag; int f_flags; int err; @@ -1143,6 +1145,12 @@ static int map_create(union bpf_attr *attr) if (err) return -EINVAL; + /* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it + * to avoid per-map type checks tripping on unknown flag + */ + token_flag = attr->map_flags & BPF_F_TOKEN_FD; + attr->map_flags &= ~BPF_F_TOKEN_FD; + if (attr->btf_vmlinux_value_type_id) { if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || attr->btf_key_type_id || attr->btf_value_type_id) @@ -1183,14 +1191,32 @@ static int map_create(union bpf_attr *attr) if (!ops->map_mem_usage) return -EINVAL; + if (token_flag) { + token = bpf_token_get_from_fd(attr->map_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + + /* if current token doesn't grant map creation permissions, + * then we can't use this token, so ignore it and rely on + * system-wide capabilities checks + */ + if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) || + !bpf_token_allow_map_type(token, attr->map_type)) { + bpf_token_put(token); + token = NULL; + } + } + + err = -EPERM; + /* Intent here is for unprivileged_bpf_disabled to block BPF map * creation for unprivileged users; other actions depend * on fd availability and access to bpffs, so are dependent on * object creation success. Even with unprivileged BPF disabled, * capability checks are still carried out. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) - return -EPERM; + if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF)) + goto put_token; /* check privileged map type permissions */ switch (map_type) { @@ -1223,25 +1249,27 @@ static int map_create(union bpf_attr *attr) case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_STRUCT_OPS: case BPF_MAP_TYPE_CPUMAP: - if (!bpf_capable()) - return -EPERM; + if (!bpf_token_capable(token, CAP_BPF)) + goto put_token; break; case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: case BPF_MAP_TYPE_DEVMAP: case BPF_MAP_TYPE_DEVMAP_HASH: case BPF_MAP_TYPE_XSKMAP: - if (!bpf_net_capable()) - return -EPERM; + if (!bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; break; default: WARN(1, "unsupported map type %d", map_type); - return -EPERM; + goto put_token; } map = ops->map_alloc(attr); - if (IS_ERR(map)) - return PTR_ERR(map); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto put_token; + } map->ops = ops; map->map_type = map_type; @@ -1278,7 +1306,7 @@ static int map_create(union bpf_attr *attr) map->btf = btf; if (attr->btf_value_type_id) { - err = map_check_btf(map, btf, attr->btf_key_type_id, + err = map_check_btf(map, token, btf, attr->btf_key_type_id, attr->btf_value_type_id); if (err) goto free_map; @@ -1299,6 +1327,7 @@ static int map_create(union bpf_attr *attr) goto free_map_sec; bpf_map_save_memcg(map); + bpf_token_put(token); err = bpf_map_new_fd(map, f_flags); if (err < 0) { @@ -1319,6 +1348,8 @@ free_map_sec: free_map: btf_put(map->btf); map->ops->map_free(map); +put_token: + bpf_token_put(token); return err; } diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index bdb6fe697568..bc86be4ca567 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -73,6 +73,13 @@ static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) seq_printf(m, "allowed_cmds:\tany\n"); else seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds); + + BUILD_BUG_ON(__MAX_BPF_MAP_TYPE >= 64); + mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; + if ((token->allowed_maps & mask) == mask) + seq_printf(m, "allowed_maps:\tany\n"); + else + seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps); } #define BPF_TOKEN_INODE_NAME "bpf-token" @@ -168,6 +175,7 @@ int bpf_token_create(union bpf_attr *attr) mnt_opts = path.dentry->d_sb->s_fs_info; token->allowed_cmds = mnt_opts->delegate_cmds; + token->allowed_maps = mnt_opts->delegate_maps; fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { @@ -215,3 +223,11 @@ bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) return false; return token->allowed_cmds & (1ULL << cmd); } + +bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type) +{ + if (!token || type >= __MAX_BPF_MAP_TYPE) + return false; + + return token->allowed_maps & (1ULL << type); +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index b9dc0cca172c..c78cab8b462d 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -983,6 +983,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -1365,6 +1366,9 @@ enum { /* Flag for value_type_btf_obj_fd, the fd is available */ BPF_F_VTYPE_BTF_OBJ_FD = (1U << 15), + +/* BPF token FD is passed in a corresponding command's token_fd field */ + BPF_F_TOKEN_FD = (1U << 16), }; /* Flags for BPF_PROG_QUERY. */ @@ -1443,6 +1447,10 @@ union bpf_attr { * type data for * btf_vmlinux_value_type_id. */ + /* BPF token FD to use with BPF_MAP_CREATE operation. + * If provided, map_flags should have BPF_F_TOKEN_FD flag set. + */ + __s32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c index 9f766ddd946a..573249a2814d 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c @@ -68,6 +68,8 @@ void test_libbpf_probe_map_types(void) if (map_type == BPF_MAP_TYPE_UNSPEC) continue; + if (strcmp(map_type_name, "__MAX_BPF_MAP_TYPE") == 0) + continue; if (!test__start_subtest(map_type_name)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c index eb34d612d6f8..1f328c0d8aff 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c @@ -132,6 +132,9 @@ static void test_libbpf_bpf_map_type_str(void) const char *map_type_str; char buf[256]; + if (map_type == __MAX_BPF_MAP_TYPE) + continue; + map_type_name = btf__str_by_offset(btf, e->name_off); map_type_str = libbpf_bpf_map_type_str(map_type); ASSERT_OK_PTR(map_type_str, map_type_name); -- cgit v1.2.3 From caf8f28e036c4ba1e823355da6c0c01c39e70ab9 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:03 -0800 Subject: bpf: Add BPF token support to BPF_PROG_LOAD command Add basic support of BPF token to BPF_PROG_LOAD. BPF_F_TOKEN_FD flag should be set in prog_flags field when providing prog_token_fd. Wire through a set of allowed BPF program types and attach types, derived from BPF FS at BPF token creation time. Then make sure we perform bpf_token_capable() checks everywhere where it's relevant. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20240124022127.2379740-7-andrii@kernel.org --- include/linux/bpf.h | 6 ++ include/uapi/linux/bpf.h | 5 ++ kernel/bpf/core.c | 1 + kernel/bpf/inode.c | 6 +- kernel/bpf/syscall.c | 90 ++++++++++++++++------ kernel/bpf/token.c | 27 +++++++ tools/include/uapi/linux/bpf.h | 5 ++ .../selftests/bpf/prog_tests/libbpf_probes.c | 2 + .../testing/selftests/bpf/prog_tests/libbpf_str.c | 3 + 9 files changed, 118 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 8252452d0c4d..d0bf37e3f166 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1489,6 +1489,7 @@ struct bpf_prog_aux { #ifdef CONFIG_SECURITY void *security; #endif + struct bpf_token *token; struct bpf_prog_offload *offload; struct btf *btf; struct bpf_func_info *func_info; @@ -1631,6 +1632,8 @@ struct bpf_token { struct user_namespace *userns; u64 allowed_cmds; u64 allowed_maps; + u64 allowed_progs; + u64 allowed_attachs; }; struct bpf_struct_ops_value; @@ -2299,6 +2302,9 @@ struct bpf_token *bpf_token_get_from_fd(u32 ufd); bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); +bool bpf_token_allow_prog_type(const struct bpf_token *token, + enum bpf_prog_type prog_type, + enum bpf_attach_type attach_type); int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index cb2c888e3bb4..d96708380e52 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1028,6 +1028,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1520,6 +1521,10 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; + /* BPF token FD to use with BPF_PROG_LOAD operation. + * If provided, prog_flags should have BPF_F_TOKEN_FD flag set. + */ + __s32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index fbb1d95a9b44..00dccba29769 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2779,6 +2779,7 @@ void bpf_prog_free(struct bpf_prog *fp) if (aux->dst_prog) bpf_prog_put(aux->dst_prog); + bpf_token_put(aux->token); INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 034b7e4d8f19..5fb10da5717f 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -626,12 +626,14 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) else if (opts->delegate_maps) seq_printf(m, ",delegate_maps=0x%llx", opts->delegate_maps); - if (opts->delegate_progs == ~0ULL) + mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; + if ((opts->delegate_progs & mask) == mask) seq_printf(m, ",delegate_progs=any"); else if (opts->delegate_progs) seq_printf(m, ",delegate_progs=0x%llx", opts->delegate_progs); - if (opts->delegate_attachs == ~0ULL) + mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; + if ((opts->delegate_attachs & mask) == mask) seq_printf(m, ",delegate_attachs=any"); else if (opts->delegate_attachs) seq_printf(m, ",delegate_attachs=0x%llx", opts->delegate_attachs); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 45b3a55896eb..61b4bf4cc287 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2626,13 +2626,15 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD log_true_size +#define BPF_PROG_LOAD_LAST_FIELD prog_token_fd static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) { enum bpf_prog_type type = attr->prog_type; struct bpf_prog *prog, *dst_prog = NULL; struct btf *attach_btf = NULL; + struct bpf_token *token = NULL; + bool bpf_cap; int err; char license[128]; @@ -2646,13 +2648,35 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) BPF_F_TEST_RND_HI32 | BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | - BPF_F_TEST_REG_INVARIANTS)) + BPF_F_TEST_REG_INVARIANTS | + BPF_F_TOKEN_FD)) return -EINVAL; + bpf_prog_load_fixup_attach_type(attr); + + if (attr->prog_flags & BPF_F_TOKEN_FD) { + token = bpf_token_get_from_fd(attr->prog_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + /* if current token doesn't grant prog loading permissions, + * then we can't use this token, so ignore it and rely on + * system-wide capabilities checks + */ + if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) || + !bpf_token_allow_prog_type(token, attr->prog_type, + attr->expected_attach_type)) { + bpf_token_put(token); + token = NULL; + } + } + + bpf_cap = bpf_token_capable(token, CAP_BPF); + err = -EPERM; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && - !bpf_capable()) - return -EPERM; + !bpf_cap) + goto put_token; /* Intent here is for unprivileged_bpf_disabled to block BPF program * creation for unprivileged users; other actions depend @@ -2661,21 +2685,23 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) * capability checks are still carried out for these * and other operations. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) - return -EPERM; + if (sysctl_unprivileged_bpf_disabled && !bpf_cap) + goto put_token; if (attr->insn_cnt == 0 || - attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) - return -E2BIG; + attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { + err = -E2BIG; + goto put_token; + } if (type != BPF_PROG_TYPE_SOCKET_FILTER && type != BPF_PROG_TYPE_CGROUP_SKB && - !bpf_capable()) - return -EPERM; + !bpf_cap) + goto put_token; - if (is_net_admin_prog_type(type) && !bpf_net_capable()) - return -EPERM; - if (is_perfmon_prog_type(type) && !perfmon_capable()) - return -EPERM; + if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; + if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) + goto put_token; /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog * or btf, we need to check which one it is @@ -2685,27 +2711,33 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (IS_ERR(dst_prog)) { dst_prog = NULL; attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); - if (IS_ERR(attach_btf)) - return -EINVAL; + if (IS_ERR(attach_btf)) { + err = -EINVAL; + goto put_token; + } if (!btf_is_kernel(attach_btf)) { /* attaching through specifying bpf_prog's BTF * objects directly might be supported eventually */ btf_put(attach_btf); - return -ENOTSUPP; + err = -ENOTSUPP; + goto put_token; } } } else if (attr->attach_btf_id) { /* fall back to vmlinux BTF, if BTF type ID is specified */ attach_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(attach_btf)) - return PTR_ERR(attach_btf); - if (!attach_btf) - return -EINVAL; + if (IS_ERR(attach_btf)) { + err = PTR_ERR(attach_btf); + goto put_token; + } + if (!attach_btf) { + err = -EINVAL; + goto put_token; + } btf_get(attach_btf); } - bpf_prog_load_fixup_attach_type(attr); if (bpf_prog_load_check_attach(type, attr->expected_attach_type, attach_btf, attr->attach_btf_id, dst_prog)) { @@ -2713,7 +2745,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - return -EINVAL; + err = -EINVAL; + goto put_token; } /* plain bpf_prog allocation */ @@ -2723,7 +2756,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - return -ENOMEM; + err = -EINVAL; + goto put_token; } prog->expected_attach_type = attr->expected_attach_type; @@ -2734,6 +2768,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; + /* move token into prog->aux, reuse taken refcnt */ + prog->aux->token = token; + token = NULL; + err = security_bpf_prog_alloc(prog->aux); if (err) goto free_prog; @@ -2851,6 +2889,8 @@ free_prog: if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); +put_token: + bpf_token_put(token); return err; } @@ -3858,7 +3898,7 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, case BPF_PROG_TYPE_SK_LOOKUP: return attach_type == prog->expected_attach_type ? 0 : -EINVAL; case BPF_PROG_TYPE_CGROUP_SKB: - if (!bpf_net_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) /* cg-skb progs can be loaded by unpriv user. * check permissions at attach time. */ diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index bc86be4ca567..c13c73788d8c 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -80,6 +80,20 @@ static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) seq_printf(m, "allowed_maps:\tany\n"); else seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps); + + BUILD_BUG_ON(__MAX_BPF_PROG_TYPE >= 64); + mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; + if ((token->allowed_progs & mask) == mask) + seq_printf(m, "allowed_progs:\tany\n"); + else + seq_printf(m, "allowed_progs:\t0x%llx\n", token->allowed_progs); + + BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE >= 64); + mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; + if ((token->allowed_attachs & mask) == mask) + seq_printf(m, "allowed_attachs:\tany\n"); + else + seq_printf(m, "allowed_attachs:\t0x%llx\n", token->allowed_attachs); } #define BPF_TOKEN_INODE_NAME "bpf-token" @@ -176,6 +190,8 @@ int bpf_token_create(union bpf_attr *attr) mnt_opts = path.dentry->d_sb->s_fs_info; token->allowed_cmds = mnt_opts->delegate_cmds; token->allowed_maps = mnt_opts->delegate_maps; + token->allowed_progs = mnt_opts->delegate_progs; + token->allowed_attachs = mnt_opts->delegate_attachs; fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { @@ -231,3 +247,14 @@ bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type t return token->allowed_maps & (1ULL << type); } + +bool bpf_token_allow_prog_type(const struct bpf_token *token, + enum bpf_prog_type prog_type, + enum bpf_attach_type attach_type) +{ + if (!token || prog_type >= __MAX_BPF_PROG_TYPE || attach_type >= __MAX_BPF_ATTACH_TYPE) + return false; + + return (token->allowed_progs & (1ULL << prog_type)) && + (token->allowed_attachs & (1ULL << attach_type)); +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index cb2c888e3bb4..d96708380e52 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1028,6 +1028,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1520,6 +1521,10 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; + /* BPF token FD to use with BPF_PROG_LOAD operation. + * If provided, prog_flags should have BPF_F_TOKEN_FD flag set. + */ + __s32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c index 573249a2814d..4ed46ed58a7b 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c @@ -30,6 +30,8 @@ void test_libbpf_probe_prog_types(void) if (prog_type == BPF_PROG_TYPE_UNSPEC) continue; + if (strcmp(prog_type_name, "__MAX_BPF_PROG_TYPE") == 0) + continue; if (!test__start_subtest(prog_type_name)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c index 1f328c0d8aff..62ea855ec4d0 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c @@ -189,6 +189,9 @@ static void test_libbpf_bpf_prog_type_str(void) const char *prog_type_str; char buf[256]; + if (prog_type == __MAX_BPF_PROG_TYPE) + continue; + prog_type_name = btf__str_by_offset(btf, e->name_off); prog_type_str = libbpf_bpf_prog_type_str(prog_type); ASSERT_OK_PTR(prog_type_str, prog_type_name); -- cgit v1.2.3 From bbc1d24724e110b86a1a7c3c1724ce0d62cc1e2e Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:04 -0800 Subject: bpf: Take into account BPF token when fetching helper protos Instead of performing unconditional system-wide bpf_capable() and perfmon_capable() calls inside bpf_base_func_proto() function (and other similar ones) to determine eligibility of a given BPF helper for a given program, use previously recorded BPF token during BPF_PROG_LOAD command handling to inform the decision. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20240124022127.2379740-8-andrii@kernel.org --- drivers/media/rc/bpf-lirc.c | 2 +- include/linux/bpf.h | 5 +++-- kernel/bpf/cgroup.c | 6 +++--- kernel/bpf/helpers.c | 6 +++--- kernel/bpf/syscall.c | 5 +++-- kernel/trace/bpf_trace.c | 2 +- net/core/filter.c | 32 ++++++++++++++++---------------- net/ipv4/bpf_tcp_ca.c | 2 +- net/netfilter/nf_bpf_link.c | 2 +- 9 files changed, 32 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index fe17c7f98e81..6d07693c6b9f 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -110,7 +110,7 @@ lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_prandom_u32: return &bpf_get_prandom_u32_proto; case BPF_FUNC_trace_printk: - if (perfmon_capable()) + if (bpf_token_capable(prog->aux->token, CAP_PERFMON)) return bpf_get_trace_printk_proto(); fallthrough; default: diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d0bf37e3f166..1325225bf602 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2550,7 +2550,8 @@ int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt, struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); -const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); +const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, + const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); @@ -2810,7 +2811,7 @@ static inline int btf_struct_access(struct bpf_verifier_log *log, } static inline const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id) +bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { return NULL; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 491d20038cbe..98e0e3835b28 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1630,7 +1630,7 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2191,7 +2191,7 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2348,7 +2348,7 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index e04ca1af8927..bcb951a2ecf4 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1680,7 +1680,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto __weak; const struct bpf_func_proto bpf_task_pt_regs_proto __weak; const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id) +bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_map_lookup_elem: @@ -1731,7 +1731,7 @@ bpf_base_func_proto(enum bpf_func_id func_id) break; } - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return NULL; switch (func_id) { @@ -1789,7 +1789,7 @@ bpf_base_func_proto(enum bpf_func_id func_id) break; } - if (!perfmon_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) return NULL; switch (func_id) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 61b4bf4cc287..f76408c957ce 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5772,7 +5772,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { const struct bpf_func_proto * __weak tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } BPF_CALL_1(bpf_sys_close, u32, fd) @@ -5822,7 +5822,8 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_sys_bpf: - return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto; + return !bpf_token_capable(prog->aux->token, CAP_PERFMON) + ? NULL : &bpf_sys_bpf_proto; case BPF_FUNC_btf_find_by_name_kind: return &bpf_btf_find_by_name_kind_proto; case BPF_FUNC_sys_close: diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index c98c20abaf99..64fdaf79d113 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1629,7 +1629,7 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_trace_vprintk: return bpf_get_trace_vprintk_proto(); default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/net/core/filter.c b/net/core/filter.c index 6a7abbaa50b8..521bcd0f5e4d 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -87,7 +87,7 @@ #include "dev.h" static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id); +bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len) { @@ -7862,7 +7862,7 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -7955,7 +7955,7 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return NULL; } default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -7974,7 +7974,7 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_skb_event_output_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8161,7 +8161,7 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8220,7 +8220,7 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } #if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) @@ -8281,7 +8281,7 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_tcp_sock_proto; #endif /* CONFIG_INET */ default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8323,7 +8323,7 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_cgroup_classid_curr_proto; #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8367,7 +8367,7 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_skc_lookup_tcp_proto; #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8378,7 +8378,7 @@ flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_load_bytes: return &bpf_flow_dissector_load_bytes_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8405,7 +8405,7 @@ lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_under_cgroup: return &bpf_skb_under_cgroup_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -11236,7 +11236,7 @@ sk_reuseport_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -11418,7 +11418,7 @@ sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_release: return &bpf_sk_release_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -11752,7 +11752,7 @@ const struct bpf_func_proto bpf_sock_from_file_proto = { }; static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id) +bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { const struct bpf_func_proto *func; @@ -11781,10 +11781,10 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } - if (!perfmon_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) return NULL; return func; diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 8e7716256d3c..834edc18463a 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -197,7 +197,7 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index 0e4beae421f8..5257d5e7eb09 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -314,7 +314,7 @@ static bool nf_is_valid_access(int off, int size, enum bpf_access_type type, static const struct bpf_func_proto * bpf_nf_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } const struct bpf_verifier_ops netfilter_verifier_ops = { -- cgit v1.2.3 From d79a3549754725bb90e58104417449edddf3da3d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:05 -0800 Subject: bpf: Consistently use BPF token throughout BPF verifier logic Remove remaining direct queries to perfmon_capable() and bpf_capable() in BPF verifier logic and instead use BPF token (if available) to make decisions about privileges. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20240124022127.2379740-9-andrii@kernel.org --- include/linux/bpf.h | 16 ++++++++-------- include/linux/filter.h | 2 +- kernel/bpf/arraymap.c | 2 +- kernel/bpf/core.c | 2 +- kernel/bpf/verifier.c | 13 ++++++------- net/core/filter.c | 4 ++-- 6 files changed, 19 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1325225bf602..4e146e9708be 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2261,24 +2261,24 @@ extern int sysctl_unprivileged_bpf_disabled; bool bpf_token_capable(const struct bpf_token *token, int cap); -static inline bool bpf_allow_ptr_leaks(void) +static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token) { - return perfmon_capable(); + return bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_allow_uninit_stack(void) +static inline bool bpf_allow_uninit_stack(const struct bpf_token *token) { - return perfmon_capable(); + return bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_bypass_spec_v1(void) +static inline bool bpf_bypass_spec_v1(const struct bpf_token *token) { - return cpu_mitigations_off() || perfmon_capable(); + return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_bypass_spec_v4(void) +static inline bool bpf_bypass_spec_v4(const struct bpf_token *token) { - return cpu_mitigations_off() || perfmon_capable(); + return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } int bpf_map_new_fd(struct bpf_map *map, int flags); diff --git a/include/linux/filter.h b/include/linux/filter.h index 35f067fd3840..fee070b9826e 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1140,7 +1140,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) return false; if (!bpf_jit_harden) return false; - if (bpf_jit_harden == 1 && bpf_capable()) + if (bpf_jit_harden == 1 && bpf_token_capable(prog->aux->token, CAP_BPF)) return false; return true; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0bdbbbeab155..13358675ff2e 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -82,7 +82,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; int numa_node = bpf_map_attr_numa_node(attr); u32 elem_size, index_mask, max_entries; - bool bypass_spec_v1 = bpf_bypass_spec_v1(); + bool bypass_spec_v1 = bpf_bypass_spec_v1(NULL); u64 array_size, mask64; struct bpf_array *array; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 00dccba29769..71c459a51d9e 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -682,7 +682,7 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) void bpf_prog_kallsyms_add(struct bpf_prog *fp) { if (!bpf_prog_kallsyms_candidate(fp) || - !bpf_capable()) + !bpf_token_capable(fp->aux->token, CAP_BPF)) return; bpf_prog_ksym_set_addr(fp); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f31868ba0c2d..fe833e831cb6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20830,7 +20830,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - is_priv = bpf_capable(); + + env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); + env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); + env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); + env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); + env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); bpf_get_btf_vmlinux(); @@ -20862,12 +20867,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) env->strict_alignment = false; - env->allow_ptr_leaks = bpf_allow_ptr_leaks(); - env->allow_uninit_stack = bpf_allow_uninit_stack(); - env->bypass_spec_v1 = bpf_bypass_spec_v1(); - env->bypass_spec_v4 = bpf_bypass_spec_v4(); - env->bpf_capable = bpf_capable(); - if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; diff --git a/net/core/filter.c b/net/core/filter.c index 521bcd0f5e4d..40121475e8d1 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -8580,7 +8580,7 @@ static bool cg_skb_is_valid_access(int off, int size, return false; case bpf_ctx_range(struct __sk_buff, data): case bpf_ctx_range(struct __sk_buff, data_end): - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return false; break; } @@ -8592,7 +8592,7 @@ static bool cg_skb_is_valid_access(int off, int size, case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): break; case bpf_ctx_range(struct __sk_buff, tstamp): - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return false; break; default: -- cgit v1.2.3 From 1b67772e4e3f16cd647b229cae95fc06d120be08 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:06 -0800 Subject: bpf,lsm: Refactor bpf_prog_alloc/bpf_prog_free LSM hooks Based on upstream discussion ([0]), rework existing bpf_prog_alloc_security LSM hook. Rename it to bpf_prog_load and instead of passing bpf_prog_aux, pass proper bpf_prog pointer for a full BPF program struct. Also, we pass bpf_attr union with all the user-provided arguments for BPF_PROG_LOAD command. This will give LSMs as much information as we can basically provide. The hook is also BPF token-aware now, and optional bpf_token struct is passed as a third argument. bpf_prog_load LSM hook is called after a bunch of sanity checks were performed, bpf_prog and bpf_prog_aux were allocated and filled out, but right before performing full-fledged BPF verification step. bpf_prog_free LSM hook is now accepting struct bpf_prog argument, for consistency. SELinux code is adjusted to all new names, types, and signatures. Note, given that bpf_prog_load (previously bpf_prog_alloc) hook can be used by some LSMs to allocate extra security blob, but also by other LSMs to reject BPF program loading, we need to make sure that bpf_prog_free LSM hook is called after bpf_prog_load/bpf_prog_alloc one *even* if the hook itself returned error. If we don't do that, we run the risk of leaking memory. This seems to be possible today when combining SELinux and BPF LSM, as one example, depending on their relative ordering. Also, for BPF LSM setup, add bpf_prog_load and bpf_prog_free to sleepable LSM hooks list, as they are both executed in sleepable context. Also drop bpf_prog_load hook from untrusted, as there is no issue with refcount or anything else anymore, that originally forced us to add it to untrusted list in c0c852dd1876 ("bpf: Do not mark certain LSM hook arguments as trusted"). We now trigger this hook much later and it should not be an issue anymore. [0] https://lore.kernel.org/bpf/9fe88aef7deabbe87d3fc38c4aea3c69.paul@paul-moore.com/ Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Paul Moore Link: https://lore.kernel.org/bpf/20240124022127.2379740-10-andrii@kernel.org --- include/linux/lsm_hook_defs.h | 5 +++-- include/linux/security.h | 12 +++++++----- kernel/bpf/bpf_lsm.c | 5 +++-- kernel/bpf/syscall.c | 25 +++++++++++++------------ security/security.c | 25 +++++++++++++++---------- security/selinux/hooks.c | 15 ++++++++------- 6 files changed, 49 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 185924c56378..370181aa685b 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -406,8 +406,9 @@ LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode) LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog) LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map) LSM_HOOK(void, LSM_RET_VOID, bpf_map_free_security, struct bpf_map *map) -LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux) -LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free_security, struct bpf_prog_aux *aux) +LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) +LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) #endif /* CONFIG_BPF_SYSCALL */ LSM_HOOK(int, 0, locked_down, enum lockdown_reason what) diff --git a/include/linux/security.h b/include/linux/security.h index d0eb20f90b26..cb2932fce448 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2064,15 +2064,16 @@ static inline void securityfs_remove(struct dentry *dentry) union bpf_attr; struct bpf_map; struct bpf_prog; -struct bpf_prog_aux; +struct bpf_token; #ifdef CONFIG_SECURITY extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size); extern int security_bpf_map(struct bpf_map *map, fmode_t fmode); extern int security_bpf_prog(struct bpf_prog *prog); extern int security_bpf_map_alloc(struct bpf_map *map); extern void security_bpf_map_free(struct bpf_map *map); -extern int security_bpf_prog_alloc(struct bpf_prog_aux *aux); -extern void security_bpf_prog_free(struct bpf_prog_aux *aux); +extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token); +extern void security_bpf_prog_free(struct bpf_prog *prog); #else static inline int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) @@ -2098,12 +2099,13 @@ static inline int security_bpf_map_alloc(struct bpf_map *map) static inline void security_bpf_map_free(struct bpf_map *map) { } -static inline int security_bpf_prog_alloc(struct bpf_prog_aux *aux) +static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { return 0; } -static inline void security_bpf_prog_free(struct bpf_prog_aux *aux) +static inline void security_bpf_prog_free(struct bpf_prog *prog) { } #endif /* CONFIG_SECURITY */ #endif /* CONFIG_BPF_SYSCALL */ diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index e8e910395bf6..7ee0dd011de4 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -263,6 +263,8 @@ BTF_ID(func, bpf_lsm_bpf_map) BTF_ID(func, bpf_lsm_bpf_map_alloc_security) BTF_ID(func, bpf_lsm_bpf_map_free_security) BTF_ID(func, bpf_lsm_bpf_prog) +BTF_ID(func, bpf_lsm_bpf_prog_load) +BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bprm_check_security) BTF_ID(func, bpf_lsm_bprm_committed_creds) BTF_ID(func, bpf_lsm_bprm_committing_creds) @@ -358,8 +360,7 @@ BTF_SET_END(sleepable_lsm_hooks) BTF_SET_START(untrusted_lsm_hooks) BTF_ID(func, bpf_lsm_bpf_map_free_security) -BTF_ID(func, bpf_lsm_bpf_prog_alloc_security) -BTF_ID(func, bpf_lsm_bpf_prog_free_security) +BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) #ifdef CONFIG_SECURITY_NETWORK diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f76408c957ce..3857a2a9ae7f 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2180,7 +2180,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); - security_bpf_prog_free(aux); + security_bpf_prog_free(aux->prog); bpf_prog_free(aux->prog); } @@ -2772,10 +2772,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->token = token; token = NULL; - err = security_bpf_prog_alloc(prog->aux); - if (err) - goto free_prog; - prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; @@ -2783,12 +2779,12 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), bpf_prog_insn_size(prog)) != 0) - goto free_prog_sec; + goto free_prog; /* copy eBPF program license from user space */ if (strncpy_from_bpfptr(license, make_bpfptr(attr->license, uattr.is_kernel), sizeof(license) - 1) < 0) - goto free_prog_sec; + goto free_prog; license[sizeof(license) - 1] = 0; /* eBPF programs must be GPL compatible to use GPL-ed functions */ @@ -2802,14 +2798,14 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (bpf_prog_is_dev_bound(prog->aux)) { err = bpf_prog_dev_bound_init(prog, attr); if (err) - goto free_prog_sec; + goto free_prog; } if (type == BPF_PROG_TYPE_EXT && dst_prog && bpf_prog_is_dev_bound(dst_prog->aux)) { err = bpf_prog_dev_bound_inherit(prog, dst_prog); if (err) - goto free_prog_sec; + goto free_prog; } /* @@ -2831,12 +2827,16 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) /* find program type: socket_filter vs tracing_filter */ err = find_prog_type(type, prog); if (err < 0) - goto free_prog_sec; + goto free_prog; prog->aux->load_time = ktime_get_boottime_ns(); err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, sizeof(attr->prog_name)); if (err < 0) + goto free_prog; + + err = security_bpf_prog_load(prog, attr, token); + if (err) goto free_prog_sec; /* run eBPF verifier */ @@ -2882,10 +2882,11 @@ free_used_maps: */ __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); return err; + free_prog_sec: - free_uid(prog->aux->user); - security_bpf_prog_free(prog->aux); + security_bpf_prog_free(prog); free_prog: + free_uid(prog->aux->user); if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); diff --git a/security/security.c b/security/security.c index 0144a98d3712..eb159da4b146 100644 --- a/security/security.c +++ b/security/security.c @@ -5423,16 +5423,21 @@ int security_bpf_map_alloc(struct bpf_map *map) } /** - * security_bpf_prog_alloc() - Allocate a bpf program LSM blob - * @aux: bpf program aux info struct + * security_bpf_prog_load() - Check if loading of BPF program is allowed + * @prog: BPF program object + * @attr: BPF syscall attributes used to create BPF program + * @token: BPF token used to grant user access to BPF subsystem * - * Initialize the security field inside bpf program. + * Perform an access control check when the kernel loads a BPF program and + * allocates associated BPF program object. This hook is also responsible for + * allocating any required LSM state for the BPF program. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_prog_alloc(struct bpf_prog_aux *aux) +int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { - return call_int_hook(bpf_prog_alloc_security, 0, aux); + return call_int_hook(bpf_prog_load, 0, prog, attr, token); } /** @@ -5447,14 +5452,14 @@ void security_bpf_map_free(struct bpf_map *map) } /** - * security_bpf_prog_free() - Free a bpf program's LSM blob - * @aux: bpf program aux info struct + * security_bpf_prog_free() - Free a BPF program's LSM blob + * @prog: BPF program struct * - * Clean up the security information stored inside bpf prog. + * Clean up the security information stored inside BPF program. */ -void security_bpf_prog_free(struct bpf_prog_aux *aux) +void security_bpf_prog_free(struct bpf_prog *prog) { - call_void_hook(bpf_prog_free_security, aux); + call_void_hook(bpf_prog_free, prog); } #endif /* CONFIG_BPF_SYSCALL */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index a6bf90ace84c..6d64fb189b1b 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6942,7 +6942,8 @@ static void selinux_bpf_map_free(struct bpf_map *map) kfree(bpfsec); } -static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) +static int selinux_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { struct bpf_security_struct *bpfsec; @@ -6951,16 +6952,16 @@ static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) return -ENOMEM; bpfsec->sid = current_sid(); - aux->security = bpfsec; + prog->aux->security = bpfsec; return 0; } -static void selinux_bpf_prog_free(struct bpf_prog_aux *aux) +static void selinux_bpf_prog_free(struct bpf_prog *prog) { - struct bpf_security_struct *bpfsec = aux->security; + struct bpf_security_struct *bpfsec = prog->aux->security; - aux->security = NULL; + prog->aux->security = NULL; kfree(bpfsec); } #endif @@ -7325,7 +7326,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(bpf_map, selinux_bpf_map), LSM_HOOK_INIT(bpf_prog, selinux_bpf_prog), LSM_HOOK_INIT(bpf_map_free_security, selinux_bpf_map_free), - LSM_HOOK_INIT(bpf_prog_free_security, selinux_bpf_prog_free), + LSM_HOOK_INIT(bpf_prog_free, selinux_bpf_prog_free), #endif #ifdef CONFIG_PERF_EVENTS @@ -7383,7 +7384,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { #endif #ifdef CONFIG_BPF_SYSCALL LSM_HOOK_INIT(bpf_map_alloc_security, selinux_bpf_map_alloc), - LSM_HOOK_INIT(bpf_prog_alloc_security, selinux_bpf_prog_alloc), + LSM_HOOK_INIT(bpf_prog_load, selinux_bpf_prog_load), #endif #ifdef CONFIG_PERF_EVENTS LSM_HOOK_INIT(perf_event_alloc, selinux_perf_event_alloc), -- cgit v1.2.3 From a2431c7eabcf9bd5a1e7a1f7ecded40fdda4a8c5 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:07 -0800 Subject: bpf,lsm: Refactor bpf_map_alloc/bpf_map_free LSM hooks Similarly to bpf_prog_alloc LSM hook, rename and extend bpf_map_alloc hook into bpf_map_create, taking not just struct bpf_map, but also bpf_attr and bpf_token, to give a fuller context to LSMs. Unlike bpf_prog_alloc, there is no need to move the hook around, as it currently is firing right before allocating BPF map ID and FD, which seems to be a sweet spot. But like bpf_prog_alloc/bpf_prog_free combo, make sure that bpf_map_free LSM hook is called even if bpf_map_create hook returned error, as if few LSMs are combined together it could be that one LSM successfully allocated security blob for its needs, while subsequent LSM rejected BPF map creation. The former LSM would still need to free up LSM blob, so we need to ensure security_bpf_map_free() is called regardless of the outcome. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Paul Moore Link: https://lore.kernel.org/bpf/20240124022127.2379740-11-andrii@kernel.org --- include/linux/lsm_hook_defs.h | 5 +++-- include/linux/security.h | 6 ++++-- kernel/bpf/bpf_lsm.c | 6 +++--- kernel/bpf/syscall.c | 4 ++-- security/security.c | 16 ++++++++++------ security/selinux/hooks.c | 7 ++++--- 6 files changed, 26 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 370181aa685b..1be4d3ca6efb 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -404,8 +404,9 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule) LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size) LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode) LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog) -LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map) -LSM_HOOK(void, LSM_RET_VOID, bpf_map_free_security, struct bpf_map *map) +LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) +LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map) LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token) LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) diff --git a/include/linux/security.h b/include/linux/security.h index cb2932fce448..83fcdc974116 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2069,7 +2069,8 @@ struct bpf_token; extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size); extern int security_bpf_map(struct bpf_map *map, fmode_t fmode); extern int security_bpf_prog(struct bpf_prog *prog); -extern int security_bpf_map_alloc(struct bpf_map *map); +extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token); extern void security_bpf_map_free(struct bpf_map *map); extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token); @@ -2091,7 +2092,8 @@ static inline int security_bpf_prog(struct bpf_prog *prog) return 0; } -static inline int security_bpf_map_alloc(struct bpf_map *map) +static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { return 0; } diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 7ee0dd011de4..76976908b302 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -260,8 +260,8 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) BTF_SET_START(sleepable_lsm_hooks) BTF_ID(func, bpf_lsm_bpf) BTF_ID(func, bpf_lsm_bpf_map) -BTF_ID(func, bpf_lsm_bpf_map_alloc_security) -BTF_ID(func, bpf_lsm_bpf_map_free_security) +BTF_ID(func, bpf_lsm_bpf_map_create) +BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) BTF_ID(func, bpf_lsm_bpf_prog_free) @@ -359,7 +359,7 @@ BTF_ID(func, bpf_lsm_userns_create) BTF_SET_END(sleepable_lsm_hooks) BTF_SET_START(untrusted_lsm_hooks) -BTF_ID(func, bpf_lsm_bpf_map_free_security) +BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 3857a2a9ae7f..b2750b79ac80 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1318,9 +1318,9 @@ static int map_create(union bpf_attr *attr) attr->btf_vmlinux_value_type_id; } - err = security_bpf_map_alloc(map); + err = security_bpf_map_create(map, attr, token); if (err) - goto free_map; + goto free_map_sec; err = bpf_map_alloc_id(map); if (err) diff --git a/security/security.c b/security/security.c index eb159da4b146..26fcab35b6cd 100644 --- a/security/security.c +++ b/security/security.c @@ -5410,16 +5410,20 @@ int security_bpf_prog(struct bpf_prog *prog) } /** - * security_bpf_map_alloc() - Allocate a bpf map LSM blob - * @map: bpf map + * security_bpf_map_create() - Check if BPF map creation is allowed + * @map: BPF map object + * @attr: BPF syscall attributes used to create BPF map + * @token: BPF token used to grant user access * - * Initialize the security field inside bpf map. + * Do a check when the kernel creates a new BPF map. This is also the + * point where LSM blob is allocated for LSMs that need them. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_map_alloc(struct bpf_map *map) +int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { - return call_int_hook(bpf_map_alloc_security, 0, map); + return call_int_hook(bpf_map_create, 0, map, attr, token); } /** @@ -5448,7 +5452,7 @@ int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, */ void security_bpf_map_free(struct bpf_map *map) { - call_void_hook(bpf_map_free_security, map); + call_void_hook(bpf_map_free, map); } /** diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 6d64fb189b1b..3d336a7952f7 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6920,7 +6920,8 @@ static int selinux_bpf_prog(struct bpf_prog *prog) BPF__PROG_RUN, NULL); } -static int selinux_bpf_map_alloc(struct bpf_map *map) +static int selinux_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { struct bpf_security_struct *bpfsec; @@ -7325,7 +7326,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(bpf, selinux_bpf), LSM_HOOK_INIT(bpf_map, selinux_bpf_map), LSM_HOOK_INIT(bpf_prog, selinux_bpf_prog), - LSM_HOOK_INIT(bpf_map_free_security, selinux_bpf_map_free), + LSM_HOOK_INIT(bpf_map_free, selinux_bpf_map_free), LSM_HOOK_INIT(bpf_prog_free, selinux_bpf_prog_free), #endif @@ -7383,7 +7384,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(audit_rule_init, selinux_audit_rule_init), #endif #ifdef CONFIG_BPF_SYSCALL - LSM_HOOK_INIT(bpf_map_alloc_security, selinux_bpf_map_alloc), + LSM_HOOK_INIT(bpf_map_create, selinux_bpf_map_create), LSM_HOOK_INIT(bpf_prog_load, selinux_bpf_prog_load), #endif #ifdef CONFIG_PERF_EVENTS -- cgit v1.2.3 From f568a3d49af9aed813a184353592efe29b0e3d16 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 23 Jan 2024 18:21:08 -0800 Subject: bpf,lsm: Add BPF token LSM hooks Wire up bpf_token_create and bpf_token_free LSM hooks, which allow to allocate LSM security blob (we add `void *security` field to struct bpf_token for that), but also control who can instantiate BPF token. This follows existing pattern for BPF map and BPF prog. Also add security_bpf_token_allow_cmd() and security_bpf_token_capable() LSM hooks that allow LSM implementation to control and negate (if necessary) BPF token's delegation of a specific bpf_cmd and capability, respectively. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Paul Moore Link: https://lore.kernel.org/bpf/20240124022127.2379740-12-andrii@kernel.org --- include/linux/bpf.h | 3 +++ include/linux/lsm_hook_defs.h | 5 ++++ include/linux/security.h | 25 ++++++++++++++++++ kernel/bpf/bpf_lsm.c | 4 +++ kernel/bpf/token.c | 12 ++++++++- security/security.c | 60 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 4e146e9708be..b86bd15a051d 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1634,6 +1634,9 @@ struct bpf_token { u64 allowed_maps; u64 allowed_progs; u64 allowed_attachs; +#ifdef CONFIG_SECURITY + void *security; +#endif }; struct bpf_struct_ops_value; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 1be4d3ca6efb..cd6fbc7af3f8 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -410,6 +410,11 @@ LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map) LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token) LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) +LSM_HOOK(int, 0, bpf_token_create, struct bpf_token *token, union bpf_attr *attr, + struct path *path) +LSM_HOOK(void, LSM_RET_VOID, bpf_token_free, struct bpf_token *token) +LSM_HOOK(int, 0, bpf_token_cmd, const struct bpf_token *token, enum bpf_cmd cmd) +LSM_HOOK(int, 0, bpf_token_capable, const struct bpf_token *token, int cap) #endif /* CONFIG_BPF_SYSCALL */ LSM_HOOK(int, 0, locked_down, enum lockdown_reason what) diff --git a/include/linux/security.h b/include/linux/security.h index 83fcdc974116..15804af54f37 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -32,6 +32,7 @@ #include #include #include +#include #include struct linux_binprm; @@ -2075,6 +2076,11 @@ extern void security_bpf_map_free(struct bpf_map *map); extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token); extern void security_bpf_prog_free(struct bpf_prog *prog); +extern int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path); +extern void security_bpf_token_free(struct bpf_token *token); +extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd); +extern int security_bpf_token_capable(const struct bpf_token *token, int cap); #else static inline int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) @@ -2109,6 +2115,25 @@ static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr * static inline void security_bpf_prog_free(struct bpf_prog *prog) { } + +static inline int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) +{ + return 0; +} + +static inline void security_bpf_token_free(struct bpf_token *token) +{ } + +static inline int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + return 0; +} + +static inline int security_bpf_token_capable(const struct bpf_token *token, int cap) +{ + return 0; +} #endif /* CONFIG_SECURITY */ #endif /* CONFIG_BPF_SYSCALL */ diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 76976908b302..63b4dc495125 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -265,6 +265,10 @@ BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) BTF_ID(func, bpf_lsm_bpf_prog_free) +BTF_ID(func, bpf_lsm_bpf_token_create) +BTF_ID(func, bpf_lsm_bpf_token_free) +BTF_ID(func, bpf_lsm_bpf_token_cmd) +BTF_ID(func, bpf_lsm_bpf_token_capable) BTF_ID(func, bpf_lsm_bprm_check_security) BTF_ID(func, bpf_lsm_bprm_committed_creds) BTF_ID(func, bpf_lsm_bprm_committing_creds) diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index c13c73788d8c..64c568f47f69 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -7,6 +7,7 @@ #include #include #include +#include static bool bpf_ns_capable(struct user_namespace *ns, int cap) { @@ -21,6 +22,8 @@ bool bpf_token_capable(const struct bpf_token *token, int cap) userns = token ? token->userns : &init_user_ns; if (!bpf_ns_capable(userns, cap)) return false; + if (token && security_bpf_token_capable(token, cap) < 0) + return false; return true; } @@ -31,6 +34,7 @@ void bpf_token_inc(struct bpf_token *token) static void bpf_token_free(struct bpf_token *token) { + security_bpf_token_free(token); put_user_ns(token->userns); kfree(token); } @@ -193,6 +197,10 @@ int bpf_token_create(union bpf_attr *attr) token->allowed_progs = mnt_opts->delegate_progs; token->allowed_attachs = mnt_opts->delegate_attachs; + err = security_bpf_token_create(token, attr, &path); + if (err) + goto out_token; + fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { err = fd; @@ -237,7 +245,9 @@ bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) { if (!token) return false; - return token->allowed_cmds & (1ULL << cmd); + if (!(token->allowed_cmds & (1ULL << cmd))) + return false; + return security_bpf_token_cmd(token, cmd) == 0; } bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type) diff --git a/security/security.c b/security/security.c index 26fcab35b6cd..73e009e3d937 100644 --- a/security/security.c +++ b/security/security.c @@ -5444,6 +5444,55 @@ int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, return call_int_hook(bpf_prog_load, 0, prog, attr, token); } +/** + * security_bpf_token_create() - Check if creating of BPF token is allowed + * @token: BPF token object + * @attr: BPF syscall attributes used to create BPF token + * @path: path pointing to BPF FS mount point from which BPF token is created + * + * Do a check when the kernel instantiates a new BPF token object from BPF FS + * instance. This is also the point where LSM blob can be allocated for LSMs. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) +{ + return call_int_hook(bpf_token_create, 0, token, attr, path); +} + +/** + * security_bpf_token_cmd() - Check if BPF token is allowed to delegate + * requested BPF syscall command + * @token: BPF token object + * @cmd: BPF syscall command requested to be delegated by BPF token + * + * Do a check when the kernel decides whether provided BPF token should allow + * delegation of requested BPF syscall command. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + return call_int_hook(bpf_token_cmd, 0, token, cmd); +} + +/** + * security_bpf_token_capable() - Check if BPF token is allowed to delegate + * requested BPF-related capability + * @token: BPF token object + * @cap: capabilities requested to be delegated by BPF token + * + * Do a check when the kernel decides whether provided BPF token should allow + * delegation of requested BPF-related capabilities. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_capable(const struct bpf_token *token, int cap) +{ + return call_int_hook(bpf_token_capable, 0, token, cap); +} + /** * security_bpf_map_free() - Free a bpf map's LSM blob * @map: bpf map @@ -5465,6 +5514,17 @@ void security_bpf_prog_free(struct bpf_prog *prog) { call_void_hook(bpf_prog_free, prog); } + +/** + * security_bpf_token_free() - Free a BPF token's LSM blob + * @token: BPF token struct + * + * Clean up the security information stored inside BPF token. + */ +void security_bpf_token_free(struct bpf_token *token) +{ + call_void_hook(bpf_token_free, token); +} #endif /* CONFIG_BPF_SYSCALL */ /** -- cgit v1.2.3 From 83e897a961b801536dd1d736e9ede5b1ddb1c188 Mon Sep 17 00:00:00 2001 From: Ayala Beker Date: Tue, 2 Jan 2024 21:35:33 +0200 Subject: wifi: ieee80211: add definitions for negotiated TID to Link map Add the relevant definitions and structures for TID to Link mapping negotiation request/response/teardown according to P802.11be_D4.0. Signed-off-by: Ayala Beker Reviewed-by: Gregory Greenman Reviewed-by: Johannes Berg Signed-off-by: Miri Korenblit Link: https://msgid.link/20240102213313.9ef2b866c8c7.Ieaf7dadea9961e0edc55d19c99f0f9fbae591de6@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 83c4d060a559..eafa70e5ba94 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -1454,6 +1454,20 @@ struct ieee80211_mgmt { u8 max_tod_error; u8 max_toa_error; } __packed wnm_timing_msr; + struct { + u8 action_code; + u8 dialog_token; + u8 variable[]; + } __packed ttlm_req; + struct { + u8 action_code; + u8 dialog_token; + u8 status_code; + u8 variable[]; + } __packed ttlm_res; + struct { + u8 action_code; + } __packed ttlm_tear_down; } u; } __packed action; DECLARE_FLEX_ARRAY(u8, body); /* Generic frame body */ @@ -3357,6 +3371,8 @@ enum ieee80211_statuscode { WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 109, WLAN_STATUS_SAE_HASH_TO_ELEMENT = 126, WLAN_STATUS_SAE_PK = 127, + WLAN_STATUS_DENIED_TID_TO_LINK_MAPPING = 133, + WLAN_STATUS_PREF_TID_TO_LINK_MAPPING_SUGGESTED = 134, }; @@ -3682,6 +3698,7 @@ enum ieee80211_category { WLAN_CATEGORY_UNPROT_DMG = 20, WLAN_CATEGORY_VHT = 21, WLAN_CATEGORY_S1G = 22, + WLAN_CATEGORY_PROTECTED_EHT = 37, WLAN_CATEGORY_VENDOR_SPECIFIC_PROTECTED = 126, WLAN_CATEGORY_VENDOR_SPECIFIC = 127, }; @@ -3745,6 +3762,13 @@ enum ieee80211_unprotected_wnm_actioncode { WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE = 1, }; +/* Protected EHT action codes */ +enum ieee80211_protected_eht_actioncode { + WLAN_PROTECTED_EHT_ACTION_TTLM_REQ = 0, + WLAN_PROTECTED_EHT_ACTION_TTLM_RES = 1, + WLAN_PROTECTED_EHT_ACTION_TTLM_TEARDOWN = 2, +}; + /* Security key length */ enum ieee80211_key_len { WLAN_KEY_LEN_WEP40 = 5, @@ -4845,6 +4869,10 @@ struct ieee80211_multi_link_elem { #define IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS 0x000f #define IEEE80211_MLD_CAP_OP_SRS_SUPPORT 0x0010 #define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP 0x0060 +#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_NO_SUPP 0 +#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME 1 +#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_RESERVED 2 +#define IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_DIFF 3 #define IEEE80211_MLD_CAP_OP_FREQ_SEP_TYPE_IND 0x0f80 #define IEEE80211_MLD_CAP_OP_AAR_SUPPORT 0x1000 -- cgit v1.2.3 From f7660b3f584aadd25dde18aa1902488577a15863 Mon Sep 17 00:00:00 2001 From: Ayala Beker Date: Tue, 2 Jan 2024 21:35:37 +0200 Subject: wifi: mac80211: add support for negotiated TTLM request Update neg_ttlm and active_links according to the new mapping, and send a negotiated TID-to-link map request with the new mapping. Signed-off-by: Ayala Beker Reviewed-by: Gregory Greenman Reviewed-by: Johannes Berg Signed-off-by: Miri Korenblit Link: https://msgid.link/20240102213313.eeb385d771df.I2a5441c14421de884dbd93d1624ce7bb2c944833@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 37 +++++++++++++++ include/net/mac80211.h | 7 ++- net/mac80211/cfg.c | 12 +++++ net/mac80211/ieee80211_i.h | 8 ++++ net/mac80211/iface.c | 4 ++ net/mac80211/mlme.c | 114 +++++++++++++++++++++++++++++++++++++++++++++ net/mac80211/rx.c | 8 ++++ 7 files changed, 188 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index eafa70e5ba94..f0c068446c79 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -5024,6 +5024,43 @@ static inline u16 ieee80211_mle_get_eml_cap(const u8 *data) return get_unaligned_le16(common); } +/** + * ieee80211_mle_get_mld_capa_op - returns the MLD capabilities and operations. + * @data: pointer to the multi link EHT IE + * + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + * + * If the MLD capabilities and operations field is not present, 0 will be + * returned. + */ +static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + /* + * common points now at the beginning of + * ieee80211_mle_basic_common_info + */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)) + return 0; + + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA) + common += 2; + + return get_unaligned_le16(common); +} + /** * ieee80211_mle_size_ok - validate multi-link element size * @data: pointer to the element data diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 6490b92d5cc1..84cc66dd93c1 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -1810,9 +1810,11 @@ enum ieee80211_offload_flags { * @ps: power-save mode (STA only). This flag is NOT affected by * offchannel/dynamic_ps operations. * @aid: association ID number, valid only when @assoc is true - * @eml_cap: EML capabilities as described in P802.11be_D2.2 Figure 9-1002k. + * @eml_cap: EML capabilities as described in P802.11be_D4.1 Figure 9-1001j. * @eml_med_sync_delay: Medium Synchronization delay as described in - * P802.11be_D2.2 Figure 9-1002j. + * P802.11be_D4.1 Figure 9-1001i. + * @mld_capa_op: MLD Capabilities and Operations per P802.11be_D4.1 + * Figure 9-1001k * @arp_addr_list: List of IPv4 addresses for hardware ARP filtering. The * may filter ARP queries targeted for other addresses than listed here. * The driver must allow ARP queries targeted for all address listed here @@ -1837,6 +1839,7 @@ struct ieee80211_vif_cfg { u16 aid; u16 eml_cap; u16 eml_med_sync_delay; + u16 mld_capa_op; __be32 arp_addr_list[IEEE80211_BSS_ARP_ADDR_LIST_LEN]; int arp_addr_cnt; diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 489dd97f5172..1e8da6372da2 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -4966,6 +4966,17 @@ static int ieee80211_set_hw_timestamp(struct wiphy *wiphy, return local->ops->set_hw_timestamp(&local->hw, &sdata->vif, hwts); } +static int +ieee80211_set_ttlm(struct wiphy *wiphy, struct net_device *dev, + struct cfg80211_ttlm_params *params) +{ + struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + + lockdep_assert_wiphy(sdata->local->hw.wiphy); + + return ieee80211_req_neg_ttlm(sdata, params); +} + const struct cfg80211_ops mac80211_config_ops = { .add_virtual_intf = ieee80211_add_iface, .del_virtual_intf = ieee80211_del_iface, @@ -5078,4 +5089,5 @@ const struct cfg80211_ops mac80211_config_ops = { .mod_link_station = ieee80211_mod_link_station, .del_link_station = ieee80211_del_link_station, .set_hw_timestamp = ieee80211_set_hw_timestamp, + .set_ttlm = ieee80211_set_ttlm, }; diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 38755f6f6fa0..79a5067ce82b 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -579,6 +579,10 @@ struct ieee80211_if_managed { /* TID-to-link mapping support */ struct wiphy_delayed_work ttlm_work; struct ieee80211_adv_ttlm_info ttlm_info; + + /* dialog token enumerator for neg TTLM request */ + u8 dialog_token_alloc; + struct wiphy_delayed_work neg_ttlm_timeout_work; }; struct ieee80211_if_ibss { @@ -2605,6 +2609,10 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata, struct link_sta_info *link_sta); void ieee80211_process_neg_ttlm_req(struct ieee80211_sub_if_data *sdata, struct ieee80211_mgmt *mgmt, size_t len); +void ieee80211_process_neg_ttlm_res(struct ieee80211_sub_if_data *sdata, + struct ieee80211_mgmt *mgmt, size_t len); +int ieee80211_req_neg_ttlm(struct ieee80211_sub_if_data *sdata, + struct cfg80211_ttlm_params *params); void ieee80211_check_wbrf_support(struct ieee80211_local *local); void ieee80211_add_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef); diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 4a87d2d336ae..df314222c2c9 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -1554,6 +1554,10 @@ static void ieee80211_iface_process_skb(struct ieee80211_local *local, ieee80211_process_neg_ttlm_req(sdata, mgmt, skb->len); break; + case WLAN_PROTECTED_EHT_ACTION_TTLM_RES: + ieee80211_process_neg_ttlm_res(sdata, mgmt, + skb->len); + break; default: break; } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index b56f7de3b53f..5279af09fd53 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -46,6 +46,8 @@ #define IEEE80211_ADV_TTLM_SAFETY_BUFFER_MS msecs_to_jiffies(100) #define IEEE80211_ADV_TTLM_ST_UNDERFLOW 0xff00 +#define IEEE80211_NEG_TTLM_REQ_TIMEOUT (HZ / 5) + static int max_nullfunc_tries = 2; module_param(max_nullfunc_tries, int, 0644); MODULE_PARM_DESC(max_nullfunc_tries, @@ -3087,6 +3089,8 @@ static void ieee80211_set_disassoc(struct ieee80211_sub_if_data *sdata, memset(&sdata->u.mgd.ttlm_info, 0, sizeof(sdata->u.mgd.ttlm_info)); wiphy_delayed_work_cancel(sdata->local->hw.wiphy, &ifmgd->ttlm_work); + wiphy_delayed_work_cancel(sdata->local->hw.wiphy, + &ifmgd->neg_ttlm_timeout_work); ieee80211_vif_set_links(sdata, 0, 0); } @@ -4984,6 +4988,8 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, ieee80211_mle_get_eml_cap(eht_ml_elem->data + 1); sdata->vif.cfg.eml_med_sync_delay = ieee80211_mle_get_eml_med_sync_delay(eht_ml_elem->data + 1); + sdata->vif.cfg.mld_capa_op = + ieee80211_mle_get_mld_capa_op(eht_ml_elem->data + 1); } } @@ -6501,6 +6507,19 @@ static void ieee80211_apply_neg_ttlm(struct ieee80211_sub_if_data *sdata, sdata->vif.neg_ttlm.valid = true; } +static void ieee80211_neg_ttlm_timeout_work(struct wiphy *wiphy, + struct wiphy_work *work) +{ + struct ieee80211_sub_if_data *sdata = + container_of(work, struct ieee80211_sub_if_data, + u.mgd.neg_ttlm_timeout_work.work); + + sdata_info(sdata, + "No negotiated TTLM response from AP, disconnecting.\n"); + + __ieee80211_disconnect(sdata); +} + static void ieee80211_neg_ttlm_add_suggested_map(struct sk_buff *skb, struct ieee80211_neg_ttlm *neg_ttlm) @@ -6547,6 +6566,74 @@ ieee80211_neg_ttlm_add_suggested_map(struct sk_buff *skb, } } +static void +ieee80211_send_neg_ttlm_req(struct ieee80211_sub_if_data *sdata, + struct ieee80211_neg_ttlm *neg_ttlm, + u8 dialog_token) +{ + struct ieee80211_local *local = sdata->local; + struct ieee80211_mgmt *mgmt; + struct sk_buff *skb; + int hdr_len = offsetofend(struct ieee80211_mgmt, u.action.u.ttlm_req); + int ttlm_max_len = 2 + 1 + sizeof(struct ieee80211_ttlm_elem) + 1 + + 2 * 2 * IEEE80211_TTLM_NUM_TIDS; + + skb = dev_alloc_skb(local->tx_headroom + hdr_len + ttlm_max_len); + if (!skb) + return; + + skb_reserve(skb, local->tx_headroom); + mgmt = skb_put_zero(skb, hdr_len); + mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | + IEEE80211_STYPE_ACTION); + memcpy(mgmt->da, sdata->vif.cfg.ap_addr, ETH_ALEN); + memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN); + memcpy(mgmt->bssid, sdata->vif.cfg.ap_addr, ETH_ALEN); + + mgmt->u.action.category = WLAN_CATEGORY_PROTECTED_EHT; + mgmt->u.action.u.ttlm_req.action_code = + WLAN_PROTECTED_EHT_ACTION_TTLM_REQ; + mgmt->u.action.u.ttlm_req.dialog_token = dialog_token; + ieee80211_neg_ttlm_add_suggested_map(skb, neg_ttlm); + ieee80211_tx_skb(sdata, skb); +} + +int ieee80211_req_neg_ttlm(struct ieee80211_sub_if_data *sdata, + struct cfg80211_ttlm_params *params) +{ + struct ieee80211_neg_ttlm neg_ttlm = {}; + u8 i; + + if (!ieee80211_vif_is_mld(&sdata->vif) || + !(sdata->vif.cfg.mld_capa_op & + IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP)) + return -EINVAL; + + for (i = 0; i < IEEE80211_TTLM_NUM_TIDS; i++) { + if ((params->dlink[i] & ~sdata->vif.valid_links) || + (params->ulink[i] & ~sdata->vif.valid_links)) + return -EINVAL; + + neg_ttlm.downlink[i] = params->dlink[i]; + neg_ttlm.uplink[i] = params->ulink[i]; + } + + if (drv_can_neg_ttlm(sdata->local, sdata, &neg_ttlm) != + NEG_TTLM_RES_ACCEPT) + return -EINVAL; + + ieee80211_apply_neg_ttlm(sdata, neg_ttlm); + sdata->u.mgd.dialog_token_alloc++; + ieee80211_send_neg_ttlm_req(sdata, &sdata->vif.neg_ttlm, + sdata->u.mgd.dialog_token_alloc); + wiphy_delayed_work_cancel(sdata->local->hw.wiphy, + &sdata->u.mgd.neg_ttlm_timeout_work); + wiphy_delayed_work_queue(sdata->local->hw.wiphy, + &sdata->u.mgd.neg_ttlm_timeout_work, + IEEE80211_NEG_TTLM_REQ_TIMEOUT); + return 0; +} + static void ieee80211_send_neg_ttlm_res(struct ieee80211_sub_if_data *sdata, enum ieee80211_neg_ttlm_res ttlm_res, @@ -6740,6 +6827,29 @@ out: ieee80211_send_neg_ttlm_res(sdata, ttlm_res, dialog_token, &neg_ttlm); } +void ieee80211_process_neg_ttlm_res(struct ieee80211_sub_if_data *sdata, + struct ieee80211_mgmt *mgmt, size_t len) +{ + if (!ieee80211_vif_is_mld(&sdata->vif) || + mgmt->u.action.u.ttlm_req.dialog_token != + sdata->u.mgd.dialog_token_alloc) + return; + + wiphy_delayed_work_cancel(sdata->local->hw.wiphy, + &sdata->u.mgd.neg_ttlm_timeout_work); + + /* MLD station sends a TID to link mapping request, mainly to handle + * BTM (BSS transition management) request, in which case it needs to + * restrict the active links set. + * In this case it's not expected that the MLD AP will reject the + * negotiated TTLM request. + * This can be better implemented in the future, to handle request + * rejections. + */ + if (mgmt->u.action.u.ttlm_res.status_code != WLAN_STATUS_SUCCESS) + __ieee80211_disconnect(sdata); +} + void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb) { @@ -7369,6 +7479,8 @@ void ieee80211_sta_setup_sdata(struct ieee80211_sub_if_data *sdata) ieee80211_sta_handle_tspec_ac_params_wk); wiphy_delayed_work_init(&ifmgd->ttlm_work, ieee80211_tid_to_link_map_work); + wiphy_delayed_work_init(&ifmgd->neg_ttlm_timeout_work, + ieee80211_neg_ttlm_timeout_work); ifmgd->flags = 0; ifmgd->powersave = sdata->wdev.ps; @@ -8459,6 +8571,8 @@ void ieee80211_mgd_stop(struct ieee80211_sub_if_data *sdata) wiphy_delayed_work_cancel(sdata->local->hw.wiphy, &ifmgd->ml_reconf_work); wiphy_delayed_work_cancel(sdata->local->hw.wiphy, &ifmgd->ttlm_work); + wiphy_delayed_work_cancel(sdata->local->hw.wiphy, + &ifmgd->neg_ttlm_timeout_work); if (ifmgd->assoc_data) ieee80211_destroy_assoc_data(sdata, ASSOC_TIMEOUT); diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 75eb3e55eaec..615795c4b052 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -3773,6 +3773,14 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx) u.action.u.ttlm_req)) goto invalid; goto queue; + case WLAN_PROTECTED_EHT_ACTION_TTLM_RES: + if (sdata->vif.type != NL80211_IFTYPE_STATION) + break; + + if (len < offsetofend(typeof(*mgmt), + u.action.u.ttlm_res)) + goto invalid; + goto queue; default: break; } -- cgit v1.2.3 From ccb964b4ab1663ce92f389b72c052fc47a0ffdb9 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Tue, 2 Jan 2024 21:35:44 +0200 Subject: wifi: cfg80211: validate MLO connections better When going into an MLO connection, validate that the link IDs match what userspace indicated, and that the AP MLD addresses and capabilities are all matching between the links. Signed-off-by: Johannes Berg Reviewed-by: Gregory Greenman Signed-off-by: Miri Korenblit Link: https://msgid.link/20240102213313.ff83c034cb9a.I9962db0bfa8c73b37b8d5b59a3fad7f02f2129ae@changeid [roll in extra fix from Miri to actually check the return value] Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 24 ++++++++ net/wireless/core.h | 3 +- net/wireless/mlme.c | 136 +++++++++++++++++++++++++++++++++++++++++----- net/wireless/nl80211.c | 3 +- net/wireless/sme.c | 3 +- 5 files changed, 152 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index f0c068446c79..a70388ae3a7b 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4935,6 +4935,30 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) return sizeof(*mle) + common + mle->variable[0]; } +/** + * ieee80211_mle_get_link_id - returns the link ID + * @data: the basic multi link element + * + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + * + * If the BSS link ID can't be found, -1 will be returned + */ +static inline int ieee80211_mle_get_link_id(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + /* common points now at the beginning of ieee80211_mle_basic_common_info */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_LINK_ID)) + return -1; + + return *common; +} + /** * ieee80211_mle_get_bss_param_ch_cnt - returns the BSS parameter change count * @mle: the basic multi link element diff --git a/net/wireless/core.h b/net/wireless/core.h index 13657a85cf61..30434551b377 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -362,7 +362,8 @@ int cfg80211_mlme_auth(struct cfg80211_registered_device *rdev, struct cfg80211_auth_request *req); int cfg80211_mlme_assoc(struct cfg80211_registered_device *rdev, struct net_device *dev, - struct cfg80211_assoc_request *req); + struct cfg80211_assoc_request *req, + struct netlink_ext_ack *extack); int cfg80211_mlme_deauth(struct cfg80211_registered_device *rdev, struct net_device *dev, const u8 *bssid, const u8 *ie, int ie_len, u16 reason, diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index 43ba7891e2a3..4052041a19ea 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -4,7 +4,7 @@ * * Copyright (c) 2009, Jouni Malinen * Copyright (c) 2015 Intel Deutschland GmbH - * Copyright (C) 2019-2020, 2022-2023 Intel Corporation + * Copyright (C) 2019-2020, 2022-2024 Intel Corporation */ #include @@ -325,28 +325,136 @@ void cfg80211_oper_and_vht_capa(struct ieee80211_vht_cap *vht_capa, p1[i] &= p2[i]; } -/* Note: caller must cfg80211_put_bss() regardless of result */ -int cfg80211_mlme_assoc(struct cfg80211_registered_device *rdev, - struct net_device *dev, - struct cfg80211_assoc_request *req) +static int +cfg80211_mlme_check_mlo_compat(const struct ieee80211_multi_link_elem *mle_a, + const struct ieee80211_multi_link_elem *mle_b, + struct netlink_ext_ack *extack) { - struct wireless_dev *wdev = dev->ieee80211_ptr; - int err, i, j; + const struct ieee80211_mle_basic_common_info *common_a, *common_b; - lockdep_assert_wiphy(wdev->wiphy); + common_a = (const void *)mle_a->variable; + common_b = (const void *)mle_b->variable; + + if (memcmp(common_a->mld_mac_addr, common_b->mld_mac_addr, ETH_ALEN)) { + NL_SET_ERR_MSG(extack, "AP MLD address mismatch"); + return -EINVAL; + } + + if (ieee80211_mle_get_eml_med_sync_delay((const u8 *)mle_a) != + ieee80211_mle_get_eml_med_sync_delay((const u8 *)mle_b)) { + NL_SET_ERR_MSG(extack, "link EML medium sync delay mismatch"); + return -EINVAL; + } + + if (ieee80211_mle_get_eml_cap((const u8 *)mle_a) != + ieee80211_mle_get_eml_cap((const u8 *)mle_b)) { + NL_SET_ERR_MSG(extack, "link EML capabilities mismatch"); + return -EINVAL; + } + + if (ieee80211_mle_get_mld_capa_op((const u8 *)mle_a) != + ieee80211_mle_get_mld_capa_op((const u8 *)mle_b)) { + NL_SET_ERR_MSG(extack, "link MLD capabilities/ops mismatch"); + return -EINVAL; + } + + return 0; +} + +static int cfg80211_mlme_check_mlo(struct net_device *dev, + struct cfg80211_assoc_request *req, + struct netlink_ext_ack *extack) +{ + const struct ieee80211_multi_link_elem *mles[ARRAY_SIZE(req->links)] = {}; + int i; + + if (req->link_id < 0) + return 0; + + if (!req->links[req->link_id].bss) { + NL_SET_ERR_MSG(extack, "no BSS for assoc link"); + return -EINVAL; + } + + rcu_read_lock(); + for (i = 0; i < ARRAY_SIZE(req->links); i++) { + const struct cfg80211_bss_ies *ies; + const struct element *ml; - for (i = 1; i < ARRAY_SIZE(req->links); i++) { if (!req->links[i].bss) continue; - for (j = 0; j < i; j++) { - if (req->links[i].bss == req->links[j].bss) - return -EINVAL; + + if (ether_addr_equal(req->links[i].bss->bssid, dev->dev_addr)) { + NL_SET_ERR_MSG(extack, "BSSID must not be our address"); + req->links[i].error = -EINVAL; + goto error; } - if (ether_addr_equal(req->links[i].bss->bssid, dev->dev_addr)) - return -EINVAL; + ies = rcu_dereference(req->links[i].bss->ies); + ml = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_MULTI_LINK, + ies->data, ies->len); + if (!ml) { + NL_SET_ERR_MSG(extack, "MLO BSS w/o ML element"); + req->links[i].error = -EINVAL; + goto error; + } + + if (!ieee80211_mle_type_ok(ml->data + 1, + IEEE80211_ML_CONTROL_TYPE_BASIC, + ml->datalen - 1)) { + NL_SET_ERR_MSG(extack, "BSS with invalid ML element"); + req->links[i].error = -EINVAL; + goto error; + } + + mles[i] = (const void *)(ml->data + 1); + + if (ieee80211_mle_get_link_id((const u8 *)mles[i]) != i) { + NL_SET_ERR_MSG(extack, "link ID mismatch"); + req->links[i].error = -EINVAL; + goto error; + } + } + + if (WARN_ON(!mles[req->link_id])) + goto error; + + for (i = 0; i < ARRAY_SIZE(req->links); i++) { + if (i == req->link_id || !req->links[i].bss) + continue; + + if (WARN_ON(!mles[i])) + goto error; + + if (cfg80211_mlme_check_mlo_compat(mles[req->link_id], mles[i], + extack)) { + req->links[i].error = -EINVAL; + goto error; + } } + rcu_read_unlock(); + return 0; +error: + rcu_read_unlock(); + return -EINVAL; +} + +/* Note: caller must cfg80211_put_bss() regardless of result */ +int cfg80211_mlme_assoc(struct cfg80211_registered_device *rdev, + struct net_device *dev, + struct cfg80211_assoc_request *req, + struct netlink_ext_ack *extack) +{ + struct wireless_dev *wdev = dev->ieee80211_ptr; + int err; + + lockdep_assert_wiphy(wdev->wiphy); + + err = cfg80211_mlme_check_mlo(dev, req, extack); + if (err) + return err; + if (wdev->connected && (!req->prev_bssid || !ether_addr_equal(wdev->u.client.connected_addr, req->prev_bssid))) diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index b267317aa33c..0809f721f045 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -11245,7 +11245,8 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info) struct nlattr *link; int rem = 0; - err = cfg80211_mlme_assoc(rdev, dev, &req); + err = cfg80211_mlme_assoc(rdev, dev, &req, + info->extack); if (!err && info->attrs[NL80211_ATTR_SOCKET_OWNER]) { dev->ieee80211_ptr->conn_owner_nlportid = diff --git a/net/wireless/sme.c b/net/wireless/sme.c index 195c8532734b..82e3ce42206c 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -209,7 +209,8 @@ static int cfg80211_conn_do_work(struct wireless_dev *wdev, if (!req.bss) { err = -ENOENT; } else { - err = cfg80211_mlme_assoc(rdev, wdev->netdev, &req); + err = cfg80211_mlme_assoc(rdev, wdev->netdev, + &req, NULL); cfg80211_put_bss(&rdev->wiphy, req.bss); } -- cgit v1.2.3 From 73697f0acc773a357946a3c5a917bfb4c85128a3 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Tue, 23 Jan 2024 09:09:11 -0600 Subject: power: supply: bq27xxx: Add devm action to free IDA Use a device lifecycle managed action to free the IDA. This helps prevent mistakes like freeing out of order in cleanup functions and forgetting to free on error paths. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240123150914.308510-2-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery_i2c.c | 35 +++++++++++++++--------------- include/linux/power/bq27xxx_battery.h | 1 - 2 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c index 86ce13a8ab9d..019f29d13d28 100644 --- a/drivers/power/supply/bq27xxx_battery_i2c.c +++ b/drivers/power/supply/bq27xxx_battery_i2c.c @@ -135,28 +135,39 @@ static int bq27xxx_battery_i2c_bulk_write(struct bq27xxx_device_info *di, return 0; } +static void bq27xxx_battery_i2c_devm_ida_free(void *data) +{ + int num = (long)data; + + ida_free(&battery_id, num); +} + static int bq27xxx_battery_i2c_probe(struct i2c_client *client) { const struct i2c_device_id *id = i2c_client_get_device_id(client); struct bq27xxx_device_info *di; int ret; char *name; - int num; + long num; /* Get new ID for the new battery device */ num = ida_alloc(&battery_id, GFP_KERNEL); if (num < 0) return num; + ret = devm_add_action_or_reset(&client->dev, + bq27xxx_battery_i2c_devm_ida_free, + (void *)num); + if (ret) + return ret; - name = devm_kasprintf(&client->dev, GFP_KERNEL, "%s-%d", id->name, num); + name = devm_kasprintf(&client->dev, GFP_KERNEL, "%s-%ld", id->name, num); if (!name) - goto err_mem; + return -ENOMEM; di = devm_kzalloc(&client->dev, sizeof(*di), GFP_KERNEL); if (!di) - goto err_mem; + return -ENOMEM; - di->id = num; di->dev = &client->dev; di->chip = id->driver_data; di->name = name; @@ -168,7 +179,7 @@ static int bq27xxx_battery_i2c_probe(struct i2c_client *client) ret = bq27xxx_battery_setup(di); if (ret) - goto err_failed; + return ret; /* Schedule a polling after about 1 min */ schedule_delayed_work(&di->work, 60 * HZ); @@ -185,19 +196,11 @@ static int bq27xxx_battery_i2c_probe(struct i2c_client *client) "Unable to register IRQ %d error %d\n", client->irq, ret); bq27xxx_battery_teardown(di); - goto err_failed; + return ret; } } return 0; - -err_mem: - ret = -ENOMEM; - -err_failed: - ida_free(&battery_id, num); - - return ret; } static void bq27xxx_battery_i2c_remove(struct i2c_client *client) @@ -206,8 +209,6 @@ static void bq27xxx_battery_i2c_remove(struct i2c_client *client) free_irq(client->irq, di); bq27xxx_battery_teardown(di); - - ida_free(&battery_id, di->id); } static const struct i2c_device_id bq27xxx_i2c_id_table[] = { diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index 7d8025fb74b7..b9e5bd2b42d3 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -61,7 +61,6 @@ struct bq27xxx_reg_cache { struct bq27xxx_device_info { struct device *dev; - int id; enum bq27xxx_chip chip; u32 opts; const char *name; -- cgit v1.2.3 From 7ae215ee7bb855f13c80565470fc7f67db4ba82f Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Thu, 25 Jan 2024 21:36:59 +0100 Subject: net: phy: add support for PHY LEDs polarity modes Add support for PHY LEDs polarity modes. Some PHY require LED to be set to active low to be turned ON. Adds support for this by declaring active-low property in DT. PHY driver needs to declare .led_polarity_set() to configure LED polarity modes. Function will pass the index with the LED index and a bitmap with all the required modes to set. Current supported modes are: - active-low with the flag PHY_LED_ACTIVE_LOW. LED is set to active-low to turn it ON. - inactive-high-impedance with the flag PHY_LED_INACTIVE_HIGH_IMPEDANCE. LED is set to high impedance to turn it OFF. Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20240125203702.4552-4-ansuelsmth@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy_device.c | 16 ++++++++++++++++ include/linux/phy.h | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 3611ea64875e..dd778c7fde1d 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -3097,6 +3097,7 @@ static int of_phy_led(struct phy_device *phydev, struct device *dev = &phydev->mdio.dev; struct led_init_data init_data = {}; struct led_classdev *cdev; + unsigned long modes = 0; struct phy_led *phyled; u32 index; int err; @@ -3114,6 +3115,21 @@ static int of_phy_led(struct phy_device *phydev, if (index > U8_MAX) return -EINVAL; + if (of_property_read_bool(led, "active-low")) + set_bit(PHY_LED_ACTIVE_LOW, &modes); + if (of_property_read_bool(led, "inactive-high-impedance")) + set_bit(PHY_LED_INACTIVE_HIGH_IMPEDANCE, &modes); + + if (modes) { + /* Return error if asked to set polarity modes but not supported */ + if (!phydev->drv->led_polarity_set) + return -EINVAL; + + err = phydev->drv->led_polarity_set(phydev, index, modes); + if (err) + return err; + } + phyled->index = index; if (phydev->drv->led_brightness_set) cdev->brightness_set_blocking = phy_led_set_brightness; diff --git a/include/linux/phy.h b/include/linux/phy.h index 684efaeca07c..c9994a59ca2e 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -852,6 +852,15 @@ struct phy_plca_status { bool pst; }; +/* Modes for PHY LED configuration */ +enum phy_led_modes { + PHY_LED_ACTIVE_LOW = 0, + PHY_LED_INACTIVE_HIGH_IMPEDANCE = 1, + + /* keep it last */ + __PHY_LED_MODES_NUM, +}; + /** * struct phy_led: An LED driven by the PHY * @@ -1145,6 +1154,19 @@ struct phy_driver { int (*led_hw_control_get)(struct phy_device *dev, u8 index, unsigned long *rules); + /** + * @led_polarity_set: Set the LED polarity modes + * @dev: PHY device which has the LED + * @index: Which LED of the PHY device + * @modes: bitmap of LED polarity modes + * + * Configure LED with all the required polarity modes in @modes + * to make it correctly turn ON or OFF. + * + * Returns 0, or an error code. + */ + int (*led_polarity_set)(struct phy_device *dev, int index, + unsigned long modes); }; #define to_phy_driver(d) container_of(to_mdio_common_driver(d), \ struct phy_driver, mdiodrv) -- cgit v1.2.3 From 4ca79255101b9edd6ac874d39361fd2b52927af0 Mon Sep 17 00:00:00 2001 From: Alexander Tsoy Date: Fri, 26 Jan 2024 00:16:35 +0300 Subject: usb: audio-v2: Correct comments for struct uac_clock_selector_descriptor This is likely a copy-paste error. Replace "Source" with "Selector" where appropriate. Fixes: 7e847894039d7 ("linux/usb/audio.h: split header") Signed-off-by: Alexander Tsoy Link: https://lore.kernel.org/r/20240125211635.30140-1-alexander@tsoy.me Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/audio-v2.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/audio-v2.h b/include/linux/usb/audio-v2.h index ca796dc1a984..6e5555610010 100644 --- a/include/linux/usb/audio-v2.h +++ b/include/linux/usb/audio-v2.h @@ -82,7 +82,7 @@ struct uac_clock_source_descriptor { #define UAC_CLOCK_SOURCE_TYPE_INT_PROG 0x3 #define UAC_CLOCK_SOURCE_SYNCED_TO_SOF (1 << 2) -/* 4.7.2.2 Clock Source Descriptor */ +/* 4.7.2.2 Clock Selector Descriptor */ struct uac_clock_selector_descriptor { __u8 bLength; @@ -91,7 +91,7 @@ struct uac_clock_selector_descriptor { __u8 bClockID; __u8 bNrInPins; __u8 baCSourceID[]; - /* bmControls and iClockSource omitted */ + /* bmControls and iClockSelector omitted */ } __attribute__((packed)); /* 4.7.2.3 Clock Multiplier Descriptor */ -- cgit v1.2.3 From 5de5f1e292e56fe4b8d28923d325f4c16f3766cf Mon Sep 17 00:00:00 2001 From: Stanley Chang Date: Wed, 13 Dec 2023 11:10:06 +0800 Subject: phy: core: add notify_connect and notify_disconnect callback In Realtek SoC, the parameter of usb phy is designed to be able to do dynamic tuning based in the port status. Therefore, add a notify callback of phy driver when usb connection/disconnection change. Signed-off-by: Stanley Chang Link: https://lore.kernel.org/r/20231213031203.4911-1-stanley_chang@realtek.com Signed-off-by: Greg Kroah-Hartman --- drivers/phy/phy-core.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/phy/phy.h | 21 +++++++++++++++++++++ 2 files changed, 68 insertions(+) (limited to 'include/linux') diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c index d9be6a4d5383..2e8b07eb637a 100644 --- a/drivers/phy/phy-core.c +++ b/drivers/phy/phy-core.c @@ -489,6 +489,53 @@ int phy_calibrate(struct phy *phy) } EXPORT_SYMBOL_GPL(phy_calibrate); +/** + * phy_notify_connect() - phy connect notification + * @phy: the phy returned by phy_get() + * @port: the port index for connect + * + * If the phy needs to get connection status, the callback can be used. + * Returns: %0 if successful, a negative error code otherwise + */ +int phy_notify_connect(struct phy *phy, int port) +{ + int ret; + + if (!phy || !phy->ops->connect) + return 0; + + mutex_lock(&phy->mutex); + ret = phy->ops->connect(phy, port); + mutex_unlock(&phy->mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(phy_notify_connect); + +/** + * phy_notify_disconnect() - phy disconnect notification + * @phy: the phy returned by phy_get() + * @port: the port index for disconnect + * + * If the phy needs to get connection status, the callback can be used. + * + * Returns: %0 if successful, a negative error code otherwise + */ +int phy_notify_disconnect(struct phy *phy, int port) +{ + int ret; + + if (!phy || !phy->ops->disconnect) + return 0; + + mutex_lock(&phy->mutex); + ret = phy->ops->disconnect(phy, port); + mutex_unlock(&phy->mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(phy_notify_disconnect); + /** * phy_configure() - Changes the phy parameters * @phy: the phy returned by phy_get() diff --git a/include/linux/phy/phy.h b/include/linux/phy/phy.h index f6d607ef0e80..aa76609ba258 100644 --- a/include/linux/phy/phy.h +++ b/include/linux/phy/phy.h @@ -122,6 +122,11 @@ struct phy_ops { union phy_configure_opts *opts); int (*reset)(struct phy *phy); int (*calibrate)(struct phy *phy); + + /* notify phy connect status change */ + int (*connect)(struct phy *phy, int port); + int (*disconnect)(struct phy *phy, int port); + void (*release)(struct phy *phy); struct module *owner; }; @@ -243,6 +248,8 @@ static inline enum phy_mode phy_get_mode(struct phy *phy) } int phy_reset(struct phy *phy); int phy_calibrate(struct phy *phy); +int phy_notify_connect(struct phy *phy, int port); +int phy_notify_disconnect(struct phy *phy, int port); static inline int phy_get_bus_width(struct phy *phy) { return phy->attrs.bus_width; @@ -396,6 +403,20 @@ static inline int phy_calibrate(struct phy *phy) return -ENOSYS; } +static inline int phy_notify_connect(struct phy *phy, int index) +{ + if (!phy) + return 0; + return -ENOSYS; +} + +static inline int phy_notify_disconnect(struct phy *phy, int index) +{ + if (!phy) + return 0; + return -ENOSYS; +} + static inline int phy_configure(struct phy *phy, union phy_configure_opts *opts) { -- cgit v1.2.3 From 7494d4bc8e32a9480fd56b018db8e404b54b24e6 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:14 +0000 Subject: usb: typec: altmodes: add typec_cable_ops to typec_altmode Add typec_cable_ops struct for enter, exit, and vdm. The struct is added to typec_altmode so port alt modes can have access to partner and cable specific callbacks, and alt mode drivers can specify operations over SOP' and SOP'' without modifying the existing API. typec_port_register_cable_ops is added as a new symbol for port drivers to use to register cable operations to their registered port alt modes. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-15-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/bus.c | 102 ++++++++++++++++++++++++++++++++++++++ drivers/usb/typec/class.c | 19 +++++++ include/linux/usb/typec.h | 4 ++ include/linux/usb/typec_altmode.h | 20 ++++++++ 4 files changed, 145 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c index e95ec7e382bb..6ea103e1abae 100644 --- a/drivers/usb/typec/bus.c +++ b/drivers/usb/typec/bus.c @@ -244,6 +244,108 @@ typec_altmode_get_partner(struct typec_altmode *adev) } EXPORT_SYMBOL_GPL(typec_altmode_get_partner); +/* -------------------------------------------------------------------------- */ +/* API for cable alternate modes */ + +/** + * typec_cable_altmode_enter - Enter Mode + * @adev: The alternate mode + * @sop: Cable plug target for Enter Mode command + * @vdo: VDO for the Enter Mode command + * + * Alternate mode drivers use this function to enter mode on the cable plug. + * If the alternate mode does not require VDO, @vdo must be NULL. + */ +int typec_cable_altmode_enter(struct typec_altmode *adev, enum typec_plug_index sop, u32 *vdo) +{ + struct altmode *partner = to_altmode(adev)->partner; + struct typec_altmode *pdev; + + if (!adev || adev->active) + return 0; + + if (!partner) + return -ENODEV; + + pdev = &partner->adev; + + if (!pdev->active) + return -EPERM; + + if (!pdev->cable_ops || !pdev->cable_ops->enter) + return -EOPNOTSUPP; + + return pdev->cable_ops->enter(pdev, sop, vdo); +} +EXPORT_SYMBOL_GPL(typec_cable_altmode_enter); + +/** + * typec_cable_altmode_exit - Exit Mode + * @adev: The alternate mode + * @sop: Cable plug target for Exit Mode command + * + * The alternate mode drivers use this function to exit mode on the cable plug. + */ +int typec_cable_altmode_exit(struct typec_altmode *adev, enum typec_plug_index sop) +{ + struct altmode *partner = to_altmode(adev)->partner; + struct typec_altmode *pdev; + + if (!adev || !adev->active) + return 0; + + if (!partner) + return -ENODEV; + + pdev = &partner->adev; + + if (!pdev->cable_ops || !pdev->cable_ops->exit) + return -EOPNOTSUPP; + + return pdev->cable_ops->exit(pdev, sop); +} +EXPORT_SYMBOL_GPL(typec_cable_altmode_exit); + +/** + * typec_cable_altmode_vdm - Send Vendor Defined Messages (VDM) between the cable plug and port. + * @adev: Alternate mode handle + * @sop: Cable plug target for VDM + * @header: VDM Header + * @vdo: Array of Vendor Defined Data Objects + * @count: Number of Data Objects + * + * The alternate mode drivers use this function for SVID specific communication + * with the cable plugs. The port drivers use it to deliver the Structured VDMs + * received from the cable plugs to the alternate mode drivers. + */ +int typec_cable_altmode_vdm(struct typec_altmode *adev, enum typec_plug_index sop, + const u32 header, const u32 *vdo, int count) +{ + struct altmode *altmode; + struct typec_altmode *pdev; + + if (!adev) + return 0; + + altmode = to_altmode(adev); + + if (is_typec_plug(adev->dev.parent)) { + if (!altmode->partner) + return -ENODEV; + pdev = &altmode->partner->adev; + } else { + if (!altmode->plug[sop]) + return -ENODEV; + pdev = &altmode->plug[sop]->adev; + } + + if (!pdev->cable_ops || !pdev->cable_ops->vdm) + return -EOPNOTSUPP; + + return pdev->cable_ops->vdm(pdev, sop, header, vdo, count); +} +EXPORT_SYMBOL_GPL(typec_cable_altmode_vdm); + /* -------------------------------------------------------------------------- */ /* API for the alternate mode drivers */ diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 015aa9253353..8fc9795d6bd4 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -2280,6 +2280,25 @@ void typec_port_register_altmodes(struct typec_port *port, } EXPORT_SYMBOL_GPL(typec_port_register_altmodes); +/** + * typec_port_register_cable_ops - Register typec_cable_ops to port altmodes + * @altmodes: USB Type-C Port's altmode vector + * @max_altmodes: The maximum number of alt modes supported by the port + * @ops: Cable alternate mode vector + */ +void typec_port_register_cable_ops(struct typec_altmode **altmodes, int max_altmodes, + const struct typec_cable_ops *ops) +{ + int i; + + for (i = 0; i < max_altmodes; i++) { + if (!altmodes[i]) + return; + altmodes[i]->cable_ops = ops; + } +} +EXPORT_SYMBOL_GPL(typec_port_register_cable_ops); + /** * typec_register_port - Register a USB Type-C Port * @parent: Parent device diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index a05d6f6f2536..38f93d72fd1b 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -18,6 +18,7 @@ struct typec_cable; struct typec_plug; struct typec_port; struct typec_altmode_ops; +struct typec_cable_ops; struct fwnode_handle; struct device; @@ -157,6 +158,9 @@ void typec_port_register_altmodes(struct typec_port *port, const struct typec_altmode_ops *ops, void *drvdata, struct typec_altmode **altmodes, size_t n); +void typec_port_register_cable_ops(struct typec_altmode **altmodes, int max_altmodes, + const struct typec_cable_ops *ops); + void typec_unregister_altmode(struct typec_altmode *altmode); struct typec_port *typec_altmode2port(struct typec_altmode *alt); diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index 28aeef8f9e7b..72ec8058543a 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -20,6 +20,7 @@ struct typec_altmode_ops; * @active: Tells has the mode been entered or not * @desc: Optional human readable description of the mode * @ops: Operations vector from the driver + * @cable_ops: Cable operations vector from the driver. */ struct typec_altmode { struct device dev; @@ -30,6 +31,7 @@ struct typec_altmode { char *desc; const struct typec_altmode_ops *ops; + const struct typec_cable_ops *cable_ops; }; #define to_typec_altmode(d) container_of(d, struct typec_altmode, dev) @@ -75,6 +77,24 @@ int typec_altmode_notify(struct typec_altmode *altmode, unsigned long conf, const struct typec_altmode * typec_altmode_get_partner(struct typec_altmode *altmode); +/** + * struct typec_cable_ops - Cable alternate mode operations vector + * @enter: Operations to be executed with Enter Mode Command + * @exit: Operations to be executed with Exit Mode Command + * @vdm: Callback for SVID specific commands + */ +struct typec_cable_ops { + int (*enter)(struct typec_altmode *altmode, enum typec_plug_index sop, u32 *vdo); + int (*exit)(struct typec_altmode *altmode, enum typec_plug_index sop); + int (*vdm)(struct typec_altmode *altmode, enum typec_plug_index sop, + const u32 hdr, const u32 *vdo, int cnt); +}; + +int typec_cable_altmode_enter(struct typec_altmode *altmode, enum typec_plug_index sop, u32 *vdo); +int typec_cable_altmode_exit(struct typec_altmode *altmode, enum typec_plug_index sop); +int typec_cable_altmode_vdm(struct typec_altmode *altmode, enum typec_plug_index sop, + const u32 header, const u32 *vdo, int count); + /* * These are the connector states (USB, Safe and Alt Mode) defined in USB Type-C * Specification. SVID specific connector states are expected to follow and -- cgit v1.2.3 From 231b7318413cef0f8e5c2ca8db1a95b666c25d70 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:15 +0000 Subject: usb: typec: altmodes: add svdm version info for typec cables Add typec_cable_set_svdm_version and typec_get_cable_svdm version symbols. Cables can operate under a lower PD revision than the port partner, and the max SVDM version is tied to the PD revision. So, typec_cable maintains its own svdm_version. Add typec_altmode_get_cable_svdm_version to return the cable's negotiated svdm_version for altmode drivers to use. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-16-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/class.c | 40 +++++++++++++++++++++++++++++++++++++++ drivers/usb/typec/class.h | 1 + include/linux/usb/typec.h | 3 +++ include/linux/usb/typec_altmode.h | 10 ++++++++++ 4 files changed, 54 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 8fc9795d6bd4..caea2b829980 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -2131,6 +2131,46 @@ int typec_get_negotiated_svdm_version(struct typec_port *port) } EXPORT_SYMBOL_GPL(typec_get_negotiated_svdm_version); +/** + * typec_get_cable_svdm_version - Get cable negotiated SVDM Version + * @port: USB Type-C Port. + * + * Get the negotiated SVDM Version for the cable. The Version is set to the port + * default value based on the PD Revision during cable registration, and updated + * after a successful Discover Identity if the negotiated value is less than the + * default. + * + * Returns usb_pd_svdm_ver if the cable has been registered otherwise -ENODEV. + */ +int typec_get_cable_svdm_version(struct typec_port *port) +{ + enum usb_pd_svdm_ver svdm_version; + struct device *cable_dev; + + cable_dev = device_find_child(&port->dev, NULL, cable_match); + if (!cable_dev) + return -ENODEV; + + svdm_version = to_typec_cable(cable_dev)->svdm_version; + put_device(cable_dev); + + return svdm_version; +} +EXPORT_SYMBOL_GPL(typec_get_cable_svdm_version); + +/** + * typec_cable_set_svdm_version - Set negotiated Structured VDM (SVDM) Version + * @cable: USB Type-C Active Cable that supports SVDM + * @svdm_version: Negotiated SVDM Version + * + * This routine is used to save the negotiated SVDM Version. + */ +void typec_cable_set_svdm_version(struct typec_cable *cable, enum usb_pd_svdm_ver svdm_version) +{ + cable->svdm_version = svdm_version; +} +EXPORT_SYMBOL_GPL(typec_cable_set_svdm_version); + /** * typec_get_drvdata - Return private driver data pointer * @port: USB Type-C port diff --git a/drivers/usb/typec/class.h b/drivers/usb/typec/class.h index c36761ba3f59..759b98355eeb 100644 --- a/drivers/usb/typec/class.h +++ b/drivers/usb/typec/class.h @@ -23,6 +23,7 @@ struct typec_cable { struct usb_pd_identity *identity; unsigned int active:1; u16 pd_revision; /* 0300H = "3.0" */ + enum usb_pd_svdm_ver svdm_version; }; struct typec_partner { diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 38f93d72fd1b..b35b427561ab 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -337,6 +337,9 @@ void typec_partner_set_svdm_version(struct typec_partner *partner, enum usb_pd_svdm_ver svdm_version); int typec_get_negotiated_svdm_version(struct typec_port *port); +int typec_get_cable_svdm_version(struct typec_port *port); +void typec_cable_set_svdm_version(struct typec_cable *cable, enum usb_pd_svdm_ver svdm_version); + struct usb_power_delivery *typec_partner_usb_power_delivery_register(struct typec_partner *partner, struct usb_power_delivery_desc *desc); diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index 72ec8058543a..b3c0866ea70f 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -95,6 +95,16 @@ int typec_cable_altmode_exit(struct typec_altmode *altmode, enum typec_plug_inde int typec_cable_altmode_vdm(struct typec_altmode *altmode, enum typec_plug_index sop, const u32 header, const u32 *vdo, int count); +/** + * typec_altmode_get_cable_svdm_version - Get negotiated SVDM version for cable plug + * @altmode: Handle to the alternate mode + */ +static inline int +typec_altmode_get_cable_svdm_version(struct typec_altmode *altmode) +{ + return typec_get_cable_svdm_version(typec_altmode2port(altmode)); +} + /* * These are the connector states (USB, Safe and Alt Mode) defined in USB Type-C * Specification. SVID specific connector states are expected to follow and -- cgit v1.2.3 From 59cd27a0cab1ceddcc4251309fd3643921ed9ab9 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:16 +0000 Subject: usb: typec: tcpci: add cable_comm_capable attribute Add cable_comm_capable to tcpci_data for tcpci drivers to indicate that the port tcpc is capable of communicating to cables over SOP. A corresponding tcpci callback tcpci_cable_comm_capable returns this value. The tcpm will primarily use this in later patches to determine if the port can transmit and receive SOP' messages. Maxim based tcpci drivers are capable of SOP' communication, so the cable_comm_capable flag is set to true. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-17-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpci.c | 8 ++++++++ drivers/usb/typec/tcpm/tcpci_maxim_core.c | 1 + include/linux/usb/tcpci.h | 3 +++ include/linux/usb/tcpm.h | 4 ++++ 4 files changed, 16 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 0ee3e6e29bb1..1ededbcecc09 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -584,6 +584,13 @@ static int tcpci_pd_transmit(struct tcpc_dev *tcpc, enum tcpm_transmit_type type return 0; } +static bool tcpci_cable_comm_capable(struct tcpc_dev *tcpc) +{ + struct tcpci *tcpci = tcpc_to_tcpci(tcpc); + + return tcpci->data->cable_comm_capable; +} + static int tcpci_init(struct tcpc_dev *tcpc) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); @@ -793,6 +800,7 @@ struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data) tcpci->tcpc.enable_frs = tcpci_enable_frs; tcpci->tcpc.frs_sourcing_vbus = tcpci_frs_sourcing_vbus; tcpci->tcpc.set_partner_usb_comm_capable = tcpci_set_partner_usb_comm_capable; + tcpci->tcpc.cable_comm_capable = tcpci_cable_comm_capable; if (tcpci->data->check_contaminant) tcpci->tcpc.check_contaminant = tcpci_check_contaminant; diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c index 7fb966fd639b..7b2d4e6e52a2 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c +++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c @@ -478,6 +478,7 @@ static int max_tcpci_probe(struct i2c_client *client) chip->data.vbus_vsafe0v = true; chip->data.set_partner_usb_comm_capable = max_tcpci_set_partner_usb_comm_capable; chip->data.check_contaminant = max_tcpci_check_contaminant; + chip->data.cable_comm_capable = true; max_tcpci_init_regs(chip); chip->tcpci = tcpci_register_port(chip->dev, &chip->data); diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index 467e8045e9f8..1d0b849defd0 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -198,12 +198,15 @@ struct tcpci; * Chip level drivers are expected to check for contaminant and call * tcpm_clean_port when the port is clean to put the port back into * toggling state. + * @cable_comm_capable + * optional; Set when TCPC can communicate with cable plugs over SOP' */ struct tcpci_data { struct regmap *regmap; unsigned char TX_BUF_BYTE_x_hidden:1; unsigned char auto_discharge_disconnect:1; unsigned char vbus_vsafe0v:1; + unsigned char cable_comm_capable:1; int (*init)(struct tcpci *tcpci, struct tcpci_data *data); int (*set_vconn)(struct tcpci *tcpci, struct tcpci_data *data, diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index 65fac5e1f317..430fa3ec69bb 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -119,6 +119,9 @@ enum tcpm_transmit_type { * at the end of the deboumce period or when the port is still * toggling. Chip level drivers are expected to check for contaminant * and call tcpm_clean_port when the port is clean. + * @cable_comm_capable + * Optional; Returns whether cable communication over SOP' is supported + * by the tcpc */ struct tcpc_dev { struct fwnode_handle *fwnode; @@ -154,6 +157,7 @@ struct tcpc_dev { bool (*is_vbus_vsafe0v)(struct tcpc_dev *dev); void (*set_partner_usb_comm_capable)(struct tcpc_dev *dev, bool enable); void (*check_contaminant)(struct tcpc_dev *dev); + bool (*cable_comm_capable)(struct tcpc_dev *dev); }; struct tcpm_port; -- cgit v1.2.3 From 3bbb9ba4f66006f27ad0d5ceaf2480117e16d489 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:17 +0000 Subject: usb: typec: tcpci: add tcpm_transmit_type to tcpm_pd_receive tcpm_pd_receive adds the SOP type as a parameter, and passes it within the pd_rx_event struct for tcpm_pd_rx_handler to use. For now, the handler drops all SOP' messages. Maxim based tcpci drivers are capable of SOP' communication, so process_rx now takes the SOP type into account and passes the value to tcpm_pd_receive. tcpci_set_pd_rx now utilizes the cable_comm_capable flag to determine if TCPC_RX_DETECT_SOP1 should be added to the bitfield when enabling PD message reception. For all other consumers of tcpm_pd_receive, default the new field to TCPC_TX_SOP. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-18-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/fusb302.c | 2 +- drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c | 2 +- drivers/usb/typec/tcpm/tcpci.c | 7 +++++-- drivers/usb/typec/tcpm/tcpci_maxim_core.c | 20 +++++++++++++++++--- drivers/usb/typec/tcpm/tcpm.c | 10 +++++++++- drivers/usb/typec/tcpm/wcove.c | 2 +- include/linux/usb/tcpci.h | 1 + include/linux/usb/tcpm.h | 3 ++- 8 files changed, 37 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index bc21006e979c..ef18a448b740 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -1467,7 +1467,7 @@ static int fusb302_pd_read_message(struct fusb302_chip *chip, if ((!len) && (pd_header_type_le(msg->header) == PD_CTRL_GOOD_CRC)) tcpm_pd_transmit_complete(chip->tcpm_port, TCPC_TX_SUCCESS); else - tcpm_pd_receive(chip->tcpm_port, msg); + tcpm_pd_receive(chip->tcpm_port, msg, TCPC_TX_SOP); return ret; } diff --git a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c index 52c81378e36e..a3154085ae32 100644 --- a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c +++ b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c @@ -299,7 +299,7 @@ done: if (!ret) { dev_vdbg(dev, "pd_receive: handing %d bytes to tcpm\n", size); - tcpm_pd_receive(pmic_typec_pdphy->tcpm_port, &msg); + tcpm_pd_receive(pmic_typec_pdphy->tcpm_port, &msg, TCPC_TX_SOP); } } diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 1ededbcecc09..8ea4ed159a13 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -445,8 +445,11 @@ static int tcpci_set_pd_rx(struct tcpc_dev *tcpc, bool enable) unsigned int reg = 0; int ret; - if (enable) + if (enable) { reg = TCPC_RX_DETECT_SOP | TCPC_RX_DETECT_HARD_RESET; + if (tcpci->data->cable_comm_capable) + reg |= TCPC_RX_DETECT_SOP1; + } ret = regmap_write(tcpci->regmap, TCPC_RX_DETECT, reg); if (ret < 0) return ret; @@ -719,7 +722,7 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci) /* Read complete, clear RX status alert bit */ tcpci_write16(tcpci, TCPC_ALERT, TCPC_ALERT_RX_STATUS); - tcpm_pd_receive(tcpci->port, &msg); + tcpm_pd_receive(tcpci->port, &msg, TCPC_TX_SOP); } if (tcpci->data->vbus_vsafe0v && (status & TCPC_ALERT_EXTENDED_STATUS)) { diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c index 7b2d4e6e52a2..f9f838df43f7 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c +++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c @@ -128,6 +128,7 @@ static void process_rx(struct max_tcpci_chip *chip, u16 status) u8 count, frame_type, rx_buf[TCPC_RECEIVE_BUFFER_LEN]; int ret, payload_index; u8 *rx_buf_ptr; + enum tcpm_transmit_type rx_type; /* * READABLE_BYTE_COUNT: Indicates the number of bytes in the RX_BUF_BYTE_x registers @@ -143,10 +144,23 @@ static void process_rx(struct max_tcpci_chip *chip, u16 status) count = rx_buf[TCPC_RECEIVE_BUFFER_COUNT_OFFSET]; frame_type = rx_buf[TCPC_RECEIVE_BUFFER_FRAME_TYPE_OFFSET]; - if (count == 0 || frame_type != TCPC_RX_BUF_FRAME_TYPE_SOP) { + switch (frame_type) { + case TCPC_RX_BUF_FRAME_TYPE_SOP1: + rx_type = TCPC_TX_SOP_PRIME; + break; + case TCPC_RX_BUF_FRAME_TYPE_SOP: + rx_type = TCPC_TX_SOP; + break; + default: + rx_type = TCPC_TX_SOP; + break; + } + + if (count == 0 || (frame_type != TCPC_RX_BUF_FRAME_TYPE_SOP && + frame_type != TCPC_RX_BUF_FRAME_TYPE_SOP1)) { max_tcpci_write16(chip, TCPC_ALERT, TCPC_ALERT_RX_STATUS); dev_err(chip->dev, "%s\n", count == 0 ? "error: count is 0" : - "error frame_type is not SOP"); + "error frame_type is not SOP/SOP'"); return; } @@ -183,7 +197,7 @@ static void process_rx(struct max_tcpci_chip *chip, u16 status) if (ret < 0) return; - tcpm_pd_receive(chip->port, &msg); + tcpm_pd_receive(chip->port, &msg, rx_type); } static int max_tcpci_set_vbus(struct tcpci *tcpci, struct tcpci_data *tdata, bool source, bool sink) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 5945e3a2b0f7..ff0fcf560c88 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -518,6 +518,7 @@ struct pd_rx_event { struct kthread_work work; struct tcpm_port *port; struct pd_message msg; + enum tcpm_transmit_type rx_sop_type; }; static const char * const pd_rev[] = { @@ -2981,12 +2982,17 @@ static void tcpm_pd_rx_handler(struct kthread_work *work) const struct pd_message *msg = &event->msg; unsigned int cnt = pd_header_cnt_le(msg->header); struct tcpm_port *port = event->port; + enum tcpm_transmit_type rx_sop_type = event->rx_sop_type; mutex_lock(&port->lock); tcpm_log(port, "PD RX, header: %#x [%d]", le16_to_cpu(msg->header), port->attached); + /* Ignore SOP' for now */ + if (rx_sop_type == TCPC_TX_SOP_PRIME) + goto done; + if (port->attached) { enum pd_ctrl_msg_type type = pd_header_type_le(msg->header); unsigned int msgid = pd_header_msgid_le(msg->header); @@ -3028,7 +3034,8 @@ done: kfree(event); } -void tcpm_pd_receive(struct tcpm_port *port, const struct pd_message *msg) +void tcpm_pd_receive(struct tcpm_port *port, const struct pd_message *msg, + enum tcpm_transmit_type rx_sop_type) { struct pd_rx_event *event; @@ -3038,6 +3045,7 @@ void tcpm_pd_receive(struct tcpm_port *port, const struct pd_message *msg) kthread_init_work(&event->work, tcpm_pd_rx_handler); event->port = port; + event->rx_sop_type = rx_sop_type; memcpy(&event->msg, msg, sizeof(*msg)); kthread_queue_work(port->wq, &event->work); } diff --git a/drivers/usb/typec/tcpm/wcove.c b/drivers/usb/typec/tcpm/wcove.c index 87d4abde0ea2..cf719307b3f6 100644 --- a/drivers/usb/typec/tcpm/wcove.c +++ b/drivers/usb/typec/tcpm/wcove.c @@ -535,7 +535,7 @@ static irqreturn_t wcove_typec_irq(int irq, void *data) goto err; } - tcpm_pd_receive(wcove->tcpm, &msg); + tcpm_pd_receive(wcove->tcpm, &msg, TCPC_TX_SOP); ret = regmap_read(wcove->regmap, USBC_RXSTATUS, &status); diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index 1d0b849defd0..9ed6d62c9c5f 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -145,6 +145,7 @@ #define TCPC_RX_BYTE_CNT 0x30 #define TCPC_RX_BUF_FRAME_TYPE 0x31 #define TCPC_RX_BUF_FRAME_TYPE_SOP 0 +#define TCPC_RX_BUF_FRAME_TYPE_SOP1 1 #define TCPC_RX_HDR 0x32 #define TCPC_RX_DATA 0x34 /* through 0x4f */ diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index 430fa3ec69bb..41d1ac9c8bbf 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -170,7 +170,8 @@ void tcpm_cc_change(struct tcpm_port *port); void tcpm_sink_frs(struct tcpm_port *port); void tcpm_sourcing_vbus(struct tcpm_port *port); void tcpm_pd_receive(struct tcpm_port *port, - const struct pd_message *msg); + const struct pd_message *msg, + enum tcpm_transmit_type rx_sop_type); void tcpm_pd_transmit_complete(struct tcpm_port *port, enum tcpm_transmit_status status); void tcpm_pd_hard_reset(struct tcpm_port *port); -- cgit v1.2.3 From 6bd181ba60e198fef6f750b543832f161fbd9f39 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:19 +0000 Subject: usb: typec: tcpm: add control message support to sop' Add tx_sop_type to tcpm_pd_send_control and rx_sop_type to tcpm_pd_ctrl_request. TCPC_TX_SOP is added to all pd_send_control calls, but TCPC_TX_SOP_PRIME is added to pd_send_control for a SOFT_RESET message sent after a Vconn swap that makes the Port the Vconn source. Likewise, tcpm_pd_ctrl_request resets the proper protocol layer depending on rx_sop_type for SOFT_RESET. VCONN_SWAP_TURN_ON_VCONN now moves to a new state, VCONN_SWAP_SEND_SOFT_RESET. This state sends SOFT_RESET over SOP' before transitioning to the ready state if applicable. It transitions after PD_T_VCONN_STABLE, definied in pd.h as the time required for Vconn to be on before transmitting messages. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-20-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 159 ++++++++++++++++++++++++++++++------------ include/linux/usb/pd.h | 1 + 2 files changed, 115 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index d2ca85c8fec6..a59927925714 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -108,6 +108,7 @@ S(VCONN_SWAP_WAIT_FOR_VCONN), \ S(VCONN_SWAP_TURN_ON_VCONN), \ S(VCONN_SWAP_TURN_OFF_VCONN), \ + S(VCONN_SWAP_SEND_SOFT_RESET), \ \ S(FR_SWAP_SEND), \ S(FR_SWAP_SEND_TIMEOUT), \ @@ -2395,7 +2396,8 @@ static inline enum tcpm_state ready_state(struct tcpm_port *port) } static int tcpm_pd_send_control(struct tcpm_port *port, - enum pd_ctrl_msg_type type); + enum pd_ctrl_msg_type type, + enum tcpm_transmit_type tx_sop_type); static void tcpm_handle_alert(struct tcpm_port *port, const __le32 *payload, int cnt) @@ -2749,10 +2751,12 @@ static void tcpm_pps_complete(struct tcpm_port *port, int result) } static void tcpm_pd_ctrl_request(struct tcpm_port *port, - const struct pd_message *msg) + const struct pd_message *msg, + enum tcpm_transmit_type rx_sop_type) { enum pd_ctrl_msg_type type = pd_header_type_le(msg->header); enum tcpm_state next_state; + unsigned int rev = pd_header_rev_le(msg->header); /* * Stop VDM state machine if interrupted by other Messages while NOT_SUPP is allowed in @@ -2917,6 +2921,16 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, case SOFT_RESET_SEND: if (port->ams == SOFT_RESET_AMS) tcpm_ams_finish(port); + /* + * SOP' Soft Reset is done after Vconn Swap, + * which returns to ready state + */ + if (rx_sop_type == TCPC_TX_SOP_PRIME) { + if (rev < port->negotiated_rev_prime) + port->negotiated_rev_prime = rev; + tcpm_set_state(port, ready_state(port), 0); + break; + } if (port->pwr_role == TYPEC_SOURCE) { port->upcoming_state = SRC_SEND_CAPABILITIES; tcpm_ams_start(port, POWER_NEGOTIATION); @@ -3116,8 +3130,7 @@ static void tcpm_pd_rx_handler(struct kthread_work *work) if (msgid == port->rx_msgid_prime) goto done; port->rx_msgid_prime = msgid; - /* Ignore SOP' for now */ - goto done; + break; case TCPC_TX_SOP: default: if (msgid == port->rx_msgid && type != PD_CTRL_SOFT_RESET) @@ -3141,7 +3154,7 @@ static void tcpm_pd_rx_handler(struct kthread_work *work) else if (cnt) tcpm_pd_data_request(port, msg); else - tcpm_pd_ctrl_request(port, msg); + tcpm_pd_ctrl_request(port, msg, rx_sop_type); } } @@ -3168,17 +3181,40 @@ void tcpm_pd_receive(struct tcpm_port *port, const struct pd_message *msg, EXPORT_SYMBOL_GPL(tcpm_pd_receive); static int tcpm_pd_send_control(struct tcpm_port *port, - enum pd_ctrl_msg_type type) + enum pd_ctrl_msg_type type, + enum tcpm_transmit_type tx_sop_type) { struct pd_message msg; memset(&msg, 0, sizeof(msg)); - msg.header = PD_HEADER_LE(type, port->pwr_role, - port->data_role, - port->negotiated_rev, - port->message_id, 0); + switch (tx_sop_type) { + case TCPC_TX_SOP_PRIME: + msg.header = PD_HEADER_LE(type, + 0, /* Cable Plug Indicator for DFP/UFP */ + 0, /* Reserved */ + port->negotiated_rev, + port->message_id_prime, + 0); + break; + case TCPC_TX_SOP: + msg.header = PD_HEADER_LE(type, + port->pwr_role, + port->data_role, + port->negotiated_rev, + port->message_id, + 0); + break; + default: + msg.header = PD_HEADER_LE(type, + port->pwr_role, + port->data_role, + port->negotiated_rev, + port->message_id, + 0); + break; + } - return tcpm_pd_transmit(port, TCPC_TX_SOP, &msg); + return tcpm_pd_transmit(port, tx_sop_type, &msg); } /* @@ -3197,13 +3233,13 @@ static bool tcpm_send_queued_message(struct tcpm_port *port) switch (queued_message) { case PD_MSG_CTRL_WAIT: - tcpm_pd_send_control(port, PD_CTRL_WAIT); + tcpm_pd_send_control(port, PD_CTRL_WAIT, TCPC_TX_SOP); break; case PD_MSG_CTRL_REJECT: - tcpm_pd_send_control(port, PD_CTRL_REJECT); + tcpm_pd_send_control(port, PD_CTRL_REJECT, TCPC_TX_SOP); break; case PD_MSG_CTRL_NOT_SUPP: - tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP); + tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP, TCPC_TX_SOP); break; case PD_MSG_DATA_SINK_CAP: ret = tcpm_pd_send_sink_caps(port); @@ -4218,7 +4254,7 @@ static void run_state_machine(struct tcpm_port *port) case SRC_NEGOTIATE_CAPABILITIES: ret = tcpm_pd_check_request(port); if (ret < 0) { - tcpm_pd_send_control(port, PD_CTRL_REJECT); + tcpm_pd_send_control(port, PD_CTRL_REJECT, TCPC_TX_SOP); if (!port->explicit_contract) { tcpm_set_state(port, SRC_WAIT_NEW_CAPABILITIES, 0); @@ -4226,7 +4262,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_state(port, SRC_READY, 0); } } else { - tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); tcpm_set_partner_usb_comm_capable(port, !!(port->sink_request & RDO_USB_COMM)); tcpm_set_state(port, SRC_TRANSITION_SUPPLY, @@ -4235,7 +4271,7 @@ static void run_state_machine(struct tcpm_port *port) break; case SRC_TRANSITION_SUPPLY: /* XXX: regulator_set_voltage(vbus, ...) */ - tcpm_pd_send_control(port, PD_CTRL_PS_RDY); + tcpm_pd_send_control(port, PD_CTRL_PS_RDY, TCPC_TX_SOP); port->explicit_contract = true; typec_set_pwr_opmode(port->typec_port, TYPEC_PWR_MODE_PD); port->pwr_opmode = TYPEC_PWR_MODE_PD; @@ -4720,7 +4756,7 @@ static void run_state_machine(struct tcpm_port *port) /* remove existing capabilities */ usb_power_delivery_unregister_capabilities(port->partner_source_caps); port->partner_source_caps = NULL; - tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); tcpm_ams_finish(port); if (port->pwr_role == TYPEC_SOURCE) { port->upcoming_state = SRC_SEND_CAPABILITIES; @@ -4737,28 +4773,41 @@ static void run_state_machine(struct tcpm_port *port) tcpm_ams_start(port, SOFT_RESET_AMS); break; case SOFT_RESET_SEND: - port->message_id = 0; - port->rx_msgid = -1; - /* remove existing capabilities */ - usb_power_delivery_unregister_capabilities(port->partner_source_caps); - port->partner_source_caps = NULL; - if (tcpm_pd_send_control(port, PD_CTRL_SOFT_RESET)) - tcpm_set_state_cond(port, hard_reset_state(port), 0); - else - tcpm_set_state_cond(port, hard_reset_state(port), - PD_T_SENDER_RESPONSE); + /* + * Power Delivery 3.0 Section 6.3.13 + * + * A Soft_Reset Message Shall be targeted at a specific entity + * depending on the type of SOP* packet used. + */ + if (port->tx_sop_type == TCPC_TX_SOP_PRIME) { + port->message_id_prime = 0; + port->rx_msgid_prime = -1; + tcpm_pd_send_control(port, PD_CTRL_SOFT_RESET, TCPC_TX_SOP_PRIME); + tcpm_set_state_cond(port, ready_state(port), PD_T_SENDER_RESPONSE); + } else { + port->message_id = 0; + port->rx_msgid = -1; + /* remove existing capabilities */ + usb_power_delivery_unregister_capabilities(port->partner_source_caps); + port->partner_source_caps = NULL; + if (tcpm_pd_send_control(port, PD_CTRL_SOFT_RESET, TCPC_TX_SOP)) + tcpm_set_state_cond(port, hard_reset_state(port), 0); + else + tcpm_set_state_cond(port, hard_reset_state(port), + PD_T_SENDER_RESPONSE); + } break; /* DR_Swap states */ case DR_SWAP_SEND: - tcpm_pd_send_control(port, PD_CTRL_DR_SWAP); + tcpm_pd_send_control(port, PD_CTRL_DR_SWAP, TCPC_TX_SOP); if (port->data_role == TYPEC_DEVICE || port->negotiated_rev > PD_REV20) port->send_discover = true; tcpm_set_state_cond(port, DR_SWAP_SEND_TIMEOUT, PD_T_SENDER_RESPONSE); break; case DR_SWAP_ACCEPT: - tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); if (port->data_role == TYPEC_DEVICE || port->negotiated_rev > PD_REV20) port->send_discover = true; tcpm_set_state_cond(port, DR_SWAP_CHANGE_DR, 0); @@ -4782,7 +4831,7 @@ static void run_state_machine(struct tcpm_port *port) break; case FR_SWAP_SEND: - if (tcpm_pd_send_control(port, PD_CTRL_FR_SWAP)) { + if (tcpm_pd_send_control(port, PD_CTRL_FR_SWAP, TCPC_TX_SOP)) { tcpm_set_state(port, ERROR_RECOVERY, 0); break; } @@ -4802,7 +4851,7 @@ static void run_state_machine(struct tcpm_port *port) break; case FR_SWAP_SNK_SRC_SOURCE_VBUS_APPLIED: tcpm_set_pwr_role(port, TYPEC_SOURCE); - if (tcpm_pd_send_control(port, PD_CTRL_PS_RDY)) { + if (tcpm_pd_send_control(port, PD_CTRL_PS_RDY, TCPC_TX_SOP)) { tcpm_set_state(port, ERROR_RECOVERY, 0); break; } @@ -4812,11 +4861,11 @@ static void run_state_machine(struct tcpm_port *port) /* PR_Swap states */ case PR_SWAP_ACCEPT: - tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); tcpm_set_state(port, PR_SWAP_START, 0); break; case PR_SWAP_SEND: - tcpm_pd_send_control(port, PD_CTRL_PR_SWAP); + tcpm_pd_send_control(port, PD_CTRL_PR_SWAP, TCPC_TX_SOP); tcpm_set_state_cond(port, PR_SWAP_SEND_TIMEOUT, PD_T_SENDER_RESPONSE); break; @@ -4858,7 +4907,7 @@ static void run_state_machine(struct tcpm_port *port) * supply is turned off" */ tcpm_set_pwr_role(port, TYPEC_SINK); - if (tcpm_pd_send_control(port, PD_CTRL_PS_RDY)) { + if (tcpm_pd_send_control(port, PD_CTRL_PS_RDY, TCPC_TX_SOP)) { tcpm_set_state(port, ERROR_RECOVERY, 0); break; } @@ -4905,17 +4954,17 @@ static void run_state_machine(struct tcpm_port *port) * Source." */ tcpm_set_pwr_role(port, TYPEC_SOURCE); - tcpm_pd_send_control(port, PD_CTRL_PS_RDY); + tcpm_pd_send_control(port, PD_CTRL_PS_RDY, TCPC_TX_SOP); tcpm_set_state(port, SRC_STARTUP, PD_T_SWAP_SRC_START); break; case VCONN_SWAP_ACCEPT: - tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); tcpm_ams_finish(port); tcpm_set_state(port, VCONN_SWAP_START, 0); break; case VCONN_SWAP_SEND: - tcpm_pd_send_control(port, PD_CTRL_VCONN_SWAP); + tcpm_pd_send_control(port, PD_CTRL_VCONN_SWAP, TCPC_TX_SOP); tcpm_set_state(port, VCONN_SWAP_SEND_TIMEOUT, PD_T_SENDER_RESPONSE); break; @@ -4934,14 +4983,34 @@ static void run_state_machine(struct tcpm_port *port) PD_T_VCONN_SOURCE_ON); break; case VCONN_SWAP_TURN_ON_VCONN: - tcpm_set_vconn(port, true); - tcpm_pd_send_control(port, PD_CTRL_PS_RDY); - tcpm_set_state(port, ready_state(port), 0); + ret = tcpm_set_vconn(port, true); + tcpm_pd_send_control(port, PD_CTRL_PS_RDY, TCPC_TX_SOP); + /* + * USB PD 3.0 Section 6.4.4.3.1 + * + * Note that a Cable Plug or VPD will not be ready for PD + * Communication until tVCONNStable after VCONN has been applied + */ + if (!ret) + tcpm_set_state(port, VCONN_SWAP_SEND_SOFT_RESET, + PD_T_VCONN_STABLE); + else + tcpm_set_state(port, ready_state(port), 0); break; case VCONN_SWAP_TURN_OFF_VCONN: tcpm_set_vconn(port, false); tcpm_set_state(port, ready_state(port), 0); break; + case VCONN_SWAP_SEND_SOFT_RESET: + tcpm_swap_complete(port, port->swap_status); + if (tcpm_can_communicate_sop_prime(port)) { + port->tx_sop_type = TCPC_TX_SOP_PRIME; + port->upcoming_state = SOFT_RESET_SEND; + tcpm_ams_start(port, SOFT_RESET_AMS); + } else { + tcpm_set_state(port, ready_state(port), 0); + } + break; case DR_SWAP_CANCEL: case PR_SWAP_CANCEL: @@ -4977,7 +5046,7 @@ static void run_state_machine(struct tcpm_port *port) } break; case GET_STATUS_SEND: - tcpm_pd_send_control(port, PD_CTRL_GET_STATUS); + tcpm_pd_send_control(port, PD_CTRL_GET_STATUS, TCPC_TX_SOP); tcpm_set_state(port, GET_STATUS_SEND_TIMEOUT, PD_T_SENDER_RESPONSE); break; @@ -4985,7 +5054,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_state(port, ready_state(port), 0); break; case GET_PPS_STATUS_SEND: - tcpm_pd_send_control(port, PD_CTRL_GET_PPS_STATUS); + tcpm_pd_send_control(port, PD_CTRL_GET_PPS_STATUS, TCPC_TX_SOP); tcpm_set_state(port, GET_PPS_STATUS_SEND_TIMEOUT, PD_T_SENDER_RESPONSE); break; @@ -4993,7 +5062,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_state(port, ready_state(port), 0); break; case GET_SINK_CAP: - tcpm_pd_send_control(port, PD_CTRL_GET_SINK_CAP); + tcpm_pd_send_control(port, PD_CTRL_GET_SINK_CAP, TCPC_TX_SOP); tcpm_set_state(port, GET_SINK_CAP_TIMEOUT, PD_T_SENDER_RESPONSE); break; case GET_SINK_CAP_TIMEOUT: @@ -5033,7 +5102,7 @@ static void run_state_machine(struct tcpm_port *port) /* Chunk state */ case CHUNK_NOT_SUPP: - tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP); + tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP, TCPC_TX_SOP); tcpm_set_state(port, port->pwr_role == TYPEC_SOURCE ? SRC_READY : SNK_READY, 0); break; default: diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index eb626af0e4e7..d50098fb16b5 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -483,6 +483,7 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_T_BIST_CONT_MODE 50 /* 30 - 60 ms */ #define PD_T_SINK_TX 16 /* 16 - 20 ms */ #define PD_T_CHUNK_NOT_SUPP 42 /* 40 - 50 ms */ +#define PD_T_VCONN_STABLE 50 #define PD_T_DRP_TRY 100 /* 75 - 150 ms */ #define PD_T_DRP_TRYWAIT 600 /* 400 - 800 ms */ -- cgit v1.2.3 From 030509ac473da439e3d5438b1cd3c5b899844046 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:20 +0000 Subject: usb: typec: tcpci: add attempt_vconn_swap_discovery callback Add attempt_vconn_swap_discovery callback to determine whether the TCPM should perform a Vconn swap following Discover Identity on SOP. The tcpci will return false unless chip level drivers implement the callback. Maxim based TCPCs will return true unless the last connection resulted in a Vconn Over Current Fault, which may be the result of the Vconn swap. In addition to the port resetting, the TCPCI will veto the next Vconn swap from occurring. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-21-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpci.c | 11 +++++++++++ drivers/usb/typec/tcpm/tcpci_maxim.h | 1 + drivers/usb/typec/tcpm/tcpci_maxim_core.c | 17 ++++++++++++++++- include/linux/usb/tcpci.h | 9 +++++++++ include/linux/usb/tcpm.h | 9 +++++++++ 5 files changed, 46 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 8ea4ed159a13..40c7b6224c74 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -594,6 +594,16 @@ static bool tcpci_cable_comm_capable(struct tcpc_dev *tcpc) return tcpci->data->cable_comm_capable; } +static bool tcpci_attempt_vconn_swap_discovery(struct tcpc_dev *tcpc) +{ + struct tcpci *tcpci = tcpc_to_tcpci(tcpc); + + if (tcpci->data->attempt_vconn_swap_discovery) + return tcpci->data->attempt_vconn_swap_discovery(tcpci, tcpci->data); + + return false; +} + static int tcpci_init(struct tcpc_dev *tcpc) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); @@ -804,6 +814,7 @@ struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data) tcpci->tcpc.frs_sourcing_vbus = tcpci_frs_sourcing_vbus; tcpci->tcpc.set_partner_usb_comm_capable = tcpci_set_partner_usb_comm_capable; tcpci->tcpc.cable_comm_capable = tcpci_cable_comm_capable; + tcpci->tcpc.attempt_vconn_swap_discovery = tcpci_attempt_vconn_swap_discovery; if (tcpci->data->check_contaminant) tcpci->tcpc.check_contaminant = tcpci_check_contaminant; diff --git a/drivers/usb/typec/tcpm/tcpci_maxim.h b/drivers/usb/typec/tcpm/tcpci_maxim.h index 2c1c4d161b0d..78ff3b73ee7e 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim.h +++ b/drivers/usb/typec/tcpm/tcpci_maxim.h @@ -62,6 +62,7 @@ struct max_tcpci_chip { struct i2c_client *client; struct tcpm_port *port; enum contamiant_state contaminant_state; + bool veto_vconn_swap; }; static inline int max_tcpci_read16(struct max_tcpci_chip *chip, unsigned int reg, u16 *val) diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c index f9f838df43f7..eec3bcec119c 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c +++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c @@ -323,8 +323,10 @@ static irqreturn_t _max_tcpci_irq(struct max_tcpci_chip *chip, u16 status) if (ret < 0) return ret; - if (reg_status & TCPC_FAULT_STATUS_VCONN_OC) + if (reg_status & TCPC_FAULT_STATUS_VCONN_OC) { + chip->veto_vconn_swap = true; tcpm_port_error_recovery(chip->port); + } } if (status & TCPC_ALERT_EXTND) { @@ -458,6 +460,18 @@ static void max_tcpci_check_contaminant(struct tcpci *tcpci, struct tcpci_data * tcpm_port_clean(chip->port); } +static bool max_tcpci_attempt_vconn_swap_discovery(struct tcpci *tcpci, struct tcpci_data *tdata) +{ + struct max_tcpci_chip *chip = tdata_to_max_tcpci(tdata); + + if (chip->veto_vconn_swap) { + chip->veto_vconn_swap = false; + return false; + } + + return true; +} + static int max_tcpci_probe(struct i2c_client *client) { int ret; @@ -493,6 +507,7 @@ static int max_tcpci_probe(struct i2c_client *client) chip->data.set_partner_usb_comm_capable = max_tcpci_set_partner_usb_comm_capable; chip->data.check_contaminant = max_tcpci_check_contaminant; chip->data.cable_comm_capable = true; + chip->data.attempt_vconn_swap_discovery = max_tcpci_attempt_vconn_swap_discovery; max_tcpci_init_regs(chip); chip->tcpci = tcpci_register_port(chip->dev, &chip->data); diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index 9ed6d62c9c5f..47a86b8a4a50 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -201,6 +201,14 @@ struct tcpci; * toggling state. * @cable_comm_capable * optional; Set when TCPC can communicate with cable plugs over SOP' + * @attempt_vconn_swap_discovery: + * Optional; The callback is called by the TCPM when the result of + * a Discover Identity request indicates that the port partner is + * a receptacle capable of modal operation. Chip level TCPCI drivers + * can implement their own policy to determine if and when a Vconn + * swap following Discover Identity on SOP' occurs. + * Return true when the TCPM is allowed to request a Vconn swap + * after Discovery Identity on SOP. */ struct tcpci_data { struct regmap *regmap; @@ -219,6 +227,7 @@ struct tcpci_data { void (*set_partner_usb_comm_capable)(struct tcpci *tcpci, struct tcpci_data *data, bool capable); void (*check_contaminant)(struct tcpci *tcpci, struct tcpci_data *data); + bool (*attempt_vconn_swap_discovery)(struct tcpci *tcpci, struct tcpci_data *data); }; struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data); diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index 41d1ac9c8bbf..6671427f7eeb 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -122,6 +122,14 @@ enum tcpm_transmit_type { * @cable_comm_capable * Optional; Returns whether cable communication over SOP' is supported * by the tcpc + * @attempt_vconn_swap_discovery: + * Optional; The callback is called by the TCPM when the result of + * a Discover Identity request indicates that the port partner is + * a receptacle capable of modal operation. Chip level TCPCI drivers + * can implement their own policy to determine if and when a Vconn + * swap following Discover Identity on SOP' occurs. + * Return true when the TCPM is allowed to request a Vconn swap + * after Discovery Identity on SOP. */ struct tcpc_dev { struct fwnode_handle *fwnode; @@ -158,6 +166,7 @@ struct tcpc_dev { void (*set_partner_usb_comm_capable)(struct tcpc_dev *dev, bool enable); void (*check_contaminant)(struct tcpc_dev *dev); bool (*cable_comm_capable)(struct tcpc_dev *dev); + bool (*attempt_vconn_swap_discovery)(struct tcpc_dev *dev); }; struct tcpm_port; -- cgit v1.2.3 From fb7ff25ae43332cb64c9e7bbbe36a6cc308d8de1 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Mon, 8 Jan 2024 19:16:21 +0000 Subject: usb: typec: tcpm: add discover identity support for SOP' Add data message handling and Discover Identity SVDM over SOP' This patch contains the following changes: 1. pd_vdo Add VDO indices for active and passive cables, documentation to reflect expected number of objects depending on PD Revision, and macro to indicate port parter is data host capable. 2. tcpm Add typec_cable and typec_plug to tcpm_port to maintain cable and plug information. tcpm_port also adds send_discover_prime to indicate that Discover Identity should be sent out of the ready state. tcpm_queue_vdm and tcpm_send_vdm now take the SOP* type when transmitting messages. tcpm_handle_vdm_request and tcpm_pd_svdm also use the SOP* type. tcpm_pd_svdm handles Discover Identity messages for SOP and SOP'. In the SOP case, the port uses tcpm_attempt_vconn_swap_discovery to determine if a Vconn swap is needed for cable communication. Otherwise, the port will send Discover Identity on SOP' if it can, or default to Discover SVIDs. svdm_consume_identity_sop_prime consumes the result of Discover Identity on SOP'. It fills out cable identity and description, and it registers the cable. The SOP' plug is registered as well. The VDM state machine is adjusted to construct messages based on the SOP* type. If a transmission error occurs after the max number of retries for Discover Identity over SOP', then the port will send Discover SVIDs over SOP. Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240108191620.987785-22-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 388 ++++++++++++++++++++++++++++++++++++------ include/linux/usb/pd_vdo.h | 8 +- 2 files changed, 347 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index a59927925714..a870fbc6bc35 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -328,6 +328,12 @@ struct tcpm_port { struct typec_partner_desc partner_desc; struct typec_partner *partner; + struct usb_pd_identity cable_ident; + struct typec_cable_desc cable_desc; + struct typec_cable *cable; + struct typec_plug_desc plug_prime_desc; + struct typec_plug *plug_prime; + enum typec_cc_status cc_req; enum typec_cc_status src_rp; /* work only if pd_supported == false */ @@ -508,6 +514,12 @@ struct tcpm_port { bool potential_contaminant; /* SOP* Related Fields */ + /* + * Flag to determine if SOP' Discover Identity is available. The flag + * is set if Discover Identity on SOP' does not immediately follow + * Discover Identity on SOP. + */ + bool send_discover_prime; /* * tx_sop_type determines which SOP* a message is being sent on. * For messages that are queued and not sent immediately such as in @@ -1508,7 +1520,7 @@ static int tcpm_ams_start(struct tcpm_port *port, enum tcpm_ams ams) * VDM/VDO handling functions */ static void tcpm_queue_vdm(struct tcpm_port *port, const u32 header, - const u32 *data, int cnt) + const u32 *data, int cnt, enum tcpm_transmit_type tx_sop_type) { u32 vdo_hdr = port->vdo_data[0]; @@ -1516,7 +1528,10 @@ static void tcpm_queue_vdm(struct tcpm_port *port, const u32 header, /* If is sending discover_identity, handle received message first */ if (PD_VDO_SVDM(vdo_hdr) && PD_VDO_CMD(vdo_hdr) == CMD_DISCOVER_IDENT) { - port->send_discover = true; + if (tx_sop_type == TCPC_TX_SOP_PRIME) + port->send_discover_prime = true; + else + port->send_discover = true; mod_send_discover_delayed_work(port, SEND_DISCOVER_RETRY_MS); } else { /* Make sure we are not still processing a previous VDM packet */ @@ -1531,6 +1546,8 @@ static void tcpm_queue_vdm(struct tcpm_port *port, const u32 header, port->vdm_state = VDM_STATE_READY; port->vdm_sm_running = true; + port->tx_sop_type = tx_sop_type; + mod_vdm_delayed_work(port, 0); } @@ -1538,7 +1555,7 @@ static void tcpm_queue_vdm_unlocked(struct tcpm_port *port, const u32 header, const u32 *data, int cnt) { mutex_lock(&port->lock); - tcpm_queue_vdm(port, header, data, cnt); + tcpm_queue_vdm(port, header, data, cnt, TCPC_TX_SOP); mutex_unlock(&port->lock); } @@ -1560,6 +1577,63 @@ static void svdm_consume_identity(struct tcpm_port *port, const u32 *p, int cnt) PD_PRODUCT_PID(product), product & 0xffff); } +static void svdm_consume_identity_sop_prime(struct tcpm_port *port, const u32 *p, int cnt) +{ + u32 idh = p[VDO_INDEX_IDH]; + u32 product = p[VDO_INDEX_PRODUCT]; + int svdm_version; + + /* + * Attempt to consume identity only if cable currently is not set + */ + if (!IS_ERR_OR_NULL(port->cable)) + goto register_plug; + + /* Reset cable identity */ + memset(&port->cable_ident, 0, sizeof(port->cable_ident)); + + /* Fill out id header, cert, product, cable VDO 1 */ + port->cable_ident.id_header = idh; + port->cable_ident.cert_stat = p[VDO_INDEX_CSTAT]; + port->cable_ident.product = product; + port->cable_ident.vdo[0] = p[VDO_INDEX_CABLE_1]; + + /* Fill out cable desc, infer svdm_version from pd revision */ + port->cable_desc.type = (enum typec_plug_type) (VDO_TYPEC_CABLE_TYPE(p[VDO_INDEX_CABLE_1]) + + USB_PLUG_TYPE_A); + port->cable_desc.active = PD_IDH_PTYPE(idh) == IDH_PTYPE_ACABLE ? 1 : 0; + /* Log PD Revision and additional cable VDO from negotiated revision */ + switch (port->negotiated_rev_prime) { + case PD_REV30: + port->cable_desc.pd_revision = 0x0300; + if (port->cable_desc.active) + port->cable_ident.vdo[1] = p[VDO_INDEX_CABLE_2]; + break; + case PD_REV20: + port->cable_desc.pd_revision = 0x0200; + break; + default: + port->cable_desc.pd_revision = 0x0200; + break; + } + port->cable_desc.identity = &port->cable_ident; + /* Register Cable, set identity and svdm_version */ + port->cable = typec_register_cable(port->typec_port, &port->cable_desc); + if (IS_ERR_OR_NULL(port->cable)) + return; + typec_cable_set_identity(port->cable); + /* Get SVDM version */ + svdm_version = PD_VDO_SVDM_VER(p[VDO_INDEX_HDR]); + typec_cable_set_svdm_version(port->cable, svdm_version); + +register_plug: + if (IS_ERR_OR_NULL(port->plug_prime)) { + port->plug_prime_desc.index = TYPEC_PLUG_SOP_P; + port->plug_prime = typec_register_plug(port->cable, + &port->plug_prime_desc); + } +} + static bool svdm_consume_svids(struct tcpm_port *port, const u32 *p, int cnt) { struct pd_mode_data *pmdata = &port->mode_data; @@ -1654,6 +1728,7 @@ static void tcpm_register_partner_altmodes(struct tcpm_port *port) } #define supports_modal(port) PD_IDH_MODAL_SUPP((port)->partner_ident.id_header) +#define supports_host(port) PD_IDH_HOST_SUPP((port->partner_ident.id_header)) /* * Helper to determine whether the port is capable of SOP' communication at the @@ -1706,9 +1781,35 @@ static bool tcpm_can_communicate_sop_prime(struct tcpm_port *port) return false; } +static bool tcpm_attempt_vconn_swap_discovery(struct tcpm_port *port) +{ + if (!port->tcpc->attempt_vconn_swap_discovery) + return false; + + /* Port is already source, no need to perform swap */ + if (port->vconn_role == TYPEC_SOURCE) + return false; + + /* + * Partner needs to support Alternate Modes with modal support. If + * partner is also capable of being a USB Host, it could be a device + * that supports Alternate Modes as the DFP. + */ + if (!supports_modal(port) || supports_host(port)) + return false; + + if ((port->negotiated_rev == PD_REV20 && port->data_role == TYPEC_HOST) || + port->negotiated_rev == PD_REV30) + return port->tcpc->attempt_vconn_swap_discovery(port->tcpc); + + return false; +} + static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, const u32 *p, int cnt, u32 *response, - enum adev_actions *adev_action) + enum adev_actions *adev_action, + enum tcpm_transmit_type rx_sop_type, + enum tcpm_transmit_type *response_tx_sop_type) { struct typec_port *typec = port->typec_port; struct typec_altmode *pdev; @@ -1718,6 +1819,7 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, int cmd_type; int cmd; int i; + int ret; cmd_type = PD_VDO_CMDT(p[0]); cmd = PD_VDO_CMD(p[0]); @@ -1730,9 +1832,25 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, pdev = typec_match_altmode(port->partner_altmode, ALTMODE_DISCOVERY_MAX, PD_VDO_VID(p[0]), PD_VDO_OPOS(p[0])); - svdm_version = typec_get_negotiated_svdm_version(typec); - if (svdm_version < 0) - return 0; + switch (rx_sop_type) { + case TCPC_TX_SOP_PRIME: + if (!IS_ERR_OR_NULL(port->cable)) { + svdm_version = typec_get_cable_svdm_version(typec); + if (PD_VDO_SVDM_VER(p[0]) < svdm_version) + typec_cable_set_svdm_version(port->cable, svdm_version); + } + break; + case TCPC_TX_SOP: + svdm_version = typec_get_negotiated_svdm_version(typec); + if (svdm_version < 0) + return 0; + break; + default: + svdm_version = typec_get_negotiated_svdm_version(typec); + if (svdm_version < 0) + return 0; + break; + } switch (cmd_type) { case CMDT_INIT: @@ -1802,22 +1920,89 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, (VDO_SVDM_VERS(typec_get_negotiated_svdm_version(typec))); break; case CMDT_RSP_ACK: - /* silently drop message if we are not connected */ - if (IS_ERR_OR_NULL(port->partner)) + /* + * Silently drop message if we are not connected, but can process + * if SOP' Discover Identity prior to explicit contract. + */ + if (IS_ERR_OR_NULL(port->partner) && + !(rx_sop_type == TCPC_TX_SOP_PRIME && cmd == CMD_DISCOVER_IDENT)) break; tcpm_ams_finish(port); switch (cmd) { + /* + * SVDM Command Flow for SOP and SOP': + * SOP Discover Identity + * SOP' Discover Identity + * SOP Discover SVIDs + * Discover Modes + * + * Perform Discover SOP' if the port can communicate with cable + * plug. + */ case CMD_DISCOVER_IDENT: - if (PD_VDO_SVDM_VER(p[0]) < svdm_version) - typec_partner_set_svdm_version(port->partner, - PD_VDO_SVDM_VER(p[0])); - /* 6.4.4.3.1 */ - svdm_consume_identity(port, p, cnt); - response[0] = VDO(USB_SID_PD, 1, typec_get_negotiated_svdm_version(typec), - CMD_DISCOVER_SVID); - rlen = 1; + switch (rx_sop_type) { + case TCPC_TX_SOP: + if (PD_VDO_SVDM_VER(p[0]) < svdm_version) { + typec_partner_set_svdm_version(port->partner, + PD_VDO_SVDM_VER(p[0])); + /* If cable is discovered before partner, downgrade svdm */ + if (!IS_ERR_OR_NULL(port->cable) && + (typec_get_cable_svdm_version(port->typec_port) > + svdm_version)) + typec_cable_set_svdm_version(port->cable, + svdm_version); + } + /* 6.4.4.3.1 */ + svdm_consume_identity(port, p, cnt); + /* Attempt Vconn swap, delay SOP' discovery if necessary */ + if (tcpm_attempt_vconn_swap_discovery(port)) { + port->send_discover_prime = true; + port->upcoming_state = VCONN_SWAP_SEND; + ret = tcpm_ams_start(port, VCONN_SWAP); + if (!ret) + return 0; + port->upcoming_state = INVALID_STATE; + port->send_discover_prime = false; + } + + /* + * Attempt Discover Identity on SOP' if the + * cable was not discovered previously, and use + * the SVDM version of the partner to probe. + */ + if (IS_ERR_OR_NULL(port->cable) && + tcpm_can_communicate_sop_prime(port)) { + *response_tx_sop_type = TCPC_TX_SOP_PRIME; + port->send_discover_prime = true; + response[0] = VDO(USB_SID_PD, 1, + typec_get_negotiated_svdm_version(typec), + CMD_DISCOVER_IDENT); + rlen = 1; + } else { + *response_tx_sop_type = TCPC_TX_SOP; + response[0] = VDO(USB_SID_PD, 1, + typec_get_negotiated_svdm_version(typec), + CMD_DISCOVER_SVID); + rlen = 1; + } + break; + case TCPC_TX_SOP_PRIME: + /* + * svdm_consume_identity_sop_prime will determine + * the svdm_version for the cable moving forward. + */ + svdm_consume_identity_sop_prime(port, p, cnt); + *response_tx_sop_type = TCPC_TX_SOP; + response[0] = VDO(USB_SID_PD, 1, + typec_get_negotiated_svdm_version(typec), + CMD_DISCOVER_SVID); + rlen = 1; + break; + default: + return 0; + } break; case CMD_DISCOVER_SVID: /* 6.4.4.3.2 */ @@ -1903,13 +2088,15 @@ static void tcpm_pd_handle_msg(struct tcpm_port *port, enum tcpm_ams ams); static void tcpm_handle_vdm_request(struct tcpm_port *port, - const __le32 *payload, int cnt) + const __le32 *payload, int cnt, + enum tcpm_transmit_type rx_sop_type) { enum adev_actions adev_action = ADEV_NONE; struct typec_altmode *adev; u32 p[PD_MAX_PAYLOAD]; u32 response[8] = { }; int i, rlen = 0; + enum tcpm_transmit_type response_tx_sop_type = TCPC_TX_SOP; for (i = 0; i < cnt; i++) p[i] = le32_to_cpu(payload[i]); @@ -1944,7 +2131,8 @@ static void tcpm_handle_vdm_request(struct tcpm_port *port, * - We will send NAK and the flag will be cleared in the state machine. */ port->vdm_sm_running = true; - rlen = tcpm_pd_svdm(port, adev, p, cnt, response, &adev_action); + rlen = tcpm_pd_svdm(port, adev, p, cnt, response, &adev_action, + rx_sop_type, &response_tx_sop_type); } else { if (port->negotiated_rev >= PD_REV30) tcpm_pd_handle_msg(port, PD_MSG_CTRL_NOT_SUPP, NONE_AMS); @@ -2012,19 +2200,38 @@ static void tcpm_handle_vdm_request(struct tcpm_port *port, mutex_lock(&port->lock); if (rlen > 0) - tcpm_queue_vdm(port, response[0], &response[1], rlen - 1); + tcpm_queue_vdm(port, response[0], &response[1], rlen - 1, response_tx_sop_type); else port->vdm_sm_running = false; } static void tcpm_send_vdm(struct tcpm_port *port, u32 vid, int cmd, - const u32 *data, int count) + const u32 *data, int count, enum tcpm_transmit_type tx_sop_type) { - int svdm_version = typec_get_negotiated_svdm_version(port->typec_port); + int svdm_version; u32 header; - if (svdm_version < 0) - return; + switch (tx_sop_type) { + case TCPC_TX_SOP_PRIME: + /* + * If the port partner is discovered, then the port partner's + * SVDM Version will be returned + */ + svdm_version = typec_get_cable_svdm_version(port->typec_port); + if (svdm_version < 0) + svdm_version = SVDM_VER_MAX; + break; + case TCPC_TX_SOP: + svdm_version = typec_get_negotiated_svdm_version(port->typec_port); + if (svdm_version < 0) + return; + break; + default: + svdm_version = typec_get_negotiated_svdm_version(port->typec_port); + if (svdm_version < 0) + return; + break; + } if (WARN_ON(count > VDO_MAX_SIZE - 1)) count = VDO_MAX_SIZE - 1; @@ -2033,7 +2240,7 @@ static void tcpm_send_vdm(struct tcpm_port *port, u32 vid, int cmd, header = VDO(vid, ((vid & USB_SID_PD) == USB_SID_PD) ? 1 : (PD_VDO_CMD(cmd) <= CMD_ATTENTION), svdm_version, cmd); - tcpm_queue_vdm(port, header, data, count); + tcpm_queue_vdm(port, header, data, count, tx_sop_type); } static unsigned int vdm_ready_timeout(u32 vdm_hdr) @@ -2067,6 +2274,7 @@ static void vdm_run_state_machine(struct tcpm_port *port) struct pd_message msg; int i, res = 0; u32 vdo_hdr = port->vdo_data[0]; + u32 response[8] = { }; switch (port->vdm_state) { case VDM_STATE_READY: @@ -2091,7 +2299,17 @@ static void vdm_run_state_machine(struct tcpm_port *port) case CMD_DISCOVER_IDENT: res = tcpm_ams_start(port, DISCOVER_IDENTITY); if (res == 0) { - port->send_discover = false; + switch (port->tx_sop_type) { + case TCPC_TX_SOP_PRIME: + port->send_discover_prime = false; + break; + case TCPC_TX_SOP: + port->send_discover = false; + break; + default: + port->send_discover = false; + break; + } } else if (res == -EAGAIN) { port->vdo_data[0] = 0; mod_send_discover_delayed_work(port, @@ -2160,19 +2378,49 @@ static void vdm_run_state_machine(struct tcpm_port *port) tcpm_ams_finish(port); } else { tcpm_ams_finish(port); + if (port->tx_sop_type == TCPC_TX_SOP) + break; + /* Handle SOP' Transmission Errors */ + switch (PD_VDO_CMD(vdo_hdr)) { + /* + * If Discover Identity fails on SOP', then resume + * discovery process on SOP only. + */ + case CMD_DISCOVER_IDENT: + port->vdo_data[0] = 0; + response[0] = VDO(USB_SID_PD, 1, + typec_get_negotiated_svdm_version( + port->typec_port), + CMD_DISCOVER_SVID); + tcpm_queue_vdm(port, response[0], &response[1], + 0, TCPC_TX_SOP); + break; + default: + break; + } } break; case VDM_STATE_SEND_MESSAGE: /* Prepare and send VDM */ memset(&msg, 0, sizeof(msg)); - msg.header = PD_HEADER_LE(PD_DATA_VENDOR_DEF, - port->pwr_role, - port->data_role, - port->negotiated_rev, - port->message_id, port->vdo_count); + if (port->tx_sop_type == TCPC_TX_SOP_PRIME) { + msg.header = PD_HEADER_LE(PD_DATA_VENDOR_DEF, + 0, /* Cable Plug Indicator for DFP/UFP */ + 0, /* Reserved */ + port->negotiated_rev_prime, + port->message_id_prime, + port->vdo_count); + } else { + msg.header = PD_HEADER_LE(PD_DATA_VENDOR_DEF, + port->pwr_role, + port->data_role, + port->negotiated_rev, + port->message_id, + port->vdo_count); + } for (i = 0; i < port->vdo_count; i++) msg.payload[i] = cpu_to_le32(port->vdo_data[i]); - res = tcpm_pd_transmit(port, TCPC_TX_SOP, &msg); + res = tcpm_pd_transmit(port, port->tx_sop_type, &msg); if (res < 0) { port->vdm_state = VDM_STATE_ERR_SEND; } else { @@ -2559,7 +2807,8 @@ static int tcpm_register_sink_caps(struct tcpm_port *port) } static void tcpm_pd_data_request(struct tcpm_port *port, - const struct pd_message *msg) + const struct pd_message *msg, + enum tcpm_transmit_type rx_sop_type) { enum pd_data_msg_type type = pd_header_type_le(msg->header); unsigned int cnt = pd_header_cnt_le(msg->header); @@ -2600,8 +2849,11 @@ static void tcpm_pd_data_request(struct tcpm_port *port, break; } - if (rev < PD_MAX_REV) + if (rev < PD_MAX_REV) { port->negotiated_rev = rev; + if (port->negotiated_rev_prime > port->negotiated_rev) + port->negotiated_rev_prime = port->negotiated_rev; + } if (port->pwr_role == TYPEC_SOURCE) { if (port->ams == GET_SOURCE_CAPABILITIES) @@ -2652,8 +2904,11 @@ static void tcpm_pd_data_request(struct tcpm_port *port, break; } - if (rev < PD_MAX_REV) + if (rev < PD_MAX_REV) { port->negotiated_rev = rev; + if (port->negotiated_rev_prime > port->negotiated_rev) + port->negotiated_rev_prime = port->negotiated_rev; + } if (port->pwr_role != TYPEC_SOURCE || cnt != 1) { tcpm_pd_handle_msg(port, @@ -2709,7 +2964,7 @@ static void tcpm_pd_data_request(struct tcpm_port *port, NONE_AMS); break; case PD_DATA_VENDOR_DEF: - tcpm_handle_vdm_request(port, msg->payload, cnt); + tcpm_handle_vdm_request(port, msg->payload, cnt, rx_sop_type); break; case PD_DATA_BIST: port->bist_request = le32_to_cpu(msg->payload[0]); @@ -3152,7 +3407,7 @@ static void tcpm_pd_rx_handler(struct kthread_work *work) if (le16_to_cpu(msg->header) & PD_HEADER_EXT_HDR) tcpm_pd_ext_msg_request(port, msg); else if (cnt) - tcpm_pd_data_request(port, msg); + tcpm_pd_data_request(port, msg, rx_sop_type); else tcpm_pd_ctrl_request(port, msg, rx_sop_type); } @@ -3809,6 +4064,7 @@ static int tcpm_src_attach(struct tcpm_port *port) port->attached = true; port->send_discover = true; + port->send_discover_prime = false; return 0; @@ -3825,6 +4081,15 @@ out_disable_mux: static void tcpm_typec_disconnect(struct tcpm_port *port) { + /* + * Unregister plug/cable outside of port->connected because cable can + * be discovered before SRC_READY/SNK_READY states where port->connected + * is set. + */ + typec_unregister_plug(port->plug_prime); + typec_unregister_cable(port->cable); + port->plug_prime = NULL; + port->cable = NULL; if (port->connected) { typec_partner_set_usb_power_delivery(port->partner, NULL); typec_unregister_partner(port->partner); @@ -3947,6 +4212,7 @@ static int tcpm_snk_attach(struct tcpm_port *port) port->attached = true; port->send_discover = true; + port->send_discover_prime = false; return 0; } @@ -4308,14 +4574,23 @@ static void run_state_machine(struct tcpm_port *port) * 6.4.4.3.1 Discover Identity * "The Discover Identity Command Shall only be sent to SOP when there is an * Explicit Contract." - * For now, this driver only supports SOP for DISCOVER_IDENTITY, thus using - * port->explicit_contract to decide whether to send the command. + * + * Discover Identity on SOP' should be discovered prior to the + * ready state, but if done after a Vconn Swap following Discover + * Identity on SOP then the discovery process can be run here + * as well. */ if (port->explicit_contract) { - tcpm_set_initial_svdm_version(port); + if (port->send_discover_prime) { + port->tx_sop_type = TCPC_TX_SOP_PRIME; + } else { + port->tx_sop_type = TCPC_TX_SOP; + tcpm_set_initial_svdm_version(port); + } mod_send_discover_delayed_work(port, 0); } else { port->send_discover = false; + port->send_discover_prime = false; } /* @@ -4607,14 +4882,23 @@ static void run_state_machine(struct tcpm_port *port) * 6.4.4.3.1 Discover Identity * "The Discover Identity Command Shall only be sent to SOP when there is an * Explicit Contract." - * For now, this driver only supports SOP for DISCOVER_IDENTITY, thus using - * port->explicit_contract. + * + * Discover Identity on SOP' should be discovered prior to the + * ready state, but if done after a Vconn Swap following Discover + * Identity on SOP then the discovery process can be run here + * as well. */ if (port->explicit_contract) { - tcpm_set_initial_svdm_version(port); + if (port->send_discover_prime) { + port->tx_sop_type = TCPC_TX_SOP_PRIME; + } else { + port->tx_sop_type = TCPC_TX_SOP; + tcpm_set_initial_svdm_version(port); + } mod_send_discover_delayed_work(port, 0); } else { port->send_discover = false; + port->send_discover_prime = false; } power_supply_changed(port->psy); @@ -4655,6 +4939,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_unregister_altmodes(port); port->nr_sink_caps = 0; port->send_discover = true; + port->send_discover_prime = false; if (port->pwr_role == TYPEC_SOURCE) tcpm_set_state(port, SRC_HARD_RESET_VBUS_OFF, PD_T_PS_HARD_RESET); @@ -4801,20 +5086,25 @@ static void run_state_machine(struct tcpm_port *port) /* DR_Swap states */ case DR_SWAP_SEND: tcpm_pd_send_control(port, PD_CTRL_DR_SWAP, TCPC_TX_SOP); - if (port->data_role == TYPEC_DEVICE || port->negotiated_rev > PD_REV20) + if (port->data_role == TYPEC_DEVICE || port->negotiated_rev > PD_REV20) { port->send_discover = true; + port->send_discover_prime = false; + } tcpm_set_state_cond(port, DR_SWAP_SEND_TIMEOUT, PD_T_SENDER_RESPONSE); break; case DR_SWAP_ACCEPT: tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); - if (port->data_role == TYPEC_DEVICE || port->negotiated_rev > PD_REV20) + if (port->data_role == TYPEC_DEVICE || port->negotiated_rev > PD_REV20) { port->send_discover = true; + port->send_discover_prime = false; + } tcpm_set_state_cond(port, DR_SWAP_CHANGE_DR, 0); break; case DR_SWAP_SEND_TIMEOUT: tcpm_swap_complete(port, -ETIMEDOUT); port->send_discover = false; + port->send_discover_prime = false; tcpm_ams_finish(port); tcpm_set_state(port, ready_state(port), 0); break; @@ -5796,7 +6086,8 @@ static void tcpm_enable_frs_work(struct kthread_work *work) goto unlock; /* Send when the state machine is idle */ - if (port->state != SNK_READY || port->vdm_sm_running || port->send_discover) + if (port->state != SNK_READY || port->vdm_sm_running || port->send_discover || + port->send_discover_prime) goto resched; port->upcoming_state = GET_SINK_CAP; @@ -5819,11 +6110,12 @@ static void tcpm_send_discover_work(struct kthread_work *work) mutex_lock(&port->lock); /* No need to send DISCOVER_IDENTITY anymore */ - if (!port->send_discover) + if (!port->send_discover && !port->send_discover_prime) goto unlock; if (port->data_role == TYPEC_DEVICE && port->negotiated_rev < PD_REV30) { port->send_discover = false; + port->send_discover_prime = false; goto unlock; } @@ -5833,7 +6125,7 @@ static void tcpm_send_discover_work(struct kthread_work *work) goto unlock; } - tcpm_send_vdm(port, USB_SID_PD, CMD_DISCOVER_IDENT, NULL, 0); + tcpm_send_vdm(port, USB_SID_PD, CMD_DISCOVER_IDENT, NULL, 0, port->tx_sop_type); unlock: mutex_unlock(&port->lock); diff --git a/include/linux/usb/pd_vdo.h b/include/linux/usb/pd_vdo.h index 3a747938cdab..c09c5a12e273 100644 --- a/include/linux/usb/pd_vdo.h +++ b/include/linux/usb/pd_vdo.h @@ -86,12 +86,15 @@ * * Request is simply properly formatted SVDM header * - * Response is 4 data objects: + * Response is 4 data objects for Power Delivery 2.0 and Passive Cables for + * Power Delivery 3.0. Active Cables in Power Delivery 3.0 have 5 data objects. * [0] :: SVDM header * [1] :: Identitiy header * [2] :: Cert Stat VDO * [3] :: (Product | Cable) VDO + * [4] :: Cable VDO 1 * [4] :: AMA VDO + * [5] :: Cable VDO 2 * */ #define VDO_INDEX_HDR 0 @@ -100,6 +103,8 @@ #define VDO_INDEX_CABLE 3 #define VDO_INDEX_PRODUCT 3 #define VDO_INDEX_AMA 4 +#define VDO_INDEX_CABLE_1 4 +#define VDO_INDEX_CABLE_2 5 /* * SVDM Identity Header @@ -150,6 +155,7 @@ #define PD_IDH_MODAL_SUPP(vdo) ((vdo) & (1 << 26)) #define PD_IDH_DFP_PTYPE(vdo) (((vdo) >> 23) & 0x7) #define PD_IDH_CONN_TYPE(vdo) (((vdo) >> 21) & 0x3) +#define PD_IDH_HOST_SUPP(vdo) ((vdo) & (1 << 31)) /* * Cert Stat VDO -- cgit v1.2.3 From 174657478cd8425288aeabf93b964b9387e096fa Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:15 +0100 Subject: vgacon: inline vc_scrolldelta_helper() into vgacon_scrolldelta() Since commit 74d58cd48a8f ("USB: sisusbvga: remove console support"), vgacon_scrolldelta() is the only user of vc_scrolldelta_helper(). Inline the helper into vgacon_scrolldelta() and drop it. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-2-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 40 ---------------------------------------- drivers/video/console/vgacon.c | 36 ++++++++++++++++++++++++++++++++++-- include/linux/vt_kern.h | 3 --- 3 files changed, 34 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 156efda7c80d..3f3f7c216819 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -4748,43 +4748,3 @@ void vcs_scr_updated(struct vc_data *vc) { notify_update(vc); } - -void vc_scrolldelta_helper(struct vc_data *c, int lines, - unsigned int rolled_over, void *base, unsigned int size) -{ - unsigned long ubase = (unsigned long)base; - ptrdiff_t scr_end = (void *)c->vc_scr_end - base; - ptrdiff_t vorigin = (void *)c->vc_visible_origin - base; - ptrdiff_t origin = (void *)c->vc_origin - base; - int margin = c->vc_size_row * 4; - int from, wrap, from_off, avail; - - /* Turn scrollback off */ - if (!lines) { - c->vc_visible_origin = c->vc_origin; - return; - } - - /* Do we have already enough to allow jumping from 0 to the end? */ - if (rolled_over > scr_end + margin) { - from = scr_end; - wrap = rolled_over + c->vc_size_row; - } else { - from = 0; - wrap = size; - } - - from_off = (vorigin - from + wrap) % wrap + lines * c->vc_size_row; - avail = (origin - from + wrap) % wrap; - - /* Only a little piece would be left? Show all incl. the piece! */ - if (avail < 2 * margin) - margin = 0; - if (from_off < margin) - from_off = 0; - if (from_off > avail - margin) - from_off = avail; - - c->vc_visible_origin = ubase + (from + from_off) % wrap; -} -EXPORT_SYMBOL_GPL(vc_scrolldelta_helper); diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 8ef1579fa57f..9176fff9ce6e 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -138,8 +138,40 @@ static inline void vga_set_mem_top(struct vc_data *c) static void vgacon_scrolldelta(struct vc_data *c, int lines) { - vc_scrolldelta_helper(c, lines, vga_rolled_over, (void *)vga_vram_base, - vga_vram_size); + unsigned long scr_end = c->vc_scr_end - vga_vram_base; + unsigned long vorigin = c->vc_visible_origin - vga_vram_base; + unsigned long origin = c->vc_origin - vga_vram_base; + int margin = c->vc_size_row * 4; + int from, wrap, from_off, avail; + + /* Turn scrollback off */ + if (!lines) { + c->vc_visible_origin = c->vc_origin; + return; + } + + /* Do we have already enough to allow jumping from 0 to the end? */ + if (vga_rolled_over > scr_end + margin) { + from = scr_end; + wrap = vga_rolled_over + c->vc_size_row; + } else { + from = 0; + wrap = vga_vram_size; + } + + from_off = (vorigin - from + wrap) % wrap + lines * c->vc_size_row; + avail = (origin - from + wrap) % wrap; + + /* Only a little piece would be left? Show all incl. the piece! */ + if (avail < 2 * margin) + margin = 0; + if (from_off < margin) + from_off = 0; + if (from_off > avail - margin) + from_off = avail; + + c->vc_visible_origin = vga_vram_base + (from + from_off) % wrap; + vga_set_mem_top(c); } diff --git a/include/linux/vt_kern.h b/include/linux/vt_kern.h index c1f5aebef170..a789ea3ed2a0 100644 --- a/include/linux/vt_kern.h +++ b/include/linux/vt_kern.h @@ -168,7 +168,4 @@ void vt_set_led_state(unsigned int console, int leds); void vt_kbd_con_start(unsigned int console); void vt_kbd_con_stop(unsigned int console); -void vc_scrolldelta_helper(struct vc_data *c, int lines, - unsigned int rolled_over, void *_base, unsigned int size); - #endif /* _VT_KERN_H */ -- cgit v1.2.3 From a0b8a1681254346010edd2f94e799fb6b6568cf1 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:19 +0100 Subject: tty: vt: pass proper pointers from tioclinux() Pass proper types and proper pointers (the data with an offset) to the TIOCL_* handlers. So that they need not to cast or add anything to the passed pointer. This makes obvious what is passed/consumed. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-6-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/selection.c | 8 ++++---- drivers/tty/vt/vt.c | 19 ++++++++++--------- include/linux/selection.h | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c index 8967c3a0d916..e172ede235a0 100644 --- a/drivers/tty/vt/selection.c +++ b/drivers/tty/vt/selection.c @@ -7,7 +7,7 @@ * 'int set_selection_kernel(struct tiocl_selection *, struct tty_struct *)' * 'void clear_selection(void)' * 'int paste_selection(struct tty_struct *)' - * 'int sel_loadlut(char __user *)' + * 'int sel_loadlut(u32 __user *)' * * Now that /dev/vcs exists, most of this can disappear again. */ @@ -111,15 +111,15 @@ static inline int inword(const u32 c) /** * sel_loadlut() - load the LUT table - * @p: user table + * @lut: user table * * Load the LUT table from user space. The caller must hold the console * lock. Make a temporary copy so a partial update doesn't make a mess. */ -int sel_loadlut(char __user *p) +int sel_loadlut(u32 __user *lut) { u32 tmplut[ARRAY_SIZE(inwordLut)]; - if (copy_from_user(tmplut, (u32 __user *)(p+4), sizeof(inwordLut))) + if (copy_from_user(tmplut, lut, sizeof(inwordLut))) return -EFAULT; memcpy(inwordLut, tmplut, sizeof(inwordLut)); return 0; diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index e131edea00da..079dbff562fd 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -145,7 +145,7 @@ static void gotoxy(struct vc_data *vc, int new_x, int new_y); static void save_cur(struct vc_data *vc); static void reset_terminal(struct vc_data *vc, int do_clear); static void con_flush_chars(struct tty_struct *tty); -static int set_vesa_blanking(char __user *p); +static int set_vesa_blanking(u8 __user *mode); static void set_cursor(struct vc_data *vc); static void hide_cursor(struct vc_data *vc); static void console_callback(struct work_struct *ignored); @@ -3134,6 +3134,8 @@ int tioclinux(struct tty_struct *tty, unsigned long arg) { char type, data; char __user *p = (char __user *)arg; + void __user *param_aligned32 = (u32 __user *)arg + 1; + void __user *param = (void __user *)arg + 1; int lines; int ret; @@ -3147,8 +3149,7 @@ int tioclinux(struct tty_struct *tty, unsigned long arg) case TIOCL_SETSEL: if (!capable(CAP_SYS_ADMIN)) return -EPERM; - return set_selection_user((struct tiocl_selection - __user *)(p+1), tty); + return set_selection_user(param, tty); case TIOCL_PASTESEL: if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -3162,7 +3163,7 @@ int tioclinux(struct tty_struct *tty, unsigned long arg) if (!capable(CAP_SYS_ADMIN)) return -EPERM; console_lock(); - ret = sel_loadlut(p); + ret = sel_loadlut(param_aligned32); console_unlock(); break; case TIOCL_GETSHIFTSTATE: @@ -3181,7 +3182,7 @@ int tioclinux(struct tty_struct *tty, unsigned long arg) return put_user(data, p); case TIOCL_SETVESABLANK: console_lock(); - ret = set_vesa_blanking(p); + ret = set_vesa_blanking(param); console_unlock(); break; case TIOCL_GETKMSGREDIRECT: @@ -3204,7 +3205,7 @@ int tioclinux(struct tty_struct *tty, unsigned long arg) */ return fg_console; case TIOCL_SCROLLCONSOLE: - if (get_user(lines, (s32 __user *)(p+4))) + if (get_user(lines, (s32 __user *)param_aligned32)) return -EFAULT; /* @@ -4262,11 +4263,11 @@ postcore_initcall(vtconsole_class_init); * Screen blanking */ -static int set_vesa_blanking(char __user *p) +static int set_vesa_blanking(u8 __user *mode_user) { - unsigned int mode; + u8 mode; - if (get_user(mode, p + 1)) + if (get_user(mode, mode_user)) return -EFAULT; vesa_blank_mode = (mode < 4) ? mode : 0; diff --git a/include/linux/selection.h b/include/linux/selection.h index 170ef28ff26b..b7cd23e56a2b 100644 --- a/include/linux/selection.h +++ b/include/linux/selection.h @@ -20,7 +20,7 @@ extern int set_selection_user(const struct tiocl_selection __user *sel, extern int set_selection_kernel(struct tiocl_selection *v, struct tty_struct *tty); extern int paste_selection(struct tty_struct *tty); -extern int sel_loadlut(char __user *p); +extern int sel_loadlut(u32 __user *lut); extern int mouse_reporting(void); extern void mouse_report(struct tty_struct * tty, int butt, int mrx, int mry); -- cgit v1.2.3 From beccdcfa15666c442ce79a5f963fcb34ec28084e Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:21 +0100 Subject: tty: vt: pass vc_resize_user as a parameter It is pretty unfortunate to set vc_data::vc_resize_user in two callers of vc_do_resize(). vc_resize_user is immediately reset there (while remembering it). So instead of this back and forth, pass 'from_user' as a parameter. Notes on 'int user': * The name changes from 'user' to 'from_user' on some places to be consistent. * The type is bool now as 'int user' might evoke user's uid or whatever. Provided vc_resize() is called on many places and they need not to care about this parameter, its prototype is kept unchanged. Instead, it is now an inline calling a new __vc_resize() which implements the above. This patch makes the situation much more obvious. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-8-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 28 +++++++++++++--------------- drivers/tty/vt/vt_ioctl.c | 6 ++---- drivers/video/console/vgacon.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 2 +- include/linux/console.h | 2 +- include/linux/console_struct.h | 1 - include/linux/vt_kern.h | 9 ++++++++- 7 files changed, 27 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 3a6f60ad2224..c87837306074 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1115,13 +1115,13 @@ err_free: } static inline int resize_screen(struct vc_data *vc, int width, int height, - int user) + bool from_user) { /* Resizes the resolution of the display adapater */ int err = 0; if (vc->vc_sw->con_resize) - err = vc->vc_sw->con_resize(vc, width, height, user); + err = vc->vc_sw->con_resize(vc, width, height, from_user); return err; } @@ -1132,6 +1132,7 @@ static inline int resize_screen(struct vc_data *vc, int width, int height, * @vc: virtual console private data * @cols: columns * @lines: lines + * @from_user: invoked by a user? * * Resize a virtual console, clipping according to the actual constraints. * If the caller passes a tty structure then update the termios winsize @@ -1142,21 +1143,17 @@ static inline int resize_screen(struct vc_data *vc, int width, int height, */ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, - unsigned int cols, unsigned int lines) + unsigned int cols, unsigned int lines, bool from_user) { unsigned long old_origin, new_origin, new_scr_end, rlth, rrem, err = 0; unsigned long end; unsigned int old_rows, old_row_size, first_copied_row; unsigned int new_cols, new_rows, new_row_size, new_screen_size; - unsigned int user; unsigned short *oldscreen, *newscreen; u32 **new_uniscr = NULL; WARN_CONSOLE_UNLOCKED(); - user = vc->vc_resize_user; - vc->vc_resize_user = 0; - if (cols > VC_MAXCOL || lines > VC_MAXROW) return -EINVAL; @@ -1182,7 +1179,7 @@ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, * to deal with possible errors from the code below, we call * the resize_screen here as well. */ - return resize_screen(vc, new_cols, new_rows, user); + return resize_screen(vc, new_cols, new_rows, from_user); } if (new_screen_size > KMALLOC_MAX_SIZE || !new_screen_size) @@ -1205,7 +1202,7 @@ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, old_rows = vc->vc_rows; old_row_size = vc->vc_size_row; - err = resize_screen(vc, new_cols, new_rows, user); + err = resize_screen(vc, new_cols, new_rows, from_user); if (err) { kfree(newscreen); vc_uniscr_free(new_uniscr); @@ -1292,22 +1289,23 @@ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, } /** - * vc_resize - resize a VT + * __vc_resize - resize a VT * @vc: virtual console * @cols: columns * @rows: rows + * @from_user: invoked by a user? * * Resize a virtual console as seen from the console end of things. We * use the common vc_do_resize methods to update the structures. The * caller must hold the console sem to protect console internals and * vc->port.tty */ - -int vc_resize(struct vc_data *vc, unsigned int cols, unsigned int rows) +int __vc_resize(struct vc_data *vc, unsigned int cols, unsigned int rows, + bool from_user) { - return vc_do_resize(vc->port.tty, vc, cols, rows); + return vc_do_resize(vc->port.tty, vc, cols, rows, from_user); } -EXPORT_SYMBOL(vc_resize); +EXPORT_SYMBOL(__vc_resize); /** * vt_resize - resize a VT @@ -1327,7 +1325,7 @@ static int vt_resize(struct tty_struct *tty, struct winsize *ws) int ret; console_lock(); - ret = vc_do_resize(tty, vc, ws->ws_col, ws->ws_row); + ret = vc_do_resize(tty, vc, ws->ws_col, ws->ws_row, false); console_unlock(); return ret; } diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 8c685b501404..4b91072f3a4e 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -714,8 +714,7 @@ static int vt_resizex(struct vc_data *vc, struct vt_consize __user *cs) vcp->vc_scan_lines = v.v_vlin; if (v.v_clin) vcp->vc_cell_height = v.v_clin; - vcp->vc_resize_user = 1; - ret = vc_resize(vcp, v.v_cols, v.v_rows); + ret = __vc_resize(vcp, v.v_cols, v.v_rows, true); if (ret) { vcp->vc_scan_lines = save_scan_lines; vcp->vc_cell_height = save_cell_height; @@ -923,9 +922,8 @@ int vt_ioctl(struct tty_struct *tty, vc = vc_cons[i].d; if (vc) { - vc->vc_resize_user = 1; /* FIXME: review v tty lock */ - vc_resize(vc_cons[i].d, cc, ll); + __vc_resize(vc_cons[i].d, cc, ll, true); } } console_unlock(); diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 9176fff9ce6e..0c76e2817b49 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -1081,12 +1081,12 @@ static int vgacon_font_get(struct vc_data *c, struct console_font *font, unsigne } static int vgacon_resize(struct vc_data *c, unsigned int width, - unsigned int height, unsigned int user) + unsigned int height, bool from_user) { if ((width << 1) * height > vga_vram_size) return -EINVAL; - if (user) { + if (from_user) { /* * Ho ho! Someone (svgatextmode, eh?) may have reprogrammed * the video mode! Set the new defaults then and go away. diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index a8c32cb4c878..dd2f4617485c 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1996,7 +1996,7 @@ static void updatescrollmode(struct fbcon_display *p, #define CALC_FONTSZ(h, p, c) ((h) * (p) * (c)) /* size = height * pitch * charcount */ static int fbcon_resize(struct vc_data *vc, unsigned int width, - unsigned int height, unsigned int user) + unsigned int height, bool from_user) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; diff --git a/include/linux/console.h b/include/linux/console.h index 779d388af8a0..38b379d6c624 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -66,7 +66,7 @@ struct consw { int (*con_font_default)(struct vc_data *vc, struct console_font *font, char *name); int (*con_resize)(struct vc_data *vc, unsigned int width, - unsigned int height, unsigned int user); + unsigned int height, bool from_user); void (*con_set_palette)(struct vc_data *vc, const unsigned char *table); void (*con_scrolldelta)(struct vc_data *vc, int lines); diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index 539f1cd45309..20f564e98552 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -151,7 +151,6 @@ struct vc_data { DECLARE_BITMAP(vc_tab_stop, VC_TABSTOPS_COUNT); /* Tab stops. 256 columns. */ unsigned char vc_palette[16*3]; /* Colour palette for VGA+ */ unsigned short * vc_translate; - unsigned int vc_resize_user; /* resize request from user */ unsigned int vc_bell_pitch; /* Console bell pitch */ unsigned int vc_bell_duration; /* Console bell duration */ unsigned short vc_cur_blink_ms; /* Cursor blink duration */ diff --git a/include/linux/vt_kern.h b/include/linux/vt_kern.h index a789ea3ed2a0..d008c3d0a9bb 100644 --- a/include/linux/vt_kern.h +++ b/include/linux/vt_kern.h @@ -25,7 +25,8 @@ extern int fg_console, last_console, want_console; int vc_allocate(unsigned int console); int vc_cons_allocated(unsigned int console); -int vc_resize(struct vc_data *vc, unsigned int cols, unsigned int lines); +int __vc_resize(struct vc_data *vc, unsigned int cols, unsigned int lines, + bool from_user); struct vc_data *vc_deallocate(unsigned int console); void reset_palette(struct vc_data *vc); void do_blank_screen(int entering_gfx); @@ -42,6 +43,12 @@ void redraw_screen(struct vc_data *vc, int is_switch); #define update_screen(x) redraw_screen(x, 0) #define switch_screen(x) redraw_screen(x, 1) +static inline int vc_resize(struct vc_data *vc, unsigned int cols, + unsigned int lines) +{ + return __vc_resize(vc, cols, lines, false); +} + struct tty_struct; int tioclinux(struct tty_struct *tty, unsigned long arg); -- cgit v1.2.3 From d4c0c481e49fdf483c43e13e4a419ea19c045023 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:22 +0100 Subject: tty: vt: make vc_is_sel()'s vc const It's only an aid to people reading the header and/or calling vc_is_sel(). vc is only tested there, so having it const makes sense. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-9-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/selection.c | 2 +- include/linux/selection.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c index 91d789c025c6..34ee09f5a8f4 100644 --- a/drivers/tty/vt/selection.c +++ b/drivers/tty/vt/selection.c @@ -88,7 +88,7 @@ void clear_selection(void) } EXPORT_SYMBOL_GPL(clear_selection); -bool vc_is_sel(struct vc_data *vc) +bool vc_is_sel(const struct vc_data *vc) { return vc == vc_sel.cons; } diff --git a/include/linux/selection.h b/include/linux/selection.h index b7cd23e56a2b..533509f6ba4f 100644 --- a/include/linux/selection.h +++ b/include/linux/selection.h @@ -24,7 +24,7 @@ extern int sel_loadlut(u32 __user *lut); extern int mouse_reporting(void); extern void mouse_report(struct tty_struct * tty, int butt, int mrx, int mry); -bool vc_is_sel(struct vc_data *vc); +bool vc_is_sel(const struct vc_data *vc); extern int console_blanked; -- cgit v1.2.3 From 649f6fbe6abe0c7749120067058709d41111f655 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:32 +0100 Subject: tty: vt: remove extern from functions in selection.h Remove unneeded 'extern' keyword from function prototypes in selection.h. This makes it more readable as no more wrapping is needed on many places. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-19-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/selection.h | 52 ++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/selection.h b/include/linux/selection.h index 533509f6ba4f..bab7d30d3446 100644 --- a/include/linux/selection.h +++ b/include/linux/selection.h @@ -14,15 +14,14 @@ struct tty_struct; struct vc_data; -extern void clear_selection(void); -extern int set_selection_user(const struct tiocl_selection __user *sel, - struct tty_struct *tty); -extern int set_selection_kernel(struct tiocl_selection *v, - struct tty_struct *tty); -extern int paste_selection(struct tty_struct *tty); -extern int sel_loadlut(u32 __user *lut); -extern int mouse_reporting(void); -extern void mouse_report(struct tty_struct * tty, int butt, int mrx, int mry); +void clear_selection(void); +int set_selection_user(const struct tiocl_selection __user *sel, + struct tty_struct *tty); +int set_selection_kernel(struct tiocl_selection *v, struct tty_struct *tty); +int paste_selection(struct tty_struct *tty); +int sel_loadlut(u32 __user *lut); +int mouse_reporting(void); +void mouse_report(struct tty_struct *tty, int butt, int mrx, int mry); bool vc_is_sel(const struct vc_data *vc); @@ -33,24 +32,21 @@ extern unsigned char default_red[]; extern unsigned char default_grn[]; extern unsigned char default_blu[]; -extern unsigned short *screen_pos(const struct vc_data *vc, int w_offset, - bool viewed); -extern u16 screen_glyph(const struct vc_data *vc, int offset); -extern u32 screen_glyph_unicode(const struct vc_data *vc, int offset); -extern void complement_pos(struct vc_data *vc, int offset); -extern void invert_screen(struct vc_data *vc, int offset, int count, bool viewed); - -extern void getconsxy(const struct vc_data *vc, unsigned char xy[static 2]); -extern void putconsxy(struct vc_data *vc, unsigned char xy[static const 2]); - -extern u16 vcs_scr_readw(const struct vc_data *vc, const u16 *org); -extern void vcs_scr_writew(struct vc_data *vc, u16 val, u16 *org); -extern void vcs_scr_updated(struct vc_data *vc); - -extern int vc_uniscr_check(struct vc_data *vc); -extern void vc_uniscr_copy_line(const struct vc_data *vc, void *dest, - bool viewed, - unsigned int row, unsigned int col, - unsigned int nr); +unsigned short *screen_pos(const struct vc_data *vc, int w_offset, bool viewed); +u16 screen_glyph(const struct vc_data *vc, int offset); +u32 screen_glyph_unicode(const struct vc_data *vc, int offset); +void complement_pos(struct vc_data *vc, int offset); +void invert_screen(struct vc_data *vc, int offset, int count, bool viewed); + +void getconsxy(const struct vc_data *vc, unsigned char xy[static 2]); +void putconsxy(struct vc_data *vc, unsigned char xy[static const 2]); + +u16 vcs_scr_readw(const struct vc_data *vc, const u16 *org); +void vcs_scr_writew(struct vc_data *vc, u16 val, u16 *org); +void vcs_scr_updated(struct vc_data *vc); + +int vc_uniscr_check(struct vc_data *vc); +void vc_uniscr_copy_line(const struct vc_data *vc, void *dest, bool viewed, + unsigned int row, unsigned int col, unsigned int nr); #endif -- cgit v1.2.3 From 7995c30d8d771c8410d7f2ba5b9d42b69e0074c8 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:33 +0100 Subject: tty: vt: make consw::con_debug_*() return void The return value of con_debug_enter() and con_debug_leave() is ignored on many fronts. So just don't propagate errors (the current implementations return 0 anyway) and make the return type a void. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Daniel Vetter Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-20-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 21 ++++----------------- drivers/video/fbdev/core/fbcon.c | 6 ++---- include/linux/console.h | 18 ++++++------------ 3 files changed, 12 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 51a2787415db..bbfda0d33ca1 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -4012,15 +4012,9 @@ EXPORT_SYMBOL(con_is_visible); * Called when the console is taken over by the kernel debugger, this * function needs to save the current console state, then put the console * into a state suitable for the kernel debugger. - * - * RETURNS: - * Zero on success, nonzero if a failure occurred when trying to prepare - * the console for the debugger. */ -int con_debug_enter(struct vc_data *vc) +void con_debug_enter(struct vc_data *vc) { - int ret = 0; - saved_fg_console = fg_console; saved_last_console = last_console; saved_want_console = want_console; @@ -4029,7 +4023,7 @@ int con_debug_enter(struct vc_data *vc) vc->vc_mode = KD_TEXT; console_blanked = 0; if (vc->vc_sw->con_debug_enter) - ret = vc->vc_sw->con_debug_enter(vc); + vc->vc_sw->con_debug_enter(vc); #ifdef CONFIG_KGDB_KDB /* Set the initial LINES variable if it is not already set */ if (vc->vc_rows < 999) { @@ -4059,7 +4053,6 @@ int con_debug_enter(struct vc_data *vc) } } #endif /* CONFIG_KGDB_KDB */ - return ret; } EXPORT_SYMBOL_GPL(con_debug_enter); @@ -4068,15 +4061,10 @@ EXPORT_SYMBOL_GPL(con_debug_enter); * * Restore the console state to what it was before the kernel debugger * was invoked. - * - * RETURNS: - * Zero on success, nonzero if a failure occurred when trying to restore - * the console. */ -int con_debug_leave(void) +void con_debug_leave(void) { struct vc_data *vc; - int ret = 0; fg_console = saved_fg_console; last_console = saved_last_console; @@ -4086,8 +4074,7 @@ int con_debug_leave(void) vc = vc_cons[fg_console].d; if (vc->vc_sw->con_debug_leave) - ret = vc->vc_sw->con_debug_leave(vc); - return ret; + vc->vc_sw->con_debug_leave(vc); } EXPORT_SYMBOL_GPL(con_debug_leave); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index dd2f4617485c..d3fb98084eda 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2243,7 +2243,7 @@ static int fbcon_blank(struct vc_data *vc, int blank, int mode_switch) return 0; } -static int fbcon_debug_enter(struct vc_data *vc) +static void fbcon_debug_enter(struct vc_data *vc) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; @@ -2253,10 +2253,9 @@ static int fbcon_debug_enter(struct vc_data *vc) if (info->fbops->fb_debug_enter) info->fbops->fb_debug_enter(info); fbcon_set_palette(vc, color_table); - return 0; } -static int fbcon_debug_leave(struct vc_data *vc) +static void fbcon_debug_leave(struct vc_data *vc) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; @@ -2264,7 +2263,6 @@ static int fbcon_debug_leave(struct vc_data *vc) ops->graphics = ops->save_graphics; if (info->fbops->fb_debug_leave) info->fbops->fb_debug_leave(info); - return 0; } static int fbcon_get_font(struct vc_data *vc, struct console_font *font, unsigned int vpitch) diff --git a/include/linux/console.h b/include/linux/console.h index 38b379d6c624..93a1db5bf3b5 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -88,11 +88,11 @@ struct consw { * limited to, unblanking the console, loading an appropriate * palette, and allowing debugger generated output. */ - int (*con_debug_enter)(struct vc_data *vc); + void (*con_debug_enter)(struct vc_data *vc); /* * Restore the console to its pre-debug state as closely as possible. */ - int (*con_debug_leave)(struct vc_data *vc); + void (*con_debug_leave)(struct vc_data *vc); }; extern const struct consw *conswitchp; @@ -113,17 +113,11 @@ int do_unregister_con_driver(const struct consw *csw); int do_take_over_console(const struct consw *sw, int first, int last, int deflt); void give_up_console(const struct consw *sw); #ifdef CONFIG_HW_CONSOLE -int con_debug_enter(struct vc_data *vc); -int con_debug_leave(void); +void con_debug_enter(struct vc_data *vc); +void con_debug_leave(void); #else -static inline int con_debug_enter(struct vc_data *vc) -{ - return 0; -} -static inline int con_debug_leave(void) -{ - return 0; -} +static inline void con_debug_enter(struct vc_data *vc) { } +static inline void con_debug_leave(void) { } #endif /* cursor */ -- cgit v1.2.3 From dae3e6b6180f1a2394b984c596d39ed2c57d25fe Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:34 +0100 Subject: tty: vt: make init parameter of consw::con_init() a bool The 'init' parameter of consw::con_init() is true for the first call of the hook on a particular console. So make the parameter a bool. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Reviewed-by: Geert Uytterhoeven Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-21-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 8 ++++---- drivers/video/console/dummycon.c | 2 +- drivers/video/console/mdacon.c | 2 +- drivers/video/console/newport_con.c | 2 +- drivers/video/console/sticon.c | 2 +- drivers/video/console/vgacon.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 2 +- include/linux/console.h | 4 +++- 8 files changed, 14 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index bbfda0d33ca1..fcb41c8724f3 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -999,7 +999,7 @@ int vc_cons_allocated(unsigned int i) return (i < MAX_NR_CONSOLES && vc_cons[i].d); } -static void visual_init(struct vc_data *vc, int num, int init) +static void visual_init(struct vc_data *vc, int num, bool init) { /* ++Geert: vc->vc_sw->con_init determines console size */ if (vc->vc_sw) @@ -1083,7 +1083,7 @@ int vc_allocate(unsigned int currcons) /* return 0 on success */ vc->port.ops = &vc_port_ops; INIT_WORK(&vc_cons[currcons].SAK_work, vc_SAK); - visual_init(vc, currcons, 1); + visual_init(vc, currcons, true); if (!*vc->uni_pagedict_loc) con_set_default_unimap(vc); @@ -3513,7 +3513,7 @@ static int __init con_init(void) vc_cons[currcons].d = vc = kzalloc(sizeof(struct vc_data), GFP_NOWAIT); INIT_WORK(&vc_cons[currcons].SAK_work, vc_SAK); tty_port_init(&vc->port); - visual_init(vc, currcons, 1); + visual_init(vc, currcons, true); /* Assuming vc->vc_{cols,rows,screenbuf_size} are sane here. */ vc->vc_screenbuf = kzalloc(vc->vc_screenbuf_size, GFP_NOWAIT); vc_init(vc, currcons || !vc->vc_sw->con_save_screen); @@ -3682,7 +3682,7 @@ static int do_bind_con_driver(const struct consw *csw, int first, int last, old_was_color = vc->vc_can_do_color; vc->vc_sw->con_deinit(vc); vc->vc_origin = (unsigned long)vc->vc_screenbuf; - visual_init(vc, i, 0); + visual_init(vc, i, false); set_origin(vc); update_attr(vc); diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index 14af5d9e13b0..f2cef9d9a4b5 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -98,7 +98,7 @@ static const char *dummycon_startup(void) return "dummy device"; } -static void dummycon_init(struct vc_data *vc, int init) +static void dummycon_init(struct vc_data *vc, bool init) { vc->vc_can_do_color = 1; if (init) { diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index ef29b321967f..c5b255c96879 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -352,7 +352,7 @@ static const char *mdacon_startup(void) return "MDA-2"; } -static void mdacon_init(struct vc_data *c, int init) +static void mdacon_init(struct vc_data *c, bool init) { c->vc_complement_mask = 0x0800; /* reverse video */ c->vc_display_fg = &mda_display_fg; diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index e8e4f82cd4a1..12c64ef47087 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -324,7 +324,7 @@ out_unmap: return NULL; } -static void newport_init(struct vc_data *vc, int init) +static void newport_init(struct vc_data *vc, bool init) { int cols, rows; diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 992a4fa431aa..0bfeabc3f7c7 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -273,7 +273,7 @@ static int sticon_font_set(struct vc_data *vc, struct console_font *font, return sticon_set_font(vc, font, vpitch); } -static void sticon_init(struct vc_data *c, int init) +static void sticon_init(struct vc_data *c, bool init) { struct sti_struct *sti = sticon_sti; int vc_cols, vc_rows; diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 0c76e2817b49..5d523753def8 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -367,7 +367,7 @@ static const char *vgacon_startup(void) return display_desc; } -static void vgacon_init(struct vc_data *c, int init) +static void vgacon_init(struct vc_data *c, bool init) { struct uni_pagedict *p; @@ -384,7 +384,7 @@ static void vgacon_init(struct vc_data *c, int init) c->vc_scan_lines = vga_scan_lines; c->vc_font.height = c->vc_cell_height = vga_video_font_height; - /* set dimensions manually if init != 0 since vc_resize() will fail */ + /* set dimensions manually if init is true since vc_resize() will fail */ if (init) { c->vc_cols = vga_video_num_columns; c->vc_rows = vga_video_num_lines; diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index d3fb98084eda..939c5d893dfb 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -988,7 +988,7 @@ static const char *fbcon_startup(void) return display_desc; } -static void fbcon_init(struct vc_data *vc, int init) +static void fbcon_init(struct vc_data *vc, bool init) { struct fb_info *info; struct fbcon_ops *ops; diff --git a/include/linux/console.h b/include/linux/console.h index 93a1db5bf3b5..fc9450e0c78f 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -36,6 +36,8 @@ enum vc_intensity; /** * struct consw - callbacks for consoles * + * @con_init: initialize the console on @vc. @init is true for the very first + * call on this @vc. * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. @@ -46,7 +48,7 @@ enum vc_intensity; struct consw { struct module *owner; const char *(*con_startup)(void); - void (*con_init)(struct vc_data *vc, int init); + void (*con_init)(struct vc_data *vc, bool init); void (*con_deinit)(struct vc_data *vc); void (*con_clear)(struct vc_data *vc, int sy, int sx, int height, int width); -- cgit v1.2.3 From 559f01a0ee6d924c6fec3eaf6a5b078b15e71070 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:35 +0100 Subject: tty: vt: sanitize arguments of consw::con_clear() In consw::con_clear(): * Height is always 1, so drop it. * Offsets and width are always unsigned values, so re-type them as such. This needs a new __fbcon_clear() in the fbcon code to still handle height which might not be 1 when called internally. Note that tests for negative count/width are left in place -- they are taken care of in the next patches. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-22-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 2 +- drivers/video/console/dummycon.c | 4 ++-- drivers/video/console/mdacon.c | 15 +++++---------- drivers/video/console/newport_con.c | 6 +++--- drivers/video/console/sticon.c | 8 ++++---- drivers/video/console/vgacon.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 32 +++++++++++++++++++------------- include/linux/console.h | 5 +++-- 8 files changed, 39 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index fcb41c8724f3..b6f1449421bc 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1585,7 +1585,7 @@ static void csi_X(struct vc_data *vc) vc_uniscr_clear_line(vc, vc->state.x, count); scr_memsetw((unsigned short *)vc->vc_pos, vc->vc_video_erase_char, 2 * count); if (con_should_update(vc)) - vc->vc_sw->con_clear(vc, vc->state.y, vc->state.x, 1, count); + vc->vc_sw->con_clear(vc, vc->state.y, vc->state.x, count); vc->vc_need_wrap = 0; } diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index f2cef9d9a4b5..0a69d5c216ee 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -109,8 +109,8 @@ static void dummycon_init(struct vc_data *vc, bool init) } static void dummycon_deinit(struct vc_data *vc) { } -static void dummycon_clear(struct vc_data *vc, int sy, int sx, int height, - int width) { } +static void dummycon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) { } static void dummycon_cursor(struct vc_data *vc, int mode) { } static bool dummycon_scroll(struct vc_data *vc, unsigned int top, diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index c5b255c96879..1ddbb6cd5b0c 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -442,23 +442,18 @@ static void mdacon_putcs(struct vc_data *c, const unsigned short *s, } } -static void mdacon_clear(struct vc_data *c, int y, int x, - int height, int width) +static void mdacon_clear(struct vc_data *c, unsigned int y, unsigned int x, + unsigned int width) { u16 *dest = mda_addr(x, y); u16 eattr = mda_convert_attr(c->vc_video_erase_char); - if (width <= 0 || height <= 0) + if (width <= 0) return; - if (x==0 && width==mda_num_columns) { - scr_memsetw(dest, eattr, height*width*2); - } else { - for (; height > 0; height--, dest+=mda_num_columns) - scr_memsetw(dest, eattr, width*2); - } + scr_memsetw(dest, eattr, width * 2); } - + static int mdacon_switch(struct vc_data *c) { return 1; /* redrawing needed */ diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index 12c64ef47087..55c6106b3507 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -346,12 +346,12 @@ static void newport_deinit(struct vc_data *c) } } -static void newport_clear(struct vc_data *vc, int sy, int sx, int height, - int width) +static void newport_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) { int xend = ((sx + width) << 3) - 1; int ystart = ((sy << 4) + topscan) & 0x3ff; - int yend = (((sy + height) << 4) + topscan - 1) & 0x3ff; + int yend = (((sy + 1) << 4) + topscan - 1) & 0x3ff; if (logo_active) return; diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 0bfeabc3f7c7..d99c2a659bfd 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -300,13 +300,13 @@ static void sticon_deinit(struct vc_data *c) sticon_set_def_font(i); } -static void sticon_clear(struct vc_data *conp, int sy, int sx, int height, - int width) +static void sticon_clear(struct vc_data *conp, unsigned int sy, unsigned int sx, + unsigned int width) { - if (!height || !width) + if (!width) return; - sti_clear(sticon_sti, sy, sx, height, width, + sti_clear(sticon_sti, sy, sx, 1, width, conp->vc_video_erase_char, font_data[conp->vc_num]); } diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 5d523753def8..85f29dec2c3d 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -1191,8 +1191,8 @@ static bool vgacon_scroll(struct vc_data *c, unsigned int t, unsigned int b, * The console `switch' structure for the VGA based console */ -static void vgacon_clear(struct vc_data *vc, int sy, int sx, int height, - int width) { } +static void vgacon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) { } static void vgacon_putc(struct vc_data *vc, int c, int ypos, int xpos) { } static void vgacon_putcs(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos) { } diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 939c5d893dfb..8a31a36483ea 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1235,8 +1235,8 @@ finished: * restriction is simplicity & efficiency at the moment. */ -static void fbcon_clear(struct vc_data *vc, int sy, int sx, int height, - int width) +static void __fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int height, unsigned int width) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; @@ -1273,6 +1273,12 @@ static void fbcon_clear(struct vc_data *vc, int sy, int sx, int height, ops->clear(vc, info, real_y(p, sy), sx, height, width); } +static void fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, + unsigned int width) +{ + __fbcon_clear(vc, sy, sx, 1, width); +} + static void fbcon_putcs(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos) { @@ -1760,7 +1766,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, case SCROLL_MOVE: fbcon_redraw_blit(vc, info, p, t, b - t - count, count); - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * (b - count)), @@ -1783,7 +1789,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_up; - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); break; case SCROLL_PAN_REDRAW: @@ -1801,7 +1807,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, vc->vc_rows - b, b); } else fbcon_redraw_move(vc, p, t + count, b - t - count, t); - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); break; case SCROLL_PAN_MOVE: @@ -1824,14 +1830,14 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_up; - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); break; case SCROLL_REDRAW: redraw_up: fbcon_redraw(vc, t, b - t - count, count * vc->vc_cols); - fbcon_clear(vc, b - count, 0, count, vc->vc_cols); + __fbcon_clear(vc, b - count, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * (b - count)), @@ -1848,7 +1854,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, case SCROLL_MOVE: fbcon_redraw_blit(vc, info, p, b - 1, b - t - count, -count); - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * t), @@ -1871,7 +1877,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_down; - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); break; case SCROLL_PAN_MOVE: @@ -1893,7 +1899,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, b - t - count, vc->vc_cols); else goto redraw_down; - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); break; case SCROLL_PAN_REDRAW: @@ -1910,14 +1916,14 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, fbcon_redraw_move(vc, p, count, t, 0); } else fbcon_redraw_move(vc, p, t, b - t - count, t + count); - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); break; case SCROLL_REDRAW: redraw_down: fbcon_redraw(vc, b - 1, b - t - count, -count * vc->vc_cols); - fbcon_clear(vc, t, 0, count, vc->vc_cols); + __fbcon_clear(vc, t, 0, count, vc->vc_cols); scr_memsetw((unsigned short *) (vc->vc_origin + vc->vc_size_row * t), @@ -2196,7 +2202,7 @@ static void fbcon_generic_blank(struct vc_data *vc, struct fb_info *info, oldc = vc->vc_video_erase_char; vc->vc_video_erase_char &= charmask; - fbcon_clear(vc, 0, 0, vc->vc_rows, vc->vc_cols); + __fbcon_clear(vc, 0, 0, vc->vc_rows, vc->vc_cols); vc->vc_video_erase_char = oldc; } } diff --git a/include/linux/console.h b/include/linux/console.h index fc9450e0c78f..8fd96a5fca5f 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -38,6 +38,7 @@ enum vc_intensity; * * @con_init: initialize the console on @vc. @init is true for the very first * call on this @vc. + * @con_clear: erase @count characters at [@x, @y] on @vc. @count >= 1. * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. @@ -50,8 +51,8 @@ struct consw { const char *(*con_startup)(void); void (*con_init)(struct vc_data *vc, bool init); void (*con_deinit)(struct vc_data *vc); - void (*con_clear)(struct vc_data *vc, int sy, int sx, int height, - int width); + void (*con_clear)(struct vc_data *vc, unsigned int y, + unsigned int x, unsigned int count); void (*con_putc)(struct vc_data *vc, int c, int ypos, int xpos); void (*con_putcs)(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos); -- cgit v1.2.3 From 338c28107b51083846afdc5fe8f7830cc8abd893 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:39 +0100 Subject: tty: vt: sanitize consw::con_putc() parameters Make parameters of consw::con_putc() saner: * x and y are unsigned now, as they cannot be negative, and * ca is made u16, as it is composed of two 8bit values (character and attribute). See the con_putcs() hook, u16/ushort is worked on there. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-26-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/video/console/dummycon.c | 6 ++++-- drivers/video/console/newport_con.c | 4 ++-- include/linux/console.h | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index 0a69d5c216ee..1874beed0325 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -50,7 +50,8 @@ void dummycon_unregister_output_notifier(struct notifier_block *nb) raw_notifier_chain_unregister(&dummycon_output_nh, nb); } -static void dummycon_putc(struct vc_data *vc, int c, int ypos, int xpos) +static void dummycon_putc(struct vc_data *vc, u16 c, unsigned int y, + unsigned int x) { WARN_CONSOLE_UNLOCKED(); @@ -84,7 +85,8 @@ static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) return 1; } #else -static void dummycon_putc(struct vc_data *vc, int c, int ypos, int xpos) { } +static void dummycon_putc(struct vc_data *vc, u16 c, unsigned int y, + unsigned int x) { } static void dummycon_putcs(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos) { } static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index 55c6106b3507..9b5c0118873e 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -367,8 +367,8 @@ static void newport_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, } } -static void newport_putc(struct vc_data *vc, int charattr, int ypos, - int xpos) +static void newport_putc(struct vc_data *vc, u16 charattr, unsigned int ypos, + unsigned int xpos) { unsigned char *p; diff --git a/include/linux/console.h b/include/linux/console.h index 8fd96a5fca5f..92d57e5b3009 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -39,6 +39,8 @@ enum vc_intensity; * @con_init: initialize the console on @vc. @init is true for the very first * call on this @vc. * @con_clear: erase @count characters at [@x, @y] on @vc. @count >= 1. + * @con_putc: emit one character with attributes @ca to [@x, @y] on @vc. + * (optional -- @con_putcs would be called instead) * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. @@ -53,7 +55,8 @@ struct consw { void (*con_deinit)(struct vc_data *vc); void (*con_clear)(struct vc_data *vc, unsigned int y, unsigned int x, unsigned int count); - void (*con_putc)(struct vc_data *vc, int c, int ypos, int xpos); + void (*con_putc)(struct vc_data *vc, u16 ca, unsigned int y, + unsigned int x); void (*con_putcs)(struct vc_data *vc, const unsigned short *s, int count, int ypos, int xpos); void (*con_cursor)(struct vc_data *vc, int mode); -- cgit v1.2.3 From bfd7de49d7444ce46a48e92ce7cb11266ce79905 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:40 +0100 Subject: tty: vt: sanitize consw::con_putcs() parameters Similar to con_putc() in the previous patch: * make the pointer to charattr a pointer to u16, and * make x, y, and count unsigned as they are strictly non-negative. And again, document that hook. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-27-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/video/console/dummycon.c | 10 +++++----- drivers/video/console/mdacon.c | 4 ++-- drivers/video/console/newport_con.c | 9 +++++---- drivers/video/console/sticon.c | 4 ++-- drivers/video/console/vgacon.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 4 ++-- include/linux/console.h | 6 ++++-- 7 files changed, 22 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index 1874beed0325..188d9f3e201c 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -59,10 +59,10 @@ static void dummycon_putc(struct vc_data *vc, u16 c, unsigned int y, raw_notifier_call_chain(&dummycon_output_nh, 0, NULL); } -static void dummycon_putcs(struct vc_data *vc, const unsigned short *s, - int count, int ypos, int xpos) +static void dummycon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, + unsigned int ypos, unsigned int xpos) { - int i; + unsigned int i; if (!dummycon_putc_called) { /* Ignore erases */ @@ -87,8 +87,8 @@ static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) #else static void dummycon_putc(struct vc_data *vc, u16 c, unsigned int y, unsigned int x) { } -static void dummycon_putcs(struct vc_data *vc, const unsigned short *s, - int count, int ypos, int xpos) { } +static void dummycon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, + unsigned int ypos, unsigned int xpos) { } static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) { return 0; diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index 01e779943c00..b8822b615b2f 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -427,8 +427,8 @@ static inline u16 *mda_addr(unsigned int x, unsigned int y) return mda_vram_base + y * mda_num_columns + x; } -static void mdacon_putcs(struct vc_data *c, const unsigned short *s, - int count, int y, int x) +static void mdacon_putcs(struct vc_data *c, const u16 *s, unsigned int count, + unsigned int y, unsigned int x) { u16 *dest = mda_addr(x, y); diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index 9b5c0118873e..5e65ee0b7c07 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -396,12 +396,13 @@ static void newport_putc(struct vc_data *vc, u16 charattr, unsigned int ypos, RENDER(npregs, p); } -static void newport_putcs(struct vc_data *vc, const unsigned short *s, - int count, int ypos, int xpos) +static void newport_putcs(struct vc_data *vc, const u16 *s, + unsigned int count, unsigned int ypos, + unsigned int xpos) { - int i; - int charattr; unsigned char *p; + unsigned int i; + u16 charattr; charattr = (scr_readw(s) >> 8) & 0xff; diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 2f87b5909d0d..906da1fde7c8 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -71,8 +71,8 @@ static const char *sticon_startup(void) return "STI console"; } -static void sticon_putcs(struct vc_data *conp, const unsigned short *s, - int count, int ypos, int xpos) +static void sticon_putcs(struct vc_data *conp, const u16 *s, unsigned int count, + unsigned int ypos, unsigned int xpos) { if (vga_is_gfx || console_blanked) return; diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 4beab11f87eb..aa0589085847 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -1193,8 +1193,8 @@ static bool vgacon_scroll(struct vc_data *c, unsigned int t, unsigned int b, static void vgacon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, unsigned int width) { } -static void vgacon_putcs(struct vc_data *vc, const unsigned short *s, - int count, int ypos, int xpos) { } +static void vgacon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, + unsigned int ypos, unsigned int xpos) { } const struct consw vga_con = { .owner = THIS_MODULE, diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 38de0f8723aa..7a7b2ac0d7a9 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1279,8 +1279,8 @@ static void fbcon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, __fbcon_clear(vc, sy, sx, 1, width); } -static void fbcon_putcs(struct vc_data *vc, const unsigned short *s, - int count, int ypos, int xpos) +static void fbcon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, + unsigned int ypos, unsigned int xpos) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_display *p = &fb_display[vc->vc_num]; diff --git a/include/linux/console.h b/include/linux/console.h index 92d57e5b3009..82d55764a66f 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -41,6 +41,7 @@ enum vc_intensity; * @con_clear: erase @count characters at [@x, @y] on @vc. @count >= 1. * @con_putc: emit one character with attributes @ca to [@x, @y] on @vc. * (optional -- @con_putcs would be called instead) + * @con_putcs: emit @count characters with attributes @s to [@x, @y] on @vc. * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. @@ -57,8 +58,9 @@ struct consw { unsigned int x, unsigned int count); void (*con_putc)(struct vc_data *vc, u16 ca, unsigned int y, unsigned int x); - void (*con_putcs)(struct vc_data *vc, const unsigned short *s, - int count, int ypos, int xpos); + void (*con_putcs)(struct vc_data *vc, const u16 *s, + unsigned int count, unsigned int ypos, + unsigned int xpos); void (*con_cursor)(struct vc_data *vc, int mode); bool (*con_scroll)(struct vc_data *vc, unsigned int top, unsigned int bottom, enum con_scroll dir, -- cgit v1.2.3 From a292e3fc94cb9795bbba4ddac075a9055cd58a5e Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:43 +0100 Subject: tty: vt: remove CM_* constants There is no difference between CM_MOVE and CM_DRAW. Either of them enables the cursor. CM_ERASE then disables cursor. So get rid of all of them and use simple "bool enable". Note that this propagates down to the fbcon code. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-30-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 4 ++-- drivers/video/console/dummycon.c | 2 +- drivers/video/console/mdacon.c | 4 ++-- drivers/video/console/newport_con.c | 4 ++-- drivers/video/console/sticon.c | 6 +++--- drivers/video/console/vgacon.c | 6 +++--- drivers/video/fbdev/core/bitblit.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 19 +++++++++---------- drivers/video/fbdev/core/fbcon.h | 4 ++-- drivers/video/fbdev/core/fbcon_ccw.c | 4 ++-- drivers/video/fbdev/core/fbcon_cw.c | 4 ++-- drivers/video/fbdev/core/fbcon_ud.c | 4 ++-- drivers/video/fbdev/core/tileblit.c | 4 ++-- include/linux/console.h | 8 ++------ 14 files changed, 36 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 6091ffcf93d8..e4edcaf9d0a3 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -860,7 +860,7 @@ static void hide_cursor(struct vc_data *vc) if (vc_is_sel(vc)) clear_selection(); - vc->vc_sw->con_cursor(vc, CM_ERASE); + vc->vc_sw->con_cursor(vc, false); hide_softcursor(vc); } @@ -873,7 +873,7 @@ static void set_cursor(struct vc_data *vc) clear_selection(); add_softcursor(vc); if (CUR_SIZE(vc->vc_cursor_type) != CUR_NONE) - vc->vc_sw->con_cursor(vc, CM_DRAW); + vc->vc_sw->con_cursor(vc, true); } else hide_cursor(vc); } diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index 188d9f3e201c..1171e27edef7 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -113,7 +113,7 @@ static void dummycon_init(struct vc_data *vc, bool init) static void dummycon_deinit(struct vc_data *vc) { } static void dummycon_clear(struct vc_data *vc, unsigned int sy, unsigned int sx, unsigned int width) { } -static void dummycon_cursor(struct vc_data *vc, int mode) { } +static void dummycon_cursor(struct vc_data *vc, bool enable) { } static bool dummycon_scroll(struct vc_data *vc, unsigned int top, unsigned int bottom, enum con_scroll dir, diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index b8822b615b2f..bc851a1d9f4d 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -470,9 +470,9 @@ static int mdacon_blank(struct vc_data *c, int blank, int mode_switch) } } -static void mdacon_cursor(struct vc_data *c, int mode) +static void mdacon_cursor(struct vc_data *c, bool enable) { - if (mode == CM_ERASE) { + if (!enable) { mda_set_cursor(mda_vram_len - 1); return; } diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index f852717b88f0..e35406dea7c7 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -438,14 +438,14 @@ static void newport_putcs(struct vc_data *vc, const u16 *s, } } -static void newport_cursor(struct vc_data *vc, int mode) +static void newport_cursor(struct vc_data *vc, bool enable) { unsigned short treg; int xcurs, ycurs; treg = newport_vc2_get(npregs, VC2_IREG_CONTROL); - if (mode == CM_ERASE) { + if (!enable) { newport_vc2_set(npregs, VC2_IREG_CONTROL, (treg & ~(VC2_CTRL_ECDISP))); return; diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 42480874db00..786e1b3a98ea 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -86,7 +86,7 @@ static void sticon_putcs(struct vc_data *conp, const u16 *s, unsigned int count, } } -static void sticon_cursor(struct vc_data *conp, int mode) +static void sticon_cursor(struct vc_data *conp, bool enable) { unsigned short car1; @@ -95,7 +95,7 @@ static void sticon_cursor(struct vc_data *conp, int mode) return; car1 = conp->vc_screenbuf[conp->state.x + conp->state.y * conp->vc_cols]; - if (mode == CM_ERASE) { + if (!enable) { sti_putc(sticon_sti, car1, conp->state.y, conp->state.x, font_data[conp->vc_num]); return; @@ -121,7 +121,7 @@ static bool sticon_scroll(struct vc_data *conp, unsigned int t, if (vga_is_gfx) return false; - sticon_cursor(conp, CM_ERASE); + sticon_cursor(conp, false); switch (dir) { case SM_UP: diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 82d01a9ccd6d..d93eb15da435 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -503,7 +503,7 @@ static void vgacon_set_cursor_size(int from, int to) raw_spin_unlock_irqrestore(&vga_lock, flags); } -static void vgacon_cursor(struct vc_data *c, int mode) +static void vgacon_cursor(struct vc_data *c, bool enable) { unsigned int c_height; @@ -516,7 +516,7 @@ static void vgacon_cursor(struct vc_data *c, int mode) write_vga(14, (c->vc_pos - vga_vram_base) / 2); - if (mode == CM_ERASE) { + if (!enable) { if (vga_video_type >= VIDEO_TYPE_VGAC) vgacon_set_cursor_size(31, 30); else @@ -1030,7 +1030,7 @@ static int vgacon_adjust_height(struct vc_data *vc, unsigned fontheight) /* void size to cause regs to be rewritten */ cursor_size_lastfrom = 0; cursor_size_lastto = 0; - c->vc_sw->con_cursor(c, CM_DRAW); + c->vc_sw->con_cursor(c, true); } c->vc_font.height = c->vc_cell_height = fontheight; vc_resize(c, 0, rows); /* Adjust console size */ diff --git a/drivers/video/fbdev/core/bitblit.c b/drivers/video/fbdev/core/bitblit.c index daff152f4c22..3ff1b2a8659e 100644 --- a/drivers/video/fbdev/core/bitblit.c +++ b/drivers/video/fbdev/core/bitblit.c @@ -233,7 +233,7 @@ static void bit_clear_margins(struct vc_data *vc, struct fb_info *info, } } -static void bit_cursor(struct vc_data *vc, struct fb_info *info, int mode, +static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable, int fg, int bg) { struct fb_cursor cursor; @@ -348,7 +348,7 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, int mode, mask[i++] = msk; } - ops->cursor_state.enable = (mode != CM_ERASE) && !use_sw; + ops->cursor_state.enable = enable && !use_sw; cursor.image.data = src; cursor.image.fg_color = ops->cursor_state.image.fg_color; diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 7a7b2ac0d7a9..c1765a6ef490 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -351,7 +351,7 @@ static void fb_flashcursor(struct work_struct *work) struct fb_info *info; struct vc_data *vc = NULL; int c; - int mode; + bool enable; int ret; /* FIXME: we should sort out the unbind locking instead */ @@ -375,9 +375,8 @@ static void fb_flashcursor(struct work_struct *work) } c = scr_readw((u16 *) vc->vc_pos); - mode = (!ops->cursor_flash || ops->cursor_state.enable) ? - CM_ERASE : CM_DRAW; - ops->cursor(vc, info, mode, get_color(vc, info, c, 1), + enable = ops->cursor_flash && !ops->cursor_state.enable; + ops->cursor(vc, info, enable, get_color(vc, info, c, 1), get_color(vc, info, c, 0)); console_unlock(); @@ -1301,7 +1300,7 @@ static void fbcon_clear_margins(struct vc_data *vc, int bottom_only) ops->clear_margins(vc, info, margin_color, bottom_only); } -static void fbcon_cursor(struct vc_data *vc, int mode) +static void fbcon_cursor(struct vc_data *vc, bool enable) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; @@ -1317,12 +1316,12 @@ static void fbcon_cursor(struct vc_data *vc, int mode) else fbcon_add_cursor_work(info); - ops->cursor_flash = (mode == CM_ERASE) ? 0 : 1; + ops->cursor_flash = enable; if (!ops->cursor) return; - ops->cursor(vc, info, mode, get_color(vc, info, c, 1), + ops->cursor(vc, info, enable, get_color(vc, info, c, 1), get_color(vc, info, c, 0)); } @@ -1742,7 +1741,7 @@ static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b, if (fbcon_is_inactive(vc, info)) return true; - fbcon_cursor(vc, CM_ERASE); + fbcon_cursor(vc, false); /* * ++Geert: Only use ywrap/ypan if the console is in text mode @@ -2221,7 +2220,7 @@ static int fbcon_blank(struct vc_data *vc, int blank, int mode_switch) if (!fbcon_is_inactive(vc, info)) { if (ops->blank_state != blank) { ops->blank_state = blank; - fbcon_cursor(vc, blank ? CM_ERASE : CM_DRAW); + fbcon_cursor(vc, !blank); ops->cursor_flash = (!blank); if (fb_blank(info, blank)) @@ -2649,7 +2648,7 @@ void fbcon_suspended(struct fb_info *info) vc = vc_cons[ops->currcon].d; /* Clear cursor, restore saved data */ - fbcon_cursor(vc, CM_ERASE); + fbcon_cursor(vc, false); } void fbcon_resumed(struct fb_info *info) diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h index 0eaf54a21151..df70ea5ec5b3 100644 --- a/drivers/video/fbdev/core/fbcon.h +++ b/drivers/video/fbdev/core/fbcon.h @@ -61,8 +61,8 @@ struct fbcon_ops { int fg, int bg); void (*clear_margins)(struct vc_data *vc, struct fb_info *info, int color, int bottom_only); - void (*cursor)(struct vc_data *vc, struct fb_info *info, int mode, - int fg, int bg); + void (*cursor)(struct vc_data *vc, struct fb_info *info, + bool enable, int fg, int bg); int (*update_start)(struct fb_info *info); int (*rotate_font)(struct fb_info *info, struct vc_data *vc); struct fb_var_screeninfo var; /* copy of the current fb_var_screeninfo */ diff --git a/drivers/video/fbdev/core/fbcon_ccw.c b/drivers/video/fbdev/core/fbcon_ccw.c index 889423d580bc..f9b794ff7d39 100644 --- a/drivers/video/fbdev/core/fbcon_ccw.c +++ b/drivers/video/fbdev/core/fbcon_ccw.c @@ -218,7 +218,7 @@ static void ccw_clear_margins(struct vc_data *vc, struct fb_info *info, } } -static void ccw_cursor(struct vc_data *vc, struct fb_info *info, int mode, +static void ccw_cursor(struct vc_data *vc, struct fb_info *info, bool enable, int fg, int bg) { struct fb_cursor cursor; @@ -349,7 +349,7 @@ static void ccw_cursor(struct vc_data *vc, struct fb_info *info, int mode, kfree(tmp); } - ops->cursor_state.enable = (mode != CM_ERASE) && !use_sw; + ops->cursor_state.enable = enable && !use_sw; cursor.image.data = src; cursor.image.fg_color = ops->cursor_state.image.fg_color; diff --git a/drivers/video/fbdev/core/fbcon_cw.c b/drivers/video/fbdev/core/fbcon_cw.c index a306ca5802e8..903f6fc174e1 100644 --- a/drivers/video/fbdev/core/fbcon_cw.c +++ b/drivers/video/fbdev/core/fbcon_cw.c @@ -201,7 +201,7 @@ static void cw_clear_margins(struct vc_data *vc, struct fb_info *info, } } -static void cw_cursor(struct vc_data *vc, struct fb_info *info, int mode, +static void cw_cursor(struct vc_data *vc, struct fb_info *info, bool enable, int fg, int bg) { struct fb_cursor cursor; @@ -332,7 +332,7 @@ static void cw_cursor(struct vc_data *vc, struct fb_info *info, int mode, kfree(tmp); } - ops->cursor_state.enable = (mode != CM_ERASE) && !use_sw; + ops->cursor_state.enable = enable && !use_sw; cursor.image.data = src; cursor.image.fg_color = ops->cursor_state.image.fg_color; diff --git a/drivers/video/fbdev/core/fbcon_ud.c b/drivers/video/fbdev/core/fbcon_ud.c index f6fc458b46c7..594331936fd3 100644 --- a/drivers/video/fbdev/core/fbcon_ud.c +++ b/drivers/video/fbdev/core/fbcon_ud.c @@ -248,7 +248,7 @@ static void ud_clear_margins(struct vc_data *vc, struct fb_info *info, } } -static void ud_cursor(struct vc_data *vc, struct fb_info *info, int mode, +static void ud_cursor(struct vc_data *vc, struct fb_info *info, bool enable, int fg, int bg) { struct fb_cursor cursor; @@ -372,7 +372,7 @@ static void ud_cursor(struct vc_data *vc, struct fb_info *info, int mode, mask[i++] = ~msk; } - ops->cursor_state.enable = (mode != CM_ERASE) && !use_sw; + ops->cursor_state.enable = enable && !use_sw; cursor.image.data = src; cursor.image.fg_color = ops->cursor_state.image.fg_color; diff --git a/drivers/video/fbdev/core/tileblit.c b/drivers/video/fbdev/core/tileblit.c index 2768eff247ba..eff7ec4da167 100644 --- a/drivers/video/fbdev/core/tileblit.c +++ b/drivers/video/fbdev/core/tileblit.c @@ -79,7 +79,7 @@ static void tile_clear_margins(struct vc_data *vc, struct fb_info *info, return; } -static void tile_cursor(struct vc_data *vc, struct fb_info *info, int mode, +static void tile_cursor(struct vc_data *vc, struct fb_info *info, bool enable, int fg, int bg) { struct fb_tilecursor cursor; @@ -87,7 +87,7 @@ static void tile_cursor(struct vc_data *vc, struct fb_info *info, int mode, cursor.sx = vc->state.x; cursor.sy = vc->state.y; - cursor.mode = (mode == CM_ERASE || use_sw) ? 0 : 1; + cursor.mode = enable && !use_sw; cursor.fg = fg; cursor.bg = bg; diff --git a/include/linux/console.h b/include/linux/console.h index 82d55764a66f..a6a46b5efd66 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -42,6 +42,7 @@ enum vc_intensity; * @con_putc: emit one character with attributes @ca to [@x, @y] on @vc. * (optional -- @con_putcs would be called instead) * @con_putcs: emit @count characters with attributes @s to [@x, @y] on @vc. + * @con_cursor: enable/disable cursor depending on @enable * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. @@ -61,7 +62,7 @@ struct consw { void (*con_putcs)(struct vc_data *vc, const u16 *s, unsigned int count, unsigned int ypos, unsigned int xpos); - void (*con_cursor)(struct vc_data *vc, int mode); + void (*con_cursor)(struct vc_data *vc, bool enable); bool (*con_scroll)(struct vc_data *vc, unsigned int top, unsigned int bottom, enum con_scroll dir, unsigned int lines); @@ -128,11 +129,6 @@ static inline void con_debug_enter(struct vc_data *vc) { } static inline void con_debug_leave(void) { } #endif -/* cursor */ -#define CM_DRAW (1) -#define CM_ERASE (2) -#define CM_MOVE (3) - /* * The interface for a console, or any other device that wants to capture * console messages (printer driver?) -- cgit v1.2.3 From 8d5cc8eed738e3202379722295c626cba0849785 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:44 +0100 Subject: tty: vt: make consw::con_switch() return a bool The non-zero (true) return value from consw::con_switch() means a redraw is needed. So make this return type a bool explicitly instead of int. The latter might imply that -Eerrors are expected. They are not. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-31-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 2 +- drivers/video/console/dummycon.c | 4 ++-- drivers/video/console/mdacon.c | 4 ++-- drivers/video/console/newport_con.c | 4 ++-- drivers/video/console/sticon.c | 4 ++-- drivers/video/console/vgacon.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 6 +++--- include/linux/console.h | 4 +++- 8 files changed, 17 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index e4edcaf9d0a3..fd868046f586 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -970,7 +970,7 @@ void redraw_screen(struct vc_data *vc, int is_switch) } if (redraw) { - int update; + bool update; int old_was_color = vc->vc_can_do_color; set_origin(vc); diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index 1171e27edef7..c8d5aa0e3ed0 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -122,9 +122,9 @@ static bool dummycon_scroll(struct vc_data *vc, unsigned int top, return false; } -static int dummycon_switch(struct vc_data *vc) +static bool dummycon_switch(struct vc_data *vc) { - return 0; + return false; } /* diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index bc851a1d9f4d..4485ef923bb3 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -446,9 +446,9 @@ static void mdacon_clear(struct vc_data *c, unsigned int y, unsigned int x, scr_memsetw(dest, eattr, width * 2); } -static int mdacon_switch(struct vc_data *c) +static bool mdacon_switch(struct vc_data *c) { - return 1; /* redrawing needed */ + return true; /* redrawing needed */ } static int mdacon_blank(struct vc_data *c, int blank, int mode_switch) diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index e35406dea7c7..039d1c9937d2 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -459,7 +459,7 @@ static void newport_cursor(struct vc_data *vc, bool enable) newport_vc2_set(npregs, VC2_IREG_CURSY, ycurs); } -static int newport_switch(struct vc_data *vc) +static bool newport_switch(struct vc_data *vc) { static int logo_drawn = 0; @@ -473,7 +473,7 @@ static int newport_switch(struct vc_data *vc) } } - return 1; + return true; } static int newport_blank(struct vc_data *c, int blank, int mode_switch) diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 786e1b3a98ea..f3bb48a0e980 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -293,9 +293,9 @@ static void sticon_clear(struct vc_data *conp, unsigned int sy, unsigned int sx, conp->vc_video_erase_char, font_data[conp->vc_num]); } -static int sticon_switch(struct vc_data *conp) +static bool sticon_switch(struct vc_data *conp) { - return 1; /* needs refreshing */ + return true; /* needs refreshing */ } static int sticon_blank(struct vc_data *c, int blank, int mode_switch) diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index d93eb15da435..f89eb53c0b79 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -614,7 +614,7 @@ static void vgacon_doresize(struct vc_data *c, raw_spin_unlock_irqrestore(&vga_lock, flags); } -static int vgacon_switch(struct vc_data *c) +static bool vgacon_switch(struct vc_data *c) { int x = c->vc_cols * VGA_FONTWIDTH; int y = c->vc_rows * c->vc_cell_height; @@ -643,7 +643,7 @@ static int vgacon_switch(struct vc_data *c) vgacon_doresize(c, c->vc_cols, c->vc_rows); } - return 0; /* Redrawing not needed */ + return false; /* Redrawing not needed */ } static void vga_set_palette(struct vc_data *vc, const unsigned char *table) diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index c1765a6ef490..d5d924225209 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2056,7 +2056,7 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width, return 0; } -static int fbcon_switch(struct vc_data *vc) +static bool fbcon_switch(struct vc_data *vc) { struct fb_info *info, *old_info = NULL; struct fbcon_ops *ops; @@ -2178,9 +2178,9 @@ static int fbcon_switch(struct vc_data *vc) vc->vc_origin + vc->vc_size_row * vc->vc_top, vc->vc_size_row * (vc->vc_bottom - vc->vc_top) / 2); - return 0; + return false; } - return 1; + return true; } static void fbcon_generic_blank(struct vc_data *vc, struct fb_info *info, diff --git a/include/linux/console.h b/include/linux/console.h index a6a46b5efd66..f7c6b5fc3a36 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -46,6 +46,8 @@ enum vc_intensity; * @con_scroll: move lines from @top to @bottom in direction @dir by @lines. * Return true if no generic handling should be done. * Invoked by csi_M and printing to the console. + * @con_switch: notifier about the console switch; it is supposed to return + * true if a redraw is needed. * @con_set_palette: sets the palette of the console to @table (optional) * @con_scrolldelta: the contents of the console should be scrolled by @lines. * Invoked by user. (optional) @@ -66,7 +68,7 @@ struct consw { bool (*con_scroll)(struct vc_data *vc, unsigned int top, unsigned int bottom, enum con_scroll dir, unsigned int lines); - int (*con_switch)(struct vc_data *vc); + bool (*con_switch)(struct vc_data *vc); int (*con_blank)(struct vc_data *vc, int blank, int mode_switch); int (*con_font_set)(struct vc_data *vc, struct console_font *font, unsigned int vpitch, unsigned int flags); -- cgit v1.2.3 From ace4ebf9b70a7daea12102c09ba5ef6bb73223aa Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:46 +0100 Subject: tty: vt: define a common enum for VESA blanking constants There are currently two places with VESA blanking constants definitions: fb.h and console.h. Extract/unify the two to a separate header (vesa.h). Given the fb's is in an uapi header, create the common header in uapi too. Note that instead of macros, an enum (vesa_blank_mode) is created. But the macros are kept too (they now expand to the enum constants), just in case someone in userspace performs some #ifdeffery. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: linux-kernel@vger.kernel.org Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: Thomas Zimmermann Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-33-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/console.h | 7 +------ include/uapi/linux/fb.h | 8 +------- include/uapi/linux/vesa.h | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 13 deletions(-) create mode 100644 include/uapi/linux/vesa.h (limited to 'include/linux') diff --git a/include/linux/console.h b/include/linux/console.h index f7c6b5fc3a36..860f82756c9c 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -18,6 +18,7 @@ #include #include #include +#include struct vc_data; struct console_font_op; @@ -520,12 +521,6 @@ void vcs_remove_sysfs(int index); */ extern atomic_t ignore_console_lock_warning; -/* VESA Blanking Levels */ -#define VESA_NO_BLANKING 0 -#define VESA_VSYNC_SUSPEND 1 -#define VESA_HSYNC_SUSPEND 2 -#define VESA_POWERDOWN 3 - extern void console_init(void); /* For deferred console takeover */ diff --git a/include/uapi/linux/fb.h b/include/uapi/linux/fb.h index 3a49913d006c..cde8f173f566 100644 --- a/include/uapi/linux/fb.h +++ b/include/uapi/linux/fb.h @@ -4,6 +4,7 @@ #include #include +#include /* Definitions of frame buffers */ @@ -293,13 +294,6 @@ struct fb_con2fbmap { __u32 framebuffer; }; -/* VESA Blanking Levels */ -#define VESA_NO_BLANKING 0 -#define VESA_VSYNC_SUSPEND 1 -#define VESA_HSYNC_SUSPEND 2 -#define VESA_POWERDOWN 3 - - enum { /* screen: unblanked, hsync: on, vsync: on */ FB_BLANK_UNBLANK = VESA_NO_BLANKING, diff --git a/include/uapi/linux/vesa.h b/include/uapi/linux/vesa.h new file mode 100644 index 000000000000..81947f5088cd --- /dev/null +++ b/include/uapi/linux/vesa.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_LINUX_VESA_H +#define _UAPI_LINUX_VESA_H + +/* VESA Blanking Levels */ +enum vesa_blank_mode { + VESA_NO_BLANKING = 0, +#define VESA_NO_BLANKING VESA_NO_BLANKING + VESA_VSYNC_SUSPEND = 1, +#define VESA_VSYNC_SUSPEND VESA_VSYNC_SUSPEND + VESA_HSYNC_SUSPEND = 2, +#define VESA_HSYNC_SUSPEND VESA_HSYNC_SUSPEND + VESA_POWERDOWN = VESA_VSYNC_SUSPEND | VESA_HSYNC_SUSPEND, +#define VESA_POWERDOWN VESA_POWERDOWN + VESA_BLANK_MAX = VESA_POWERDOWN, +}; + +#endif -- cgit v1.2.3 From 0a58d83dfb14ac30126c37b18d4578e5b261459d Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:48 +0100 Subject: tty: vt: use enum constants for VESA blanking modes Use the new enum for VESA constants. This improves type checking in consw::con_blank(). Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-35-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 4 ++-- drivers/video/console/dummycon.c | 6 ++++-- drivers/video/console/mdacon.c | 3 ++- drivers/video/console/newport_con.c | 3 ++- drivers/video/console/sticon.c | 3 ++- drivers/video/console/vgacon.c | 7 ++++--- drivers/video/fbdev/core/fbcon.c | 3 ++- include/linux/console.h | 3 ++- 8 files changed, 20 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 0d5d7b5074a4..de9148094c2d 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -175,7 +175,7 @@ int do_poke_blanked_console; int console_blanked; EXPORT_SYMBOL(console_blanked); -static int vesa_blank_mode; /* 0:none 1:suspendV 2:suspendH 3:powerdown */ +static enum vesa_blank_mode vesa_blank_mode; static int vesa_off_interval; static int blankinterval; core_param(consoleblank, blankinterval, int, 0444); @@ -4320,7 +4320,7 @@ static int set_vesa_blanking(u8 __user *mode_user) return -EFAULT; console_lock(); - vesa_blank_mode = (mode < 4) ? mode : VESA_NO_BLANKING; + vesa_blank_mode = (mode <= VESA_BLANK_MAX) ? mode : VESA_NO_BLANKING; console_unlock(); return 0; diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index c8d5aa0e3ed0..d86c1d798690 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -79,7 +79,8 @@ static void dummycon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, raw_notifier_call_chain(&dummycon_output_nh, 0, NULL); } -static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) +static int dummycon_blank(struct vc_data *vc, enum vesa_blank_mode blank, + int mode_switch) { /* Redraw, so that we get putc(s) for output done while blanked */ return 1; @@ -89,7 +90,8 @@ static void dummycon_putc(struct vc_data *vc, u16 c, unsigned int y, unsigned int x) { } static void dummycon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, unsigned int ypos, unsigned int xpos) { } -static int dummycon_blank(struct vc_data *vc, int blank, int mode_switch) +static int dummycon_blank(struct vc_data *vc, enum vesa_blank_mode blank, + int mode_switch) { return 0; } diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index 4485ef923bb3..63e3ce678aab 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -451,7 +451,8 @@ static bool mdacon_switch(struct vc_data *c) return true; /* redrawing needed */ } -static int mdacon_blank(struct vc_data *c, int blank, int mode_switch) +static int mdacon_blank(struct vc_data *c, enum vesa_blank_mode blank, + int mode_switch) { if (mda_type == TYPE_MDA) { if (blank) diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index ad3a09142770..38437a53b7f1 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -476,7 +476,8 @@ static bool newport_switch(struct vc_data *vc) return true; } -static int newport_blank(struct vc_data *c, int blank, int mode_switch) +static int newport_blank(struct vc_data *c, enum vesa_blank_mode blank, + int mode_switch) { unsigned short treg; diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 817b89c45e81..e9d5d1f92883 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -298,7 +298,8 @@ static bool sticon_switch(struct vc_data *conp) return true; /* needs refreshing */ } -static int sticon_blank(struct vc_data *c, int blank, int mode_switch) +static int sticon_blank(struct vc_data *c, enum vesa_blank_mode blank, + int mode_switch) { if (blank == VESA_NO_BLANKING) { if (mode_switch) diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 02eccd9b3542..84f3682704c7 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -81,7 +81,7 @@ static unsigned int vga_video_num_lines; /* Number of text lines */ static bool vga_can_do_color; /* Do we support colors? */ static unsigned int vga_default_font_height __read_mostly; /* Height of default screen font */ static unsigned char vga_video_type __read_mostly; /* Card type */ -static int vga_vesa_blanked; +static enum vesa_blank_mode vga_vesa_blanked; static bool vga_palette_blanked; static bool vga_is_gfx; static bool vga_512_chars; @@ -683,7 +683,7 @@ static struct { unsigned char ClockingMode; /* Seq-Controller:01h */ } vga_state; -static void vga_vesa_blank(struct vgastate *state, int mode) +static void vga_vesa_blank(struct vgastate *state, enum vesa_blank_mode mode) { /* save original values of VGA controller registers */ if (!vga_vesa_blanked) { @@ -797,7 +797,8 @@ static void vga_pal_blank(struct vgastate *state) } } -static int vgacon_blank(struct vc_data *c, int blank, int mode_switch) +static int vgacon_blank(struct vc_data *c, enum vesa_blank_mode blank, + int mode_switch) { switch (blank) { case VESA_NO_BLANKING: /* Unblank */ diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index d5d924225209..69be5f2106bc 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2198,7 +2198,8 @@ static void fbcon_generic_blank(struct vc_data *vc, struct fb_info *info, } } -static int fbcon_blank(struct vc_data *vc, int blank, int mode_switch) +static int fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank, + int mode_switch) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; diff --git a/include/linux/console.h b/include/linux/console.h index 860f82756c9c..69040d7c8f97 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -70,7 +70,8 @@ struct consw { unsigned int bottom, enum con_scroll dir, unsigned int lines); bool (*con_switch)(struct vc_data *vc); - int (*con_blank)(struct vc_data *vc, int blank, int mode_switch); + int (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank, + int mode_switch); int (*con_font_set)(struct vc_data *vc, struct console_font *font, unsigned int vpitch, unsigned int flags); int (*con_font_get)(struct vc_data *vc, struct console_font *font, -- cgit v1.2.3 From 77e110936a42b212c0fb576356ed274eb1d90c54 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:49 +0100 Subject: tty: vt: make types around consw::con_blank() bool Both the mode_switch parameter and the return value (a redraw needed) are true/false. So switch them to bool, so that users won't return -Eerrors or anything else. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-36-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/video/console/dummycon.c | 12 ++++++------ drivers/video/console/mdacon.c | 8 ++++---- drivers/video/console/newport_con.c | 7 ++++--- drivers/video/console/sticon.c | 9 +++++---- drivers/video/console/vgacon.c | 4 ++-- drivers/video/fbdev/core/fbcon.c | 6 +++--- include/linux/console.h | 7 +++++-- 7 files changed, 29 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/console/dummycon.c b/drivers/video/console/dummycon.c index d86c1d798690..139049368fdc 100644 --- a/drivers/video/console/dummycon.c +++ b/drivers/video/console/dummycon.c @@ -79,21 +79,21 @@ static void dummycon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, raw_notifier_call_chain(&dummycon_output_nh, 0, NULL); } -static int dummycon_blank(struct vc_data *vc, enum vesa_blank_mode blank, - int mode_switch) +static bool dummycon_blank(struct vc_data *vc, enum vesa_blank_mode blank, + bool mode_switch) { /* Redraw, so that we get putc(s) for output done while blanked */ - return 1; + return true; } #else static void dummycon_putc(struct vc_data *vc, u16 c, unsigned int y, unsigned int x) { } static void dummycon_putcs(struct vc_data *vc, const u16 *s, unsigned int count, unsigned int ypos, unsigned int xpos) { } -static int dummycon_blank(struct vc_data *vc, enum vesa_blank_mode blank, - int mode_switch) +static bool dummycon_blank(struct vc_data *vc, enum vesa_blank_mode blank, + bool mode_switch) { - return 0; + return false; } #endif diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c index 63e3ce678aab..c0e1f4554a44 100644 --- a/drivers/video/console/mdacon.c +++ b/drivers/video/console/mdacon.c @@ -451,8 +451,8 @@ static bool mdacon_switch(struct vc_data *c) return true; /* redrawing needed */ } -static int mdacon_blank(struct vc_data *c, enum vesa_blank_mode blank, - int mode_switch) +static bool mdacon_blank(struct vc_data *c, enum vesa_blank_mode blank, + bool mode_switch) { if (mda_type == TYPE_MDA) { if (blank) @@ -460,14 +460,14 @@ static int mdacon_blank(struct vc_data *c, enum vesa_blank_mode blank, mda_convert_attr(c->vc_video_erase_char), c->vc_screenbuf_size); /* Tell console.c that it has to restore the screen itself */ - return 1; + return true; } else { if (blank) outb_p(0x00, mda_mode_port); /* disable video */ else outb_p(MDA_MODE_VIDEO_EN | MDA_MODE_BLINK_EN, mda_mode_port); - return 0; + return false; } } diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index 38437a53b7f1..dbb31bf87bf1 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -476,8 +476,8 @@ static bool newport_switch(struct vc_data *vc) return true; } -static int newport_blank(struct vc_data *c, enum vesa_blank_mode blank, - int mode_switch) +static bool newport_blank(struct vc_data *c, enum vesa_blank_mode blank, + bool mode_switch) { unsigned short treg; @@ -492,7 +492,8 @@ static int newport_blank(struct vc_data *c, enum vesa_blank_mode blank, newport_vc2_set(npregs, VC2_IREG_CONTROL, (treg & ~(VC2_CTRL_EDISP))); } - return 1; + + return true; } static int newport_set_font(int unit, struct console_font *op, unsigned int vpitch) diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index e9d5d1f92883..cbb9ef438214 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -298,19 +298,20 @@ static bool sticon_switch(struct vc_data *conp) return true; /* needs refreshing */ } -static int sticon_blank(struct vc_data *c, enum vesa_blank_mode blank, - int mode_switch) +static bool sticon_blank(struct vc_data *c, enum vesa_blank_mode blank, + bool mode_switch) { if (blank == VESA_NO_BLANKING) { if (mode_switch) vga_is_gfx = 0; - return 1; + return true; } sti_clear(sticon_sti, 0, 0, c->vc_rows, c->vc_cols, BLANK, font_data[c->vc_num]); if (mode_switch) vga_is_gfx = 1; - return 1; + + return true; } static u8 sticon_build_attr(struct vc_data *conp, u8 color, diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 84f3682704c7..c9a22118102f 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -797,8 +797,8 @@ static void vga_pal_blank(struct vgastate *state) } } -static int vgacon_blank(struct vc_data *c, enum vesa_blank_mode blank, - int mode_switch) +static bool vgacon_blank(struct vc_data *c, enum vesa_blank_mode blank, + bool mode_switch) { switch (blank) { case VESA_NO_BLANKING: /* Unblank */ diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 69be5f2106bc..eee2adf5c682 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2198,8 +2198,8 @@ static void fbcon_generic_blank(struct vc_data *vc, struct fb_info *info, } } -static int fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank, - int mode_switch) +static bool fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank, + bool mode_switch) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; @@ -2238,7 +2238,7 @@ static int fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank, else fbcon_add_cursor_work(info); - return 0; + return false; } static void fbcon_debug_enter(struct vc_data *vc) diff --git a/include/linux/console.h b/include/linux/console.h index 69040d7c8f97..6392bcd2fe7c 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -49,6 +49,9 @@ enum vc_intensity; * Invoked by csi_M and printing to the console. * @con_switch: notifier about the console switch; it is supposed to return * true if a redraw is needed. + * @con_blank: blank/unblank the console. The target mode is passed in @blank. + * @mode_switch is set if changing from/to text/graphics. The hook + * is supposed to return true if a redraw is needed. * @con_set_palette: sets the palette of the console to @table (optional) * @con_scrolldelta: the contents of the console should be scrolled by @lines. * Invoked by user. (optional) @@ -70,8 +73,8 @@ struct consw { unsigned int bottom, enum con_scroll dir, unsigned int lines); bool (*con_switch)(struct vc_data *vc); - int (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank, - int mode_switch); + bool (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank, + bool mode_switch); int (*con_font_set)(struct vc_data *vc, struct console_font *font, unsigned int vpitch, unsigned int flags); int (*con_font_get)(struct vc_data *vc, struct console_font *font, -- cgit v1.2.3 From fd0f631fffa87f1c26045c3c88c0c4a7706d14de Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:50 +0100 Subject: tty: vt: make font of consw::con_font_set() const Provided the font parameter of consw::con_font_set() is not supposed to be changed, make it const. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-37-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 2 +- drivers/video/console/newport_con.c | 5 +++-- drivers/video/console/sticon.c | 4 ++-- drivers/video/console/vgacon.c | 2 +- drivers/video/fbdev/core/fbcon.c | 2 +- include/linux/console.h | 5 +++-- 6 files changed, 11 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index de9148094c2d..0ac537f82f7a 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -4625,7 +4625,7 @@ out: return rc; } -static int con_font_set(struct vc_data *vc, struct console_font_op *op) +static int con_font_set(struct vc_data *vc, const struct console_font_op *op) { struct console_font font; int rc = -EINVAL; diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index dbb31bf87bf1..4203bd5fd0a1 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -496,7 +496,8 @@ static bool newport_blank(struct vc_data *c, enum vesa_blank_mode blank, return true; } -static int newport_set_font(int unit, struct console_font *op, unsigned int vpitch) +static int newport_set_font(int unit, const struct console_font *op, + unsigned int vpitch) { int w = op->width; int h = op->height; @@ -568,7 +569,7 @@ static int newport_font_default(struct vc_data *vc, struct console_font *op, cha return newport_set_def_font(vc->vc_num, op); } -static int newport_font_set(struct vc_data *vc, struct console_font *font, +static int newport_font_set(struct vc_data *vc, const struct console_font *font, unsigned int vpitch, unsigned int flags) { return newport_set_font(vc->vc_num, font, vpitch); diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index cbb9ef438214..710201fb8ce4 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -153,7 +153,7 @@ static void sticon_set_def_font(int unit) } } -static int sticon_set_font(struct vc_data *vc, struct console_font *op, +static int sticon_set_font(struct vc_data *vc, const struct console_font *op, unsigned int vpitch) { struct sti_struct *sti = sticon_sti; @@ -253,7 +253,7 @@ static int sticon_font_default(struct vc_data *vc, struct console_font *op, char return 0; } -static int sticon_font_set(struct vc_data *vc, struct console_font *font, +static int sticon_font_set(struct vc_data *vc, const struct console_font *font, unsigned int vpitch, unsigned int flags) { return sticon_set_font(vc, font, vpitch); diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index c9a22118102f..4d1c8f5863af 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -1039,7 +1039,7 @@ static int vgacon_adjust_height(struct vc_data *vc, unsigned fontheight) return 0; } -static int vgacon_font_set(struct vc_data *c, struct console_font *font, +static int vgacon_font_set(struct vc_data *c, const struct console_font *font, unsigned int vpitch, unsigned int flags) { unsigned charcount = font->charcount; diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index eee2adf5c682..62474630c4d4 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2460,7 +2460,7 @@ err_out: * but lets not assume that, since charcount of 512 is small for unicode support. */ -static int fbcon_set_font(struct vc_data *vc, struct console_font *font, +static int fbcon_set_font(struct vc_data *vc, const struct console_font *font, unsigned int vpitch, unsigned int flags) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); diff --git a/include/linux/console.h b/include/linux/console.h index 6392bcd2fe7c..0a9f4cbdde83 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -75,8 +75,9 @@ struct consw { bool (*con_switch)(struct vc_data *vc); bool (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank, bool mode_switch); - int (*con_font_set)(struct vc_data *vc, struct console_font *font, - unsigned int vpitch, unsigned int flags); + int (*con_font_set)(struct vc_data *vc, + const struct console_font *font, + unsigned int vpitch, unsigned int flags); int (*con_font_get)(struct vc_data *vc, struct console_font *font, unsigned int vpitch); int (*con_font_default)(struct vc_data *vc, -- cgit v1.2.3 From 4f59617065592c446cd8450e9e6bac229cbc1383 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:51 +0100 Subject: tty: vt: make consw::con_font_default()'s name const It's a name after all and that is not supposed to be changed. So make it const to make this obvious. Signed-off-by: "Jiri Slaby (SUSE)" Cc: "James E.J. Bottomley" Cc: Daniel Vetter Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linux-parisc@vger.kernel.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-38-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/video/console/newport_con.c | 3 ++- drivers/video/console/sticon.c | 3 ++- drivers/video/fbdev/core/fbcon.c | 3 ++- include/linux/console.h | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c index 4203bd5fd0a1..a51cfc1d560e 100644 --- a/drivers/video/console/newport_con.c +++ b/drivers/video/console/newport_con.c @@ -564,7 +564,8 @@ static int newport_set_def_font(int unit, struct console_font *op) return 0; } -static int newport_font_default(struct vc_data *vc, struct console_font *op, char *name) +static int newport_font_default(struct vc_data *vc, struct console_font *op, + const char *name) { return newport_set_def_font(vc->vc_num, op); } diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c index 710201fb8ce4..4c7b4959a1aa 100644 --- a/drivers/video/console/sticon.c +++ b/drivers/video/console/sticon.c @@ -246,7 +246,8 @@ static int sticon_set_font(struct vc_data *vc, const struct console_font *op, return 0; } -static int sticon_font_default(struct vc_data *vc, struct console_font *op, char *name) +static int sticon_font_default(struct vc_data *vc, struct console_font *op, + const char *name) { sticon_set_def_font(vc->vc_num); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 62474630c4d4..657160eec0a5 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2533,7 +2533,8 @@ static int fbcon_set_font(struct vc_data *vc, const struct console_font *font, return fbcon_do_set_font(vc, font->width, font->height, charcount, new_data, 1); } -static int fbcon_set_def_font(struct vc_data *vc, struct console_font *font, char *name) +static int fbcon_set_def_font(struct vc_data *vc, struct console_font *font, + const char *name) { struct fb_info *info = fbcon_info_from_console(vc->vc_num); const struct font_desc *f; diff --git a/include/linux/console.h b/include/linux/console.h index 0a9f4cbdde83..6bb7e5e37ae4 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -81,7 +81,7 @@ struct consw { int (*con_font_get)(struct vc_data *vc, struct console_font *font, unsigned int vpitch); int (*con_font_default)(struct vc_data *vc, - struct console_font *font, char *name); + struct console_font *font, const char *name); int (*con_resize)(struct vc_data *vc, unsigned int width, unsigned int height, bool from_user); void (*con_set_palette)(struct vc_data *vc, -- cgit v1.2.3 From 42822fabfc24f4fc8d5404d9359fa17a0bcfcea8 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:52 +0100 Subject: tty: vt: change consw::con_set_origin() return type The return value of consw::con_set_origin() is only true/false, meaining if vc->vc_origin is set to vc->vc_screenbuf or not. So switch the type and returned values accordingly. And document the hook. Signed-off-by: "Jiri Slaby (SUSE)" Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-39-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/video/console/vgacon.c | 8 ++++---- include/linux/console.h | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 4d1c8f5863af..7597f04b0dc7 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -65,7 +65,7 @@ static struct vgastate vgastate; * Interface used by the world */ -static int vgacon_set_origin(struct vc_data *c); +static bool vgacon_set_origin(struct vc_data *c); static struct uni_pagedict *vgacon_uni_pagedir; static int vgacon_refcount; @@ -1100,15 +1100,15 @@ static int vgacon_resize(struct vc_data *c, unsigned int width, return 0; } -static int vgacon_set_origin(struct vc_data *c) +static bool vgacon_set_origin(struct vc_data *c) { if (vga_is_gfx || /* We don't play origin tricks in graphic modes */ (console_blanked && !vga_palette_blanked)) /* Nor we write to blanked screens */ - return 0; + return false; c->vc_origin = c->vc_visible_origin = vga_vram_base; vga_set_mem_top(c); vga_rolled_over = 0; - return 1; + return true; } static void vgacon_save_screen(struct vc_data *c) diff --git a/include/linux/console.h b/include/linux/console.h index 6bb7e5e37ae4..82e4b554a801 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -55,6 +55,9 @@ enum vc_intensity; * @con_set_palette: sets the palette of the console to @table (optional) * @con_scrolldelta: the contents of the console should be scrolled by @lines. * Invoked by user. (optional) + * @con_set_origin: set origin (see &vc_data::vc_origin) of the @vc. If not + * provided or returns false, the origin is set to + * @vc->vc_screenbuf. (optional) */ struct consw { struct module *owner; @@ -87,7 +90,7 @@ struct consw { void (*con_set_palette)(struct vc_data *vc, const unsigned char *table); void (*con_scrolldelta)(struct vc_data *vc, int lines); - int (*con_set_origin)(struct vc_data *vc); + bool (*con_set_origin)(struct vc_data *vc); void (*con_save_screen)(struct vc_data *vc); u8 (*con_build_attr)(struct vc_data *vc, u8 color, enum vc_intensity intensity, -- cgit v1.2.3 From 7cf01c92addb73c3055ff0fc596441c80ce82113 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:54 +0100 Subject: tty: vt: remove consw::con_screen_pos() After the previous patch, nobody sets that hook. So drop it completely. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-41-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 12 +++--------- include/linux/console.h | 1 - 2 files changed, 3 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 0ac537f82f7a..4e9d689143e0 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -289,15 +289,9 @@ static inline bool con_should_update(const struct vc_data *vc) static inline unsigned short *screenpos(const struct vc_data *vc, int offset, bool viewed) { - unsigned short *p; - - if (!viewed) - p = (unsigned short *)(vc->vc_origin + offset); - else if (!vc->vc_sw->con_screen_pos) - p = (unsigned short *)(vc->vc_visible_origin + offset); - else - p = vc->vc_sw->con_screen_pos(vc, offset); - return p; + unsigned long origin = viewed ? vc->vc_visible_origin : vc->vc_origin; + + return (unsigned short *)(origin + offset); } static void con_putc(struct vc_data *vc, u16 ca, unsigned int y, unsigned int x) diff --git a/include/linux/console.h b/include/linux/console.h index 82e4b554a801..b2d8621cea57 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -96,7 +96,6 @@ struct consw { enum vc_intensity intensity, bool blink, bool underline, bool reverse, bool italic); void (*con_invert_region)(struct vc_data *vc, u16 *p, int count); - u16 *(*con_screen_pos)(const struct vc_data *vc, int offset); unsigned long (*con_getxy)(struct vc_data *vc, unsigned long position, int *px, int *py); /* -- cgit v1.2.3 From f441aa3b441306e35e8fcbec5ac13c68b5f48245 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:57 +0100 Subject: tty: vt: remove consw::con_getxy() After the previous patch, nobody sets that hook. So drop it completely. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-44-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 20 +++++--------------- include/linux/console.h | 2 -- 2 files changed, 5 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index f6fa76c0eb5b..cbe1a1106d53 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -593,18 +593,12 @@ static void con_scroll(struct vc_data *vc, unsigned int top, static void do_update_region(struct vc_data *vc, unsigned long start, int count) { unsigned int xx, yy, offset; - u16 *p; + u16 *p = (u16 *)start; + + offset = (start - vc->vc_origin) / 2; + xx = offset % vc->vc_cols; + yy = offset / vc->vc_cols; - p = (u16 *) start; - if (!vc->vc_sw->con_getxy) { - offset = (start - vc->vc_origin) / 2; - xx = offset % vc->vc_cols; - yy = offset / vc->vc_cols; - } else { - int nxx, nyy; - start = vc->vc_sw->con_getxy(vc, start, &nxx, &nyy); - xx = nxx; yy = nyy; - } for(;;) { u16 attrib = scr_readw(p) & 0xff00; int startx = xx; @@ -627,10 +621,6 @@ static void do_update_region(struct vc_data *vc, unsigned long start, int count) break; xx = 0; yy++; - if (vc->vc_sw->con_getxy) { - p = (u16 *)start; - start = vc->vc_sw->con_getxy(vc, start, NULL, NULL); - } } } diff --git a/include/linux/console.h b/include/linux/console.h index b2d8621cea57..fa2cd81102b8 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -96,8 +96,6 @@ struct consw { enum vc_intensity intensity, bool blink, bool underline, bool reverse, bool italic); void (*con_invert_region)(struct vc_data *vc, u16 *p, int count); - unsigned long (*con_getxy)(struct vc_data *vc, unsigned long position, - int *px, int *py); /* * Flush the video console driver's scrollback buffer */ -- cgit v1.2.3 From b23bf1a43bdbce1a281f11169dd9d426018b00c9 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:58 +0100 Subject: tty: vt: remove unused consw::con_flush_scrollback() consw::con_flush_scrollback() is unused since commit 973c096f6a85 (vgacon: remove software scrollback support). Drop it. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-45-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 27 ++++++++++++--------------- include/linux/console.h | 4 ---- 2 files changed, 12 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index cbe1a1106d53..bfe51af9a0f3 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -888,21 +888,18 @@ static void flush_scrollback(struct vc_data *vc) WARN_CONSOLE_UNLOCKED(); set_origin(vc); - if (vc->vc_sw->con_flush_scrollback) { - vc->vc_sw->con_flush_scrollback(vc); - } else if (con_is_visible(vc)) { - /* - * When no con_flush_scrollback method is provided then the - * legacy way for flushing the scrollback buffer is to use - * a side effect of the con_switch method. We do it only on - * the foreground console as background consoles have no - * scrollback buffers in that case and we obviously don't - * want to switch to them. - */ - hide_cursor(vc); - vc->vc_sw->con_switch(vc); - set_cursor(vc); - } + if (!con_is_visible(vc)) + return; + + /* + * The legacy way for flushing the scrollback buffer is to use a side + * effect of the con_switch method. We do it only on the foreground + * console as background consoles have no scrollback buffers in that + * case and we obviously don't want to switch to them. + */ + hide_cursor(vc); + vc->vc_sw->con_switch(vc); + set_cursor(vc); } /* diff --git a/include/linux/console.h b/include/linux/console.h index fa2cd81102b8..1eac3e6e32a2 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -96,10 +96,6 @@ struct consw { enum vc_intensity intensity, bool blink, bool underline, bool reverse, bool italic); void (*con_invert_region)(struct vc_data *vc, u16 *p, int count); - /* - * Flush the video console driver's scrollback buffer - */ - void (*con_flush_scrollback)(struct vc_data *vc); /* * Prepare the console for the debugger. This includes, but is not * limited to, unblanking the console, loading an appropriate -- cgit v1.2.3 From d1e2221644c490a73d2968fe316f0af170e0ebcf Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:03:59 +0100 Subject: tty: vt: document the rest of struct consw There are still members of struct consw which are not documented yet. Fix that up, so we can generate kernel-doc for that struct. Signed-off-by: "Jiri Slaby (SUSE)" Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-46-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/console.h | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/console.h b/include/linux/console.h index 1eac3e6e32a2..f1a334ad268d 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -37,8 +37,11 @@ enum vc_intensity; /** * struct consw - callbacks for consoles * + * @owner: the module to get references of when this console is used + * @con_startup: set up the console and return its name (like VGA, EGA, ...) * @con_init: initialize the console on @vc. @init is true for the very first * call on this @vc. + * @con_deinit: deinitialize the console from @vc. * @con_clear: erase @count characters at [@x, @y] on @vc. @count >= 1. * @con_putc: emit one character with attributes @ca to [@x, @y] on @vc. * (optional -- @con_putcs would be called instead) @@ -52,12 +55,33 @@ enum vc_intensity; * @con_blank: blank/unblank the console. The target mode is passed in @blank. * @mode_switch is set if changing from/to text/graphics. The hook * is supposed to return true if a redraw is needed. - * @con_set_palette: sets the palette of the console to @table (optional) + * @con_font_set: set console @vc font to @font with height @vpitch. @flags can + * be %KD_FONT_FLAG_DONT_RECALC. (optional) + * @con_font_get: fetch the current font on @vc of height @vpitch into @font. + * (optional) + * @con_font_default: set default font on @vc. @name can be %NULL or font name + * to search for. @font can be filled back. (optional) + * @con_resize: resize the @vc console to @width x @height. @from_user is true + * when this change comes from the user space. + * @con_set_palette: sets the palette of the console @vc to @table (optional) * @con_scrolldelta: the contents of the console should be scrolled by @lines. * Invoked by user. (optional) * @con_set_origin: set origin (see &vc_data::vc_origin) of the @vc. If not * provided or returns false, the origin is set to * @vc->vc_screenbuf. (optional) + * @con_save_screen: save screen content into @vc->vc_screenbuf. Called e.g. + * upon entering graphics. (optional) + * @con_build_attr: build attributes based on @color, @intensity and other + * parameters. The result is used for both normal and erase + * characters. (optional) + * @con_invert_region: invert a region of length @count on @vc starting at @p. + * (optional) + * @con_debug_enter: prepare the console for the debugger. This includes, but + * is not limited to, unblanking the console, loading an + * appropriate palette, and allowing debugger generated output. + * (optional) + * @con_debug_leave: restore the console to its pre-debug state as closely as + * possible. (optional) */ struct consw { struct module *owner; @@ -96,15 +120,7 @@ struct consw { enum vc_intensity intensity, bool blink, bool underline, bool reverse, bool italic); void (*con_invert_region)(struct vc_data *vc, u16 *p, int count); - /* - * Prepare the console for the debugger. This includes, but is not - * limited to, unblanking the console, loading an appropriate - * palette, and allowing debugger generated output. - */ void (*con_debug_enter)(struct vc_data *vc); - /* - * Restore the console to its pre-debug state as closely as possible. - */ void (*con_debug_leave)(struct vc_data *vc); }; -- cgit v1.2.3 From 60234365aee22c9ac576491f787f20a17279d28e Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 22 Jan 2024 12:04:00 +0100 Subject: tty: vt: fix up kernel-doc selection.c and vt.c still uses tabs in the kernel-doc. This misrenders the functions in the output -- sphinx misinterprets the description. So remove these tabs, incl. those around dashes. 'enum' keyword is needed before enum names. Fix that. Superfluous \n after the comments are also removed. They are not completely faulty, but this unifies all the kernel-doc in the files. Finally fix up the cross references. Signed-off-by: "Jiri Slaby (SUSE)" Reviewed-by: Randy Dunlap Tested-by: Helge Deller # parisc STI console Link: https://lore.kernel.org/r/20240122110401.7289-47-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/selection.c | 30 ++++++++-------- drivers/tty/vt/vt.c | 86 +++++++++++++++++++++++----------------------- include/linux/console.h | 6 ++-- 3 files changed, 62 insertions(+), 60 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c index 34ee09f5a8f4..564341f1a74f 100644 --- a/drivers/tty/vt/selection.c +++ b/drivers/tty/vt/selection.c @@ -73,10 +73,12 @@ sel_pos(int n, bool unicode) } /** - * clear_selection - remove current selection + * clear_selection - remove current selection * - * Remove the current selection highlight, if any from the console - * holding the selection. The caller must hold the console lock. + * Remove the current selection highlight, if any from the console holding the + * selection. + * + * Locking: The caller must hold the console lock. */ void clear_selection(void) { @@ -110,13 +112,13 @@ static inline int inword(const u32 c) } /** - * sel_loadlut() - load the LUT table - * @lut: user table + * sel_loadlut() - load the LUT table + * @lut: user table * - * Load the LUT table from user space. Make a temporary copy so a partial - * update doesn't make a mess. + * Load the LUT table from user space. Make a temporary copy so a partial + * update doesn't make a mess. * - * Locking: The console lock is acquired. + * Locking: The console lock is acquired. */ int sel_loadlut(u32 __user *lut) { @@ -173,14 +175,14 @@ static int store_utf8(u32 c, char *p) } /** - * set_selection_user - set the current selection. - * @sel: user selection info - * @tty: the console tty + * set_selection_user - set the current selection. + * @sel: user selection info + * @tty: the console tty * - * Invoked by the ioctl handle for the vt layer. + * Invoked by the ioctl handle for the vt layer. * - * The entire selection process is managed under the console_lock. It's - * a lot under the lock but its hardly a performance path + * Locking: The entire selection process is managed under the console_lock. + * It's a lot under the lock but its hardly a performance path. */ int set_selection_user(const struct tiocl_selection __user *sel, struct tty_struct *tty) diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index bfe51af9a0f3..e9cdcf40fe14 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1115,21 +1115,20 @@ static inline int resize_screen(struct vc_data *vc, int width, int height, } /** - * vc_do_resize - resizing method for the tty - * @tty: tty being resized - * @vc: virtual console private data - * @cols: columns - * @lines: lines - * @from_user: invoked by a user? + * vc_do_resize - resizing method for the tty + * @tty: tty being resized + * @vc: virtual console private data + * @cols: columns + * @lines: lines + * @from_user: invoked by a user? * - * Resize a virtual console, clipping according to the actual constraints. - * If the caller passes a tty structure then update the termios winsize - * information and perform any necessary signal handling. + * Resize a virtual console, clipping according to the actual constraints. If + * the caller passes a tty structure then update the termios winsize + * information and perform any necessary signal handling. * - * Caller must hold the console semaphore. Takes the termios rwsem and - * ctrl.lock of the tty IFF a tty is passed. + * Locking: Caller must hold the console semaphore. Takes the termios rwsem and + * ctrl.lock of the tty IFF a tty is passed. */ - static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, unsigned int cols, unsigned int lines, bool from_user) { @@ -1277,16 +1276,17 @@ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, } /** - * __vc_resize - resize a VT - * @vc: virtual console - * @cols: columns - * @rows: rows - * @from_user: invoked by a user? + * __vc_resize - resize a VT + * @vc: virtual console + * @cols: columns + * @rows: rows + * @from_user: invoked by a user? + * + * Resize a virtual console as seen from the console end of things. We use the + * common vc_do_resize() method to update the structures. * - * Resize a virtual console as seen from the console end of things. We - * use the common vc_do_resize methods to update the structures. The - * caller must hold the console sem to protect console internals and - * vc->port.tty + * Locking: The caller must hold the console sem to protect console internals + * and @vc->port.tty. */ int __vc_resize(struct vc_data *vc, unsigned int cols, unsigned int rows, bool from_user) @@ -1296,16 +1296,15 @@ int __vc_resize(struct vc_data *vc, unsigned int cols, unsigned int rows, EXPORT_SYMBOL(__vc_resize); /** - * vt_resize - resize a VT - * @tty: tty to resize - * @ws: winsize attributes + * vt_resize - resize a VT + * @tty: tty to resize + * @ws: winsize attributes * - * Resize a virtual terminal. This is called by the tty layer as we - * register our own handler for resizing. The mutual helper does all - * the actual work. + * Resize a virtual terminal. This is called by the tty layer as we register + * our own handler for resizing. The mutual helper does all the actual work. * - * Takes the console sem and the called methods then take the tty - * termios_rwsem and the tty ctrl.lock in that order. + * Locking: Takes the console sem and the called methods then take the tty + * termios_rwsem and the tty ctrl.lock in that order. */ static int vt_resize(struct tty_struct *tty, struct winsize *ws) { @@ -2633,8 +2632,8 @@ static inline int vc_translate_ascii(const struct vc_data *vc, int c) /** - * vc_sanitize_unicode - Replace invalid Unicode code points with U+FFFD - * @c: the received character, or U+FFFD for invalid sequences. + * vc_sanitize_unicode - Replace invalid Unicode code points with ``U+FFFD`` + * @c: the received character, or ``U+FFFD`` for invalid sequences. */ static inline int vc_sanitize_unicode(const int c) { @@ -2645,14 +2644,15 @@ static inline int vc_sanitize_unicode(const int c) } /** - * vc_translate_unicode - Combine UTF-8 into Unicode in @vc_utf_char + * vc_translate_unicode - Combine UTF-8 into Unicode in &vc_data.vc_utf_char * @vc: virtual console * @c: character to translate * @rescan: we return true if we need more (continuation) data * - * @vc_utf_char is the being-constructed unicode character. - * @vc_utf_count is the number of continuation bytes still expected to arrive. - * @vc_npar is the number of continuation bytes arrived so far. + * * &vc_data.vc_utf_char is the being-constructed unicode character. + * * &vc_data.vc_utf_count is the number of continuation bytes still expected to + * arrive. + * * &vc_data.vc_npar is the number of continuation bytes arrived so far. */ static int vc_translate_unicode(struct vc_data *vc, int c, bool *rescan) { @@ -3030,16 +3030,16 @@ struct tty_driver *console_driver; #ifdef CONFIG_VT_CONSOLE /** - * vt_kmsg_redirect() - Sets/gets the kernel message console - * @new: The new virtual terminal number or -1 if the console should stay - * unchanged + * vt_kmsg_redirect() - sets/gets the kernel message console + * @new: the new virtual terminal number or -1 if the console should stay + * unchanged * * By default, the kernel messages are always printed on the current virtual * console. However, the user may modify that default with the - * TIOCL_SETKMSGREDIRECT ioctl call. + * %TIOCL_SETKMSGREDIRECT ioctl call. * * This function sets the kernel message console to be @new. It returns the old - * virtual console number. The virtual terminal number 0 (both as parameter and + * virtual console number. The virtual terminal number %0 (both as parameter and * return value) means no redirection (i.e. always printed on the currently * active console). * @@ -3047,8 +3047,8 @@ struct tty_driver *console_driver; * value is not modified. You may use the macro vt_get_kmsg_redirect() in that * case to make the code more understandable. * - * When the kernel is compiled without CONFIG_VT_CONSOLE, this function ignores - * the parameter and always returns 0. + * When the kernel is compiled without %CONFIG_VT_CONSOLE, this function ignores + * the parameter and always returns %0. */ int vt_kmsg_redirect(int new) { @@ -3960,7 +3960,7 @@ static void vtconsole_deinit_device(struct con_driver *con) * RETURNS: zero if unbound, nonzero if bound * * Drivers can call this and if zero, they should release - * all resources allocated on con_startup() + * all resources allocated on &consw.con_startup() */ int con_is_bound(const struct consw *csw) { diff --git a/include/linux/console.h b/include/linux/console.h index f1a334ad268d..d6d8b7e6b93b 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -155,7 +155,7 @@ static inline void con_debug_leave(void) { } */ /** - * cons_flags - General console flags + * enum cons_flags - General console flags * @CON_PRINTBUFFER: Used by newly registered consoles to avoid duplicate * output of messages that were already shown by boot * consoles or read by userspace via syslog() syscall. @@ -236,7 +236,7 @@ struct nbcon_state { static_assert(sizeof(struct nbcon_state) <= sizeof(int)); /** - * nbcon_prio - console owner priority for nbcon consoles + * enum nbcon_prio - console owner priority for nbcon consoles * @NBCON_PRIO_NONE: Unused * @NBCON_PRIO_NORMAL: Normal (non-emergency) usage * @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...) @@ -468,7 +468,7 @@ static inline bool console_is_registered(const struct console *con) * for_each_console() - Iterator over registered consoles * @con: struct console pointer used as loop cursor * - * The console list and the console->flags are immutable while iterating. + * The console list and the &console.flags are immutable while iterating. * * Requires console_list_lock to be held. */ -- cgit v1.2.3 From fed99212acae832607817b24fa589f8aaf03103f Mon Sep 17 00:00:00 2001 From: Francesco Dolcini Date: Mon, 22 Jan 2024 19:05:51 +0100 Subject: treewide, serdev: change receive_buf() return type to size_t MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit receive_buf() is called from ttyport_receive_buf() that expects values ">= 0" from serdev_controller_receive_buf(), change its return type from ssize_t to size_t. The need for this clean-up was noticed while fixing a warning, see commit 94d053942544 ("Bluetooth: btnxpuart: fix recv_buf() return value"). Changing the callback prototype to return an unsigned seems the best way to document the API and ensure that is properly used. GNSS drivers implementation of serdev receive_buf() callback return directly the return value of gnss_insert_raw(). gnss_insert_raw() returns a signed int, however this is not an issue since the value returned is always positive, because of the kfifo_in() implementation. gnss_insert_raw() could be changed to return also an unsigned, however this is not implemented here as request by the GNSS maintainer Johan Hovold. Suggested-by: Jiri Slaby Link: https://lore.kernel.org/all/087be419-ec6b-47ad-851a-5e1e3ea5cfcc@kernel.org/ Signed-off-by: Francesco Dolcini Acked-by: Jonathan Cameron #for-iio Reviewed-by: Johan Hovold Reviewed-by: Rob Herring Reviewed-by: Alex Elder Acked-by: Maximilian Luz # for platform/surface Acked-by: Lee Jones Acked-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20240122180551.34429-1-francesco@dolcini.it Signed-off-by: Greg Kroah-Hartman --- drivers/bluetooth/btmtkuart.c | 4 ++-- drivers/bluetooth/btnxpuart.c | 4 ++-- drivers/bluetooth/hci_serdev.c | 4 ++-- drivers/gnss/serial.c | 2 +- drivers/gnss/sirf.c | 2 +- drivers/greybus/gb-beagleplay.c | 6 +++--- drivers/iio/chemical/pms7003.c | 4 ++-- drivers/iio/chemical/scd30_serial.c | 4 ++-- drivers/iio/chemical/sps30_serial.c | 4 ++-- drivers/iio/imu/bno055/bno055_ser_core.c | 4 ++-- drivers/mfd/rave-sp.c | 4 ++-- drivers/net/ethernet/qualcomm/qca_uart.c | 2 +- drivers/nfc/pn533/uart.c | 4 ++-- drivers/nfc/s3fwrn5/uart.c | 4 ++-- drivers/platform/chrome/cros_ec_uart.c | 4 ++-- drivers/platform/surface/aggregator/core.c | 4 ++-- drivers/tty/serdev/serdev-ttyport.c | 10 ++++------ include/linux/serdev.h | 8 ++++---- sound/drivers/serial-generic.c | 4 ++-- 19 files changed, 40 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/drivers/bluetooth/btmtkuart.c b/drivers/bluetooth/btmtkuart.c index 3c84fcbda01a..e6bc4a73c9fc 100644 --- a/drivers/bluetooth/btmtkuart.c +++ b/drivers/bluetooth/btmtkuart.c @@ -383,8 +383,8 @@ static void btmtkuart_recv(struct hci_dev *hdev, const u8 *data, size_t count) } } -static ssize_t btmtkuart_receive_buf(struct serdev_device *serdev, - const u8 *data, size_t count) +static size_t btmtkuart_receive_buf(struct serdev_device *serdev, + const u8 *data, size_t count) { struct btmtkuart_dev *bdev = serdev_device_get_drvdata(serdev); diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index 1d592ac413d1..056bef5b2919 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -1264,8 +1264,8 @@ static const struct h4_recv_pkt nxp_recv_pkts[] = { { NXP_RECV_FW_REQ_V3, .recv = nxp_recv_fw_req_v3 }, }; -static ssize_t btnxpuart_receive_buf(struct serdev_device *serdev, - const u8 *data, size_t count) +static size_t btnxpuart_receive_buf(struct serdev_device *serdev, + const u8 *data, size_t count) { struct btnxpuart_dev *nxpdev = serdev_device_get_drvdata(serdev); diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index 39c8b567da3c..a3c3beb2806d 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -271,8 +271,8 @@ static void hci_uart_write_wakeup(struct serdev_device *serdev) * * Return: number of processed bytes */ -static ssize_t hci_uart_receive_buf(struct serdev_device *serdev, - const u8 *data, size_t count) +static size_t hci_uart_receive_buf(struct serdev_device *serdev, + const u8 *data, size_t count) { struct hci_uart *hu = serdev_device_get_drvdata(serdev); diff --git a/drivers/gnss/serial.c b/drivers/gnss/serial.c index baa956494e79..0e43bf6294f8 100644 --- a/drivers/gnss/serial.c +++ b/drivers/gnss/serial.c @@ -80,7 +80,7 @@ static const struct gnss_operations gnss_serial_gnss_ops = { .write_raw = gnss_serial_write_raw, }; -static ssize_t gnss_serial_receive_buf(struct serdev_device *serdev, +static size_t gnss_serial_receive_buf(struct serdev_device *serdev, const u8 *buf, size_t count) { struct gnss_serial *gserial = serdev_device_get_drvdata(serdev); diff --git a/drivers/gnss/sirf.c b/drivers/gnss/sirf.c index 6801a8fb2040..79375d14bbb6 100644 --- a/drivers/gnss/sirf.c +++ b/drivers/gnss/sirf.c @@ -160,7 +160,7 @@ static const struct gnss_operations sirf_gnss_ops = { .write_raw = sirf_write_raw, }; -static ssize_t sirf_receive_buf(struct serdev_device *serdev, +static size_t sirf_receive_buf(struct serdev_device *serdev, const u8 *buf, size_t count) { struct sirf_data *data = serdev_device_get_drvdata(serdev); diff --git a/drivers/greybus/gb-beagleplay.c b/drivers/greybus/gb-beagleplay.c index c3e90025064b..33f8fad70260 100644 --- a/drivers/greybus/gb-beagleplay.c +++ b/drivers/greybus/gb-beagleplay.c @@ -271,7 +271,7 @@ static void hdlc_rx_frame(struct gb_beagleplay *bg) } } -static ssize_t hdlc_rx(struct gb_beagleplay *bg, const u8 *data, size_t count) +static size_t hdlc_rx(struct gb_beagleplay *bg, const u8 *data, size_t count) { size_t i; u8 c; @@ -331,8 +331,8 @@ static void hdlc_deinit(struct gb_beagleplay *bg) flush_work(&bg->tx_work); } -static ssize_t gb_tty_receive(struct serdev_device *sd, const u8 *data, - size_t count) +static size_t gb_tty_receive(struct serdev_device *sd, const u8 *data, + size_t count) { struct gb_beagleplay *bg = serdev_device_get_drvdata(sd); diff --git a/drivers/iio/chemical/pms7003.c b/drivers/iio/chemical/pms7003.c index b5cf15a515d2..43025866d5b7 100644 --- a/drivers/iio/chemical/pms7003.c +++ b/drivers/iio/chemical/pms7003.c @@ -211,8 +211,8 @@ static bool pms7003_frame_is_okay(struct pms7003_frame *frame) return checksum == pms7003_calc_checksum(frame); } -static ssize_t pms7003_receive_buf(struct serdev_device *serdev, const u8 *buf, - size_t size) +static size_t pms7003_receive_buf(struct serdev_device *serdev, const u8 *buf, + size_t size) { struct iio_dev *indio_dev = serdev_device_get_drvdata(serdev); struct pms7003_state *state = iio_priv(indio_dev); diff --git a/drivers/iio/chemical/scd30_serial.c b/drivers/iio/chemical/scd30_serial.c index a47654591e55..2adb76dbb020 100644 --- a/drivers/iio/chemical/scd30_serial.c +++ b/drivers/iio/chemical/scd30_serial.c @@ -174,8 +174,8 @@ static int scd30_serdev_command(struct scd30_state *state, enum scd30_cmd cmd, u return 0; } -static ssize_t scd30_serdev_receive_buf(struct serdev_device *serdev, - const u8 *buf, size_t size) +static size_t scd30_serdev_receive_buf(struct serdev_device *serdev, + const u8 *buf, size_t size) { struct iio_dev *indio_dev = serdev_device_get_drvdata(serdev); struct scd30_serdev_priv *priv; diff --git a/drivers/iio/chemical/sps30_serial.c b/drivers/iio/chemical/sps30_serial.c index 3afa89f8acc3..a6dfbe28c914 100644 --- a/drivers/iio/chemical/sps30_serial.c +++ b/drivers/iio/chemical/sps30_serial.c @@ -210,8 +210,8 @@ static int sps30_serial_command(struct sps30_state *state, unsigned char cmd, return rsp_size; } -static ssize_t sps30_serial_receive_buf(struct serdev_device *serdev, - const u8 *buf, size_t size) +static size_t sps30_serial_receive_buf(struct serdev_device *serdev, + const u8 *buf, size_t size) { struct iio_dev *indio_dev = dev_get_drvdata(&serdev->dev); struct sps30_serial_priv *priv; diff --git a/drivers/iio/imu/bno055/bno055_ser_core.c b/drivers/iio/imu/bno055/bno055_ser_core.c index 5677bdf4f846..694ff14a3aa2 100644 --- a/drivers/iio/imu/bno055/bno055_ser_core.c +++ b/drivers/iio/imu/bno055/bno055_ser_core.c @@ -378,8 +378,8 @@ static void bno055_ser_handle_rx(struct bno055_ser_priv *priv, int status) * Also, we assume to RX one pkt per time (i.e. the HW doesn't send anything * unless we require to AND we don't queue more than one request per time). */ -static ssize_t bno055_ser_receive_buf(struct serdev_device *serdev, - const u8 *buf, size_t size) +static size_t bno055_ser_receive_buf(struct serdev_device *serdev, + const u8 *buf, size_t size) { int status; struct bno055_ser_priv *priv = serdev_device_get_drvdata(serdev); diff --git a/drivers/mfd/rave-sp.c b/drivers/mfd/rave-sp.c index 6ff84b2600c5..62a6613fb070 100644 --- a/drivers/mfd/rave-sp.c +++ b/drivers/mfd/rave-sp.c @@ -471,8 +471,8 @@ static void rave_sp_receive_frame(struct rave_sp *sp, rave_sp_receive_reply(sp, data, length); } -static ssize_t rave_sp_receive_buf(struct serdev_device *serdev, - const u8 *buf, size_t size) +static size_t rave_sp_receive_buf(struct serdev_device *serdev, + const u8 *buf, size_t size) { struct device *dev = &serdev->dev; struct rave_sp *sp = dev_get_drvdata(dev); diff --git a/drivers/net/ethernet/qualcomm/qca_uart.c b/drivers/net/ethernet/qualcomm/qca_uart.c index 223321897b96..20f50bde82ac 100644 --- a/drivers/net/ethernet/qualcomm/qca_uart.c +++ b/drivers/net/ethernet/qualcomm/qca_uart.c @@ -58,7 +58,7 @@ struct qcauart { unsigned char *tx_buffer; }; -static ssize_t +static size_t qca_tty_receive(struct serdev_device *serdev, const u8 *data, size_t count) { struct qcauart *qca = serdev_device_get_drvdata(serdev); diff --git a/drivers/nfc/pn533/uart.c b/drivers/nfc/pn533/uart.c index 2eb5978bd79e..cfbbe0713317 100644 --- a/drivers/nfc/pn533/uart.c +++ b/drivers/nfc/pn533/uart.c @@ -203,8 +203,8 @@ static int pn532_uart_rx_is_frame(struct sk_buff *skb) return 0; } -static ssize_t pn532_receive_buf(struct serdev_device *serdev, - const u8 *data, size_t count) +static size_t pn532_receive_buf(struct serdev_device *serdev, + const u8 *data, size_t count) { struct pn532_uart_phy *dev = serdev_device_get_drvdata(serdev); size_t i; diff --git a/drivers/nfc/s3fwrn5/uart.c b/drivers/nfc/s3fwrn5/uart.c index 456d3947116c..9c09c10c2a46 100644 --- a/drivers/nfc/s3fwrn5/uart.c +++ b/drivers/nfc/s3fwrn5/uart.c @@ -51,8 +51,8 @@ static const struct s3fwrn5_phy_ops uart_phy_ops = { .write = s3fwrn82_uart_write, }; -static ssize_t s3fwrn82_uart_read(struct serdev_device *serdev, - const u8 *data, size_t count) +static size_t s3fwrn82_uart_read(struct serdev_device *serdev, + const u8 *data, size_t count) { struct s3fwrn82_uart_phy *phy = serdev_device_get_drvdata(serdev); size_t i; diff --git a/drivers/platform/chrome/cros_ec_uart.c b/drivers/platform/chrome/cros_ec_uart.c index 68d80559fddc..8ea867c2a01a 100644 --- a/drivers/platform/chrome/cros_ec_uart.c +++ b/drivers/platform/chrome/cros_ec_uart.c @@ -81,8 +81,8 @@ struct cros_ec_uart { struct response_info response; }; -static ssize_t cros_ec_uart_rx_bytes(struct serdev_device *serdev, - const u8 *data, size_t count) +static size_t cros_ec_uart_rx_bytes(struct serdev_device *serdev, + const u8 *data, size_t count) { struct ec_host_response *host_response; struct cros_ec_device *ec_dev = serdev_device_get_drvdata(serdev); diff --git a/drivers/platform/surface/aggregator/core.c b/drivers/platform/surface/aggregator/core.c index 9591a28bc38a..ba550eaa06fc 100644 --- a/drivers/platform/surface/aggregator/core.c +++ b/drivers/platform/surface/aggregator/core.c @@ -227,8 +227,8 @@ EXPORT_SYMBOL_GPL(ssam_client_bind); /* -- Glue layer (serdev_device -> ssam_controller). ------------------------ */ -static ssize_t ssam_receive_buf(struct serdev_device *dev, const u8 *buf, - size_t n) +static size_t ssam_receive_buf(struct serdev_device *dev, const u8 *buf, + size_t n) { struct ssam_controller *ctrl; int ret; diff --git a/drivers/tty/serdev/serdev-ttyport.c b/drivers/tty/serdev/serdev-ttyport.c index e94e090cf0a1..3d7ae7fa5018 100644 --- a/drivers/tty/serdev/serdev-ttyport.c +++ b/drivers/tty/serdev/serdev-ttyport.c @@ -27,19 +27,17 @@ static size_t ttyport_receive_buf(struct tty_port *port, const u8 *cp, { struct serdev_controller *ctrl = port->client_data; struct serport *serport = serdev_controller_get_drvdata(ctrl); - int ret; + size_t ret; if (!test_bit(SERPORT_ACTIVE, &serport->flags)) return 0; ret = serdev_controller_receive_buf(ctrl, cp, count); - dev_WARN_ONCE(&ctrl->dev, ret < 0 || ret > count, - "receive_buf returns %d (count = %zu)\n", + dev_WARN_ONCE(&ctrl->dev, ret > count, + "receive_buf returns %zu (count = %zu)\n", ret, count); - if (ret < 0) - return 0; - else if (ret > count) + if (ret > count) return count; return ret; diff --git a/include/linux/serdev.h b/include/linux/serdev.h index 3fab88ba265e..ff78efc1f60d 100644 --- a/include/linux/serdev.h +++ b/include/linux/serdev.h @@ -27,7 +27,7 @@ struct serdev_device; * not sleep. */ struct serdev_device_ops { - ssize_t (*receive_buf)(struct serdev_device *, const u8 *, size_t); + size_t (*receive_buf)(struct serdev_device *, const u8 *, size_t); void (*write_wakeup)(struct serdev_device *); }; @@ -185,9 +185,9 @@ static inline void serdev_controller_write_wakeup(struct serdev_controller *ctrl serdev->ops->write_wakeup(serdev); } -static inline ssize_t serdev_controller_receive_buf(struct serdev_controller *ctrl, - const u8 *data, - size_t count) +static inline size_t serdev_controller_receive_buf(struct serdev_controller *ctrl, + const u8 *data, + size_t count) { struct serdev_device *serdev = ctrl->serdev; diff --git a/sound/drivers/serial-generic.c b/sound/drivers/serial-generic.c index d6e5aafd697c..36409a56c675 100644 --- a/sound/drivers/serial-generic.c +++ b/sound/drivers/serial-generic.c @@ -100,8 +100,8 @@ static void snd_serial_generic_write_wakeup(struct serdev_device *serdev) snd_serial_generic_tx_wakeup(drvdata); } -static ssize_t snd_serial_generic_receive_buf(struct serdev_device *serdev, - const u8 *buf, size_t count) +static size_t snd_serial_generic_receive_buf(struct serdev_device *serdev, + const u8 *buf, size_t count) { int ret; struct snd_serial_generic *drvdata = serdev_device_get_drvdata(serdev); -- cgit v1.2.3 From 85725449f3e5faf385210d535f266430be71cebb Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 10 Jan 2024 14:21:46 +0100 Subject: serial: 8250: Move hp300_setup_serial_console() to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If CONFIG_SERIAL_8250_HP300=y and CONFIG_SERIAL_8250_CONSOLE=y (e.g. m68k/allyesconfig): drivers/tty/serial/8250/8250_hp300.c:91:12: error: no previous prototype for ‘hp300_setup_serial_console’ [-Werror=missing-prototypes] 91 | int __init hp300_setup_serial_console(void) | ^~~~~~~~~~~~~~~~~~~~~~~~~~ Fix this by moving the existing prototype in arch/m68k/hp300/config.c to , so it is visible to both caller and implementor. While at it, provide a dummy in case CONFIG_SERIAL_8250_CONSOLE is not enabled, to reduce #ifdef clutter in the caller. Exposed by commit 0fcb70851fbfea17 ("Makefile.extrawarn: turn on missing-prototypes globally"). Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/c17469f8e47b2ef49234a85a7a14882ddf374e41.1704892597.git.geert@linux-m68k.org Signed-off-by: Greg Kroah-Hartman --- arch/m68k/hp300/config.c | 6 +----- include/linux/serial_8250.h | 6 ++++++ 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/arch/m68k/hp300/config.c b/arch/m68k/hp300/config.c index e4bd6913f50e..1a2739852351 100644 --- a/arch/m68k/hp300/config.c +++ b/arch/m68k/hp300/config.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -67,9 +68,6 @@ static char *hp300_models[] __initdata = { static char hp300_model_name[13] = "HP9000/"; extern void hp300_reset(void); -#ifdef CONFIG_SERIAL_8250_CONSOLE -extern int hp300_setup_serial_console(void) __init; -#endif int __init hp300_parse_bootinfo(const struct bi_record *record) { @@ -263,7 +261,5 @@ void __init config_hp300(void) } else { panic("Unknown HP9000 Model"); } -#ifdef CONFIG_SERIAL_8250_CONSOLE hp300_setup_serial_console(); -#endif } diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index be65de65fe61..fd59ed2cca53 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -210,6 +210,12 @@ int serial8250_console_exit(struct uart_port *port); void serial8250_set_isa_configurator(void (*v)(int port, struct uart_port *up, u32 *capabilities)); +#ifdef CONFIG_SERIAL_8250_CONSOLE +extern int hp300_setup_serial_console(void) __init; +#else +static inline int hp300_setup_serial_console(void) { return 0; } +#endif + #ifdef CONFIG_SERIAL_8250_RT288X int rt288x_setup(struct uart_port *p); int au_platform_setup(struct plat_serial8250_port *p); -- cgit v1.2.3 From 486676116f4852d4198690c2c98af060cd96ab83 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 12 Jan 2024 15:03:07 -0800 Subject: soc: qcom: geni-se: Add M_TX_FIFO_NOT_EMPTY bit definition According to the docs I have, bit 21 of the status register is asserted when the FIFO is _not_ empty. Add the definition. Signed-off-by: Douglas Anderson Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240112150307.1.I7dc0993c1e758a1efedd651e7e1670deb1b430fb@changeid Signed-off-by: Greg Kroah-Hartman --- include/linux/soc/qcom/geni-se.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 29e06905bc1f..0f038a1a0330 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -178,6 +178,7 @@ struct geni_se { #define M_GP_IRQ_3_EN BIT(12) #define M_GP_IRQ_4_EN BIT(13) #define M_GP_IRQ_5_EN BIT(14) +#define M_TX_FIFO_NOT_EMPTY_EN BIT(21) #define M_IO_DATA_DEASSERT_EN BIT(22) #define M_IO_DATA_ASSERT_EN BIT(23) #define M_RX_FIFO_RD_ERR_EN BIT(24) -- cgit v1.2.3 From 5c49b6a4a4bcf368f85cfe7a0e5ac3a7016f30fd Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 8 Jan 2024 14:41:02 +0100 Subject: vt: remove superfluous CONFIG_HW_CONSOLE The config HW_CONSOLE is always identical to the config VT and is not visible in the kernel's build menuconfig. So, CONFIG_HW_CONSOLE is redundant. Replace all references to CONFIG_HW_CONSOLE with CONFIG_VT and remove CONFIG_HW_CONSOLE. Signed-off-by: Lukas Bulwahn Reviewed-by: Javier Martinez Canillas Acked-by: Dmitry Torokhov Reviewed-by: Geert Uytterhoeven Acked-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240108134102.601-1-lukas.bulwahn@gmail.com Signed-off-by: Greg Kroah-Hartman --- arch/m68k/amiga/config.c | 2 +- drivers/input/keyboard/amikbd.c | 6 +++--- drivers/tty/Kconfig | 7 +------ drivers/tty/vt/Makefile | 4 ++-- drivers/video/fbdev/tgafb.c | 2 +- include/linux/console.h | 2 +- lib/Kconfig.kgdb | 2 +- 7 files changed, 10 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/arch/m68k/amiga/config.c b/arch/m68k/amiga/config.c index 7791673e547b..99718f3dc686 100644 --- a/arch/m68k/amiga/config.c +++ b/arch/m68k/amiga/config.c @@ -846,6 +846,6 @@ static void amiga_get_hardware_list(struct seq_file *m) * The Amiga keyboard driver needs key_maps, but we cannot export it in * drivers/char/defkeymap.c, as it is autogenerated */ -#ifdef CONFIG_HW_CONSOLE +#ifdef CONFIG_VT EXPORT_SYMBOL_GPL(key_maps); #endif diff --git a/drivers/input/keyboard/amikbd.c b/drivers/input/keyboard/amikbd.c index e305c44cd0aa..ecfae0b0b6aa 100644 --- a/drivers/input/keyboard/amikbd.c +++ b/drivers/input/keyboard/amikbd.c @@ -26,7 +26,7 @@ MODULE_AUTHOR("Vojtech Pavlik "); MODULE_DESCRIPTION("Amiga keyboard driver"); MODULE_LICENSE("GPL"); -#ifdef CONFIG_HW_CONSOLE +#ifdef CONFIG_VT static unsigned char amikbd_keycode[0x78] __initdata = { [0] = KEY_GRAVE, [1] = KEY_1, @@ -148,9 +148,9 @@ static void __init amikbd_init_console_keymaps(void) memcpy(key_maps[i], temp_map, sizeof(temp_map)); } } -#else /* !CONFIG_HW_CONSOLE */ +#else /* !CONFIG_VT */ static inline void amikbd_init_console_keymaps(void) {} -#endif /* !CONFIG_HW_CONSOLE */ +#endif /* !CONFIG_VT */ static const char *amikbd_messages[8] = { [0] = KERN_ALERT "amikbd: Ctrl-Amiga-Amiga reset warning!!\n", diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig index 5646dc6242cd..a45d423ad10f 100644 --- a/drivers/tty/Kconfig +++ b/drivers/tty/Kconfig @@ -75,14 +75,9 @@ config VT_CONSOLE_SLEEP def_bool y depends on VT_CONSOLE && PM_SLEEP -config HW_CONSOLE - bool - depends on VT - default y - config VT_HW_CONSOLE_BINDING bool "Support for binding and unbinding console drivers" - depends on HW_CONSOLE + depends on VT help The virtual terminal is the device that interacts with the physical terminal through console drivers. On these systems, at least one diff --git a/drivers/tty/vt/Makefile b/drivers/tty/vt/Makefile index b3dfe9d5717e..2c8ce8b592ed 100644 --- a/drivers/tty/vt/Makefile +++ b/drivers/tty/vt/Makefile @@ -5,9 +5,9 @@ FONTMAPFILE = cp437.uni obj-$(CONFIG_VT) += vt_ioctl.o vc_screen.o \ - selection.o keyboard.o + selection.o keyboard.o \ + vt.o defkeymap.o obj-$(CONFIG_CONSOLE_TRANSLATIONS) += consolemap.o consolemap_deftbl.o -obj-$(CONFIG_HW_CONSOLE) += vt.o defkeymap.o # Files generated that shall be removed upon make clean clean-files := consolemap_deftbl.c defkeymap.c diff --git a/drivers/video/fbdev/tgafb.c b/drivers/video/fbdev/tgafb.c index ca43774f3156..dccfc38cfbd5 100644 --- a/drivers/video/fbdev/tgafb.c +++ b/drivers/video/fbdev/tgafb.c @@ -380,7 +380,7 @@ tgafb_set_par(struct fb_info *info) BT463_LOAD_ADDR(par, 0x0000); TGA_WRITE_REG(par, BT463_PALETTE << 2, TGA_RAMDAC_SETUP_REG); -#ifdef CONFIG_HW_CONSOLE +#ifdef CONFIG_VT for (i = 0; i < 16; i++) { int j = color_table[i]; diff --git a/include/linux/console.h b/include/linux/console.h index d6d8b7e6b93b..31a8f5b85f5d 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -141,7 +141,7 @@ int con_is_bound(const struct consw *csw); int do_unregister_con_driver(const struct consw *csw); int do_take_over_console(const struct consw *sw, int first, int last, int deflt); void give_up_console(const struct consw *sw); -#ifdef CONFIG_HW_CONSOLE +#ifdef CONFIG_VT void con_debug_enter(struct vc_data *vc); void con_debug_leave(void); #else diff --git a/lib/Kconfig.kgdb b/lib/Kconfig.kgdb index 3b9a44008433..b5c0e6576749 100644 --- a/lib/Kconfig.kgdb +++ b/lib/Kconfig.kgdb @@ -43,7 +43,7 @@ config KGDB_SERIAL_CONSOLE tristate "KGDB: use kgdb over the serial console" select CONSOLE_POLL select MAGIC_SYSRQ - depends on TTY && HW_CONSOLE + depends on TTY && VT default y help Share a serial console with kgdb. Sysrq-g must be used -- cgit v1.2.3 From 88b7049635dc5d0e2a7dfaaf89e70a9654ed6561 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 11:06:06 +0100 Subject: gpio: unexport GPIO irq domain functions only used internally There are no external users for the irq domain helpers so unexport them and remove the prototypes from the driver header. Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 93 ++++++++++++++++++++------------------------- include/linux/gpio/driver.h | 12 ------ 2 files changed, 42 insertions(+), 63 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 44c8f5743a24..d50a786f8176 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1254,8 +1254,8 @@ static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc) gpiochip_free_mask(&gc->irq.valid_mask); } -bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, - unsigned int offset) +static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, + unsigned int offset) { if (!gpiochip_line_is_valid(gc, offset)) return false; @@ -1264,7 +1264,6 @@ bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, return true; return test_bit(offset, gc->irq.valid_mask); } -EXPORT_SYMBOL_GPL(gpiochip_irqchip_irq_valid); #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY @@ -1439,6 +1438,43 @@ static unsigned int gpiochip_child_offset_to_irq_noop(struct gpio_chip *gc, return offset; } +/** + * gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ + * @domain: The IRQ domain used by this IRQ chip + * @data: Outermost irq_data associated with the IRQ + * @reserve: If set, only reserve an interrupt vector instead of assigning one + * + * This function is a wrapper that calls gpiochip_lock_as_irq() and is to be + * used as the activate function for the &struct irq_domain_ops. The host_data + * for the IRQ domain must be the &struct gpio_chip. + */ +static int gpiochip_irq_domain_activate(struct irq_domain *domain, + struct irq_data *data, bool reserve) +{ + struct gpio_chip *gc = domain->host_data; + unsigned int hwirq = irqd_to_hwirq(data); + + return gpiochip_lock_as_irq(gc, hwirq); +} + +/** + * gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ + * @domain: The IRQ domain used by this IRQ chip + * @data: Outermost irq_data associated with the IRQ + * + * This function is a wrapper that will call gpiochip_unlock_as_irq() and is to + * be used as the deactivate function for the &struct irq_domain_ops. The + * host_data for the IRQ domain must be the &struct gpio_chip. + */ +static void gpiochip_irq_domain_deactivate(struct irq_domain *domain, + struct irq_data *data) +{ + struct gpio_chip *gc = domain->host_data; + unsigned int hwirq = irqd_to_hwirq(data); + + return gpiochip_unlock_as_irq(gc, hwirq); +} + static void gpiochip_hierarchy_setup_domain_ops(struct irq_domain_ops *ops) { ops->activate = gpiochip_irq_domain_activate; @@ -1556,7 +1592,8 @@ static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc) * gpiochip by assigning the gpiochip as chip data, and using the irqchip * stored inside the gpiochip. */ -int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) +static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) { struct gpio_chip *gc = d->host_data; int ret = 0; @@ -1593,9 +1630,8 @@ int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwi return 0; } -EXPORT_SYMBOL_GPL(gpiochip_irq_map); -void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) +static void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) { struct gpio_chip *gc = d->host_data; @@ -1604,7 +1640,6 @@ void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) irq_set_chip_and_handler(irq, NULL, NULL); irq_set_chip_data(irq, NULL); } -EXPORT_SYMBOL_GPL(gpiochip_irq_unmap); static const struct irq_domain_ops gpiochip_domain_ops = { .map = gpiochip_irq_map, @@ -1626,50 +1661,6 @@ static struct irq_domain *gpiochip_simple_create_domain(struct gpio_chip *gc) return domain; } -/* - * TODO: move these activate/deactivate in under the hierarchicial - * irqchip implementation as static once SPMI and SSBI (all external - * users) are phased over. - */ -/** - * gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ - * @domain: The IRQ domain used by this IRQ chip - * @data: Outermost irq_data associated with the IRQ - * @reserve: If set, only reserve an interrupt vector instead of assigning one - * - * This function is a wrapper that calls gpiochip_lock_as_irq() and is to be - * used as the activate function for the &struct irq_domain_ops. The host_data - * for the IRQ domain must be the &struct gpio_chip. - */ -int gpiochip_irq_domain_activate(struct irq_domain *domain, - struct irq_data *data, bool reserve) -{ - struct gpio_chip *gc = domain->host_data; - unsigned int hwirq = irqd_to_hwirq(data); - - return gpiochip_lock_as_irq(gc, hwirq); -} -EXPORT_SYMBOL_GPL(gpiochip_irq_domain_activate); - -/** - * gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ - * @domain: The IRQ domain used by this IRQ chip - * @data: Outermost irq_data associated with the IRQ - * - * This function is a wrapper that will call gpiochip_unlock_as_irq() and is to - * be used as the deactivate function for the &struct irq_domain_ops. The - * host_data for the IRQ domain must be the &struct gpio_chip. - */ -void gpiochip_irq_domain_deactivate(struct irq_domain *domain, - struct irq_data *data) -{ - struct gpio_chip *gc = domain->host_data; - unsigned int hwirq = irqd_to_hwirq(data); - - return gpiochip_unlock_as_irq(gc, hwirq); -} -EXPORT_SYMBOL_GPL(gpiochip_irq_domain_deactivate); - static int gpiochip_to_irq(struct gpio_chip *gc, unsigned int offset) { struct irq_domain *domain = gc->irq.domain; diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 9a5c6c76e653..363d06c7b637 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -704,18 +704,6 @@ int bgpio_init(struct gpio_chip *gc, struct device *dev, #define BGPIOF_NO_OUTPUT BIT(5) /* only input */ #define BGPIOF_NO_SET_ON_INPUT BIT(6) -int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, - irq_hw_number_t hwirq); -void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq); - -int gpiochip_irq_domain_activate(struct irq_domain *domain, - struct irq_data *data, bool reserve); -void gpiochip_irq_domain_deactivate(struct irq_domain *domain, - struct irq_data *data); - -bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, - unsigned int offset); - #ifdef CONFIG_GPIOLIB_IRQCHIP int gpiochip_irqchip_add_domain(struct gpio_chip *gc, struct irq_domain *domain); -- cgit v1.2.3 From 6933ba529d06afdd3faf5501855e410b46b77160 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 25 Jan 2024 09:35:07 +0100 Subject: gpio: improve the API contract for setting direction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a GPIO driver returns a positive integer from one of the direction setter callbacks, we'll end up propagating it to user-space. Whether we should sanitize the values returned by callbacks is a different question but let's first improve the documentation and fortify the contract with GPIO providers. Reported-by: José Guilherme de Castro Rodrigues Signed-off-by: Bartosz Golaszewski Reviewed-by: Kent Gibson --- include/linux/gpio/driver.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 363d06c7b637..3a37d058cfcf 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -335,10 +335,12 @@ struct gpio_irq_chip { * (same as GPIO_LINE_DIRECTION_OUT / GPIO_LINE_DIRECTION_IN), * or negative error. It is recommended to always implement this * function, even on input-only or output-only gpio chips. - * @direction_input: configures signal "offset" as input, or returns error - * This can be omitted on input-only or output-only gpio chips. - * @direction_output: configures signal "offset" as output, or returns error - * This can be omitted on input-only or output-only gpio chips. + * @direction_input: configures signal "offset" as input, returns 0 on success + * or a negative error number. This can be omitted on input-only or + * output-only gpio chips. + * @direction_output: configures signal "offset" as output, returns 0 on + * success or a negative error number. This can be omitted on input-only + * or output-only gpio chips. * @get: returns value for signal "offset", 0=low, 1=high, or negative error * @get_multiple: reads values for multiple signals defined by "mask" and * stores them in "bits", returns 0 on success or negative error -- cgit v1.2.3 From ea1cc3ee34a5f3144f6c2cdc07c19c914ccb9526 Mon Sep 17 00:00:00 2001 From: Min Li Date: Wed, 24 Jan 2024 13:49:46 -0500 Subject: ptp: introduce PTP_CLOCK_EXTOFF event for the measured external offset This change is for the PHC devices that can measure the phase offset between PHC signal and the external signal, such as the 1PPS signal of GNSS. Reporting PTP_CLOCK_EXTOFF to user space will be piggy-backed to the existing ptp_extts_event so that application such as ts2phc can poll the external offset the same way as extts. Hence, ts2phc can use the offset to achieve the alignment between PHC and the external signal by the help of either SW or HW filters. Signed-off-by: Min Li Acked-by: Richard Cochran Signed-off-by: David S. Miller --- drivers/ptp/ptp_clock.c | 16 +++++++++++++++- include/linux/ptp_clock_kernel.h | 3 +++ include/uapi/linux/ptp_clock.h | 13 ++++++++++--- 3 files changed, 28 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c index 15b804ba4868..3aaf1a3430c5 100644 --- a/drivers/ptp/ptp_clock.c +++ b/drivers/ptp/ptp_clock.c @@ -44,18 +44,31 @@ static void enqueue_external_timestamp(struct timestamp_event_queue *queue, struct ptp_clock_event *src) { struct ptp_extts_event *dst; + struct timespec64 offset_ts; unsigned long flags; s64 seconds; u32 remainder; - seconds = div_u64_rem(src->timestamp, 1000000000, &remainder); + if (src->type == PTP_CLOCK_EXTTS) { + seconds = div_u64_rem(src->timestamp, 1000000000, &remainder); + } else if (src->type == PTP_CLOCK_EXTOFF) { + offset_ts = ns_to_timespec64(src->offset); + seconds = offset_ts.tv_sec; + remainder = offset_ts.tv_nsec; + } else { + WARN(1, "%s: unknown type %d\n", __func__, src->type); + return; + } spin_lock_irqsave(&queue->lock, flags); dst = &queue->buf[queue->tail]; dst->index = src->index; + dst->flags = PTP_EXTTS_EVENT_VALID; dst->t.sec = seconds; dst->t.nsec = remainder; + if (src->type == PTP_CLOCK_EXTOFF) + dst->flags |= PTP_EXT_OFFSET; /* Both WRITE_ONCE() are paired with READ_ONCE() in queue_cnt() */ if (!queue_free(queue)) @@ -417,6 +430,7 @@ void ptp_clock_event(struct ptp_clock *ptp, struct ptp_clock_event *event) break; case PTP_CLOCK_EXTTS: + case PTP_CLOCK_EXTOFF: /* Enqueue timestamp on selected queues */ spin_lock_irqsave(&ptp->tsevqs_lock, flags); list_for_each_entry(tsevq, &ptp->tsevqs, qlist) { diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h index 1ef4e0f9bd2a..6e4b8206c7d0 100644 --- a/include/linux/ptp_clock_kernel.h +++ b/include/linux/ptp_clock_kernel.h @@ -200,6 +200,7 @@ struct ptp_clock; enum ptp_clock_events { PTP_CLOCK_ALARM, PTP_CLOCK_EXTTS, + PTP_CLOCK_EXTOFF, PTP_CLOCK_PPS, PTP_CLOCK_PPSUSR, }; @@ -210,6 +211,7 @@ enum ptp_clock_events { * @type: One of the ptp_clock_events enumeration values. * @index: Identifies the source of the event. * @timestamp: When the event occurred (%PTP_CLOCK_EXTTS only). + * @offset: When the event occurred (%PTP_CLOCK_EXTOFF only). * @pps_times: When the event occurred (%PTP_CLOCK_PPSUSR only). */ @@ -218,6 +220,7 @@ struct ptp_clock_event { int index; union { u64 timestamp; + s64 offset; struct pps_event_time pps_times; }; }; diff --git a/include/uapi/linux/ptp_clock.h b/include/uapi/linux/ptp_clock.h index da700999cad4..053b40d642de 100644 --- a/include/uapi/linux/ptp_clock.h +++ b/include/uapi/linux/ptp_clock.h @@ -32,6 +32,7 @@ #define PTP_RISING_EDGE (1<<1) #define PTP_FALLING_EDGE (1<<2) #define PTP_STRICT_FLAGS (1<<3) +#define PTP_EXT_OFFSET (1<<4) #define PTP_EXTTS_EDGES (PTP_RISING_EDGE | PTP_FALLING_EDGE) /* @@ -40,7 +41,8 @@ #define PTP_EXTTS_VALID_FLAGS (PTP_ENABLE_FEATURE | \ PTP_RISING_EDGE | \ PTP_FALLING_EDGE | \ - PTP_STRICT_FLAGS) + PTP_STRICT_FLAGS | \ + PTP_EXT_OFFSET) /* * flag fields valid for the original PTP_EXTTS_REQUEST ioctl. @@ -50,6 +52,11 @@ PTP_RISING_EDGE | \ PTP_FALLING_EDGE) +/* + * flag fields valid for the ptp_extts_event report. + */ +#define PTP_EXTTS_EVENT_VALID (PTP_ENABLE_FEATURE) + /* * Bits of the ptp_perout_request.flags field: */ @@ -228,9 +235,9 @@ struct ptp_pin_desc { #define PTP_MASK_EN_SINGLE _IOW(PTP_CLK_MAGIC, 20, unsigned int) struct ptp_extts_event { - struct ptp_clock_time t; /* Time event occured. */ + struct ptp_clock_time t; /* Time event occurred. */ unsigned int index; /* Which channel produced the event. */ - unsigned int flags; /* Reserved for future use. */ + unsigned int flags; /* Event type. */ unsigned int rsv[2]; /* Reserved for future use. */ }; -- cgit v1.2.3 From 1ddfecafabf71e0e5345dff877d2680083c7e078 Mon Sep 17 00:00:00 2001 From: Min Li Date: Wed, 24 Jan 2024 13:49:47 -0500 Subject: ptp: add FemtoClock3 Wireless as ptp hardware clock The RENESAS FemtoClock3 Wireless is a high-performance jitter attenuator, frequency translator, and clock synthesizer. The device is comprised of 3 digital PLLs (DPLL) to track CLKIN inputs and three independent low phase noise fractional output dividers (FOD) that output low phase noise clocks. FemtoClock3 supports one Time Synchronization (Time Sync) channel to enable an external processor to control the phase and frequency of the Time Sync channel and to take phase measurements using the TDC. Intended applications are synchronization using the precision time protocol (PTP) and synchronization with 0.5 Hz and 1 Hz signals from GNSS. Signed-off-by: Min Li Acked-by: Lee Jones Signed-off-by: David S. Miller --- drivers/ptp/Kconfig | 12 + drivers/ptp/Makefile | 1 + drivers/ptp/ptp_fc3.c | 1016 ++++++++++++++++++++++++++++++++++++ drivers/ptp/ptp_fc3.h | 45 ++ include/linux/mfd/idtRC38xxx_reg.h | 273 ++++++++++ 5 files changed, 1347 insertions(+) create mode 100644 drivers/ptp/ptp_fc3.c create mode 100644 drivers/ptp/ptp_fc3.h create mode 100644 include/linux/mfd/idtRC38xxx_reg.h (limited to 'include/linux') diff --git a/drivers/ptp/Kconfig b/drivers/ptp/Kconfig index 5dd5f188e14f..604541dcb320 100644 --- a/drivers/ptp/Kconfig +++ b/drivers/ptp/Kconfig @@ -155,6 +155,18 @@ config PTP_1588_CLOCK_IDTCM To compile this driver as a module, choose M here: the module will be called ptp_clockmatrix. +config PTP_1588_CLOCK_FC3W + tristate "RENESAS FemtoClock3 Wireless as PTP clock" + depends on PTP_1588_CLOCK && I2C + default n + help + This driver adds support for using Renesas FemtoClock3 Wireless + as a PTP clock. This clock is only useful if your time stamping + MAC is connected to the RENESAS chip. + + To compile this driver as a module, choose M here: the module + will be called ptp_fc3. + config PTP_1588_CLOCK_MOCK tristate "Mock-up PTP clock" depends on PTP_1588_CLOCK diff --git a/drivers/ptp/Makefile b/drivers/ptp/Makefile index dea0cebd2303..68bf02078053 100644 --- a/drivers/ptp/Makefile +++ b/drivers/ptp/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_PTP_1588_CLOCK_QORIQ) += ptp-qoriq.o ptp-qoriq-y += ptp_qoriq.o ptp-qoriq-$(CONFIG_DEBUG_FS) += ptp_qoriq_debugfs.o obj-$(CONFIG_PTP_1588_CLOCK_IDTCM) += ptp_clockmatrix.o +obj-$(CONFIG_PTP_1588_CLOCK_FC3W) += ptp_fc3.o obj-$(CONFIG_PTP_1588_CLOCK_IDT82P33) += ptp_idt82p33.o obj-$(CONFIG_PTP_1588_CLOCK_MOCK) += ptp_mock.o obj-$(CONFIG_PTP_1588_CLOCK_VMW) += ptp_vmw.o diff --git a/drivers/ptp/ptp_fc3.c b/drivers/ptp/ptp_fc3.c new file mode 100644 index 000000000000..0e2286ba088a --- /dev/null +++ b/drivers/ptp/ptp_fc3.c @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PTP hardware clock driver for the FemtoClock3 family of timing and + * synchronization devices. + * + * Copyright (C) 2023 Integrated Device Technology, Inc., a Renesas Company. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ptp_private.h" +#include "ptp_fc3.h" + +MODULE_DESCRIPTION("Driver for IDT FemtoClock3(TM) family"); +MODULE_AUTHOR("IDT support-1588 "); +MODULE_VERSION("1.0"); +MODULE_LICENSE("GPL"); + +/* + * The name of the firmware file to be loaded + * over-rides any automatic selection + */ +static char *firmware; +module_param(firmware, charp, 0); + +static s64 ns2counters(struct idtfc3 *idtfc3, s64 nsec, u32 *sub_ns) +{ + s64 sync; + s32 rem; + + if (likely(nsec >= 0)) { + sync = div_u64_rem(nsec, idtfc3->ns_per_sync, &rem); + *sub_ns = rem; + } else { + sync = -div_u64_rem(-nsec - 1, idtfc3->ns_per_sync, &rem) - 1; + *sub_ns = idtfc3->ns_per_sync - rem - 1; + } + + return sync * idtfc3->ns_per_sync; +} + +static s64 tdc_meas2offset(struct idtfc3 *idtfc3, u64 meas_read) +{ + s64 coarse, fine; + + fine = sign_extend64(FIELD_GET(FINE_MEAS_MASK, meas_read), 12); + coarse = sign_extend64(FIELD_GET(COARSE_MEAS_MASK, meas_read), (39 - 13)); + + fine = div64_s64(fine * NSEC_PER_SEC, idtfc3->tdc_apll_freq * 62LL); + coarse = div64_s64(coarse * NSEC_PER_SEC, idtfc3->time_ref_freq); + + return coarse + fine; +} + +static s64 tdc_offset2phase(struct idtfc3 *idtfc3, s64 offset_ns) +{ + if (offset_ns > idtfc3->ns_per_sync / 2) + offset_ns -= idtfc3->ns_per_sync; + + return offset_ns * idtfc3->tdc_offset_sign; +} + +static int idtfc3_set_lpf_mode(struct idtfc3 *idtfc3, u8 mode) +{ + int err; + + if (mode >= LPF_INVALID) + return -EINVAL; + + if (idtfc3->lpf_mode == mode) + return 0; + + err = regmap_bulk_write(idtfc3->regmap, LPF_MODE_CNFG, &mode, sizeof(mode)); + if (err) + return err; + + idtfc3->lpf_mode = mode; + + return 0; +} + +static int idtfc3_enable_lpf(struct idtfc3 *idtfc3, bool enable) +{ + u8 val; + int err; + + err = regmap_bulk_read(idtfc3->regmap, LPF_CTRL, &val, sizeof(val)); + if (err) + return err; + + if (enable == true) + val |= LPF_EN; + else + val &= ~LPF_EN; + + return regmap_bulk_write(idtfc3->regmap, LPF_CTRL, &val, sizeof(val)); +} + +static int idtfc3_get_time_ref_freq(struct idtfc3 *idtfc3) +{ + int err; + u8 buf[4]; + u8 time_ref_div; + u8 time_clk_div; + + err = regmap_bulk_read(idtfc3->regmap, TIME_CLOCK_MEAS_DIV_CNFG, buf, sizeof(buf)); + if (err) + return err; + time_ref_div = FIELD_GET(TIME_REF_DIV_MASK, get_unaligned_le32(buf)) + 1; + + err = regmap_bulk_read(idtfc3->regmap, TIME_CLOCK_COUNT, buf, 1); + if (err) + return err; + time_clk_div = (buf[0] & TIME_CLOCK_COUNT_MASK) + 1; + idtfc3->time_ref_freq = idtfc3->hw_param.time_clk_freq * + time_clk_div / time_ref_div; + + return 0; +} + +static int idtfc3_get_tdc_offset_sign(struct idtfc3 *idtfc3) +{ + int err; + u8 buf[4]; + u32 val; + u8 sig1, sig2; + + err = regmap_bulk_read(idtfc3->regmap, TIME_CLOCK_TDC_FANOUT_CNFG, buf, sizeof(buf)); + if (err) + return err; + + val = get_unaligned_le32(buf); + if ((val & TIME_SYNC_TO_TDC_EN) != TIME_SYNC_TO_TDC_EN) { + dev_err(idtfc3->dev, "TIME_SYNC_TO_TDC_EN is off !!!"); + return -EINVAL; + } + + sig1 = FIELD_GET(SIG1_MUX_SEL_MASK, val); + sig2 = FIELD_GET(SIG2_MUX_SEL_MASK, val); + + if ((sig1 == sig2) || ((sig1 != TIME_SYNC) && (sig2 != TIME_SYNC))) { + dev_err(idtfc3->dev, "Invalid tdc_mux_sel sig1=%d sig2=%d", sig1, sig2); + return -EINVAL; + } else if (sig1 == TIME_SYNC) { + idtfc3->tdc_offset_sign = 1; + } else if (sig2 == TIME_SYNC) { + idtfc3->tdc_offset_sign = -1; + } + + return 0; +} + +static int idtfc3_lpf_bw(struct idtfc3 *idtfc3, u8 shift, u8 mult) +{ + u8 val = FIELD_PREP(LPF_BW_SHIFT, shift) | FIELD_PREP(LPF_BW_MULT, mult); + + return regmap_bulk_write(idtfc3->regmap, LPF_BW_CNFG, &val, sizeof(val)); +} + +static int idtfc3_enable_tdc(struct idtfc3 *idtfc3, bool enable, u8 meas_mode) +{ + int err; + u8 val = 0; + + /* Disable TDC first */ + err = regmap_bulk_write(idtfc3->regmap, TIME_CLOCK_MEAS_CTRL, &val, sizeof(val)); + if (err) + return err; + + if (enable == false) + return idtfc3_lpf_bw(idtfc3, LPF_BW_SHIFT_DEFAULT, LPF_BW_MULT_DEFAULT); + + if (meas_mode >= MEAS_MODE_INVALID) + return -EINVAL; + + /* Change TDC meas mode */ + err = regmap_bulk_write(idtfc3->regmap, TIME_CLOCK_MEAS_CNFG, + &meas_mode, sizeof(meas_mode)); + if (err) + return err; + + /* Enable TDC */ + val = TDC_MEAS_EN; + if (meas_mode == CONTINUOUS) + val |= TDC_MEAS_START; + err = regmap_bulk_write(idtfc3->regmap, TIME_CLOCK_MEAS_CTRL, &val, sizeof(val)); + if (err) + return err; + + return idtfc3_lpf_bw(idtfc3, LPF_BW_SHIFT_1PPS, LPF_BW_MULT_DEFAULT); +} + +static bool get_tdc_meas(struct idtfc3 *idtfc3, s64 *offset_ns) +{ + bool valid = false; + u8 buf[9]; + u8 val; + int err; + + while (true) { + err = regmap_bulk_read(idtfc3->regmap, TDC_FIFO_STS, + &val, sizeof(val)); + if (err) + return false; + + if (val & FIFO_EMPTY) + break; + + err = regmap_bulk_read(idtfc3->regmap, TDC_FIFO_READ_REQ, + &buf, sizeof(buf)); + if (err) + return false; + + valid = true; + } + + if (valid) + *offset_ns = tdc_meas2offset(idtfc3, get_unaligned_le64(&buf[1])); + + return valid; +} + +static int check_tdc_fifo_overrun(struct idtfc3 *idtfc3) +{ + u8 val; + int err; + + /* Check if FIFO is overrun */ + err = regmap_bulk_read(idtfc3->regmap, TDC_FIFO_STS, &val, sizeof(val)); + if (err) + return err; + + if (!(val & FIFO_FULL)) + return 0; + + dev_warn(idtfc3->dev, "TDC FIFO overrun !!!"); + + err = idtfc3_enable_tdc(idtfc3, true, CONTINUOUS); + if (err) + return err; + + return 0; +} + +static int get_tdc_meas_continuous(struct idtfc3 *idtfc3) +{ + int err; + s64 offset_ns; + struct ptp_clock_event event; + + err = check_tdc_fifo_overrun(idtfc3); + if (err) + return err; + + if (get_tdc_meas(idtfc3, &offset_ns) && offset_ns >= 0) { + event.index = 0; + event.offset = tdc_offset2phase(idtfc3, offset_ns); + event.type = PTP_CLOCK_EXTOFF; + ptp_clock_event(idtfc3->ptp_clock, &event); + } + + return 0; +} + +static int idtfc3_read_subcounter(struct idtfc3 *idtfc3) +{ + u8 buf[5] = {0}; + int err; + + err = regmap_bulk_read(idtfc3->regmap, TOD_COUNTER_READ_REQ, + &buf, sizeof(buf)); + if (err) + return err; + + /* sync_counter_value is [31:82] and sub_sync_counter_value is [0:30] */ + return get_unaligned_le32(&buf[1]) & SUB_SYNC_COUNTER_MASK; +} + +static int idtfc3_tod_update_is_done(struct idtfc3 *idtfc3) +{ + int err; + u8 req; + + err = read_poll_timeout_atomic(regmap_bulk_read, err, !req, USEC_PER_MSEC, + idtfc3->tc_write_timeout, true, idtfc3->regmap, + TOD_SYNC_LOAD_REQ_CTRL, &req, 1); + if (err) + dev_err(idtfc3->dev, "TOD counter write timeout !!!"); + + return err; +} + +static int idtfc3_write_subcounter(struct idtfc3 *idtfc3, u32 counter) +{ + u8 buf[18] = {0}; + int err; + + /* sync_counter_value is [31:82] and sub_sync_counter_value is [0:30] */ + put_unaligned_le32(counter & SUB_SYNC_COUNTER_MASK, &buf[0]); + + buf[16] = SUB_SYNC_LOAD_ENABLE | SYNC_LOAD_ENABLE; + buf[17] = SYNC_LOAD_REQ; + + err = regmap_bulk_write(idtfc3->regmap, TOD_SYNC_LOAD_VAL_CTRL, + &buf, sizeof(buf)); + if (err) + return err; + + return idtfc3_tod_update_is_done(idtfc3); +} + +static int idtfc3_timecounter_update(struct idtfc3 *idtfc3, u32 counter, s64 ns) +{ + int err; + + err = idtfc3_write_subcounter(idtfc3, counter); + if (err) + return err; + + /* Update time counter */ + idtfc3->ns = ns; + idtfc3->last_counter = counter; + + return 0; +} + +static int idtfc3_timecounter_read(struct idtfc3 *idtfc3) +{ + int now, delta; + + now = idtfc3_read_subcounter(idtfc3); + if (now < 0) + return now; + + /* calculate the delta since the last idtfc3_timecounter_read(): */ + if (now >= idtfc3->last_counter) + delta = now - idtfc3->last_counter; + else + delta = idtfc3->sub_sync_count - idtfc3->last_counter + now; + + /* Update time counter */ + idtfc3->ns += delta * idtfc3->ns_per_counter; + idtfc3->last_counter = now; + + return 0; +} + +static int _idtfc3_gettime(struct idtfc3 *idtfc3, struct timespec64 *ts) +{ + int err; + + err = idtfc3_timecounter_read(idtfc3); + if (err) + return err; + + *ts = ns_to_timespec64(idtfc3->ns); + + return 0; +} + +static int idtfc3_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + int err; + + mutex_lock(idtfc3->lock); + err = _idtfc3_gettime(idtfc3, ts); + mutex_unlock(idtfc3->lock); + + return err; +} + +static int _idtfc3_settime(struct idtfc3 *idtfc3, const struct timespec64 *ts) +{ + s64 offset_ns, now_ns; + u32 counter, sub_ns; + int now; + + if (timespec64_valid(ts) == false) { + dev_err(idtfc3->dev, "%s: invalid timespec", __func__); + return -EINVAL; + } + + now = idtfc3_read_subcounter(idtfc3); + if (now < 0) + return now; + + offset_ns = (idtfc3->sub_sync_count - now) * idtfc3->ns_per_counter; + now_ns = timespec64_to_ns(ts); + (void)ns2counters(idtfc3, offset_ns + now_ns, &sub_ns); + + counter = sub_ns / idtfc3->ns_per_counter; + return idtfc3_timecounter_update(idtfc3, counter, now_ns); +} + +static int idtfc3_settime(struct ptp_clock_info *ptp, const struct timespec64 *ts) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + int err; + + mutex_lock(idtfc3->lock); + err = _idtfc3_settime(idtfc3, ts); + mutex_unlock(idtfc3->lock); + + return err; +} + +static int _idtfc3_adjtime(struct idtfc3 *idtfc3, s64 delta) +{ + /* + * The TOD counter can be synchronously loaded with any value, + * to be loaded on the next Time Sync pulse + */ + s64 sync_ns; + u32 sub_ns; + u32 counter; + + if (idtfc3->ns + delta < 0) { + dev_err(idtfc3->dev, "%lld ns adj is too large", delta); + return -EINVAL; + } + + sync_ns = ns2counters(idtfc3, delta + idtfc3->ns_per_sync, &sub_ns); + + counter = sub_ns / idtfc3->ns_per_counter; + return idtfc3_timecounter_update(idtfc3, counter, idtfc3->ns + sync_ns + + counter * idtfc3->ns_per_counter); +} + +static int idtfc3_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + int err; + + mutex_lock(idtfc3->lock); + err = _idtfc3_adjtime(idtfc3, delta); + mutex_unlock(idtfc3->lock); + + return err; +} + +static int _idtfc3_adjphase(struct idtfc3 *idtfc3, s32 delta) +{ + u8 buf[8] = {0}; + int err; + s64 pcw; + + err = idtfc3_set_lpf_mode(idtfc3, LPF_WP); + if (err) + return err; + + /* + * Phase Control Word unit is: 10^9 / (TDC_APLL_FREQ * 124) + * + * delta * TDC_APLL_FREQ * 124 + * PCW = --------------------------- + * 10^9 + * + */ + pcw = div_s64((s64)delta * idtfc3->tdc_apll_freq * 124, NSEC_PER_SEC); + + put_unaligned_le64(pcw, buf); + + return regmap_bulk_write(idtfc3->regmap, LPF_WR_PHASE_CTRL, buf, sizeof(buf)); +} + +static int idtfc3_adjphase(struct ptp_clock_info *ptp, s32 delta) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + int err; + + mutex_lock(idtfc3->lock); + err = _idtfc3_adjphase(idtfc3, delta); + mutex_unlock(idtfc3->lock); + + return err; +} + +static int _idtfc3_adjfine(struct idtfc3 *idtfc3, long scaled_ppm) +{ + u8 buf[8] = {0}; + int err; + s64 fcw; + + err = idtfc3_set_lpf_mode(idtfc3, LPF_WF); + if (err) + return err; + + /* + * Frequency Control Word unit is: 2^-44 * 10^6 ppm + * + * adjfreq: + * ppb * 2^44 + * FCW = ---------- + * 10^9 + * + * adjfine: + * ppm_16 * 2^28 + * FCW = ------------- + * 10^6 + */ + fcw = scaled_ppm * BIT(28); + fcw = div_s64(fcw, 1000000); + + put_unaligned_le64(fcw, buf); + + return regmap_bulk_write(idtfc3->regmap, LPF_WR_FREQ_CTRL, buf, sizeof(buf)); +} + +static int idtfc3_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + int err; + + mutex_lock(idtfc3->lock); + err = _idtfc3_adjfine(idtfc3, scaled_ppm); + mutex_unlock(idtfc3->lock); + + return err; +} + +static int idtfc3_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + int err = -EOPNOTSUPP; + + mutex_lock(idtfc3->lock); + switch (rq->type) { + case PTP_CLK_REQ_PEROUT: + if (!on) + err = 0; + /* Only accept a 1-PPS aligned to the second. */ + else if (rq->perout.start.nsec || rq->perout.period.sec != 1 || + rq->perout.period.nsec) + err = -ERANGE; + else + err = 0; + break; + case PTP_CLK_REQ_EXTTS: + if (on) { + /* Only accept requests for external phase offset */ + if ((rq->extts.flags & PTP_EXT_OFFSET) != (PTP_EXT_OFFSET)) + err = -EOPNOTSUPP; + else + err = idtfc3_enable_tdc(idtfc3, true, CONTINUOUS); + } else { + err = idtfc3_enable_tdc(idtfc3, false, MEAS_MODE_INVALID); + } + break; + default: + break; + } + mutex_unlock(idtfc3->lock); + + if (err) + dev_err(idtfc3->dev, "Failed in %s with err %d!", __func__, err); + + return err; +} + +static long idtfc3_aux_work(struct ptp_clock_info *ptp) +{ + struct idtfc3 *idtfc3 = container_of(ptp, struct idtfc3, caps); + static int tdc_get; + + mutex_lock(idtfc3->lock); + tdc_get %= TDC_GET_PERIOD; + if ((tdc_get == 0) || (tdc_get == TDC_GET_PERIOD / 2)) + idtfc3_timecounter_read(idtfc3); + get_tdc_meas_continuous(idtfc3); + tdc_get++; + mutex_unlock(idtfc3->lock); + + return idtfc3->tc_update_period; +} + +static const struct ptp_clock_info idtfc3_caps = { + .owner = THIS_MODULE, + .max_adj = MAX_FFO_PPB, + .n_per_out = 1, + .n_ext_ts = 1, + .adjphase = &idtfc3_adjphase, + .adjfine = &idtfc3_adjfine, + .adjtime = &idtfc3_adjtime, + .gettime64 = &idtfc3_gettime, + .settime64 = &idtfc3_settime, + .enable = &idtfc3_enable, + .do_aux_work = &idtfc3_aux_work, +}; + +static int idtfc3_hw_calibrate(struct idtfc3 *idtfc3) +{ + int err = 0; + u8 val; + + mdelay(10); + /* + * Toggle TDC_DAC_RECAL_REQ: + * (1) set tdc_en to 1 + * (2) set tdc_dac_recal_req to 0 + * (3) set tdc_dac_recal_req to 1 + */ + val = TDC_EN; + err = regmap_bulk_write(idtfc3->regmap, TDC_CTRL, + &val, sizeof(val)); + if (err) + return err; + val = TDC_EN | TDC_DAC_RECAL_REQ; + err = regmap_bulk_write(idtfc3->regmap, TDC_CTRL, + &val, sizeof(val)); + if (err) + return err; + mdelay(10); + + /* + * Toggle APLL_REINIT: + * (1) set apll_reinit to 0 + * (2) set apll_reinit to 1 + */ + val = 0; + err = regmap_bulk_write(idtfc3->regmap, SOFT_RESET_CTRL, + &val, sizeof(val)); + if (err) + return err; + val = APLL_REINIT; + err = regmap_bulk_write(idtfc3->regmap, SOFT_RESET_CTRL, + &val, sizeof(val)); + if (err) + return err; + mdelay(10); + + return err; +} + +static int idtfc3_init_timecounter(struct idtfc3 *idtfc3) +{ + int err; + u32 period_ms; + + period_ms = idtfc3->sub_sync_count * MSEC_PER_SEC / + idtfc3->hw_param.time_clk_freq; + + idtfc3->tc_update_period = msecs_to_jiffies(period_ms / TDC_GET_PERIOD); + idtfc3->tc_write_timeout = period_ms * USEC_PER_MSEC; + + err = idtfc3_timecounter_update(idtfc3, 0, 0); + if (err) + return err; + + err = idtfc3_timecounter_read(idtfc3); + if (err) + return err; + + ptp_schedule_worker(idtfc3->ptp_clock, idtfc3->tc_update_period); + + return 0; +} + +static int idtfc3_get_tdc_apll_freq(struct idtfc3 *idtfc3) +{ + int err; + u8 tdc_fb_div_int; + u8 tdc_ref_div; + struct idtfc3_hw_param *param = &idtfc3->hw_param; + + err = regmap_bulk_read(idtfc3->regmap, TDC_REF_DIV_CNFG, + &tdc_ref_div, sizeof(tdc_ref_div)); + if (err) + return err; + + err = regmap_bulk_read(idtfc3->regmap, TDC_FB_DIV_INT_CNFG, + &tdc_fb_div_int, sizeof(tdc_fb_div_int)); + if (err) + return err; + + tdc_fb_div_int &= TDC_FB_DIV_INT_MASK; + tdc_ref_div &= TDC_REF_DIV_CONFIG_MASK; + + idtfc3->tdc_apll_freq = div_u64(param->xtal_freq * (u64)tdc_fb_div_int, + 1 << tdc_ref_div); + + return 0; +} + +static int idtfc3_get_fod(struct idtfc3 *idtfc3) +{ + int err; + u8 fod; + + err = regmap_bulk_read(idtfc3->regmap, TIME_CLOCK_SRC, &fod, sizeof(fod)); + if (err) + return err; + + switch (fod) { + case 0: + idtfc3->fod_n = FOD_0; + break; + case 1: + idtfc3->fod_n = FOD_1; + break; + case 2: + idtfc3->fod_n = FOD_2; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int idtfc3_get_sync_count(struct idtfc3 *idtfc3) +{ + int err; + u8 buf[4]; + + err = regmap_bulk_read(idtfc3->regmap, SUB_SYNC_GEN_CNFG, buf, sizeof(buf)); + if (err) + return err; + + idtfc3->sub_sync_count = (get_unaligned_le32(buf) & SUB_SYNC_COUNTER_MASK) + 1; + idtfc3->ns_per_counter = NSEC_PER_SEC / idtfc3->hw_param.time_clk_freq; + idtfc3->ns_per_sync = idtfc3->sub_sync_count * idtfc3->ns_per_counter; + + return 0; +} + +static int idtfc3_setup_hw_param(struct idtfc3 *idtfc3) +{ + int err; + + err = idtfc3_get_fod(idtfc3); + if (err) + return err; + + err = idtfc3_get_sync_count(idtfc3); + if (err) + return err; + + err = idtfc3_get_time_ref_freq(idtfc3); + if (err) + return err; + + return idtfc3_get_tdc_apll_freq(idtfc3); +} + +static int idtfc3_configure_hw(struct idtfc3 *idtfc3) +{ + int err = 0; + + err = idtfc3_hw_calibrate(idtfc3); + if (err) + return err; + + err = idtfc3_enable_lpf(idtfc3, true); + if (err) + return err; + + err = idtfc3_enable_tdc(idtfc3, false, MEAS_MODE_INVALID); + if (err) + return err; + + err = idtfc3_get_tdc_offset_sign(idtfc3); + if (err) + return err; + + return idtfc3_setup_hw_param(idtfc3); +} + +static int idtfc3_set_overhead(struct idtfc3 *idtfc3) +{ + s64 current_ns = 0; + s64 lowest_ns = 0; + int err; + u8 i; + ktime_t start; + ktime_t stop; + ktime_t diff; + + char buf[18] = {0}; + + for (i = 0; i < 5; i++) { + start = ktime_get_raw(); + + err = regmap_bulk_write(idtfc3->regmap, TOD_SYNC_LOAD_VAL_CTRL, + &buf, sizeof(buf)); + if (err) + return err; + + stop = ktime_get_raw(); + + diff = ktime_sub(stop, start); + + current_ns = ktime_to_ns(diff); + + if (i == 0) { + lowest_ns = current_ns; + } else { + if (current_ns < lowest_ns) + lowest_ns = current_ns; + } + } + + idtfc3->tod_write_overhead = lowest_ns; + + return err; +} + +static int idtfc3_enable_ptp(struct idtfc3 *idtfc3) +{ + int err; + + idtfc3->caps = idtfc3_caps; + snprintf(idtfc3->caps.name, sizeof(idtfc3->caps.name), "IDT FC3W"); + idtfc3->ptp_clock = ptp_clock_register(&idtfc3->caps, NULL); + + if (IS_ERR(idtfc3->ptp_clock)) { + err = PTR_ERR(idtfc3->ptp_clock); + idtfc3->ptp_clock = NULL; + return err; + } + + err = idtfc3_set_overhead(idtfc3); + if (err) + return err; + + err = idtfc3_init_timecounter(idtfc3); + if (err) + return err; + + dev_info(idtfc3->dev, "TIME_SYNC_CHANNEL registered as ptp%d", + idtfc3->ptp_clock->index); + + return 0; +} + +static int idtfc3_load_firmware(struct idtfc3 *idtfc3) +{ + char fname[128] = FW_FILENAME; + const struct firmware *fw; + struct idtfc3_fwrc *rec; + u16 addr; + u8 val; + int err; + s32 len; + + idtfc3_default_hw_param(&idtfc3->hw_param); + + if (firmware) /* module parameter */ + snprintf(fname, sizeof(fname), "%s", firmware); + + dev_info(idtfc3->dev, "requesting firmware '%s'\n", fname); + + err = request_firmware(&fw, fname, idtfc3->dev); + + if (err) { + dev_err(idtfc3->dev, + "requesting firmware failed with err %d!\n", err); + return err; + } + + dev_dbg(idtfc3->dev, "firmware size %zu bytes\n", fw->size); + + rec = (struct idtfc3_fwrc *)fw->data; + + for (len = fw->size; len > 0; len -= sizeof(*rec)) { + if (rec->reserved) { + dev_err(idtfc3->dev, + "bad firmware, reserved field non-zero\n"); + err = -EINVAL; + } else { + val = rec->value; + addr = rec->hiaddr << 8 | rec->loaddr; + + rec++; + + err = idtfc3_set_hw_param(&idtfc3->hw_param, addr, val); + } + + if (err != -EINVAL) { + err = 0; + + /* Max register */ + if (addr >= 0xE88) + continue; + + err = regmap_bulk_write(idtfc3->regmap, addr, + &val, sizeof(val)); + } + + if (err) + goto out; + } + + err = idtfc3_configure_hw(idtfc3); +out: + release_firmware(fw); + return err; +} + +static int idtfc3_read_device_id(struct idtfc3 *idtfc3, u16 *device_id) +{ + int err; + u8 buf[2] = {0}; + + err = regmap_bulk_read(idtfc3->regmap, DEVICE_ID, + &buf, sizeof(buf)); + if (err) { + dev_err(idtfc3->dev, "%s failed with %d", __func__, err); + return err; + } + + *device_id = get_unaligned_le16(buf); + + return 0; +} + +static int idtfc3_check_device_compatibility(struct idtfc3 *idtfc3) +{ + int err; + u16 device_id; + + err = idtfc3_read_device_id(idtfc3, &device_id); + if (err) + return err; + + if ((device_id & DEVICE_ID_MASK) == 0) { + dev_err(idtfc3->dev, "invalid device"); + return -EINVAL; + } + + return 0; +} + +static int idtfc3_probe(struct platform_device *pdev) +{ + struct rsmu_ddata *ddata = dev_get_drvdata(pdev->dev.parent); + struct idtfc3 *idtfc3; + int err; + + idtfc3 = devm_kzalloc(&pdev->dev, sizeof(struct idtfc3), GFP_KERNEL); + + if (!idtfc3) + return -ENOMEM; + + idtfc3->dev = &pdev->dev; + idtfc3->mfd = pdev->dev.parent; + idtfc3->lock = &ddata->lock; + idtfc3->regmap = ddata->regmap; + + mutex_lock(idtfc3->lock); + + err = idtfc3_check_device_compatibility(idtfc3); + if (err) { + mutex_unlock(idtfc3->lock); + return err; + } + + err = idtfc3_load_firmware(idtfc3); + if (err) { + if (err == -ENOENT) { + mutex_unlock(idtfc3->lock); + return -EPROBE_DEFER; + } + dev_warn(idtfc3->dev, "loading firmware failed with %d", err); + } + + err = idtfc3_enable_ptp(idtfc3); + if (err) { + dev_err(idtfc3->dev, "idtfc3_enable_ptp failed with %d", err); + mutex_unlock(idtfc3->lock); + return err; + } + + mutex_unlock(idtfc3->lock); + + if (err) { + ptp_clock_unregister(idtfc3->ptp_clock); + return err; + } + + platform_set_drvdata(pdev, idtfc3); + + return 0; +} + +static int idtfc3_remove(struct platform_device *pdev) +{ + struct idtfc3 *idtfc3 = platform_get_drvdata(pdev); + + ptp_clock_unregister(idtfc3->ptp_clock); + + return 0; +} + +static struct platform_driver idtfc3_driver = { + .driver = { + .name = "rc38xxx-phc", + }, + .probe = idtfc3_probe, + .remove = idtfc3_remove, +}; + +module_platform_driver(idtfc3_driver); diff --git a/drivers/ptp/ptp_fc3.h b/drivers/ptp/ptp_fc3.h new file mode 100644 index 000000000000..897101579207 --- /dev/null +++ b/drivers/ptp/ptp_fc3.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * PTP hardware clock driver for the FemtoClock3 family of timing and + * synchronization devices. + * + * Copyright (C) 2023 Integrated Device Technology, Inc., a Renesas Company. + */ +#ifndef PTP_IDTFC3_H +#define PTP_IDTFC3_H + +#include +#include +#include + +#define FW_FILENAME "idtfc3.bin" + +#define MAX_FFO_PPB (244000) +#define TDC_GET_PERIOD (10) + +struct idtfc3 { + struct ptp_clock_info caps; + struct ptp_clock *ptp_clock; + struct device *dev; + /* Mutex to protect operations from being interrupted */ + struct mutex *lock; + struct device *mfd; + struct regmap *regmap; + struct idtfc3_hw_param hw_param; + u32 sub_sync_count; + u32 ns_per_sync; + int tdc_offset_sign; + u64 tdc_apll_freq; + u32 time_ref_freq; + u16 fod_n; + u8 lpf_mode; + /* Time counter */ + u32 last_counter; + s64 ns; + u32 ns_per_counter; + u32 tc_update_period; + u32 tc_write_timeout; + s64 tod_write_overhead; +}; + +#endif /* PTP_IDTFC3_H */ diff --git a/include/linux/mfd/idtRC38xxx_reg.h b/include/linux/mfd/idtRC38xxx_reg.h new file mode 100644 index 000000000000..ec11872f51ad --- /dev/null +++ b/include/linux/mfd/idtRC38xxx_reg.h @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Register Map - Based on PolarBear_CSRs.RevA.xlsx (2023-04-21) + * + * Copyright (C) 2023 Integrated Device Technology, Inc., a Renesas Company. + */ +#ifndef MFD_IDTRC38XXX_REG +#define MFD_IDTRC38XXX_REG + +/* GLOBAL */ +#define SOFT_RESET_CTRL (0x15) /* Specific to FC3W */ +#define MISC_CTRL (0x14) /* Specific to FC3A */ +#define APLL_REINIT BIT(1) +#define APLL_REINIT_VFC3A BIT(2) + +#define DEVICE_ID (0x2) +#define DEVICE_ID_MASK (0x1000) /* Bit 12 is 1 if FC3W and 0 if FC3A */ +#define DEVICE_ID_SHIFT (12) + +/* FOD */ +#define FOD_0 (0x300) +#define FOD_0_VFC3A (0x400) +#define FOD_1 (0x340) +#define FOD_1_VFC3A (0x440) +#define FOD_2 (0x380) +#define FOD_2_VFC3A (0x480) + +/* TDCAPLL */ +#define TDC_CTRL (0x44a) /* Specific to FC3W */ +#define TDC_ENABLE_CTRL (0x169) /* Specific to FC3A */ +#define TDC_DAC_CAL_CTRL (0x16a) /* Specific to FC3A */ +#define TDC_EN BIT(0) +#define TDC_DAC_RECAL_REQ BIT(1) +#define TDC_DAC_RECAL_REQ_VFC3A BIT(0) + +#define TDC_FB_DIV_INT_CNFG (0x442) +#define TDC_FB_DIV_INT_CNFG_VFC3A (0x162) +#define TDC_FB_DIV_INT_MASK GENMASK(7, 0) +#define TDC_REF_DIV_CNFG (0x443) +#define TDC_REF_DIV_CNFG_VFC3A (0x163) +#define TDC_REF_DIV_CONFIG_MASK GENMASK(2, 0) + +/* TIME SYNC CHANNEL */ +#define TIME_CLOCK_SRC (0xa01) /* Specific to FC3W */ +#define TIME_CLOCK_COUNT (0xa00) /* Specific to FC3W */ +#define TIME_CLOCK_COUNT_MASK GENMASK(5, 0) + +#define SUB_SYNC_GEN_CNFG (0xa04) + +#define TOD_COUNTER_READ_REQ (0xa5f) +#define TOD_COUNTER_READ_REQ_VFC3A (0x6df) +#define TOD_SYNC_LOAD_VAL_CTRL (0xa10) +#define TOD_SYNC_LOAD_VAL_CTRL_VFC3A (0x690) +#define SYNC_COUNTER_MASK GENMASK_ULL(51, 0) +#define SUB_SYNC_COUNTER_MASK GENMASK(30, 0) +#define TOD_SYNC_LOAD_REQ_CTRL (0xa21) +#define TOD_SYNC_LOAD_REQ_CTRL_VFC3A (0x6a1) +#define SYNC_LOAD_ENABLE BIT(1) +#define SUB_SYNC_LOAD_ENABLE BIT(0) +#define SYNC_LOAD_REQ BIT(0) + +#define LPF_MODE_CNFG (0xa80) +#define LPF_MODE_CNFG_VFC3A (0x700) +enum lpf_mode { + LPF_DISABLED = 0, + LPF_WP = 1, + LPF_HOLDOVER = 2, + LPF_WF = 3, + LPF_INVALID = 4 +}; +#define LPF_CTRL (0xa98) +#define LPF_CTRL_VFC3A (0x718) +#define LPF_EN BIT(0) + +#define LPF_BW_CNFG (0xa81) +#define LPF_BW_SHIFT GENMASK(7, 3) +#define LPF_BW_MULT GENMASK(2, 0) +#define LPF_BW_SHIFT_DEFAULT (0xb) +#define LPF_BW_MULT_DEFAULT (0x0) +#define LPF_BW_SHIFT_1PPS (0x5) + +#define LPF_WR_PHASE_CTRL (0xaa8) +#define LPF_WR_PHASE_CTRL_VFC3A (0x728) +#define LPF_WR_FREQ_CTRL (0xab0) +#define LPF_WR_FREQ_CTRL_VFC3A (0x730) + +#define TIME_CLOCK_TDC_FANOUT_CNFG (0xB00) +#define TIME_SYNC_TO_TDC_EN BIT(0) +#define SIG1_MUX_SEL_MASK GENMASK(7, 4) +#define SIG2_MUX_SEL_MASK GENMASK(11, 8) +enum tdc_mux_sel { + REF0 = 0, + REF1 = 1, + REF2 = 2, + REF3 = 3, + REF_CLK5 = 4, + REF_CLK6 = 5, + DPLL_FB_TO_TDC = 6, + DPLL_FB_DIVIDED_TO_TDC = 7, + TIME_CLK_DIVIDED = 8, + TIME_SYNC = 9, +}; + +#define TIME_CLOCK_MEAS_CNFG (0xB04) +#define TDC_MEAS_MODE BIT(0) +enum tdc_meas_mode { + CONTINUOUS = 0, + ONE_SHOT = 1, + MEAS_MODE_INVALID = 2, +}; + +#define TIME_CLOCK_MEAS_DIV_CNFG (0xB08) +#define TIME_REF_DIV_MASK GENMASK(29, 24) + +#define TIME_CLOCK_MEAS_CTRL (0xB10) +#define TDC_MEAS_EN BIT(0) +#define TDC_MEAS_START BIT(1) + +#define TDC_FIFO_READ_REQ (0xB2F) +#define TDC_FIFO_READ (0xB30) +#define COARSE_MEAS_MASK GENMASK_ULL(39, 13) +#define FINE_MEAS_MASK GENMASK(12, 0) + +#define TDC_FIFO_CTRL (0xB12) +#define FIFO_CLEAR BIT(0) +#define TDC_FIFO_STS (0xB38) +#define FIFO_FULL BIT(1) +#define FIFO_EMPTY BIT(0) +#define TDC_FIFO_EVENT (0xB39) +#define FIFO_OVERRUN BIT(1) + +/* DPLL */ +#define MAX_REFERENCE_INDEX (3) +#define MAX_NUM_REF_PRIORITY (4) + +#define MAX_DPLL_INDEX (2) + +#define DPLL_STS (0x580) +#define DPLL_STS_VFC3A (0x571) +#define DPLL_STATE_STS_MASK (0x70) +#define DPLL_STATE_STS_SHIFT (4) +#define DPLL_REF_SEL_STS_MASK (0x6) +#define DPLL_REF_SEL_STS_SHIFT (1) + +#define DPLL_REF_PRIORITY_CNFG (0x502) +#define DPLL_REFX_PRIORITY_DISABLE_MASK (0xf) +#define DPLL_REF0_PRIORITY_ENABLE_AND_SET_MASK (0x31) +#define DPLL_REF1_PRIORITY_ENABLE_AND_SET_MASK (0xc2) +#define DPLL_REF2_PRIORITY_ENABLE_AND_SET_MASK (0x304) +#define DPLL_REF3_PRIORITY_ENABLE_AND_SET_MASK (0xc08) +#define DPLL_REF0_PRIORITY_SHIFT (4) +#define DPLL_REF1_PRIORITY_SHIFT (6) +#define DPLL_REF2_PRIORITY_SHIFT (8) +#define DPLL_REF3_PRIORITY_SHIFT (10) + +enum dpll_state { + DPLL_STATE_MIN = 0, + DPLL_STATE_FREERUN = DPLL_STATE_MIN, + DPLL_STATE_LOCKED = 1, + DPLL_STATE_HOLDOVER = 2, + DPLL_STATE_WRITE_FREQUENCY = 3, + DPLL_STATE_ACQUIRE = 4, + DPLL_STATE_HITLESS_SWITCH = 5, + DPLL_STATE_MAX = DPLL_STATE_HITLESS_SWITCH +}; + +/* REFMON */ +#define LOSMON_STS_0 (0x81e) +#define LOSMON_STS_0_VFC3A (0x18e) +#define LOSMON_STS_1 (0x82e) +#define LOSMON_STS_1_VFC3A (0x19e) +#define LOSMON_STS_2 (0x83e) +#define LOSMON_STS_2_VFC3A (0x1ae) +#define LOSMON_STS_3 (0x84e) +#define LOSMON_STS_3_VFC3A (0x1be) +#define LOS_STS_MASK (0x1) + +#define FREQMON_STS_0 (0x874) +#define FREQMON_STS_0_VFC3A (0x1d4) +#define FREQMON_STS_1 (0x894) +#define FREQMON_STS_1_VFC3A (0x1f4) +#define FREQMON_STS_2 (0x8b4) +#define FREQMON_STS_2_VFC3A (0x214) +#define FREQMON_STS_3 (0x8d4) +#define FREQMON_STS_3_VFC3A (0x234) +#define FREQ_FAIL_STS_SHIFT (31) + +/* Firmware interface */ +#define TIME_CLK_FREQ_ADDR (0xffa0) +#define XTAL_FREQ_ADDR (0xffa1) + +/* + * Return register address and field mask based on passed in firmware version + */ +#define IDTFC3_FW_REG(FW, VER, REG) (((FW) < (VER)) ? (REG) : (REG##_##VER)) +#define IDTFC3_FW_FIELD(FW, VER, FIELD) (((FW) < (VER)) ? (FIELD) : (FIELD##_##VER)) +enum fw_version { + V_DEFAULT = 0, + VFC3W = 1, + VFC3A = 2 +}; + +/* XTAL_FREQ_ADDR/TIME_CLK_FREQ_ADDR */ +enum { + FREQ_MIN = 0, + FREQ_25M = 1, + FREQ_49_152M = 2, + FREQ_50M = 3, + FREQ_100M = 4, + FREQ_125M = 5, + FREQ_250M = 6, + FREQ_MAX +}; + +struct idtfc3_hw_param { + u32 xtal_freq; + u32 time_clk_freq; +}; + +struct idtfc3_fwrc { + u8 hiaddr; + u8 loaddr; + u8 value; + u8 reserved; +} __packed; + +static inline void idtfc3_default_hw_param(struct idtfc3_hw_param *hw_param) +{ + hw_param->xtal_freq = 49152000; + hw_param->time_clk_freq = 25000000; +} + +static inline int idtfc3_set_hw_param(struct idtfc3_hw_param *hw_param, + u16 addr, u8 val) +{ + if (addr == XTAL_FREQ_ADDR) + switch (val) { + case FREQ_49_152M: + hw_param->xtal_freq = 49152000; + break; + case FREQ_50M: + hw_param->xtal_freq = 50000000; + break; + default: + return -EINVAL; + } + else if (addr == TIME_CLK_FREQ_ADDR) + switch (val) { + case FREQ_25M: + hw_param->time_clk_freq = 25000000; + break; + case FREQ_50M: + hw_param->time_clk_freq = 50000000; + break; + case FREQ_100M: + hw_param->time_clk_freq = 100000000; + break; + case FREQ_125M: + hw_param->time_clk_freq = 125000000; + break; + case FREQ_250M: + hw_param->time_clk_freq = 250000000; + break; + default: + return -EINVAL; + } + else + return -EFAULT; + + return 0; +} + +#endif -- cgit v1.2.3 From efaa47db92451608499ab7edf108bf30141c33db Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Sun, 28 Jan 2024 13:54:43 +0800 Subject: bpf: Remove unused field "mod" in struct bpf_trampoline It seems that the field "mod" in struct bpf_trampoline is not used anywhere after the commit 31bf1dbccfb0 ("bpf: Fix attaching fentry/fexit/fmod_ret/lsm to modules"). So we can just remove it now. Fixes: 31bf1dbccfb0 ("bpf: Fix attaching fentry/fexit/fmod_ret/lsm to modules") Signed-off-by: Menglong Dong Signed-off-by: Daniel Borkmann Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20240128055443.413291-1-dongmenglong.8@bytedance.com --- include/linux/bpf.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b86bd15a051d..1ebbee1d648e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1189,7 +1189,6 @@ struct bpf_trampoline { int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; - struct module *mod; }; struct bpf_attach_target_info { -- cgit v1.2.3 From ed0ef85795b58134172e8c82ab2f1b869cd501a6 Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Mon, 29 Jan 2024 11:21:35 +0530 Subject: ASoC/soundwire: implement generic api for scanning amd soundwire controller Implement generic function for scanning SoundWire controller. Same function will be used for legacy and sof stack for AMD platforms. Signed-off-by: Vijendar Mukunda Acked-by: Vinod Koul Link: https://msgid.link/r/20240129055147.1493853-2-Vijendar.Mukunda@amd.com Signed-off-by: Mark Brown --- include/linux/soundwire/sdw_amd.h | 15 ++++++++++ sound/soc/amd/acp/Kconfig | 7 +++++ sound/soc/amd/acp/Makefile | 2 ++ sound/soc/amd/acp/amd-sdw-acpi.c | 62 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 sound/soc/amd/acp/amd-sdw-acpi.c (limited to 'include/linux') diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index ceecad74aef9..41dd64941cef 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -6,6 +6,7 @@ #ifndef __SDW_AMD_H #define __SDW_AMD_H +#include #include /* AMD pm_runtime quirk definitions */ @@ -106,4 +107,18 @@ struct amd_sdw_manager { struct sdw_amd_dai_runtime **dai_runtime_array; }; + +/** + * struct sdw_amd_acpi_info - Soundwire AMD information found in ACPI tables + * @handle: ACPI controller handle + * @count: maximum no of soundwire manager links supported on AMD platform. + * @link_mask: bit-wise mask listing links enabled by BIOS menu + */ +struct sdw_amd_acpi_info { + acpi_handle handle; + int count; + u32 link_mask; +}; + +int amd_sdw_scan_controller(struct sdw_amd_acpi_info *info); #endif diff --git a/sound/soc/amd/acp/Kconfig b/sound/soc/amd/acp/Kconfig index 84c963241dc5..b3105ba9c3a3 100644 --- a/sound/soc/amd/acp/Kconfig +++ b/sound/soc/amd/acp/Kconfig @@ -101,6 +101,13 @@ config SND_SOC_AMD_MACH_COMMON help This option enables common Machine driver module for ACP. +config SND_AMD_SOUNDWIRE_ACPI + tristate "AMD SoundWire ACPI Support" + depends on ACPI + help + This options enables ACPI helper functions for SoundWire + interface for AMD platforms. + config SND_SOC_AMD_LEGACY_MACH tristate "AMD Legacy Machine Driver Support" depends on X86 && PCI && I2C diff --git a/sound/soc/amd/acp/Makefile b/sound/soc/amd/acp/Makefile index ff5f7893b81e..1fd581a2aa33 100644 --- a/sound/soc/amd/acp/Makefile +++ b/sound/soc/amd/acp/Makefile @@ -10,6 +10,7 @@ snd-acp-i2s-objs := acp-i2s.o snd-acp-pdm-objs := acp-pdm.o snd-acp-legacy-common-objs := acp-legacy-common.o snd-acp-pci-objs := acp-pci.o +snd-amd-sdw-acpi-objs := amd-sdw-acpi.o #platform specific driver snd-acp-renoir-objs := acp-renoir.o @@ -33,6 +34,7 @@ obj-$(CONFIG_SND_AMD_ASOC_REMBRANDT) += snd-acp-rembrandt.o obj-$(CONFIG_SND_AMD_ASOC_ACP63) += snd-acp63.o obj-$(CONFIG_SND_AMD_ASOC_ACP70) += snd-acp70.o +obj-$(CONFIG_SND_AMD_SOUNDWIRE_ACPI) += snd-amd-sdw-acpi.o obj-$(CONFIG_SND_SOC_AMD_MACH_COMMON) += snd-acp-mach.o obj-$(CONFIG_SND_SOC_AMD_LEGACY_MACH) += snd-acp-legacy-mach.o obj-$(CONFIG_SND_SOC_AMD_SOF_MACH) += snd-acp-sof-mach.o diff --git a/sound/soc/amd/acp/amd-sdw-acpi.c b/sound/soc/amd/acp/amd-sdw-acpi.c new file mode 100644 index 000000000000..babd841d3296 --- /dev/null +++ b/sound/soc/amd/acp/amd-sdw-acpi.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Authors: Vijendar Mukunda + +/* + * SDW AMD ACPI scan helper function + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int amd_sdw_scan_controller(struct sdw_amd_acpi_info *info) +{ + struct acpi_device *adev = acpi_fetch_acpi_dev(info->handle); + u32 sdw_bitmap = 0; + u8 count = 0; + int ret; + + if (!adev) + return -EINVAL; + + /* Found controller, find links supported */ + ret = fwnode_property_read_u32_array(acpi_fwnode_handle(adev), + "mipi-sdw-manager-list", &sdw_bitmap, 1); + if (ret) { + dev_err(&adev->dev, + "Failed to read mipi-sdw-manager-list: %d\n", ret); + return -EINVAL; + } + count = hweight32(sdw_bitmap); + /* Check count is within bounds */ + if (count > info->count) { + dev_err(&adev->dev, "Manager count %d exceeds max %d\n", + count, info->count); + return -EINVAL; + } + + if (!count) { + dev_dbg(&adev->dev, "No SoundWire Managers detected\n"); + return -EINVAL; + } + dev_dbg(&adev->dev, "ACPI reports %d SoundWire Manager devices\n", count); + info->link_mask = sdw_bitmap; + return 0; +} +EXPORT_SYMBOL_NS(amd_sdw_scan_controller, SND_AMD_SOUNDWIRE_ACPI); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("AMD SoundWire ACPI helpers"); -- cgit v1.2.3 From a47746428cf5762290d0c55f6ef82067af04d165 Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Mon, 29 Jan 2024 11:21:36 +0530 Subject: soundwire: amd: update license Update license to dual license to align with Sound Open Firmware (SOF) driver as SOF uses dual license. Signed-off-by: Vijendar Mukunda Acked-by: Vinod Koul Link: https://msgid.link/r/20240129055147.1493853-3-Vijendar.Mukunda@amd.com Signed-off-by: Mark Brown --- drivers/soundwire/amd_manager.c | 4 ++-- drivers/soundwire/amd_manager.h | 2 +- include/linux/soundwire/sdw_amd.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index f54bb4dd2d10..f2c66b165be8 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * SoundWire AMD Manager driver * @@ -1215,5 +1215,5 @@ module_platform_driver(amd_sdw_driver); MODULE_AUTHOR("Vijendar.Mukunda@amd.com"); MODULE_DESCRIPTION("AMD SoundWire driver"); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("Dual BSD/GPL"); MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h index 5f040151a259..f57165bbb9d9 100644 --- a/drivers/soundwire/amd_manager.h +++ b/drivers/soundwire/amd_manager.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ /* * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. */ diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index 41dd64941cef..56b4117c087a 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ /* * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. */ -- cgit v1.2.3 From ed5e8741b8db908d51a26e368c18573ee1b9e208 Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Mon, 29 Jan 2024 11:21:37 +0530 Subject: soundwire: amd: refactor amd soundwire manager device node creation Refactor amd SoundWire manager device node creation logic and implement generic functions to have a common functionality for SoundWire manager platform device creation, start and exit sequence for both legacy(NO DSP) and SOF stack for AMD platforms. These functions will be invoked from legacy and SOF stack. Signed-off-by: Vijendar Mukunda Acked-by: Vinod Koul Link: https://msgid.link/r/20240129055147.1493853-4-Vijendar.Mukunda@amd.com Signed-off-by: Mark Brown --- drivers/soundwire/Makefile | 2 +- drivers/soundwire/amd_init.c | 147 ++++++++++++++++++++++++++++++++++++++ drivers/soundwire/amd_init.h | 13 ++++ drivers/soundwire/amd_manager.c | 18 ++--- include/linux/soundwire/sdw_amd.h | 56 ++++++++++++++- 5 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 drivers/soundwire/amd_init.c create mode 100644 drivers/soundwire/amd_init.h (limited to 'include/linux') diff --git a/drivers/soundwire/Makefile b/drivers/soundwire/Makefile index 657f5888a77b..e80a2c2cf3e7 100644 --- a/drivers/soundwire/Makefile +++ b/drivers/soundwire/Makefile @@ -20,7 +20,7 @@ soundwire-bus-y += irq.o endif #AMD driver -soundwire-amd-y := amd_manager.o +soundwire-amd-y := amd_init.o amd_manager.o obj-$(CONFIG_SOUNDWIRE_AMD) += soundwire-amd.o #Cadence Objs diff --git a/drivers/soundwire/amd_init.c b/drivers/soundwire/amd_init.c new file mode 100644 index 000000000000..699391d9acba --- /dev/null +++ b/drivers/soundwire/amd_init.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * SoundWire AMD Manager Initialize routines + * + * Initializes and creates SDW devices based on ACPI and Hardware values + * + * Copyright 2024 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include + +#include "amd_init.h" + +static int sdw_amd_cleanup(struct sdw_amd_ctx *ctx) +{ + int i; + + for (i = 0; i < ctx->count; i++) { + if (!(ctx->link_mask & BIT(i))) + continue; + platform_device_unregister(ctx->pdev[i]); + } + + return 0; +} + +static struct sdw_amd_ctx *sdw_amd_probe_controller(struct sdw_amd_res *res) +{ + struct sdw_amd_ctx *ctx; + struct acpi_device *adev; + struct resource *sdw_res; + struct acp_sdw_pdata sdw_pdata[2]; + struct platform_device_info pdevinfo[2]; + u32 link_mask; + int count, index; + + if (!res) + return NULL; + + adev = acpi_fetch_acpi_dev(res->handle); + if (!adev) + return NULL; + + if (!res->count) + return NULL; + + count = res->count; + dev_dbg(&adev->dev, "Creating %d SDW Link devices\n", count); + + /* + * we need to alloc/free memory manually and can't use devm: + * this routine may be called from a workqueue, and not from + * the parent .probe. + * If devm_ was used, the memory might never be freed on errors. + */ + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->count = count; + ctx->link_mask = res->link_mask; + sdw_res = kzalloc(sizeof(*sdw_res), GFP_KERNEL); + if (!sdw_res) { + kfree(ctx); + return NULL; + } + sdw_res->flags = IORESOURCE_MEM; + sdw_res->start = res->addr; + sdw_res->end = res->addr + res->reg_range; + memset(&pdevinfo, 0, sizeof(pdevinfo)); + link_mask = ctx->link_mask; + for (index = 0; index < count; index++) { + if (!(link_mask & BIT(index))) + continue; + + sdw_pdata[index].instance = index; + sdw_pdata[index].acp_sdw_lock = res->acp_lock; + pdevinfo[index].name = "amd_sdw_manager"; + pdevinfo[index].id = index; + pdevinfo[index].parent = res->parent; + pdevinfo[index].num_res = 1; + pdevinfo[index].res = sdw_res; + pdevinfo[index].data = &sdw_pdata[index]; + pdevinfo[index].size_data = sizeof(struct acp_sdw_pdata); + pdevinfo[index].fwnode = acpi_fwnode_handle(adev); + ctx->pdev[index] = platform_device_register_full(&pdevinfo[index]); + if (IS_ERR(ctx->pdev[index])) + goto err; + } + kfree(sdw_res); + return ctx; +err: + while (index--) { + if (!(link_mask & BIT(index))) + continue; + + platform_device_unregister(ctx->pdev[index]); + } + + kfree(sdw_res); + kfree(ctx); + return NULL; +} + +static int sdw_amd_startup(struct sdw_amd_ctx *ctx) +{ + struct amd_sdw_manager *amd_manager; + int i, ret; + + /* Startup SDW Manager devices */ + for (i = 0; i < ctx->count; i++) { + if (!(ctx->link_mask & BIT(i))) + continue; + amd_manager = dev_get_drvdata(&ctx->pdev[i]->dev); + ret = amd_sdw_manager_start(amd_manager); + if (ret) + return ret; + } + + return 0; +} + +int sdw_amd_probe(struct sdw_amd_res *res, struct sdw_amd_ctx **sdw_ctx) +{ + *sdw_ctx = sdw_amd_probe_controller(res); + if (!*sdw_ctx) + return -ENODEV; + + return sdw_amd_startup(*sdw_ctx); +} +EXPORT_SYMBOL_NS(sdw_amd_probe, SOUNDWIRE_AMD_INIT); + +void sdw_amd_exit(struct sdw_amd_ctx *ctx) +{ + sdw_amd_cleanup(ctx); + kfree(ctx->ids); + kfree(ctx); +} +EXPORT_SYMBOL_NS(sdw_amd_exit, SOUNDWIRE_AMD_INIT); + +MODULE_AUTHOR("Vijendar.Mukunda@amd.com"); +MODULE_DESCRIPTION("AMD SoundWire Init Library"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/soundwire/amd_init.h b/drivers/soundwire/amd_init.h new file mode 100644 index 000000000000..928b0c707162 --- /dev/null +++ b/drivers/soundwire/amd_init.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ +/* + * Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef __AMD_INIT_H +#define __AMD_INIT_H + +#include + +int amd_sdw_manager_start(struct amd_sdw_manager *amd_manager); + +#endif diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index f2c66b165be8..efc000334aa0 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -2,7 +2,7 @@ /* * SoundWire AMD Manager driver * - * Copyright 2023 Advanced Micro Devices, Inc. + * Copyright 2023-24 Advanced Micro Devices, Inc. */ #include @@ -19,6 +19,7 @@ #include #include #include "bus.h" +#include "amd_init.h" #include "amd_manager.h" #define DRV_NAME "amd_sdw_manager" @@ -864,10 +865,8 @@ static void amd_sdw_irq_thread(struct work_struct *work) writel(0x00, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_0TO7); } -static void amd_sdw_probe_work(struct work_struct *work) +int amd_sdw_manager_start(struct amd_sdw_manager *amd_manager) { - struct amd_sdw_manager *amd_manager = container_of(work, struct amd_sdw_manager, - probe_work); struct sdw_master_prop *prop; int ret; @@ -876,11 +875,11 @@ static void amd_sdw_probe_work(struct work_struct *work) amd_enable_sdw_pads(amd_manager); ret = amd_init_sdw_manager(amd_manager); if (ret) - return; + return ret; amd_enable_sdw_interrupts(amd_manager); ret = amd_enable_sdw_manager(amd_manager); if (ret) - return; + return ret; amd_sdw_set_frameshape(amd_manager); } /* Enable runtime PM */ @@ -889,6 +888,7 @@ static void amd_sdw_probe_work(struct work_struct *work) pm_runtime_mark_last_busy(amd_manager->dev); pm_runtime_set_active(amd_manager->dev); pm_runtime_enable(amd_manager->dev); + return 0; } static int amd_sdw_manager_probe(struct platform_device *pdev) @@ -972,11 +972,6 @@ static int amd_sdw_manager_probe(struct platform_device *pdev) dev_set_drvdata(dev, amd_manager); INIT_WORK(&amd_manager->amd_sdw_irq_thread, amd_sdw_irq_thread); INIT_WORK(&amd_manager->amd_sdw_work, amd_sdw_update_slave_status_work); - INIT_WORK(&amd_manager->probe_work, amd_sdw_probe_work); - /* - * Instead of having lengthy probe sequence, use deferred probe. - */ - schedule_work(&amd_manager->probe_work); return 0; } @@ -986,7 +981,6 @@ static void amd_sdw_manager_remove(struct platform_device *pdev) int ret; pm_runtime_disable(&pdev->dev); - cancel_work_sync(&amd_manager->probe_work); amd_disable_sdw_interrupts(amd_manager); sdw_bus_master_delete(&amd_manager->bus); ret = amd_disable_sdw_manager(amd_manager); diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index 56b4117c087a..54735fa49759 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ /* - * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2023-24 Advanced Micro Devices, Inc. All rights reserved. */ #ifndef __SDW_AMD_H @@ -26,6 +26,7 @@ #define AMD_SDW_POWER_OFF_MODE 2 #define ACP_SDW0 0 #define ACP_SDW1 1 +#define AMD_SDW_MAX_MANAGER_COUNT 2 struct acp_sdw_pdata { u16 instance; @@ -63,7 +64,6 @@ struct sdw_amd_dai_runtime { * @reg_mask: register mask structure per manager instance * @amd_sdw_irq_thread: SoundWire manager irq workqueue * @amd_sdw_work: peripheral status work queue - * @probe_work: SoundWire manager probe workqueue * @acp_sdw_lock: mutex to protect acp share register access * @status: peripheral devices status array * @num_din_ports: number of input ports @@ -87,7 +87,6 @@ struct amd_sdw_manager { struct sdw_manager_reg_mask *reg_mask; struct work_struct amd_sdw_irq_thread; struct work_struct amd_sdw_work; - struct work_struct probe_work; /* mutex to protect acp common register access */ struct mutex *acp_sdw_lock; @@ -120,5 +119,56 @@ struct sdw_amd_acpi_info { u32 link_mask; }; +/** + * struct sdw_amd_ctx - context allocated by the controller driver probe + * + * @count: link count + * @num_slaves: total number of devices exposed across all enabled links + * @link_mask: bit-wise mask listing SoundWire links reported by the + * Controller + * @ids: array of slave_id, representing Slaves exposed across all enabled + * links + * @pdev: platform device structure + */ +struct sdw_amd_ctx { + int count; + int num_slaves; + u32 link_mask; + struct sdw_extended_slave_id *ids; + struct platform_device *pdev[AMD_SDW_MAX_MANAGER_COUNT]; +}; + +/** + * struct sdw_amd_res - Soundwire AMD global resource structure, + * typically populated by the DSP driver/Legacy driver + * + * @addr: acp pci device resource start address + * @reg_range: ACP register range + * @link_mask: bit-wise mask listing links selected by the DSP driver/ + * legacy driver + * @count: link count + * @mmio_base: mmio base of SoundWire registers + * @handle: ACPI parent handle + * @parent: parent device + * @dev: device implementing hwparams and free callbacks + * @acp_lock: mutex protecting acp common registers access + */ +struct sdw_amd_res { + u32 addr; + u32 reg_range; + u32 link_mask; + int count; + void __iomem *mmio_base; + acpi_handle handle; + struct device *parent; + struct device *dev; + /* use to protect acp common registers access */ + struct mutex *acp_lock; +}; + +int sdw_amd_probe(struct sdw_amd_res *res, struct sdw_amd_ctx **ctx); + +void sdw_amd_exit(struct sdw_amd_ctx *ctx); + int amd_sdw_scan_controller(struct sdw_amd_acpi_info *info); #endif -- cgit v1.2.3 From aff9d088a306541117e420d96ed6b6f1215a7e2d Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Mon, 29 Jan 2024 11:21:38 +0530 Subject: soundwire: amd: implement function to extract slave information Implement function to extract slaves information connected on the bus. This information is required during machine select logic. This function will be called from machine select logic code. Signed-off-by: Vijendar Mukunda Acked-by: Vinod Koul Link: https://msgid.link/r/20240129055147.1493853-5-Vijendar.Mukunda@amd.com Signed-off-by: Mark Brown --- drivers/soundwire/amd_init.c | 43 +++++++++++++++++++++++++++++++++++++++ include/linux/soundwire/sdw_amd.h | 2 ++ 2 files changed, 45 insertions(+) (limited to 'include/linux') diff --git a/drivers/soundwire/amd_init.c b/drivers/soundwire/amd_init.c index 699391d9acba..46a0538d7fc7 100644 --- a/drivers/soundwire/amd_init.c +++ b/drivers/soundwire/amd_init.c @@ -142,6 +142,49 @@ void sdw_amd_exit(struct sdw_amd_ctx *ctx) } EXPORT_SYMBOL_NS(sdw_amd_exit, SOUNDWIRE_AMD_INIT); +int sdw_amd_get_slave_info(struct sdw_amd_ctx *ctx) +{ + struct amd_sdw_manager *amd_manager; + struct sdw_bus *bus; + struct sdw_slave *slave; + struct list_head *node; + int index; + int i = 0; + int num_slaves = 0; + + for (index = 0; index < ctx->count; index++) { + if (!(ctx->link_mask & BIT(index))) + continue; + amd_manager = dev_get_drvdata(&ctx->pdev[index]->dev); + if (!amd_manager) + return -ENODEV; + bus = &amd_manager->bus; + /* Calculate number of slaves */ + list_for_each(node, &bus->slaves) + num_slaves++; + } + + ctx->ids = kcalloc(num_slaves, sizeof(*ctx->ids), GFP_KERNEL); + if (!ctx->ids) + return -ENOMEM; + ctx->num_slaves = num_slaves; + for (index = 0; index < ctx->count; index++) { + if (!(ctx->link_mask & BIT(index))) + continue; + amd_manager = dev_get_drvdata(&ctx->pdev[index]->dev); + if (amd_manager) { + bus = &amd_manager->bus; + list_for_each_entry(slave, &bus->slaves, node) { + ctx->ids[i].id = slave->id; + ctx->ids[i].link_id = bus->link_id; + i++; + } + } + } + return 0; +} +EXPORT_SYMBOL_NS(sdw_amd_get_slave_info, SOUNDWIRE_AMD_INIT); + MODULE_AUTHOR("Vijendar.Mukunda@amd.com"); MODULE_DESCRIPTION("AMD SoundWire Init Library"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index 54735fa49759..9103772c2497 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -170,5 +170,7 @@ int sdw_amd_probe(struct sdw_amd_res *res, struct sdw_amd_ctx **ctx); void sdw_amd_exit(struct sdw_amd_ctx *ctx); +int sdw_amd_get_slave_info(struct sdw_amd_ctx *ctx); + int amd_sdw_scan_controller(struct sdw_amd_acpi_info *info); #endif -- cgit v1.2.3 From c1263c75294cc8178ca964e0220b35518d6fb38d Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Mon, 29 Jan 2024 11:21:40 +0530 Subject: soundwire: amd: refactor register mask structure Register mask array structure is no longer needed as except interrupt control masks, rest of the register masks are not used in code. Use array for interrupt masks instead of structure. Signed-off-by: Vijendar Mukunda Acked-by: Vinod Koul Link: https://msgid.link/r/20240129055147.1493853-7-Vijendar.Mukunda@amd.com Signed-off-by: Mark Brown --- drivers/soundwire/amd_manager.c | 7 ++----- drivers/soundwire/amd_manager.h | 14 +++----------- include/linux/soundwire/sdw_amd.h | 8 -------- 3 files changed, 5 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index 44d682a83e23..7cd24bd8e224 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -86,12 +86,11 @@ static int amd_disable_sdw_manager(struct amd_sdw_manager *amd_manager) static void amd_enable_sdw_interrupts(struct amd_sdw_manager *amd_manager) { - struct sdw_manager_reg_mask *reg_mask = amd_manager->reg_mask; u32 val; mutex_lock(amd_manager->acp_sdw_lock); val = readl(amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); - val |= reg_mask->acp_sdw_intr_mask; + val |= sdw_manager_reg_mask_array[amd_manager->instance]; writel(val, amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); mutex_unlock(amd_manager->acp_sdw_lock); @@ -104,12 +103,11 @@ static void amd_enable_sdw_interrupts(struct amd_sdw_manager *amd_manager) static void amd_disable_sdw_interrupts(struct amd_sdw_manager *amd_manager) { - struct sdw_manager_reg_mask *reg_mask = amd_manager->reg_mask; u32 val; mutex_lock(amd_manager->acp_sdw_lock); val = readl(amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); - val &= ~reg_mask->acp_sdw_intr_mask; + val &= ~sdw_manager_reg_mask_array[amd_manager->instance]; writel(val, amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); mutex_unlock(amd_manager->acp_sdw_lock); @@ -930,7 +928,6 @@ static int amd_sdw_manager_probe(struct platform_device *pdev) return -EINVAL; } - amd_manager->reg_mask = &sdw_manager_reg_mask_array[amd_manager->instance]; params = &amd_manager->bus.params; params->col = AMD_SDW_DEFAULT_COLUMNS; diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h index f57165bbb9d9..418b679e0b1a 100644 --- a/drivers/soundwire/amd_manager.h +++ b/drivers/soundwire/amd_manager.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ /* - * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2023-24 Advanced Micro Devices, Inc. All rights reserved. */ #ifndef __AMD_MANAGER_H @@ -243,16 +243,8 @@ static struct sdw_manager_dp_reg sdw1_manager_dp_reg[AMD_SDW1_MAX_DAI] = { ACP_SW_AUDIO1_RX_OFFSET, ACP_SW_AUDIO1_RX_CHANNEL_ENABLE_DP0} }; -static struct sdw_manager_reg_mask sdw_manager_reg_mask_array[2] = { - { - AMD_SDW0_PAD_KEEPER_EN_MASK, - AMD_SDW0_PAD_PULLDOWN_CTRL_ENABLE_MASK, - AMD_SDW0_EXT_INTR_MASK - }, - { - AMD_SDW1_PAD_KEEPER_EN_MASK, - AMD_SDW1_PAD_PULLDOWN_CTRL_ENABLE_MASK, +static u32 sdw_manager_reg_mask_array[AMD_SDW_MAX_MANAGER_COUNT] = { + AMD_SDW0_EXT_INTR_MASK, AMD_SDW1_EXT_INTR_MASK - } }; #endif diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index 9103772c2497..28a4eb77717f 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -34,12 +34,6 @@ struct acp_sdw_pdata { struct mutex *acp_sdw_lock; }; -struct sdw_manager_reg_mask { - u32 sw_pad_enable_mask; - u32 sw_pad_pulldown_mask; - u32 acp_sdw_intr_mask; -}; - /** * struct sdw_amd_dai_runtime: AMD sdw dai runtime data * @@ -61,7 +55,6 @@ struct sdw_amd_dai_runtime { * @dev: linux device * @mmio: SoundWire registers mmio base * @acp_mmio: acp registers mmio base - * @reg_mask: register mask structure per manager instance * @amd_sdw_irq_thread: SoundWire manager irq workqueue * @amd_sdw_work: peripheral status work queue * @acp_sdw_lock: mutex to protect acp share register access @@ -84,7 +77,6 @@ struct amd_sdw_manager { void __iomem *mmio; void __iomem *acp_mmio; - struct sdw_manager_reg_mask *reg_mask; struct work_struct amd_sdw_irq_thread; struct work_struct amd_sdw_work; /* mutex to protect acp common register access */ -- cgit v1.2.3 From 7cbf7f4bf71a054d687c8860380c655a36d0f369 Mon Sep 17 00:00:00 2001 From: Siddharth Vadapalli Date: Wed, 24 Jan 2024 18:13:18 +0530 Subject: dmaengine: ti: k3-udma-glue: Add function to request TX chan for thread ID The existing function k3_udma_glue_request_tx_chn() supports requesting a TX DMA channel by its name. Add a new function to request TX DMA channel for a given thread ID, named k3_udma_glue_request_tx_chn_for_thread_id(). Also, export it for use by drivers which are probed by alternate methods (non device-tree) but still wish to make use of the existing DMA APIs. Such drivers could be informed about the thread ID corresponding to the TX DMA channel by RPMsg for example. Since the new function k3_udma_glue_request_tx_chn_for_thread_id() reuses most of the code in k3_udma_glue_request_tx_chn(), create a new function for the common code, named k3_udma_glue_request_tx_chn_common(). Signed-off-by: Siddharth Vadapalli Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20240124124319.820002-4-s-vadapalli@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 102 +++++++++++++++++++++++++++++---------- include/linux/dma/k3-udma-glue.h | 5 ++ 2 files changed, 81 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index eff1ae3d3efe..a475bbea35ee 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -274,29 +274,13 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); } -struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, - const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) +static int +k3_udma_glue_request_tx_chn_common(struct device *dev, + struct k3_udma_glue_tx_channel *tx_chn, + struct k3_udma_glue_tx_channel_cfg *cfg) { - struct k3_udma_glue_tx_channel *tx_chn; int ret; - tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); - if (!tx_chn) - return ERR_PTR(-ENOMEM); - - tx_chn->common.dev = dev; - tx_chn->common.swdata_size = cfg->swdata_size; - tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; - tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; - tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; - tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; - - /* parse of udmap channel */ - ret = of_k3_udma_glue_parse_chn(dev->of_node, name, - &tx_chn->common, true); - if (ret) - goto err; - tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, tx_chn->common.psdata_size, tx_chn->common.swdata_size); @@ -312,7 +296,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, if (IS_ERR(tx_chn->udma_tchanx)) { ret = PTR_ERR(tx_chn->udma_tchanx); dev_err(dev, "UDMAX tchanx get err %d\n", ret); - goto err; + return ret; } tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); @@ -325,7 +309,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, dev_err(dev, "Channel Device registration failed %d\n", ret); put_device(&tx_chn->common.chan_dev); tx_chn->common.chan_dev.parent = NULL; - goto err; + return ret; } if (xudma_is_pktdma(tx_chn->common.udmax)) { @@ -349,7 +333,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, &tx_chn->ringtxcq); if (ret) { dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret); - goto err; + return ret; } /* Set the dma_dev for the rings to be configured */ @@ -365,13 +349,13 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); - goto err; + return ret; } ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); - goto err; + return ret; } /* request and cfg psi-l */ @@ -382,11 +366,42 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = k3_udma_glue_cfg_tx_chn(tx_chn); if (ret) { dev_err(dev, "Failed to cfg tchan %d\n", ret); - goto err; + return ret; } k3_udma_glue_dump_tx_chn(tx_chn); + return 0; +} + +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn(struct device *dev, const char *name, + struct k3_udma_glue_tx_channel_cfg *cfg) +{ + struct k3_udma_glue_tx_channel *tx_chn; + int ret; + + tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); + if (!tx_chn) + return ERR_PTR(-ENOMEM); + + tx_chn->common.dev = dev; + tx_chn->common.swdata_size = cfg->swdata_size; + tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; + tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; + tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; + tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; + + /* parse of udmap channel */ + ret = of_k3_udma_glue_parse_chn(dev->of_node, name, + &tx_chn->common, true); + if (ret) + goto err; + + ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); + if (ret) + goto err; + return tx_chn; err: @@ -395,6 +410,41 @@ err: } EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_tx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id) +{ + struct k3_udma_glue_tx_channel *tx_chn; + int ret; + + tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); + if (!tx_chn) + return ERR_PTR(-ENOMEM); + + tx_chn->common.dev = dev; + tx_chn->common.swdata_size = cfg->swdata_size; + tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; + tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; + tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; + tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; + + ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, &tx_chn->common, true, thread_id); + if (ret) + goto err; + + ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); + if (ret) + goto err; + + return tx_chn; + +err: + k3_udma_glue_release_tx_chn(tx_chn); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn_for_thread_id); + void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) { if (tx_chn->psil_paired) { diff --git a/include/linux/dma/k3-udma-glue.h b/include/linux/dma/k3-udma-glue.h index e443be4d3b4b..c81386ceb1c1 100644 --- a/include/linux/dma/k3-udma-glue.h +++ b/include/linux/dma/k3-udma-glue.h @@ -26,6 +26,11 @@ struct k3_udma_glue_tx_channel; struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, const char *name, struct k3_udma_glue_tx_channel_cfg *cfg); +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_tx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id); + void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn); int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, struct cppi5_host_desc_t *desc_tx, -- cgit v1.2.3 From e54df52312fed462a005706d5d7ed6250da91d1e Mon Sep 17 00:00:00 2001 From: Siddharth Vadapalli Date: Wed, 24 Jan 2024 18:13:19 +0530 Subject: dmaengine: ti: k3-udma-glue: Add function to request RX chan for thread ID The existing function k3_udma_glue_request_remote_rx_chn() supports requesting an RX DMA channel and flow by the name of the RX DMA channel. Add support to request RX DMA channel for a given thread ID in the form of a new function named k3_udma_glue_request_remote_rx_chn_for_thread_id(). Also, export it for use by drivers which are probed by alternate methods (non device-tree) but still wish to make use of the existing DMA APIs. Such drivers could be informed about the thread ID corresponding to the RX DMA channel by RPMsg for example. Since the new function k3_udma_glue_request_remote_rx_chn_for_thread_id() reuses most of the code in k3_udma_glue_request_remote_rx_chn(), create a new function named k3_udma_glue_request_remote_rx_chn_common() for the common code. Signed-off-by: Siddharth Vadapalli Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20240124124319.820002-5-s-vadapalli@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 123 +++++++++++++++++++++++++++++---------- include/linux/dma/k3-udma-glue.h | 5 ++ 2 files changed, 96 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index a475bbea35ee..c9b93055dc9d 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -1073,12 +1073,59 @@ err: return ERR_PTR(ret); } +static int +k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device *dev) +{ + int ret, i; + + rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, + rx_chn->common.psdata_size, + rx_chn->common.swdata_size); + + rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, + sizeof(*rx_chn->flows), GFP_KERNEL); + if (!rx_chn->flows) + return -ENOMEM; + + rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; + rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); + dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x-0x%02x", + rx_chn->common.src_thread, rx_chn->flow_id_base); + ret = device_register(&rx_chn->common.chan_dev); + if (ret) { + dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&rx_chn->common.chan_dev); + rx_chn->common.chan_dev.parent = NULL; + return ret; + } + + if (xudma_is_pktdma(rx_chn->common.udmax)) { + /* prepare the channel device as coherent */ + rx_chn->common.chan_dev.dma_coherent = true; + dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, + DMA_BIT_MASK(48)); + } + + ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); + if (ret) + return ret; + + for (i = 0; i < rx_chn->flow_num; i++) + rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; + + k3_udma_glue_dump_rx_chn(rx_chn); + + return 0; +} + static struct k3_udma_glue_rx_channel * k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, struct k3_udma_glue_rx_channel_cfg *cfg) { struct k3_udma_glue_rx_channel *rx_chn; - int ret, i; + int ret; if (cfg->flow_id_num <= 0 || cfg->flow_id_use_rxchan_id || @@ -1109,44 +1156,55 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, if (ret) goto err; - rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, - rx_chn->common.psdata_size, - rx_chn->common.swdata_size); - - rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, - sizeof(*rx_chn->flows), GFP_KERNEL); - if (!rx_chn->flows) { - ret = -ENOMEM; + ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); + if (ret) goto err; - } - rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; - rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); - dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x-0x%02x", - rx_chn->common.src_thread, rx_chn->flow_id_base); - ret = device_register(&rx_chn->common.chan_dev); - if (ret) { - dev_err(dev, "Channel Device registration failed %d\n", ret); - put_device(&rx_chn->common.chan_dev); - rx_chn->common.chan_dev.parent = NULL; - goto err; - } + return rx_chn; - if (xudma_is_pktdma(rx_chn->common.udmax)) { - /* prepare the channel device as coherent */ - rx_chn->common.chan_dev.dma_coherent = true; - dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, - DMA_BIT_MASK(48)); - } +err: + k3_udma_glue_release_rx_chn(rx_chn); + return ERR_PTR(ret); +} - ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); +struct k3_udma_glue_rx_channel * +k3_udma_glue_request_remote_rx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id) +{ + struct k3_udma_glue_rx_channel *rx_chn; + int ret; + + if (cfg->flow_id_num <= 0 || + cfg->flow_id_use_rxchan_id || + cfg->def_flow_cfg || + cfg->flow_id_base < 0) + return ERR_PTR(-EINVAL); + + /* + * Remote RX channel is under control of Remote CPU core, so + * Linux can only request and manipulate by dedicated RX flows + */ + + rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); + if (!rx_chn) + return ERR_PTR(-ENOMEM); + + rx_chn->common.dev = dev; + rx_chn->common.swdata_size = cfg->swdata_size; + rx_chn->remote = true; + rx_chn->udma_rchan_id = -1; + rx_chn->flow_num = cfg->flow_id_num; + rx_chn->flow_id_base = cfg->flow_id_base; + rx_chn->psil_paired = false; + + ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, &rx_chn->common, false, thread_id); if (ret) goto err; - for (i = 0; i < rx_chn->flow_num; i++) - rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; - - k3_udma_glue_dump_rx_chn(rx_chn); + ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); + if (ret) + goto err; return rx_chn; @@ -1154,6 +1212,7 @@ err: k3_udma_glue_release_rx_chn(rx_chn); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(k3_udma_glue_request_remote_rx_chn_for_thread_id); struct k3_udma_glue_rx_channel * k3_udma_glue_request_rx_chn(struct device *dev, const char *name, diff --git a/include/linux/dma/k3-udma-glue.h b/include/linux/dma/k3-udma-glue.h index c81386ceb1c1..1e491c5dcac2 100644 --- a/include/linux/dma/k3-udma-glue.h +++ b/include/linux/dma/k3-udma-glue.h @@ -114,6 +114,11 @@ struct k3_udma_glue_rx_channel *k3_udma_glue_request_rx_chn( const char *name, struct k3_udma_glue_rx_channel_cfg *cfg); +struct k3_udma_glue_rx_channel * +k3_udma_glue_request_remote_rx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id); + void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn); int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn); void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn); -- cgit v1.2.3 From e2b3c4ff5d183da6d1863c2321413406a2752e7a Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 29 Jan 2024 16:06:45 -0800 Subject: bpf: add __arg_trusted global func arg tag Add support for passing PTR_TO_BTF_ID registers to global subprogs. Currently only PTR_TRUSTED flavor of PTR_TO_BTF_ID is supported. Non-NULL semantics is assumed, so caller will be forced to prove PTR_TO_BTF_ID can't be NULL. Note, we disallow global subprogs to destroy passed in PTR_TO_BTF_ID arguments, even the trusted one. We achieve that by not setting ref_obj_id when validating subprog code. This basically enforces (in Rust terms) borrowing semantics vs move semantics. Borrowing semantics seems to be a better fit for isolated global subprog validation approach. Implementation-wise, we utilize existing logic for matching user-provided BTF type to kernel-side BTF type, used by BPF CO-RE logic and following same matching rules. We enforce a unique match for types. Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20240130000648.2144827-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 1 + kernel/bpf/btf.c | 99 ++++++++++++++++++++++++++++++++++++++------ kernel/bpf/verifier.c | 24 +++++++++++ 3 files changed, 111 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7f5816482a10..0dcde339dc7e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -610,6 +610,7 @@ struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { u32 mem_size; + u32 btf_id; }; }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9ec08cfb2967..ed7a05815984 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6985,9 +6985,77 @@ static bool btf_is_dynptr_ptr(const struct btf *btf, const struct btf_type *t) return false; } +struct bpf_cand_cache { + const char *name; + u32 name_len; + u16 kind; + u16 cnt; + struct { + const struct btf *btf; + u32 id; + } cands[]; +}; + +static DEFINE_MUTEX(cand_cache_mutex); + +static struct bpf_cand_cache * +bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id); + +static int btf_get_ptr_to_btf_id(struct bpf_verifier_log *log, int arg_idx, + const struct btf *btf, const struct btf_type *t) +{ + struct bpf_cand_cache *cc; + struct bpf_core_ctx ctx = { + .btf = btf, + .log = log, + }; + u32 kern_type_id, type_id; + int err = 0; + + /* skip PTR and modifiers */ + type_id = t->type; + t = btf_type_by_id(btf, t->type); + while (btf_type_is_modifier(t)) { + type_id = t->type; + t = btf_type_by_id(btf, t->type); + } + + mutex_lock(&cand_cache_mutex); + cc = bpf_core_find_cands(&ctx, type_id); + if (IS_ERR(cc)) { + err = PTR_ERR(cc); + bpf_log(log, "arg#%d reference type('%s %s') candidate matching error: %d\n", + arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), + err); + goto cand_cache_unlock; + } + if (cc->cnt != 1) { + bpf_log(log, "arg#%d reference type('%s %s') %s\n", + arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), + cc->cnt == 0 ? "has no matches" : "is ambiguous"); + err = cc->cnt == 0 ? -ENOENT : -ESRCH; + goto cand_cache_unlock; + } + if (btf_is_module(cc->cands[0].btf)) { + bpf_log(log, "arg#%d reference type('%s %s') points to kernel module type (unsupported)\n", + arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off)); + err = -EOPNOTSUPP; + goto cand_cache_unlock; + } + kern_type_id = cc->cands[0].id; + +cand_cache_unlock: + mutex_unlock(&cand_cache_mutex); + if (err) + return err; + + return kern_type_id; +} + enum btf_arg_tag { ARG_TAG_CTX = 0x1, ARG_TAG_NONNULL = 0x2, + ARG_TAG_TRUSTED = 0x4, }; /* Process BTF of a function to produce high-level expectation of function @@ -7089,6 +7157,8 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (strcmp(tag, "ctx") == 0) { tags |= ARG_TAG_CTX; + } else if (strcmp(tag, "trusted") == 0) { + tags |= ARG_TAG_TRUSTED; } else if (strcmp(tag, "nonnull") == 0) { tags |= ARG_TAG_NONNULL; } else { @@ -7127,6 +7197,22 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) sub->args[i].arg_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY; continue; } + if (tags & ARG_TAG_TRUSTED) { + int kern_type_id; + + if (tags & ARG_TAG_NONNULL) { + bpf_log(log, "arg#%d has invalid combination of tags\n", i); + return -EINVAL; + } + + kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t); + if (kern_type_id < 0) + return kern_type_id; + + sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; + sub->args[i].btf_id = kern_type_id; + continue; + } if (is_global) { /* generic user data pointer */ u32 mem_size; @@ -8229,17 +8315,6 @@ size_t bpf_core_essential_name_len(const char *name) return n; } -struct bpf_cand_cache { - const char *name; - u32 name_len; - u16 kind; - u16 cnt; - struct { - const struct btf *btf; - u32 id; - } cands[]; -}; - static void bpf_free_cands(struct bpf_cand_cache *cands) { if (!cands->cnt) @@ -8260,8 +8335,6 @@ static struct bpf_cand_cache *vmlinux_cand_cache[VMLINUX_CAND_CACHE_SIZE]; #define MODULE_CAND_CACHE_SIZE 31 static struct bpf_cand_cache *module_cand_cache[MODULE_CAND_CACHE_SIZE]; -static DEFINE_MUTEX(cand_cache_mutex); - static void __print_cand_cache(struct bpf_verifier_log *log, struct bpf_cand_cache **cache, int cache_size) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c5d68a9d8acc..cd4d780e5400 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9336,6 +9336,18 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0); if (ret) return ret; + } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { + struct bpf_call_arg_meta meta; + int err; + + if (register_is_null(reg) && type_may_be_null(arg->arg_type)) + continue; + + memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ + err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta); + err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); + if (err) + return err; } else { bpf_log(log, "verifier bug: unrecognized arg#%d type %d\n", i, arg->arg_type); @@ -20137,6 +20149,18 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) mark_reg_known_zero(env, regs, i); reg->mem_size = arg->mem_size; reg->id = ++env->id_gen; + } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { + reg->type = PTR_TO_BTF_ID; + if (arg->arg_type & PTR_MAYBE_NULL) + reg->type |= PTR_MAYBE_NULL; + if (arg->arg_type & PTR_UNTRUSTED) + reg->type |= PTR_UNTRUSTED; + if (arg->arg_type & PTR_TRUSTED) + reg->type |= PTR_TRUSTED; + mark_reg_known_zero(env, regs, i); + reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */ + reg->btf_id = arg->btf_id; + reg->id = ++env->id_gen; } else { WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n", i - BPF_REG_1, arg->arg_type); -- cgit v1.2.3 From 8ddf54a32111f6dbe06cd318af443c6545a6c037 Mon Sep 17 00:00:00 2001 From: Jeffrey Hugo Date: Fri, 5 Jan 2024 10:42:53 -0700 Subject: bus: mhi: host: Read PK HASH dynamically The OEM PK HASH registers in the BHI region are read once during firmware load (boot), cached, and displayed on demand via sysfs. This has a few problems - if firmware load is skipped, the registers will not be read and if the register values change over the life of the device the local cache will be out of sync. Qualcomm Cloud AI 100 can expose both these problems. It is possible for mhi_async_power_up() to be invoked while the device is in AMSS EE, which would bypass firmware loading. Also, Qualcomm Cloud AI 100 has 5 PK HASH slots which can be dynamically provisioned while the device is active, which would result in the values changing and users may want to know what keys are active. Address these concerns by reading the PK HASH registers on-demand during the sysfs read. This will result in showing the most current information. Signed-off-by: Jeffrey Hugo Reviewed-by: Pranjal Ramajor Asha Kanojiya Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20240105174253.863388-1-quic_jhugo@quicinc.com Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/host/boot.c | 11 +---------- drivers/bus/mhi/host/init.c | 16 ++++++++++++---- include/linux/mhi.h | 2 -- 3 files changed, 13 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/mhi/host/boot.c b/drivers/bus/mhi/host/boot.c index edc0ec5a0933..dedd29ca8db3 100644 --- a/drivers/bus/mhi/host/boot.c +++ b/drivers/bus/mhi/host/boot.c @@ -395,7 +395,7 @@ void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl) void *buf; dma_addr_t dma_addr; size_t size, fw_sz; - int i, ret; + int ret; if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { dev_err(dev, "Device MHI is not in valid state\n"); @@ -408,15 +408,6 @@ void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl) if (ret) dev_err(dev, "Could not capture serial number via BHI\n"); - for (i = 0; i < ARRAY_SIZE(mhi_cntrl->oem_pk_hash); i++) { - ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_OEMPKHASH(i), - &mhi_cntrl->oem_pk_hash[i]); - if (ret) { - dev_err(dev, "Could not capture OEM PK HASH via BHI\n"); - break; - } - } - /* wait for ready on pass through or any other execution environment */ if (!MHI_FW_LOAD_CAPABLE(mhi_cntrl->ee)) goto fw_load_ready_state; diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c index 8e5ec1a409b8..6d3b045ab259 100644 --- a/drivers/bus/mhi/host/init.c +++ b/drivers/bus/mhi/host/init.c @@ -98,11 +98,19 @@ static ssize_t oem_pk_hash_show(struct device *dev, { struct mhi_device *mhi_dev = to_mhi_device(dev); struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; - int i, cnt = 0; + u32 hash_segment[MHI_MAX_OEM_PK_HASH_SEGMENTS]; + int i, cnt = 0, ret; - for (i = 0; i < ARRAY_SIZE(mhi_cntrl->oem_pk_hash); i++) - cnt += sysfs_emit_at(buf, cnt, "OEMPKHASH[%d]: 0x%x\n", - i, mhi_cntrl->oem_pk_hash[i]); + for (i = 0; i < MHI_MAX_OEM_PK_HASH_SEGMENTS; i++) { + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_OEMPKHASH(i), &hash_segment[i]); + if (ret) { + dev_err(dev, "Could not capture OEM PK HASH\n"); + return ret; + } + } + + for (i = 0; i < MHI_MAX_OEM_PK_HASH_SEGMENTS; i++) + cnt += sysfs_emit_at(buf, cnt, "OEMPKHASH[%d]: 0x%x\n", i, hash_segment[i]); return cnt; } diff --git a/include/linux/mhi.h b/include/linux/mhi.h index d0f9b522f328..474d32cb0520 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -325,7 +325,6 @@ struct mhi_controller_config { * @major_version: MHI controller major revision number * @minor_version: MHI controller minor revision number * @serial_number: MHI controller serial number obtained from BHI - * @oem_pk_hash: MHI controller OEM PK Hash obtained from BHI * @mhi_event: MHI event ring configurations table * @mhi_cmd: MHI command ring configurations table * @mhi_ctxt: MHI device context, shared memory between host and device @@ -413,7 +412,6 @@ struct mhi_controller { u32 major_version; u32 minor_version; u32 serial_number; - u32 oem_pk_hash[MHI_MAX_OEM_PK_HASH_SEGMENTS]; struct mhi_event *mhi_event; struct mhi_cmd *mhi_cmd; -- cgit v1.2.3 From 1c9f2c7606afe149800986182638f636646dd824 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 10 Jan 2024 08:28:16 -1000 Subject: kernfs: Rearrange kernfs_node fields to reduce its size on 64bit Moving .flags and .mode right below .hash makes kernfs_node smaller by 8 bytes on 64bit. To avoid creating a hole from 8 bytes alignment on 32bit archs, .priv is moved below so that there are two 32bit pointers after the 64bit .id field. v2: Updated to avoid size increase on 32bit noticed by Geert. Signed-off-by: Tejun Heo Cc: Geert Uytterhoeven Link: https://lore.kernel.org/r/ZZ7hwA18nfmFjYpj@slm.duckdns.org Signed-off-by: Greg Kroah-Hartman --- include/linux/kernfs.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 99aaa050ccb7..82e1ce79a70c 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -206,22 +206,22 @@ struct kernfs_node { const void *ns; /* namespace tag */ unsigned int hash; /* ns + name hash */ + unsigned short flags; + umode_t mode; + union { struct kernfs_elem_dir dir; struct kernfs_elem_symlink symlink; struct kernfs_elem_attr attr; }; - void *priv; - /* * 64bit unique ID. On 64bit ino setups, id is the ino. On 32bit, * the low 32bits are ino and upper generation. */ u64 id; - unsigned short flags; - umode_t mode; + void *priv; struct kernfs_iattrs *iattr; }; -- cgit v1.2.3 From 4207b556e62f0a8915afc5da4c5d5ad915a253a5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 9 Jan 2024 11:48:04 -1000 Subject: kernfs: RCU protect kernfs_nodes and avoid kernfs_idr_lock in kernfs_find_and_get_node_by_id() The BPF helper bpf_cgroup_from_id() calls kernfs_find_and_get_node_by_id() which acquires kernfs_idr_lock, which is an non-raw non-IRQ-safe lock. This can lead to deadlocks as bpf_cgroup_from_id() can be called from any BPF programs including e.g. the ones that attach to functions which are holding the scheduler rq lock. Consider the following BPF program: SEC("fentry/__set_cpus_allowed_ptr_locked") int BPF_PROG(__set_cpus_allowed_ptr_locked, struct task_struct *p, struct affinity_context *affn_ctx, struct rq *rq, struct rq_flags *rf) { struct cgroup *cgrp = bpf_cgroup_from_id(p->cgroups->dfl_cgrp->kn->id); if (cgrp) { bpf_printk("%d[%s] in %s", p->pid, p->comm, cgrp->kn->name); bpf_cgroup_release(cgrp); } return 0; } __set_cpus_allowed_ptr_locked() is called with rq lock held and the above BPF program calls bpf_cgroup_from_id() within leading to the following lockdep warning: ===================================================== WARNING: HARDIRQ-safe -> HARDIRQ-unsafe lock order detected 6.7.0-rc3-work-00053-g07124366a1d7-dirty #147 Not tainted ----------------------------------------------------- repro/1620 [HC0[0]:SC0[0]:HE0:SE1] is trying to acquire: ffffffff833b3688 (kernfs_idr_lock){+.+.}-{2:2}, at: kernfs_find_and_get_node_by_id+0x1e/0x70 and this task is already holding: ffff888237ced698 (&rq->__lock){-.-.}-{2:2}, at: task_rq_lock+0x4e/0xf0 which would create a new lock dependency: (&rq->__lock){-.-.}-{2:2} -> (kernfs_idr_lock){+.+.}-{2:2} ... Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(kernfs_idr_lock); local_irq_disable(); lock(&rq->__lock); lock(kernfs_idr_lock); lock(&rq->__lock); *** DEADLOCK *** ... Call Trace: dump_stack_lvl+0x55/0x70 dump_stack+0x10/0x20 __lock_acquire+0x781/0x2a40 lock_acquire+0xbf/0x1f0 _raw_spin_lock+0x2f/0x40 kernfs_find_and_get_node_by_id+0x1e/0x70 cgroup_get_from_id+0x21/0x240 bpf_cgroup_from_id+0xe/0x20 bpf_prog_98652316e9337a5a___set_cpus_allowed_ptr_locked+0x96/0x11a bpf_trampoline_6442545632+0x4f/0x1000 __set_cpus_allowed_ptr_locked+0x5/0x5a0 sched_setaffinity+0x1b3/0x290 __x64_sys_sched_setaffinity+0x4f/0x60 do_syscall_64+0x40/0xe0 entry_SYSCALL_64_after_hwframe+0x46/0x4e Let's fix it by protecting kernfs_node and kernfs_root with RCU and making kernfs_find_and_get_node_by_id() acquire rcu_read_lock() instead of kernfs_idr_lock. This adds an rcu_head to kernfs_node making it larger by 16 bytes on 64bit. Combined with the preceding rearrange patch, the net increase is 8 bytes. Signed-off-by: Tejun Heo Cc: Andrea Righi Cc: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240109214828.252092-4-tj@kernel.org Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/dir.c | 31 ++++++++++++++++++++----------- fs/kernfs/kernfs-internal.h | 2 ++ include/linux/kernfs.h | 2 ++ 3 files changed, 24 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index bce1d7ac95ca..458519e416fe 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -529,6 +529,20 @@ void kernfs_get(struct kernfs_node *kn) } EXPORT_SYMBOL_GPL(kernfs_get); +static void kernfs_free_rcu(struct rcu_head *rcu) +{ + struct kernfs_node *kn = container_of(rcu, struct kernfs_node, rcu); + + kfree_const(kn->name); + + if (kn->iattr) { + simple_xattrs_free(&kn->iattr->xattrs, NULL); + kmem_cache_free(kernfs_iattrs_cache, kn->iattr); + } + + kmem_cache_free(kernfs_node_cache, kn); +} + /** * kernfs_put - put a reference count on a kernfs_node * @kn: the target kernfs_node @@ -557,16 +571,11 @@ void kernfs_put(struct kernfs_node *kn) if (kernfs_type(kn) == KERNFS_LINK) kernfs_put(kn->symlink.target_kn); - kfree_const(kn->name); - - if (kn->iattr) { - simple_xattrs_free(&kn->iattr->xattrs, NULL); - kmem_cache_free(kernfs_iattrs_cache, kn->iattr); - } spin_lock(&kernfs_idr_lock); idr_remove(&root->ino_idr, (u32)kernfs_ino(kn)); spin_unlock(&kernfs_idr_lock); - kmem_cache_free(kernfs_node_cache, kn); + + call_rcu(&kn->rcu, kernfs_free_rcu); kn = parent; if (kn) { @@ -575,7 +584,7 @@ void kernfs_put(struct kernfs_node *kn) } else { /* just released the root kn, free @root too */ idr_destroy(&root->ino_idr); - kfree(root); + kfree_rcu(root, rcu); } } EXPORT_SYMBOL_GPL(kernfs_put); @@ -715,7 +724,7 @@ struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root, ino_t ino = kernfs_id_ino(id); u32 gen = kernfs_id_gen(id); - spin_lock(&kernfs_idr_lock); + rcu_read_lock(); kn = idr_find(&root->ino_idr, (u32)ino); if (!kn) @@ -739,10 +748,10 @@ struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root, if (unlikely(!__kernfs_active(kn) || !atomic_inc_not_zero(&kn->count))) goto err_unlock; - spin_unlock(&kernfs_idr_lock); + rcu_read_unlock(); return kn; err_unlock: - spin_unlock(&kernfs_idr_lock); + rcu_read_unlock(); return NULL; } diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 237f2764b941..b42ee6547cdc 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -49,6 +49,8 @@ struct kernfs_root { struct rw_semaphore kernfs_rwsem; struct rw_semaphore kernfs_iattr_rwsem; struct rw_semaphore kernfs_supers_rwsem; + + struct rcu_head rcu; }; /* +1 to avoid triggering overflow warning when negating it */ diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 82e1ce79a70c..87c79d076d6d 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -223,6 +223,8 @@ struct kernfs_node { void *priv; struct kernfs_iattrs *iattr; + + struct rcu_head rcu; }; /* -- cgit v1.2.3 From 3a480d4bb5b1e1f09426223e68acaa90da32e384 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 5 Jan 2024 11:26:48 +0100 Subject: driver core: cpu: make cpu_subsys const Now that the driver core can properly handle constant struct bus_type, move the cpu_subsys variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/2024010548-crane-snooze-a871@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/base/cpu.c | 2 +- include/linux/cpu.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 47de0f140ba6..ac84854c85d7 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -366,7 +366,7 @@ static int cpu_uevent(const struct device *dev, struct kobj_uevent_env *env) } #endif -struct bus_type cpu_subsys = { +const struct bus_type cpu_subsys = { .name = "cpu", .dev_name = "cpu", .match = cpu_subsys_match, diff --git a/include/linux/cpu.h b/include/linux/cpu.h index dcb89c987164..0b993a140946 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -128,7 +128,7 @@ static inline void cpu_maps_update_done(void) static inline int add_cpu(unsigned int cpu) { return 0;} #endif /* CONFIG_SMP */ -extern struct bus_type cpu_subsys; +extern const struct bus_type cpu_subsys; extern int lockdep_is_cpus_held(void); -- cgit v1.2.3 From c5c3e1bfc9e0ee72af528df8d773980f4855938a Mon Sep 17 00:00:00 2001 From: Rohan G Thomas Date: Sat, 27 Jan 2024 12:04:41 +0800 Subject: net: stmmac: Offload queueMaxSDU from tc-taprio Add support for configuring queueMaxSDU. As DWMAC IPs doesn't support queueMaxSDU table handle this in the SW. The maximum 802.3 frame size that is allowed to be transmitted by any queue is queueMaxSDU + 16 bytes (i.e. 6 bytes SA + 6 bytes DA + 4 bytes FCS). Inspired from intel i225 driver. Signed-off-by: Rohan G Thomas Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/common.h | 1 + drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 22 ++++++++++++++++++++ drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c | 25 +++++++++++++++++++++++ include/linux/stmmac.h | 1 + 4 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 721c1f8e892f..d8d2a90fd228 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -202,6 +202,7 @@ struct stmmac_extra_stats { unsigned long mtl_est_hlbf; unsigned long mtl_est_btre; unsigned long mtl_est_btrlm; + unsigned long max_sdu_txq_drop[MTL_MAX_TX_QUEUES]; /* per queue statistics */ struct stmmac_txq_stats txq_stats[MTL_MAX_TX_QUEUES]; struct stmmac_rxq_stats rxq_stats[MTL_MAX_RX_QUEUES]; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index b334eb16da23..33509237fe60 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2507,6 +2507,13 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) if (!xsk_tx_peek_desc(pool, &xdp_desc)) break; + if (priv->plat->est && priv->plat->est->enable && + priv->plat->est->max_sdu[queue] && + xdp_desc.len > priv->plat->est->max_sdu[queue]) { + priv->xstats.max_sdu_txq_drop[queue]++; + continue; + } + if (likely(priv->extend_desc)) tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); else if (tx_q->tbs & STMMAC_TBS_AVAIL) @@ -4498,6 +4505,13 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) return stmmac_tso_xmit(skb, dev); } + if (priv->plat->est && priv->plat->est->enable && + priv->plat->est->max_sdu[queue] && + skb->len > priv->plat->est->max_sdu[queue]){ + priv->xstats.max_sdu_txq_drop[queue]++; + goto max_sdu_err; + } + if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, @@ -4715,6 +4729,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) dma_map_err: netdev_err(priv->dev, "Tx DMA map failed\n"); +max_sdu_err: dev_kfree_skb(skb); priv->xstats.tx_dropped++; return NETDEV_TX_OK; @@ -4871,6 +4886,13 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) return STMMAC_XDP_CONSUMED; + if (priv->plat->est && priv->plat->est->enable && + priv->plat->est->max_sdu[queue] && + xdpf->len > priv->plat->est->max_sdu[queue]) { + priv->xstats.max_sdu_txq_drop[queue]++; + return STMMAC_XDP_CONSUMED; + } + if (likely(priv->extend_desc)) tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); else if (tx_q->tbs & STMMAC_TBS_AVAIL) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 26fa33e5ec34..07aa3a3089dc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -915,6 +915,28 @@ struct timespec64 stmmac_calc_tas_basetime(ktime_t old_base_time, return time; } +static void tc_taprio_map_maxsdu_txq(struct stmmac_priv *priv, + struct tc_taprio_qopt_offload *qopt) +{ + struct plat_stmmacenet_data *plat = priv->plat; + u32 num_tc = qopt->mqprio.qopt.num_tc; + u32 offset, count, i, j; + + /* QueueMaxSDU received from the driver corresponds to the Linux traffic + * class. Map queueMaxSDU per Linux traffic class to DWMAC Tx queues. + */ + for (i = 0; i < num_tc; i++) { + if (!qopt->max_sdu[i]) + continue; + + offset = qopt->mqprio.qopt.offset[i]; + count = qopt->mqprio.qopt.count[i]; + + for (j = offset; j < offset + count; j++) + plat->est->max_sdu[j] = qopt->max_sdu[i] + ETH_HLEN - ETH_TLEN; + } +} + static int tc_setup_taprio(struct stmmac_priv *priv, struct tc_taprio_qopt_offload *qopt) { @@ -1045,6 +1067,8 @@ static int tc_setup_taprio(struct stmmac_priv *priv, priv->plat->est->ter = qopt->cycle_time_extension; + tc_taprio_map_maxsdu_txq(priv, qopt); + if (fpe && !priv->dma_cap.fpesel) { mutex_unlock(&priv->plat->est->lock); return -EOPNOTSUPP; @@ -1126,6 +1150,7 @@ static int tc_query_caps(struct stmmac_priv *priv, return -EOPNOTSUPP; caps->gate_mask_per_txq = true; + caps->supports_queue_max_sdu = true; return 0; } diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index dee5ad6e48c5..dfa1828cd756 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -127,6 +127,7 @@ struct stmmac_est { u32 gcl_unaligned[EST_GCL]; u32 gcl[EST_GCL]; u32 gcl_size; + u32 max_sdu[MTL_MAX_TX_QUEUES]; }; struct stmmac_rxq_cfg { -- cgit v1.2.3 From d80a52335374e484a4ff2afdc9af843e73273945 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 27 Jan 2024 14:25:09 +0100 Subject: ethtool: replace struct ethtool_eee with a new struct ethtool_keee on kernel side In order to pass EEE link modes beyond bit 32 to userspace we have to complement the 32 bit bitmaps in struct ethtool_eee with linkmode bitmaps. Therefore, similar to ethtool_link_settings and ethtool_link_ksettings, add a struct ethtool_keee. In a first step it's an identical copy of ethtool_eee. This patch simply does a s/ethtool_eee/ethtool_keee/g for all users. No functional change intended. Suggested-by: Andrew Lunn Signed-off-by: Heiner Kallweit Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/dsa/b53/b53_common.c | 10 +++++----- drivers/net/dsa/b53/b53_priv.h | 6 +++--- drivers/net/dsa/bcm_sf2.c | 2 +- drivers/net/dsa/microchip/ksz_common.c | 4 ++-- drivers/net/dsa/mt7530.c | 4 ++-- drivers/net/dsa/mv88e6xxx/chip.c | 4 ++-- drivers/net/dsa/qca/qca8k-common.c | 4 ++-- drivers/net/dsa/qca/qca8k.h | 4 ++-- drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c | 4 ++-- drivers/net/ethernet/broadcom/asp2/bcmasp.h | 2 +- drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c | 8 ++++---- drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c | 4 ++-- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 8 ++++---- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 2 +- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 6 +++--- drivers/net/ethernet/broadcom/genet/bcmgenet.c | 8 ++++---- drivers/net/ethernet/broadcom/genet/bcmgenet.h | 2 +- drivers/net/ethernet/broadcom/tg3.c | 10 +++++----- drivers/net/ethernet/broadcom/tg3.h | 2 +- drivers/net/ethernet/engleder/tsnep_main.c | 6 +++--- drivers/net/ethernet/freescale/enetc/enetc.c | 4 ++-- drivers/net/ethernet/freescale/fec.h | 2 +- drivers/net/ethernet/freescale/fec_main.c | 10 +++++----- drivers/net/ethernet/freescale/gianfar.c | 4 ++-- drivers/net/ethernet/intel/e1000e/ethtool.c | 6 +++--- drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 6 +++--- drivers/net/ethernet/intel/igb/igb_ethtool.c | 8 ++++---- drivers/net/ethernet/intel/igc/igc.h | 2 +- drivers/net/ethernet/intel/igc/igc_ethtool.c | 8 ++++---- drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 10 +++++----- drivers/net/ethernet/marvell/mvneta.c | 4 ++-- drivers/net/ethernet/microchip/lan743x_ethtool.c | 4 ++-- drivers/net/ethernet/qlogic/qede/qede_ethtool.c | 4 ++-- drivers/net/ethernet/realtek/r8169_main.c | 4 ++-- drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c | 4 ++-- drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c | 4 ++-- drivers/net/ethernet/ti/am65-cpsw-ethtool.c | 4 ++-- drivers/net/ethernet/ti/cpsw_ethtool.c | 4 ++-- drivers/net/ethernet/ti/cpsw_priv.h | 4 ++-- drivers/net/ethernet/ti/icssg/icssg_ethtool.c | 4 ++-- drivers/net/phy/marvell.c | 2 +- drivers/net/phy/phy-c45.c | 8 ++++---- drivers/net/phy/phy.c | 8 ++++---- drivers/net/phy/phylink.c | 8 ++++---- drivers/net/usb/ax88179_178a.c | 10 +++++----- drivers/net/usb/lan78xx.c | 4 ++-- drivers/net/usb/r8152.c | 14 +++++++------- include/linux/ethtool.h | 16 ++++++++++++++-- include/linux/phy.h | 8 ++++---- include/linux/phylink.h | 4 ++-- include/net/dsa.h | 4 ++-- net/dsa/user.c | 4 ++-- net/ethtool/eee.c | 10 +++++----- net/ethtool/ioctl.c | 6 +++--- 54 files changed, 159 insertions(+), 147 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 0d628b35fd5c..adc93abf4551 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -1257,7 +1257,7 @@ static void b53_adjust_link(struct dsa_switch *ds, int port, struct phy_device *phydev) { struct b53_device *dev = ds->priv; - struct ethtool_eee *p = &dev->ports[port].eee; + struct ethtool_keee *p = &dev->ports[port].eee; u8 rgmii_ctrl = 0, reg = 0, off; bool tx_pause = false; bool rx_pause = false; @@ -2224,10 +2224,10 @@ int b53_eee_init(struct dsa_switch *ds, int port, struct phy_device *phy) } EXPORT_SYMBOL(b53_eee_init); -int b53_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e) +int b53_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e) { struct b53_device *dev = ds->priv; - struct ethtool_eee *p = &dev->ports[port].eee; + struct ethtool_keee *p = &dev->ports[port].eee; u16 reg; if (is5325(dev) || is5365(dev)) @@ -2241,10 +2241,10 @@ int b53_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e) } EXPORT_SYMBOL(b53_get_mac_eee); -int b53_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e) +int b53_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e) { struct b53_device *dev = ds->priv; - struct ethtool_eee *p = &dev->ports[port].eee; + struct ethtool_keee *p = &dev->ports[port].eee; if (is5325(dev) || is5365(dev)) return -EOPNOTSUPP; diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h index fdcfd5081c28..c26a03755e83 100644 --- a/drivers/net/dsa/b53/b53_priv.h +++ b/drivers/net/dsa/b53/b53_priv.h @@ -95,7 +95,7 @@ struct b53_pcs { struct b53_port { u16 vlan_ctl_mask; - struct ethtool_eee eee; + struct ethtool_keee eee; }; struct b53_vlan { @@ -397,7 +397,7 @@ void b53_disable_port(struct dsa_switch *ds, int port); void b53_brcm_hdr_setup(struct dsa_switch *ds, int port); void b53_eee_enable_set(struct dsa_switch *ds, int port, bool enable); int b53_eee_init(struct dsa_switch *ds, int port, struct phy_device *phy); -int b53_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e); -int b53_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e); +int b53_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e); +int b53_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e); #endif diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index 4a52ccbe393f..bc77ee9e6d0a 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -835,7 +835,7 @@ static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port, bool tx_pause, bool rx_pause) { struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); - struct ethtool_eee *p = &priv->dev->ports[port].eee; + struct ethtool_keee *p = &priv->dev->ports[port].eee; u32 reg_rgmii_ctrl = 0; u32 reg, offset; diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 245dfb7a7a31..a7b5ddb8656e 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -2852,7 +2852,7 @@ static int ksz_validate_eee(struct dsa_switch *ds, int port) } static int ksz_get_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { int ret; @@ -2872,7 +2872,7 @@ static int ksz_get_mac_eee(struct dsa_switch *ds, int port, } static int ksz_set_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { struct ksz_device *dev = ds->priv; int ret; diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 68be38ae66e0..98a73a62f2ee 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -3048,7 +3048,7 @@ mt753x_setup(struct dsa_switch *ds) } static int mt753x_get_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { struct mt7530_priv *priv = ds->priv; u32 eeecr = mt7530_read(priv, MT7530_PMEEECR_P(port)); @@ -3060,7 +3060,7 @@ static int mt753x_get_mac_eee(struct dsa_switch *ds, int port, } static int mt753x_set_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { struct mt7530_priv *priv = ds->priv; u32 set, mask = LPI_THRESH_MASK | LPI_MODE_EN; diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 383b3c4d6f59..8b0079b8e0e0 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -1451,14 +1451,14 @@ static void mv88e6xxx_get_regs(struct dsa_switch *ds, int port, } static int mv88e6xxx_get_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { /* Nothing to do on the port's MAC */ return 0; } static int mv88e6xxx_set_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { /* Nothing to do on the port's MAC */ return 0; diff --git a/drivers/net/dsa/qca/qca8k-common.c b/drivers/net/dsa/qca/qca8k-common.c index 2358cd399c7e..7f80035c5441 100644 --- a/drivers/net/dsa/qca/qca8k-common.c +++ b/drivers/net/dsa/qca/qca8k-common.c @@ -534,7 +534,7 @@ int qca8k_get_sset_count(struct dsa_switch *ds, int port, int sset) } int qca8k_set_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *eee) + struct ethtool_keee *eee) { u32 lpi_en = QCA8K_REG_EEE_CTRL_LPI_EN(port); struct qca8k_priv *priv = ds->priv; @@ -558,7 +558,7 @@ exit: } int qca8k_get_mac_eee(struct dsa_switch *ds, int port, - struct ethtool_eee *e) + struct ethtool_keee *e) { /* Nothing to do on the port's MAC */ return 0; diff --git a/drivers/net/dsa/qca/qca8k.h b/drivers/net/dsa/qca/qca8k.h index c8785c36c54e..2184d8d2d5a9 100644 --- a/drivers/net/dsa/qca/qca8k.h +++ b/drivers/net/dsa/qca/qca8k.h @@ -518,8 +518,8 @@ void qca8k_get_ethtool_stats(struct dsa_switch *ds, int port, int qca8k_get_sset_count(struct dsa_switch *ds, int port, int sset); /* Common eee function */ -int qca8k_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *eee); -int qca8k_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e); +int qca8k_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *eee); +int qca8k_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e); /* Common bridge function */ void qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state); diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index 18a6c8d99fa0..be865776de55 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -697,7 +697,7 @@ static u32 eee_mask_to_ethtool_mask(u32 speed) return rate; } -static int aq_ethtool_get_eee(struct net_device *ndev, struct ethtool_eee *eee) +static int aq_ethtool_get_eee(struct net_device *ndev, struct ethtool_keee *eee) { struct aq_nic_s *aq_nic = netdev_priv(ndev); u32 rate, supported_rates; @@ -729,7 +729,7 @@ static int aq_ethtool_get_eee(struct net_device *ndev, struct ethtool_eee *eee) return 0; } -static int aq_ethtool_set_eee(struct net_device *ndev, struct ethtool_eee *eee) +static int aq_ethtool_set_eee(struct net_device *ndev, struct ethtool_keee *eee) { struct aq_nic_s *aq_nic = netdev_priv(ndev); u32 rate, supported_rates; diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.h b/drivers/net/ethernet/broadcom/asp2/bcmasp.h index ec90add6b03e..312bf9b6576e 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp.h +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.h @@ -337,7 +337,7 @@ struct bcmasp_intf { int wol_irq; unsigned int wol_irq_enabled:1; - struct ethtool_eee eee; + struct ethtool_keee eee; }; #define NUM_NET_FILTERS 32 diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c index ce6a3d56fb23..2851bed153e6 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_ethtool.c @@ -363,10 +363,10 @@ void bcmasp_eee_enable_set(struct bcmasp_intf *intf, bool enable) intf->eee.eee_active = enable; } -static int bcmasp_get_eee(struct net_device *dev, struct ethtool_eee *e) +static int bcmasp_get_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmasp_intf *intf = netdev_priv(dev); - struct ethtool_eee *p = &intf->eee; + struct ethtool_keee *p = &intf->eee; if (!dev->phydev) return -ENODEV; @@ -379,10 +379,10 @@ static int bcmasp_get_eee(struct net_device *dev, struct ethtool_eee *e) return phy_ethtool_get_eee(dev->phydev, e); } -static int bcmasp_set_eee(struct net_device *dev, struct ethtool_eee *e) +static int bcmasp_set_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmasp_intf *intf = netdev_priv(dev); - struct ethtool_eee *p = &intf->eee; + struct ethtool_keee *p = &intf->eee; int ret; if (!dev->phydev) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index 0bc7690cdee1..12d2785eeb8a 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -2108,7 +2108,7 @@ static u32 bnx2x_adv_to_eee(u32 modes, u32 shift) return eee_adv << shift; } -static int bnx2x_get_eee(struct net_device *dev, struct ethtool_eee *edata) +static int bnx2x_get_eee(struct net_device *dev, struct ethtool_keee *edata) { struct bnx2x *bp = netdev_priv(dev); u32 eee_cfg; @@ -2141,7 +2141,7 @@ static int bnx2x_get_eee(struct net_device *dev, struct ethtool_eee *edata) return 0; } -static int bnx2x_set_eee(struct net_device *dev, struct ethtool_eee *edata) +static int bnx2x_set_eee(struct net_device *dev, struct ethtool_keee *edata) { struct bnx2x *bp = netdev_priv(dev); u32 eee_cfg; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 39845d556baf..d7626c26f9a9 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -10621,7 +10621,7 @@ static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8); if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { - struct ethtool_eee *eee = &bp->eee; + struct ethtool_keee *eee = &bp->eee; u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); @@ -10766,7 +10766,7 @@ int bnxt_update_link(struct bnxt *bp, bool chng_link_state) link_info->module_status = resp->module_status; if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { - struct ethtool_eee *eee = &bp->eee; + struct ethtool_keee *eee = &bp->eee; u16 fw_speeds; eee->eee_active = 0; @@ -10957,7 +10957,7 @@ int bnxt_hwrm_set_pause(struct bnxt *bp) static void bnxt_hwrm_set_eee(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) { - struct ethtool_eee *eee = &bp->eee; + struct ethtool_keee *eee = &bp->eee; if (eee->eee_enabled) { u16 eee_speeds; @@ -11322,7 +11322,7 @@ static void bnxt_get_wol_settings(struct bnxt *bp) static bool bnxt_eee_config_ok(struct bnxt *bp) { - struct ethtool_eee *eee = &bp->eee; + struct ethtool_keee *eee = &bp->eee; struct bnxt_link_info *link_info = &bp->link_info; if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 47338b48ca20..b2cb3e77559d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2442,7 +2442,7 @@ struct bnxt { */ struct mutex link_lock; struct bnxt_link_info link_info; - struct ethtool_eee eee; + struct ethtool_keee eee; u32 lpi_tmr_lo; u32 lpi_tmr_hi; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index dc4ca706b0e2..d6a8577d68af 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -3884,10 +3884,10 @@ static int bnxt_set_eeprom(struct net_device *dev, eeprom->len); } -static int bnxt_set_eee(struct net_device *dev, struct ethtool_eee *edata) +static int bnxt_set_eee(struct net_device *dev, struct ethtool_keee *edata) { struct bnxt *bp = netdev_priv(dev); - struct ethtool_eee *eee = &bp->eee; + struct ethtool_keee *eee = &bp->eee; struct bnxt_link_info *link_info = &bp->link_info; u32 advertising; int rc = 0; @@ -3942,7 +3942,7 @@ eee_exit: return rc; } -static int bnxt_get_eee(struct net_device *dev, struct ethtool_eee *edata) +static int bnxt_get_eee(struct net_device *dev, struct ethtool_keee *edata) { struct bnxt *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 2d7ae71287b1..051c31fb17c2 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1317,10 +1317,10 @@ void bcmgenet_eee_enable_set(struct net_device *dev, bool enable, priv->eee.tx_lpi_enabled = tx_lpi_enabled; } -static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e) +static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmgenet_priv *priv = netdev_priv(dev); - struct ethtool_eee *p = &priv->eee; + struct ethtool_keee *p = &priv->eee; if (GENET_IS_V1(priv)) return -EOPNOTSUPP; @@ -1336,10 +1336,10 @@ static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e) return phy_ethtool_get_eee(dev->phydev, e); } -static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e) +static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e) { struct bcmgenet_priv *priv = netdev_priv(dev); - struct ethtool_eee *p = &priv->eee; + struct ethtool_keee *p = &priv->eee; if (GENET_IS_V1(priv)) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 1985c0ec4da2..7523b60b3c1c 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -645,7 +645,7 @@ struct bcmgenet_priv { struct bcmgenet_mib_counters mib; - struct ethtool_eee eee; + struct ethtool_keee eee; }; #define GENET_IO_MACRO(name, offset) \ diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 04964bbe08cf..11054177c2ec 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -2338,10 +2338,10 @@ static void tg3_phy_apply_otp(struct tg3 *tp) tg3_phy_toggle_auxctl_smdsp(tp, false); } -static void tg3_eee_pull_config(struct tg3 *tp, struct ethtool_eee *eee) +static void tg3_eee_pull_config(struct tg3 *tp, struct ethtool_keee *eee) { u32 val; - struct ethtool_eee *dest = &tp->eee; + struct ethtool_keee *dest = &tp->eee; if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) return; @@ -4618,7 +4618,7 @@ static int tg3_init_5401phy_dsp(struct tg3 *tp) static bool tg3_phy_eee_config_ok(struct tg3 *tp) { - struct ethtool_eee eee; + struct ethtool_keee eee; if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) return true; @@ -14180,7 +14180,7 @@ static int tg3_set_coalesce(struct net_device *dev, return 0; } -static int tg3_set_eee(struct net_device *dev, struct ethtool_eee *edata) +static int tg3_set_eee(struct net_device *dev, struct ethtool_keee *edata) { struct tg3 *tp = netdev_priv(dev); @@ -14217,7 +14217,7 @@ static int tg3_set_eee(struct net_device *dev, struct ethtool_eee *edata) return 0; } -static int tg3_get_eee(struct net_device *dev, struct ethtool_eee *edata) +static int tg3_get_eee(struct net_device *dev, struct ethtool_keee *edata) { struct tg3 *tp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/tg3.h b/drivers/net/ethernet/broadcom/tg3.h index 5016475e5005..cf1b2b123c7e 100644 --- a/drivers/net/ethernet/broadcom/tg3.h +++ b/drivers/net/ethernet/broadcom/tg3.h @@ -3419,7 +3419,7 @@ struct tg3 { unsigned int irq_cnt; struct ethtool_coalesce coal; - struct ethtool_eee eee; + struct ethtool_keee eee; /* firmware info */ const char *fw_needed; diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index ae0b8b37b9bf..eb64118f5b18 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -240,7 +240,7 @@ static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) static int tsnep_phy_open(struct tsnep_adapter *adapter) { struct phy_device *phydev; - struct ethtool_eee ethtool_eee; + struct ethtool_keee ethtool_keee; int retval; retval = phy_connect_direct(adapter->netdev, adapter->phydev, @@ -259,8 +259,8 @@ static int tsnep_phy_open(struct tsnep_adapter *adapter) phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); /* disable EEE autoneg, EEE not supported by TSNEP */ - memset(ðtool_eee, 0, sizeof(ethtool_eee)); - phy_ethtool_set_eee(adapter->phydev, ðtool_eee); + memset(ðtool_keee, 0, sizeof(ethtool_keee)); + phy_ethtool_set_eee(adapter->phydev, ðtool_keee); adapter->phydev->irq = PHY_MAC_INTERRUPT; phy_start(adapter->phydev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index bfdbdab443ae..9f07f4947b63 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -2402,7 +2402,7 @@ static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) static int enetc_phylink_connect(struct net_device *ndev) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct ethtool_eee edata; + struct ethtool_keee edata; int err; if (!priv->phylink) { @@ -2418,7 +2418,7 @@ static int enetc_phylink_connect(struct net_device *ndev) } /* disable EEE autoneg, until ENETC driver supports it */ - memset(&edata, 0, sizeof(struct ethtool_eee)); + memset(&edata, 0, sizeof(struct ethtool_keee)); phylink_ethtool_set_eee(priv->phylink, &edata); phylink_start(priv->phylink); diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h index a8fbcada6b01..a19cb2a786fd 100644 --- a/drivers/net/ethernet/freescale/fec.h +++ b/drivers/net/ethernet/freescale/fec.h @@ -672,7 +672,7 @@ struct fec_enet_private { unsigned int itr_clk_rate; /* tx lpi eee mode */ - struct ethtool_eee eee; + struct ethtool_keee eee; unsigned int clk_ref_rate; /* ptp clock period in ns*/ diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 432523b2c789..63707e065141 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -3122,7 +3122,7 @@ static int fec_enet_us_to_tx_cycle(struct net_device *ndev, int us) static int fec_enet_eee_mode_set(struct net_device *ndev, bool enable) { struct fec_enet_private *fep = netdev_priv(ndev); - struct ethtool_eee *p = &fep->eee; + struct ethtool_keee *p = &fep->eee; unsigned int sleep_cycle, wake_cycle; int ret = 0; @@ -3149,10 +3149,10 @@ static int fec_enet_eee_mode_set(struct net_device *ndev, bool enable) } static int -fec_enet_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +fec_enet_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct fec_enet_private *fep = netdev_priv(ndev); - struct ethtool_eee *p = &fep->eee; + struct ethtool_keee *p = &fep->eee; if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) return -EOPNOTSUPP; @@ -3169,10 +3169,10 @@ fec_enet_get_eee(struct net_device *ndev, struct ethtool_eee *edata) } static int -fec_enet_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +fec_enet_set_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct fec_enet_private *fep = netdev_priv(ndev); - struct ethtool_eee *p = &fep->eee; + struct ethtool_keee *p = &fep->eee; int ret = 0; if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c index e3dfbd7a4236..a811238c018d 100644 --- a/drivers/net/ethernet/freescale/gianfar.c +++ b/drivers/net/ethernet/freescale/gianfar.c @@ -1649,7 +1649,7 @@ static int init_phy(struct net_device *dev) struct gfar_private *priv = netdev_priv(dev); phy_interface_t interface = priv->interface; struct phy_device *phydev; - struct ethtool_eee edata; + struct ethtool_keee edata; linkmode_set_bit_array(phy_10_100_features_array, ARRAY_SIZE(phy_10_100_features_array), @@ -1681,7 +1681,7 @@ static int init_phy(struct net_device *dev) phy_support_asym_pause(phydev); /* disable EEE autoneg, EEE not supported by eTSEC */ - memset(&edata, 0, sizeof(struct ethtool_eee)); + memset(&edata, 0, sizeof(struct ethtool_keee)); phy_ethtool_set_eee(phydev, &edata); return 0; diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index fc0f98ea6133..343f54b2b6ae 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -2186,7 +2186,7 @@ static int e1000_get_rxnfc(struct net_device *netdev, } } -static int e1000e_get_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int e1000e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; @@ -2262,11 +2262,11 @@ release: return ret_val; } -static int e1000e_set_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int e1000e_set_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; - struct ethtool_eee eee_curr; + struct ethtool_keee eee_curr; s32 ret_val; ret_val = e1000e_get_eee(netdev, &eee_curr); diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index c841779713f6..9dfda3c48af3 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -5644,7 +5644,7 @@ static int i40e_get_module_eeprom(struct net_device *netdev, return 0; } -static int i40e_get_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int i40e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_aq_get_phy_abilities_resp phy_cfg; @@ -5682,7 +5682,7 @@ static int i40e_get_eee(struct net_device *netdev, struct ethtool_eee *edata) } static int i40e_is_eee_param_supported(struct net_device *netdev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_vsi *vsi = np->vsi; @@ -5709,7 +5709,7 @@ static int i40e_is_eee_param_supported(struct net_device *netdev, return 0; } -static int i40e_set_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int i40e_set_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_aq_get_phy_abilities_resp abilities; diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index b66199c9bb3a..778d1e6cfc45 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -3027,7 +3027,7 @@ static int igb_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) return ret; } -static int igb_get_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int igb_get_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct igb_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; @@ -3106,11 +3106,11 @@ static int igb_get_eee(struct net_device *netdev, struct ethtool_eee *edata) } static int igb_set_eee(struct net_device *netdev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct igb_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; - struct ethtool_eee eee_curr; + struct ethtool_keee eee_curr; bool adv1g_eee = true, adv100m_eee = true; s32 ret_val; @@ -3118,7 +3118,7 @@ static int igb_set_eee(struct net_device *netdev, (hw->phy.media_type != e1000_media_type_copper)) return -EOPNOTSUPP; - memset(&eee_curr, 0, sizeof(struct ethtool_eee)); + memset(&eee_curr, 0, sizeof(struct ethtool_keee)); ret_val = igb_get_eee(netdev, &eee_curr); if (ret_val) diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 45430e246e9c..75f7c5ba65e0 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -168,7 +168,7 @@ struct igc_ring { struct igc_adapter { struct net_device *netdev; - struct ethtool_eee eee; + struct ethtool_keee eee; u16 eee_advert; unsigned long state; diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index b95d2c86e803..f2dcfe920f4f 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -1623,7 +1623,7 @@ static int igc_ethtool_set_priv_flags(struct net_device *netdev, u32 priv_flags) } static int igc_ethtool_get_eee(struct net_device *netdev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; @@ -1664,14 +1664,14 @@ static int igc_ethtool_get_eee(struct net_device *netdev, } static int igc_ethtool_set_eee(struct net_device *netdev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; - struct ethtool_eee eee_curr; + struct ethtool_keee eee_curr; s32 ret_val; - memset(&eee_curr, 0, sizeof(struct ethtool_eee)); + memset(&eee_curr, 0, sizeof(struct ethtool_keee)); ret_val = igc_ethtool_get_eee(netdev, &eee_curr); if (ret_val) { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 9a63457712c7..0aa73519a29d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -3425,7 +3425,7 @@ static const struct { }; static int -ixgbe_get_eee_fw(struct ixgbe_adapter *adapter, struct ethtool_eee *edata) +ixgbe_get_eee_fw(struct ixgbe_adapter *adapter, struct ethtool_keee *edata) { u32 info[FW_PHY_ACT_DATA_COUNT] = { 0 }; struct ixgbe_hw *hw = &adapter->hw; @@ -3462,7 +3462,7 @@ ixgbe_get_eee_fw(struct ixgbe_adapter *adapter, struct ethtool_eee *edata) return 0; } -static int ixgbe_get_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int ixgbe_get_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -3476,17 +3476,17 @@ static int ixgbe_get_eee(struct net_device *netdev, struct ethtool_eee *edata) return -EOPNOTSUPP; } -static int ixgbe_set_eee(struct net_device *netdev, struct ethtool_eee *edata) +static int ixgbe_set_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; - struct ethtool_eee eee_data; + struct ethtool_keee eee_data; s32 ret_val; if (!(adapter->flags2 & IXGBE_FLAG2_EEE_CAPABLE)) return -EOPNOTSUPP; - memset(&eee_data, 0, sizeof(struct ethtool_eee)); + memset(&eee_data, 0, sizeof(struct ethtool_keee)); ret_val = ixgbe_get_eee(netdev, &eee_data); if (ret_val) diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index a641b3534ca3..40a5f1431e4e 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -5097,7 +5097,7 @@ static int mvneta_ethtool_set_wol(struct net_device *dev, } static int mvneta_ethtool_get_eee(struct net_device *dev, - struct ethtool_eee *eee) + struct ethtool_keee *eee) { struct mvneta_port *pp = netdev_priv(dev); u32 lpi_ctl0; @@ -5113,7 +5113,7 @@ static int mvneta_ethtool_get_eee(struct net_device *dev, } static int mvneta_ethtool_set_eee(struct net_device *dev, - struct ethtool_eee *eee) + struct ethtool_keee *eee) { struct mvneta_port *pp = netdev_priv(dev); u32 lpi_ctl0; diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index a2b3f4433ca8..8a6ae171e375 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -1055,7 +1055,7 @@ static int lan743x_ethtool_get_ts_info(struct net_device *netdev, } static int lan743x_ethtool_get_eee(struct net_device *netdev, - struct ethtool_eee *eee) + struct ethtool_keee *eee) { struct lan743x_adapter *adapter = netdev_priv(netdev); struct phy_device *phydev = netdev->phydev; @@ -1092,7 +1092,7 @@ static int lan743x_ethtool_get_eee(struct net_device *netdev, } static int lan743x_ethtool_set_eee(struct net_device *netdev, - struct ethtool_eee *eee) + struct ethtool_keee *eee) { struct lan743x_adapter *adapter; struct phy_device *phydev; diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 0e240b5ab8d4..77491fb64039 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -1776,7 +1776,7 @@ static int qede_get_tunable(struct net_device *dev, return 0; } -static int qede_get_eee(struct net_device *dev, struct ethtool_eee *edata) +static int qede_get_eee(struct net_device *dev, struct ethtool_keee *edata) { struct qede_dev *edev = netdev_priv(dev); struct qed_link_output current_link; @@ -1810,7 +1810,7 @@ static int qede_get_eee(struct net_device *dev, struct ethtool_eee *edata) return 0; } -static int qede_set_eee(struct net_device *dev, struct ethtool_eee *edata) +static int qede_set_eee(struct net_device *dev, struct ethtool_keee *edata) { struct qede_dev *edev = netdev_priv(dev); struct qed_link_output current_link; diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index dd73df6b17b0..3d30d4499791 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -1974,7 +1974,7 @@ static int rtl_set_coalesce(struct net_device *dev, return 0; } -static int rtl8169_get_eee(struct net_device *dev, struct ethtool_eee *data) +static int rtl8169_get_eee(struct net_device *dev, struct ethtool_keee *data) { struct rtl8169_private *tp = netdev_priv(dev); @@ -1984,7 +1984,7 @@ static int rtl8169_get_eee(struct net_device *dev, struct ethtool_eee *data) return phy_ethtool_get_eee(tp->phydev, data); } -static int rtl8169_set_eee(struct net_device *dev, struct ethtool_eee *data) +static int rtl8169_set_eee(struct net_device *dev, struct ethtool_keee *data) { struct rtl8169_private *tp = netdev_priv(dev); int ret; diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c index 8ba017ec9849..d93b628b7046 100644 --- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c +++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c @@ -133,7 +133,7 @@ static const struct sxgbe_stats sxgbe_gstrings_stats[] = { #define SXGBE_STATS_LEN ARRAY_SIZE(sxgbe_gstrings_stats) static int sxgbe_get_eee(struct net_device *dev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct sxgbe_priv_data *priv = netdev_priv(dev); @@ -148,7 +148,7 @@ static int sxgbe_get_eee(struct net_device *dev, } static int sxgbe_set_eee(struct net_device *dev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct sxgbe_priv_data *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 42d27b97dd1d..bbecb3b89535 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -852,7 +852,7 @@ static int stmmac_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) } static int stmmac_ethtool_op_get_eee(struct net_device *dev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct stmmac_priv *priv = netdev_priv(dev); @@ -868,7 +868,7 @@ static int stmmac_ethtool_op_get_eee(struct net_device *dev, } static int stmmac_ethtool_op_set_eee(struct net_device *dev, - struct ethtool_eee *edata) + struct ethtool_keee *edata) { struct stmmac_priv *priv = netdev_priv(dev); int ret; diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c index 35fceba01ea4..d6ce2c9f0a8d 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c +++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c @@ -514,14 +514,14 @@ am65_cpsw_set_link_ksettings(struct net_device *ndev, return phylink_ethtool_ksettings_set(salve->phylink, ecmd); } -static int am65_cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +static int am65_cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev); return phylink_ethtool_get_eee(salve->phylink, edata); } -static int am65_cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +static int am65_cpsw_set_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct am65_cpsw_slave_data *salve = am65_ndev_to_slave(ndev); diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c index a557a477d039..f7b283353ba2 100644 --- a/drivers/net/ethernet/ti/cpsw_ethtool.c +++ b/drivers/net/ethernet/ti/cpsw_ethtool.c @@ -422,7 +422,7 @@ int cpsw_set_link_ksettings(struct net_device *ndev, return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy, ecmd); } -int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +int cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct cpsw_priv *priv = netdev_priv(ndev); struct cpsw_common *cpsw = priv->cpsw; @@ -434,7 +434,7 @@ int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata) return -EOPNOTSUPP; } -int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +int cpsw_set_eee(struct net_device *ndev, struct ethtool_keee *edata) { struct cpsw_priv *priv = netdev_priv(ndev); struct cpsw_common *cpsw = priv->cpsw; diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h index 0e27c433098d..7efa72502c86 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.h +++ b/drivers/net/ethernet/ti/cpsw_priv.h @@ -496,8 +496,8 @@ int cpsw_get_link_ksettings(struct net_device *ndev, struct ethtool_link_ksettings *ecmd); int cpsw_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *ecmd); -int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata); -int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata); +int cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata); +int cpsw_set_eee(struct net_device *ndev, struct ethtool_keee *edata); int cpsw_nway_reset(struct net_device *ndev); void cpsw_get_ringparam(struct net_device *ndev, struct ethtool_ringparam *ering, diff --git a/drivers/net/ethernet/ti/icssg/icssg_ethtool.c b/drivers/net/ethernet/ti/icssg/icssg_ethtool.c index a27ec1dcc8d5..9a7dd7efcf69 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_ethtool.c +++ b/drivers/net/ethernet/ti/icssg/icssg_ethtool.c @@ -45,7 +45,7 @@ static int emac_set_link_ksettings(struct net_device *ndev, return phy_ethtool_set_link_ksettings(ndev, ecmd); } -static int emac_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +static int emac_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { if (!ndev->phydev) return -EOPNOTSUPP; @@ -53,7 +53,7 @@ static int emac_get_eee(struct net_device *ndev, struct ethtool_eee *edata) return phy_ethtool_get_eee(ndev->phydev, edata); } -static int emac_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +static int emac_set_eee(struct net_device *ndev, struct ethtool_keee *edata) { if (!ndev->phydev) return -EOPNOTSUPP; diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index eba652a4c1d8..1faa22f58366 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -1461,7 +1461,7 @@ static int m88e1540_get_fld(struct phy_device *phydev, u8 *msecs) static int m88e1540_set_fld(struct phy_device *phydev, const u8 *msecs) { - struct ethtool_eee eee; + struct ethtool_keee eee; int val, ret; if (*msecs == ETHTOOL_PHY_FAST_LINK_DOWN_OFF) diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c index 747d14bf152c..adee5e712871 100644 --- a/drivers/net/phy/phy-c45.c +++ b/drivers/net/phy/phy-c45.c @@ -1443,13 +1443,13 @@ EXPORT_SYMBOL(genphy_c45_eee_is_active); /** * genphy_c45_ethtool_get_eee - get EEE supported and status * @phydev: target phy_device struct - * @data: ethtool_eee data + * @data: ethtool_keee data * * Description: it reports the Supported/Advertisement/LP Advertisement * capabilities. */ int genphy_c45_ethtool_get_eee(struct phy_device *phydev, - struct ethtool_eee *data) + struct ethtool_keee *data) { __ETHTOOL_DECLARE_LINK_MODE_MASK(adv) = {}; __ETHTOOL_DECLARE_LINK_MODE_MASK(lp) = {}; @@ -1481,7 +1481,7 @@ EXPORT_SYMBOL(genphy_c45_ethtool_get_eee); /** * genphy_c45_ethtool_set_eee - set EEE supported and status * @phydev: target phy_device struct - * @data: ethtool_eee data + * @data: ethtool_keee data * * Description: sets the Supported/Advertisement/LP Advertisement * capabilities. If eee_enabled is false, no links modes are @@ -1490,7 +1490,7 @@ EXPORT_SYMBOL(genphy_c45_ethtool_get_eee); * non-destructive way. */ int genphy_c45_ethtool_set_eee(struct phy_device *phydev, - struct ethtool_eee *data) + struct ethtool_keee *data) { int ret; diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 3376e58e2b88..3b9531143be1 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -1632,12 +1632,12 @@ EXPORT_SYMBOL(phy_get_eee_err); /** * phy_ethtool_get_eee - get EEE supported and status * @phydev: target phy_device struct - * @data: ethtool_eee data + * @data: ethtool_keee data * * Description: it reportes the Supported/Advertisement/LP Advertisement * capabilities. */ -int phy_ethtool_get_eee(struct phy_device *phydev, struct ethtool_eee *data) +int phy_ethtool_get_eee(struct phy_device *phydev, struct ethtool_keee *data) { int ret; @@ -1655,11 +1655,11 @@ EXPORT_SYMBOL(phy_ethtool_get_eee); /** * phy_ethtool_set_eee - set EEE supported and status * @phydev: target phy_device struct - * @data: ethtool_eee data + * @data: ethtool_keee data * * Description: it is to program the Advertisement EEE register. */ -int phy_ethtool_set_eee(struct phy_device *phydev, struct ethtool_eee *data) +int phy_ethtool_set_eee(struct phy_device *phydev, struct ethtool_keee *data) { int ret; diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index ed0b4ccaa6a6..503fd7c40523 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -2764,9 +2764,9 @@ EXPORT_SYMBOL_GPL(phylink_init_eee); /** * phylink_ethtool_get_eee() - read the energy efficient ethernet parameters * @pl: a pointer to a &struct phylink returned from phylink_create() - * @eee: a pointer to a &struct ethtool_eee for the read parameters + * @eee: a pointer to a &struct ethtool_keee for the read parameters */ -int phylink_ethtool_get_eee(struct phylink *pl, struct ethtool_eee *eee) +int phylink_ethtool_get_eee(struct phylink *pl, struct ethtool_keee *eee) { int ret = -EOPNOTSUPP; @@ -2782,9 +2782,9 @@ EXPORT_SYMBOL_GPL(phylink_ethtool_get_eee); /** * phylink_ethtool_set_eee() - set the energy efficient ethernet parameters * @pl: a pointer to a &struct phylink returned from phylink_create() - * @eee: a pointer to a &struct ethtool_eee for the desired parameters + * @eee: a pointer to a &struct ethtool_keee for the desired parameters */ -int phylink_ethtool_set_eee(struct phylink *pl, struct ethtool_eee *eee) +int phylink_ethtool_set_eee(struct phylink *pl, struct ethtool_keee *eee) { int ret = -EOPNOTSUPP; diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index d837c1887416..3922a9afdd1f 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -667,7 +667,7 @@ static int ax88179_set_link_ksettings(struct net_device *net, } static int -ax88179_ethtool_get_eee(struct usbnet *dev, struct ethtool_eee *data) +ax88179_ethtool_get_eee(struct usbnet *dev, struct ethtool_keee *data) { int val; @@ -696,7 +696,7 @@ ax88179_ethtool_get_eee(struct usbnet *dev, struct ethtool_eee *data) } static int -ax88179_ethtool_set_eee(struct usbnet *dev, struct ethtool_eee *data) +ax88179_ethtool_set_eee(struct usbnet *dev, struct ethtool_keee *data) { u16 tmp16 = ethtool_adv_to_mmd_eee_adv_t(data->advertised); @@ -807,7 +807,7 @@ static void ax88179_enable_eee(struct usbnet *dev) GMII_PHY_PAGE_SELECT, 2, &tmp16); } -static int ax88179_get_eee(struct net_device *net, struct ethtool_eee *edata) +static int ax88179_get_eee(struct net_device *net, struct ethtool_keee *edata) { struct usbnet *dev = netdev_priv(net); struct ax88179_data *priv = dev->driver_priv; @@ -818,7 +818,7 @@ static int ax88179_get_eee(struct net_device *net, struct ethtool_eee *edata) return ax88179_ethtool_get_eee(dev, edata); } -static int ax88179_set_eee(struct net_device *net, struct ethtool_eee *edata) +static int ax88179_set_eee(struct net_device *net, struct ethtool_keee *edata) { struct usbnet *dev = netdev_priv(net); struct ax88179_data *priv = dev->driver_priv; @@ -1587,7 +1587,7 @@ static int ax88179_reset(struct usbnet *dev) u16 *tmp16; u8 *tmp; struct ax88179_data *ax179_data = dev->driver_priv; - struct ethtool_eee eee_data; + struct ethtool_keee eee_data; tmp16 = (u16 *)buf; tmp = (u8 *)buf; diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index a6d653ff552a..106282612bc2 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -1673,7 +1673,7 @@ static int lan78xx_set_wol(struct net_device *netdev, return ret; } -static int lan78xx_get_eee(struct net_device *net, struct ethtool_eee *edata) +static int lan78xx_get_eee(struct net_device *net, struct ethtool_keee *edata) { struct lan78xx_net *dev = netdev_priv(net); struct phy_device *phydev = net->phydev; @@ -1709,7 +1709,7 @@ exit: return ret; } -static int lan78xx_set_eee(struct net_device *net, struct ethtool_eee *edata) +static int lan78xx_set_eee(struct net_device *net, struct ethtool_keee *edata) { struct lan78xx_net *dev = netdev_priv(net); int ret; diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 0d0672d2a654..dc163b766a89 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -891,8 +891,8 @@ struct r8152 { void (*up)(struct r8152 *tp); void (*down)(struct r8152 *tp); void (*unload)(struct r8152 *tp); - int (*eee_get)(struct r8152 *tp, struct ethtool_eee *eee); - int (*eee_set)(struct r8152 *tp, struct ethtool_eee *eee); + int (*eee_get)(struct r8152 *tp, struct ethtool_keee *eee); + int (*eee_set)(struct r8152 *tp, struct ethtool_keee *eee); bool (*in_nway)(struct r8152 *tp); void (*hw_phy_cfg)(struct r8152 *tp); void (*autosuspend_en)(struct r8152 *tp, bool enable); @@ -8922,7 +8922,7 @@ static void rtl8152_get_strings(struct net_device *dev, u32 stringset, u8 *data) } } -static int r8152_get_eee(struct r8152 *tp, struct ethtool_eee *eee) +static int r8152_get_eee(struct r8152 *tp, struct ethtool_keee *eee) { u32 lp, adv, supported = 0; u16 val; @@ -8945,7 +8945,7 @@ static int r8152_get_eee(struct r8152 *tp, struct ethtool_eee *eee) return 0; } -static int r8152_set_eee(struct r8152 *tp, struct ethtool_eee *eee) +static int r8152_set_eee(struct r8152 *tp, struct ethtool_keee *eee) { u16 val = ethtool_adv_to_mmd_eee_adv_t(eee->advertised); @@ -8957,7 +8957,7 @@ static int r8152_set_eee(struct r8152 *tp, struct ethtool_eee *eee) return 0; } -static int r8153_get_eee(struct r8152 *tp, struct ethtool_eee *eee) +static int r8153_get_eee(struct r8152 *tp, struct ethtool_keee *eee) { u32 lp, adv, supported = 0; u16 val; @@ -8981,7 +8981,7 @@ static int r8153_get_eee(struct r8152 *tp, struct ethtool_eee *eee) } static int -rtl_ethtool_get_eee(struct net_device *net, struct ethtool_eee *edata) +rtl_ethtool_get_eee(struct net_device *net, struct ethtool_keee *edata) { struct r8152 *tp = netdev_priv(net); int ret; @@ -9008,7 +9008,7 @@ out: } static int -rtl_ethtool_set_eee(struct net_device *net, struct ethtool_eee *edata) +rtl_ethtool_set_eee(struct net_device *net, struct ethtool_keee *edata) { struct r8152 *tp = netdev_priv(net); int ret; diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 325e0778e937..a850bab8489f 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -222,6 +222,18 @@ extern int __ethtool_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *link_ksettings); +struct ethtool_keee { + u32 cmd; + u32 supported; + u32 advertised; + u32 lp_advertised; + u32 eee_active; + u32 eee_enabled; + u32 tx_lpi_enabled; + u32 tx_lpi_timer; + u32 reserved[2]; +}; + struct kernel_ethtool_coalesce { u8 use_cqe_mode_tx; u8 use_cqe_mode_rx; @@ -892,8 +904,8 @@ struct ethtool_ops { struct ethtool_modinfo *); int (*get_module_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); - int (*get_eee)(struct net_device *, struct ethtool_eee *); - int (*set_eee)(struct net_device *, struct ethtool_eee *); + int (*get_eee)(struct net_device *dev, struct ethtool_keee *eee); + int (*set_eee)(struct net_device *dev, struct ethtool_keee *eee); int (*get_tunable)(struct net_device *, const struct ethtool_tunable *, void *); int (*set_tunable)(struct net_device *, diff --git a/include/linux/phy.h b/include/linux/phy.h index c9994a59ca2e..a66f07d3f5f4 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1908,9 +1908,9 @@ int genphy_c45_plca_get_status(struct phy_device *phydev, int genphy_c45_eee_is_active(struct phy_device *phydev, unsigned long *adv, unsigned long *lp, bool *is_enabled); int genphy_c45_ethtool_get_eee(struct phy_device *phydev, - struct ethtool_eee *data); + struct ethtool_keee *data); int genphy_c45_ethtool_set_eee(struct phy_device *phydev, - struct ethtool_eee *data); + struct ethtool_keee *data); int genphy_c45_write_eee_adv(struct phy_device *phydev, unsigned long *adv); int genphy_c45_an_config_eee_aneg(struct phy_device *phydev); int genphy_c45_read_eee_adv(struct phy_device *phydev, unsigned long *adv); @@ -1988,8 +1988,8 @@ int phy_unregister_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask); int phy_init_eee(struct phy_device *phydev, bool clk_stop_enable); int phy_get_eee_err(struct phy_device *phydev); -int phy_ethtool_set_eee(struct phy_device *phydev, struct ethtool_eee *data); -int phy_ethtool_get_eee(struct phy_device *phydev, struct ethtool_eee *data); +int phy_ethtool_set_eee(struct phy_device *phydev, struct ethtool_keee *data); +int phy_ethtool_get_eee(struct phy_device *phydev, struct ethtool_keee *data); int phy_ethtool_set_wol(struct phy_device *phydev, struct ethtool_wolinfo *wol); void phy_ethtool_get_wol(struct phy_device *phydev, struct ethtool_wolinfo *wol); diff --git a/include/linux/phylink.h b/include/linux/phylink.h index d589f89c612c..6ba411732a0d 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -584,8 +584,8 @@ int phylink_ethtool_set_pauseparam(struct phylink *, struct ethtool_pauseparam *); int phylink_get_eee_err(struct phylink *); int phylink_init_eee(struct phylink *, bool); -int phylink_ethtool_get_eee(struct phylink *, struct ethtool_eee *); -int phylink_ethtool_set_eee(struct phylink *, struct ethtool_eee *); +int phylink_ethtool_get_eee(struct phylink *link, struct ethtool_keee *eee); +int phylink_ethtool_set_eee(struct phylink *link, struct ethtool_keee *eee); int phylink_mii_ioctl(struct phylink *, struct ifreq *, int); int phylink_speed_down(struct phylink *pl, bool sync); int phylink_speed_up(struct phylink *pl); diff --git a/include/net/dsa.h b/include/net/dsa.h index 82135fbdb1e6..7c0da9effe4e 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -991,9 +991,9 @@ struct dsa_switch_ops { * Port's MAC EEE settings */ int (*set_mac_eee)(struct dsa_switch *ds, int port, - struct ethtool_eee *e); + struct ethtool_keee *e); int (*get_mac_eee)(struct dsa_switch *ds, int port, - struct ethtool_eee *e); + struct ethtool_keee *e); /* EEPROM access */ int (*get_eeprom_len)(struct dsa_switch *ds); diff --git a/net/dsa/user.c b/net/dsa/user.c index b15e71cc342c..e03da3a4ac38 100644 --- a/net/dsa/user.c +++ b/net/dsa/user.c @@ -1222,7 +1222,7 @@ static int dsa_user_set_wol(struct net_device *dev, struct ethtool_wolinfo *w) return ret; } -static int dsa_user_set_eee(struct net_device *dev, struct ethtool_eee *e) +static int dsa_user_set_eee(struct net_device *dev, struct ethtool_keee *e) { struct dsa_port *dp = dsa_user_to_port(dev); struct dsa_switch *ds = dp->ds; @@ -1242,7 +1242,7 @@ static int dsa_user_set_eee(struct net_device *dev, struct ethtool_eee *e) return phylink_ethtool_set_eee(dp->pl, e); } -static int dsa_user_get_eee(struct net_device *dev, struct ethtool_eee *e) +static int dsa_user_get_eee(struct net_device *dev, struct ethtool_keee *e) { struct dsa_port *dp = dsa_user_to_port(dev); struct dsa_switch *ds = dp->ds; diff --git a/net/ethtool/eee.c b/net/ethtool/eee.c index 2853394d06a8..21b0e845a531 100644 --- a/net/ethtool/eee.c +++ b/net/ethtool/eee.c @@ -5,7 +5,7 @@ #include "bitset.h" #define EEE_MODES_COUNT \ - (sizeof_field(struct ethtool_eee, supported) * BITS_PER_BYTE) + (sizeof_field(struct ethtool_keee, supported) * BITS_PER_BYTE) struct eee_req_info { struct ethnl_req_info base; @@ -13,7 +13,7 @@ struct eee_req_info { struct eee_reply_data { struct ethnl_reply_data base; - struct ethtool_eee eee; + struct ethtool_keee eee; }; #define EEE_REPDATA(__reply_base) \ @@ -48,7 +48,7 @@ static int eee_reply_size(const struct ethnl_req_info *req_base, { bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS; const struct eee_reply_data *data = EEE_REPDATA(reply_base); - const struct ethtool_eee *eee = &data->eee; + const struct ethtool_keee *eee = &data->eee; int len = 0; int ret; @@ -84,7 +84,7 @@ static int eee_fill_reply(struct sk_buff *skb, { bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS; const struct eee_reply_data *data = EEE_REPDATA(reply_base); - const struct ethtool_eee *eee = &data->eee; + const struct ethtool_keee *eee = &data->eee; int ret; ret = ethnl_put_bitset32(skb, ETHTOOL_A_EEE_MODES_OURS, @@ -132,7 +132,7 @@ ethnl_set_eee(struct ethnl_req_info *req_info, struct genl_info *info) { struct net_device *dev = req_info->dev; struct nlattr **tb = info->attrs; - struct ethtool_eee eee = {}; + struct ethtool_keee eee = {}; bool mod = false; int ret; diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 7519b0818b91..b02ca72f438e 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1510,13 +1510,13 @@ static int ethtool_set_wol(struct net_device *dev, char __user *useraddr) static int ethtool_get_eee(struct net_device *dev, char __user *useraddr) { - struct ethtool_eee edata; + struct ethtool_keee edata; int rc; if (!dev->ethtool_ops->get_eee) return -EOPNOTSUPP; - memset(&edata, 0, sizeof(struct ethtool_eee)); + memset(&edata, 0, sizeof(struct ethtool_keee)); edata.cmd = ETHTOOL_GEEE; rc = dev->ethtool_ops->get_eee(dev, &edata); @@ -1531,7 +1531,7 @@ static int ethtool_get_eee(struct net_device *dev, char __user *useraddr) static int ethtool_set_eee(struct net_device *dev, char __user *useraddr) { - struct ethtool_eee edata; + struct ethtool_keee edata; int ret; if (!dev->ethtool_ops->set_eee) -- cgit v1.2.3 From 285cc15cc555b4f05ebf2556bc6e85a6d36b790a Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 27 Jan 2024 14:26:50 +0100 Subject: ethtool: adjust struct ethtool_keee to kernel needs This patch changes the following in struct ethtool_keee - remove member cmd, it's not needed on kernel side - remove reserved fields - switch the semantically boolean members to type bool We don't have to change any user of the boolean members due to the implicit casting from/to bool. A small change is needed where a pointer to bool members is used, in addition remove few now unneeded double negations. Reviewed-by: Andrew Lunn Signed-off-by: Heiner Kallweit Signed-off-by: David S. Miller --- include/linux/ethtool.h | 8 +++----- net/ethtool/eee.c | 12 ++++++------ 2 files changed, 9 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index a850bab8489f..14549cb9e2b2 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -223,15 +223,13 @@ __ethtool_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *link_ksettings); struct ethtool_keee { - u32 cmd; u32 supported; u32 advertised; u32 lp_advertised; - u32 eee_active; - u32 eee_enabled; - u32 tx_lpi_enabled; u32 tx_lpi_timer; - u32 reserved[2]; + bool tx_lpi_enabled; + bool eee_active; + bool eee_enabled; }; struct kernel_ethtool_coalesce { diff --git a/net/ethtool/eee.c b/net/ethtool/eee.c index 21b0e845a531..ac9f694ffe20 100644 --- a/net/ethtool/eee.c +++ b/net/ethtool/eee.c @@ -98,10 +98,10 @@ static int eee_fill_reply(struct sk_buff *skb, if (ret < 0) return ret; - if (nla_put_u8(skb, ETHTOOL_A_EEE_ACTIVE, !!eee->eee_active) || - nla_put_u8(skb, ETHTOOL_A_EEE_ENABLED, !!eee->eee_enabled) || + if (nla_put_u8(skb, ETHTOOL_A_EEE_ACTIVE, eee->eee_active) || + nla_put_u8(skb, ETHTOOL_A_EEE_ENABLED, eee->eee_enabled) || nla_put_u8(skb, ETHTOOL_A_EEE_TX_LPI_ENABLED, - !!eee->tx_lpi_enabled) || + eee->tx_lpi_enabled) || nla_put_u32(skb, ETHTOOL_A_EEE_TX_LPI_TIMER, eee->tx_lpi_timer)) return -EMSGSIZE; @@ -145,9 +145,9 @@ ethnl_set_eee(struct ethnl_req_info *req_info, struct genl_info *info) link_mode_names, info->extack, &mod); if (ret < 0) return ret; - ethnl_update_bool32(&eee.eee_enabled, tb[ETHTOOL_A_EEE_ENABLED], &mod); - ethnl_update_bool32(&eee.tx_lpi_enabled, - tb[ETHTOOL_A_EEE_TX_LPI_ENABLED], &mod); + ethnl_update_bool(&eee.eee_enabled, tb[ETHTOOL_A_EEE_ENABLED], &mod); + ethnl_update_bool(&eee.tx_lpi_enabled, tb[ETHTOOL_A_EEE_TX_LPI_ENABLED], + &mod); ethnl_update_u32(&eee.tx_lpi_timer, tb[ETHTOOL_A_EEE_TX_LPI_TIMER], &mod); if (!mod) -- cgit v1.2.3 From 1d756ff13da6a2222ac4387511f2a0e2e83ce670 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 27 Jan 2024 14:28:47 +0100 Subject: ethtool: add suffix _u32 to legacy bitmap members of struct ethtool_keee This is in preparation of using the existing names for linkmode bitmaps. Suggested-by: Andrew Lunn Signed-off-by: Heiner Kallweit Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- .../net/ethernet/aquantia/atlantic/aq_ethtool.c | 8 +++---- .../net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c | 8 +++---- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 12 +++++----- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 14 +++++------ drivers/net/ethernet/broadcom/tg3.c | 22 ++++++++--------- drivers/net/ethernet/intel/e1000e/ethtool.c | 10 ++++---- drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 10 ++++---- drivers/net/ethernet/intel/igb/igb_ethtool.c | 20 ++++++++-------- drivers/net/ethernet/intel/igc/igc_ethtool.c | 12 +++++----- drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 18 +++++++------- drivers/net/ethernet/qlogic/qede/qede_ethtool.c | 28 +++++++++++----------- drivers/net/phy/phy-c45.c | 12 +++++----- drivers/net/usb/ax88179_178a.c | 10 ++++---- drivers/net/usb/r8152.c | 14 +++++------ include/linux/ethtool.h | 6 ++--- net/ethtool/eee.c | 16 ++++++------- net/ethtool/ioctl.c | 12 +++++----- 17 files changed, 116 insertions(+), 116 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index be865776de55..0bd1a0a1ae6a 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -713,14 +713,14 @@ static int aq_ethtool_get_eee(struct net_device *ndev, struct ethtool_keee *eee) if (err < 0) return err; - eee->supported = eee_mask_to_ethtool_mask(supported_rates); + eee->supported_u32 = eee_mask_to_ethtool_mask(supported_rates); if (aq_nic->aq_nic_cfg.eee_speeds) - eee->advertised = eee->supported; + eee->advertised_u32 = eee->supported_u32; - eee->lp_advertised = eee_mask_to_ethtool_mask(rate); + eee->lp_advertised_u32 = eee_mask_to_ethtool_mask(rate); - eee->eee_enabled = !!eee->advertised; + eee->eee_enabled = !!eee->advertised_u32; eee->tx_lpi_enabled = eee->eee_enabled; if ((supported_rates & rate) & AQ_NIC_RATE_EEE_MSK) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index 12d2785eeb8a..5f0e1759d078 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -2120,14 +2120,14 @@ static int bnx2x_get_eee(struct net_device *dev, struct ethtool_keee *edata) eee_cfg = bp->link_vars.eee_status; - edata->supported = + edata->supported_u32 = bnx2x_eee_to_adv((eee_cfg & SHMEM_EEE_SUPPORTED_MASK) >> SHMEM_EEE_SUPPORTED_SHIFT); - edata->advertised = + edata->advertised_u32 = bnx2x_eee_to_adv((eee_cfg & SHMEM_EEE_ADV_STATUS_MASK) >> SHMEM_EEE_ADV_STATUS_SHIFT); - edata->lp_advertised = + edata->lp_advertised_u32 = bnx2x_eee_to_adv((eee_cfg & SHMEM_EEE_LP_ADV_STATUS_MASK) >> SHMEM_EEE_LP_ADV_STATUS_SHIFT); @@ -2162,7 +2162,7 @@ static int bnx2x_set_eee(struct net_device *dev, struct ethtool_keee *edata) return -EOPNOTSUPP; } - advertised = bnx2x_adv_to_eee(edata->advertised, + advertised = bnx2x_adv_to_eee(edata->advertised_u32, SHMEM_EEE_ADV_STATUS_SHIFT); if ((advertised != (eee_cfg & SHMEM_EEE_ADV_STATUS_MASK))) { DP(BNX2X_MSG_ETHTOOL, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index d7626c26f9a9..fde32b32fa81 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -10624,7 +10624,7 @@ static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) struct ethtool_keee *eee = &bp->eee; u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); - eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); + eee->supported_u32 = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & @@ -10775,7 +10775,7 @@ int bnxt_update_link(struct bnxt *bp, bool chng_link_state) eee->eee_active = 1; fw_speeds = le16_to_cpu( resp->link_partner_adv_eee_link_speed_mask); - eee->lp_advertised = + eee->lp_advertised_u32 = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); } @@ -10786,7 +10786,7 @@ int bnxt_update_link(struct bnxt *bp, bool chng_link_state) eee->eee_enabled = 1; fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); - eee->advertised = + eee->advertised_u32 = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); if (resp->eee_config_phy_addr & @@ -10969,7 +10969,7 @@ static void bnxt_hwrm_set_eee(struct bnxt *bp, flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; req->flags |= cpu_to_le32(flags); - eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); + eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised_u32); req->eee_link_speed_mask = cpu_to_le16(eee_speeds); req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); } else { @@ -11336,8 +11336,8 @@ static bool bnxt_eee_config_ok(struct bnxt *bp) eee->eee_enabled = 0; return false; } - if (eee->advertised & ~advertising) { - eee->advertised = advertising & eee->supported; + if (eee->advertised_u32 & ~advertising) { + eee->advertised_u32 = advertising & eee->supported_u32; return false; } } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index d6a8577d68af..481b835a7703 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -3919,16 +3919,16 @@ static int bnxt_set_eee(struct net_device *dev, struct ethtool_keee *edata) edata->tx_lpi_timer = eee->tx_lpi_timer; } } - if (!edata->advertised) { - edata->advertised = advertising & eee->supported; - } else if (edata->advertised & ~advertising) { + if (!edata->advertised_u32) { + edata->advertised_u32 = advertising & eee->supported_u32; + } else if (edata->advertised_u32 & ~advertising) { netdev_warn(dev, "EEE advertised %x must be a subset of autoneg advertised speeds %x\n", - edata->advertised, advertising); + edata->advertised_u32, advertising); rc = -EINVAL; goto eee_exit; } - eee->advertised = edata->advertised; + eee->advertised_u32 = edata->advertised_u32; eee->tx_lpi_enabled = edata->tx_lpi_enabled; eee->tx_lpi_timer = edata->tx_lpi_timer; eee_ok: @@ -3954,12 +3954,12 @@ static int bnxt_get_eee(struct net_device *dev, struct ethtool_keee *edata) /* Preserve tx_lpi_timer so that the last value will be used * by default when it is re-enabled. */ - edata->advertised = 0; + edata->advertised_u32 = 0; edata->tx_lpi_enabled = 0; } if (!bp->eee.eee_active) - edata->lp_advertised = 0; + edata->lp_advertised_u32 = 0; return 0; } diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 11054177c2ec..f644a91317c9 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -2362,13 +2362,13 @@ static void tg3_eee_pull_config(struct tg3 *tp, struct ethtool_keee *eee) /* Pull lp advertised settings */ if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_LPABLE, &val)) return; - dest->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(val); + dest->lp_advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(val); /* Pull advertised and eee_enabled settings */ if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, &val)) return; dest->eee_enabled = !!val; - dest->advertised = mmd_eee_adv_to_ethtool_adv_t(val); + dest->advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(val); /* Pull tx_lpi_enabled */ val = tr32(TG3_CPMU_EEE_MODE); @@ -4364,9 +4364,9 @@ static int tg3_phy_autoneg_cfg(struct tg3 *tp, u32 advertise, u32 flowctrl) if (!tp->eee.eee_enabled) { val = 0; - tp->eee.advertised = 0; + tp->eee.advertised_u32 = 0; } else { - tp->eee.advertised = advertise & + tp->eee.advertised_u32 = advertise & (ADVERTISED_100baseT_Full | ADVERTISED_1000baseT_Full); } @@ -4626,13 +4626,13 @@ static bool tg3_phy_eee_config_ok(struct tg3 *tp) tg3_eee_pull_config(tp, &eee); if (tp->eee.eee_enabled) { - if (tp->eee.advertised != eee.advertised || + if (tp->eee.advertised_u32 != eee.advertised_u32 || tp->eee.tx_lpi_timer != eee.tx_lpi_timer || tp->eee.tx_lpi_enabled != eee.tx_lpi_enabled) return false; } else { /* EEE is disabled but we're advertising */ - if (eee.advertised) + if (eee.advertised_u32) return false; } @@ -14189,7 +14189,7 @@ static int tg3_set_eee(struct net_device *dev, struct ethtool_keee *edata) return -EOPNOTSUPP; } - if (edata->advertised != tp->eee.advertised) { + if (edata->advertised_u32 != tp->eee.advertised_u32) { netdev_warn(tp->dev, "Direct manipulation of EEE advertisement is not supported\n"); return -EINVAL; @@ -15655,10 +15655,10 @@ static int tg3_phy_probe(struct tg3 *tp) tg3_chip_rev_id(tp) != CHIPREV_ID_57765_A0))) { tp->phy_flags |= TG3_PHYFLG_EEE_CAP; - tp->eee.supported = SUPPORTED_100baseT_Full | - SUPPORTED_1000baseT_Full; - tp->eee.advertised = ADVERTISED_100baseT_Full | - ADVERTISED_1000baseT_Full; + tp->eee.supported_u32 = SUPPORTED_100baseT_Full | + SUPPORTED_1000baseT_Full; + tp->eee.advertised_u32 = ADVERTISED_100baseT_Full | + ADVERTISED_1000baseT_Full; tp->eee.eee_enabled = 1; tp->eee.tx_lpi_enabled = 1; tp->eee.tx_lpi_timer = TG3_CPMU_DBTMR1_LNKIDLE_2047US; diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index 343f54b2b6ae..ff243ae71b78 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -2223,16 +2223,16 @@ static int e1000e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) ret_val = e1000_read_emi_reg_locked(hw, cap_addr, &phy_data); if (ret_val) goto release; - edata->supported = mmd_eee_cap_to_ethtool_sup_t(phy_data); + edata->supported_u32 = mmd_eee_cap_to_ethtool_sup_t(phy_data); /* EEE Advertised */ - edata->advertised = mmd_eee_adv_to_ethtool_adv_t(adapter->eee_advert); + edata->advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(adapter->eee_advert); /* EEE Link Partner Advertised */ ret_val = e1000_read_emi_reg_locked(hw, lpa_addr, &phy_data); if (ret_val) goto release; - edata->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(phy_data); + edata->lp_advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(phy_data); /* EEE PCS Status */ ret_val = e1000_read_emi_reg_locked(hw, pcs_stat_addr, &phy_data); @@ -2283,12 +2283,12 @@ static int e1000e_set_eee(struct net_device *netdev, struct ethtool_keee *edata) return -EINVAL; } - if (edata->advertised & ~(ADVERTISE_100_FULL | ADVERTISE_1000_FULL)) { + if (edata->advertised_u32 & ~(ADVERTISE_100_FULL | ADVERTISE_1000_FULL)) { e_err("EEE advertisement supports only 100TX and/or 1000T full-duplex\n"); return -EINVAL; } - adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised); + adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised_u32); hw->dev_spec.ich8lan.eee_disable = !edata->eee_enabled; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 9dfda3c48af3..1b5473358e1a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -5664,16 +5664,16 @@ static int i40e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) if (phy_cfg.eee_capability == 0) return -EOPNOTSUPP; - edata->supported = SUPPORTED_Autoneg; - edata->lp_advertised = edata->supported; + edata->supported_u32 = SUPPORTED_Autoneg; + edata->lp_advertised_u32 = edata->supported_u32; /* Get current configuration */ status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_cfg, NULL); if (status) return -EAGAIN; - edata->advertised = phy_cfg.eee_capability ? SUPPORTED_Autoneg : 0U; - edata->eee_enabled = !!edata->advertised; + edata->advertised_u32 = phy_cfg.eee_capability ? SUPPORTED_Autoneg : 0U; + edata->eee_enabled = !!edata->advertised_u32; edata->tx_lpi_enabled = pf->stats.tx_lpi_status; edata->eee_active = pf->stats.tx_lpi_status && pf->stats.rx_lpi_status; @@ -5691,7 +5691,7 @@ static int i40e_is_eee_param_supported(struct net_device *netdev, u32 value; const char *name; } param[] = { - {edata->advertised & ~SUPPORTED_Autoneg, "advertise"}, + {edata->advertised_u32 & ~SUPPORTED_Autoneg, "advertise"}, {edata->tx_lpi_timer, "tx-timer"}, {edata->tx_lpi_enabled != pf->stats.tx_lpi_status, "tx-lpi"} }; diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 778d1e6cfc45..b87b23d2151c 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -3038,10 +3038,10 @@ static int igb_get_eee(struct net_device *netdev, struct ethtool_keee *edata) (hw->phy.media_type != e1000_media_type_copper)) return -EOPNOTSUPP; - edata->supported = (SUPPORTED_1000baseT_Full | - SUPPORTED_100baseT_Full); + edata->supported_u32 = (SUPPORTED_1000baseT_Full | + SUPPORTED_100baseT_Full); if (!hw->dev_spec._82575.eee_disable) - edata->advertised = + edata->advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(adapter->eee_advert); /* The IPCNFG and EEER registers are not supported on I354. */ @@ -3068,7 +3068,7 @@ static int igb_get_eee(struct net_device *netdev, struct ethtool_keee *edata) if (ret_val) return -ENODATA; - edata->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(phy_data); + edata->lp_advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(phy_data); break; case e1000_i354: case e1000_i210: @@ -3079,7 +3079,7 @@ static int igb_get_eee(struct net_device *netdev, struct ethtool_keee *edata) if (ret_val) return -ENODATA; - edata->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(phy_data); + edata->lp_advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(phy_data); break; default: @@ -3099,7 +3099,7 @@ static int igb_get_eee(struct net_device *netdev, struct ethtool_keee *edata) edata->eee_enabled = false; edata->eee_active = false; edata->tx_lpi_enabled = false; - edata->advertised &= ~edata->advertised; + edata->advertised_u32 &= ~edata->advertised_u32; } return 0; @@ -3138,14 +3138,14 @@ static int igb_set_eee(struct net_device *netdev, return -EINVAL; } - if (!edata->advertised || (edata->advertised & + if (!edata->advertised_u32 || (edata->advertised_u32 & ~(ADVERTISE_100_FULL | ADVERTISE_1000_FULL))) { dev_err(&adapter->pdev->dev, "EEE Advertisement supports only 100Tx and/or 100T full duplex\n"); return -EINVAL; } - adv100m_eee = !!(edata->advertised & ADVERTISE_100_FULL); - adv1g_eee = !!(edata->advertised & ADVERTISE_1000_FULL); + adv100m_eee = !!(edata->advertised_u32 & ADVERTISE_100_FULL); + adv1g_eee = !!(edata->advertised_u32 & ADVERTISE_1000_FULL); } else if (!edata->eee_enabled) { dev_err(&adapter->pdev->dev, @@ -3153,7 +3153,7 @@ static int igb_set_eee(struct net_device *netdev, return -EINVAL; } - adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised); + adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised_u32); if (hw->dev_spec._82575.eee_disable != !edata->eee_enabled) { hw->dev_spec._82575.eee_disable = !edata->eee_enabled; adapter->flags |= IGB_FLAG_EEE; diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index f2dcfe920f4f..7f844e967421 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -1630,11 +1630,11 @@ static int igc_ethtool_get_eee(struct net_device *netdev, u32 eeer; if (hw->dev_spec._base.eee_enable) - edata->advertised = + edata->advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(adapter->eee_advert); *edata = adapter->eee; - edata->supported = SUPPORTED_Autoneg; + edata->supported_u32 = SUPPORTED_Autoneg; eeer = rd32(IGC_EEER); @@ -1647,8 +1647,8 @@ static int igc_ethtool_get_eee(struct net_device *netdev, edata->eee_enabled = hw->dev_spec._base.eee_enable; - edata->advertised = SUPPORTED_Autoneg; - edata->lp_advertised = SUPPORTED_Autoneg; + edata->advertised_u32 = SUPPORTED_Autoneg; + edata->lp_advertised_u32 = SUPPORTED_Autoneg; /* Report correct negotiated EEE status for devices that * wrongly report EEE at half-duplex @@ -1657,7 +1657,7 @@ static int igc_ethtool_get_eee(struct net_device *netdev, edata->eee_enabled = false; edata->eee_active = false; edata->tx_lpi_enabled = false; - edata->advertised &= ~edata->advertised; + edata->advertised_u32 &= ~edata->advertised_u32; } return 0; @@ -1699,7 +1699,7 @@ static int igc_ethtool_set_eee(struct net_device *netdev, return -EINVAL; } - adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised); + adapter->eee_advert = ethtool_adv_to_mmd_eee_adv_t(edata->advertised_u32); if (hw->dev_spec._base.eee_enable != edata->eee_enabled) { hw->dev_spec._base.eee_enable = edata->eee_enabled; adapter->flags |= IGC_FLAG_EEE; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 0aa73519a29d..ca69a8221793 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -3436,27 +3436,27 @@ ixgbe_get_eee_fw(struct ixgbe_adapter *adapter, struct ethtool_keee *edata) if (rc) return rc; - edata->lp_advertised = 0; + edata->lp_advertised_u32 = 0; for (i = 0; i < ARRAY_SIZE(ixgbe_lp_map); ++i) { if (info[0] & ixgbe_lp_map[i].lp_advertised) - edata->lp_advertised |= ixgbe_lp_map[i].mac_speed; + edata->lp_advertised_u32 |= ixgbe_lp_map[i].mac_speed; } - edata->supported = 0; + edata->supported_u32 = 0; for (i = 0; i < ARRAY_SIZE(ixgbe_ls_map); ++i) { if (hw->phy.eee_speeds_supported & ixgbe_ls_map[i].mac_speed) - edata->supported |= ixgbe_ls_map[i].supported; + edata->supported_u32 |= ixgbe_ls_map[i].supported; } - edata->advertised = 0; + edata->advertised_u32 = 0; for (i = 0; i < ARRAY_SIZE(ixgbe_ls_map); ++i) { if (hw->phy.eee_speeds_advertised & ixgbe_ls_map[i].mac_speed) - edata->advertised |= ixgbe_ls_map[i].supported; + edata->advertised_u32 |= ixgbe_ls_map[i].supported; } - edata->eee_enabled = !!edata->advertised; + edata->eee_enabled = !!edata->advertised_u32; edata->tx_lpi_enabled = edata->eee_enabled; - if (edata->advertised & edata->lp_advertised) + if (edata->advertised_u32 & edata->lp_advertised_u32) edata->eee_active = true; return 0; @@ -3504,7 +3504,7 @@ static int ixgbe_set_eee(struct net_device *netdev, struct ethtool_keee *edata) return -EINVAL; } - if (eee_data.advertised != edata->advertised) { + if (eee_data.advertised_u32 != edata->advertised_u32) { e_err(drv, "Setting EEE advertised speeds is not supported\n"); return -EINVAL; diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 77491fb64039..dfa15619fd78 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -1790,17 +1790,17 @@ static int qede_get_eee(struct net_device *dev, struct ethtool_keee *edata) } if (current_link.eee.adv_caps & QED_EEE_1G_ADV) - edata->advertised = ADVERTISED_1000baseT_Full; + edata->advertised_u32 = ADVERTISED_1000baseT_Full; if (current_link.eee.adv_caps & QED_EEE_10G_ADV) - edata->advertised |= ADVERTISED_10000baseT_Full; + edata->advertised_u32 |= ADVERTISED_10000baseT_Full; if (current_link.sup_caps & QED_EEE_1G_ADV) - edata->supported = ADVERTISED_1000baseT_Full; + edata->supported_u32 = ADVERTISED_1000baseT_Full; if (current_link.sup_caps & QED_EEE_10G_ADV) - edata->supported |= ADVERTISED_10000baseT_Full; + edata->supported_u32 |= ADVERTISED_10000baseT_Full; if (current_link.eee.lp_adv_caps & QED_EEE_1G_ADV) - edata->lp_advertised = ADVERTISED_1000baseT_Full; + edata->lp_advertised_u32 = ADVERTISED_1000baseT_Full; if (current_link.eee.lp_adv_caps & QED_EEE_10G_ADV) - edata->lp_advertised |= ADVERTISED_10000baseT_Full; + edata->lp_advertised_u32 |= ADVERTISED_10000baseT_Full; edata->tx_lpi_timer = current_link.eee.tx_lpi_timer; edata->eee_enabled = current_link.eee.enable; @@ -1832,20 +1832,20 @@ static int qede_set_eee(struct net_device *dev, struct ethtool_keee *edata) memset(¶ms, 0, sizeof(params)); params.override_flags |= QED_LINK_OVERRIDE_EEE_CONFIG; - if (!(edata->advertised & (ADVERTISED_1000baseT_Full | - ADVERTISED_10000baseT_Full)) || - ((edata->advertised & (ADVERTISED_1000baseT_Full | - ADVERTISED_10000baseT_Full)) != - edata->advertised)) { + if (!(edata->advertised_u32 & (ADVERTISED_1000baseT_Full | + ADVERTISED_10000baseT_Full)) || + ((edata->advertised_u32 & (ADVERTISED_1000baseT_Full | + ADVERTISED_10000baseT_Full)) != + edata->advertised_u32)) { DP_VERBOSE(edev, QED_MSG_DEBUG, "Invalid advertised capabilities %d\n", - edata->advertised); + edata->advertised_u32); return -EINVAL; } - if (edata->advertised & ADVERTISED_1000baseT_Full) + if (edata->advertised_u32 & ADVERTISED_1000baseT_Full) params.eee.adv_caps = QED_EEE_1G_ADV; - if (edata->advertised & ADVERTISED_10000baseT_Full) + if (edata->advertised_u32 & ADVERTISED_10000baseT_Full) params.eee.adv_caps |= QED_EEE_10G_ADV; params.eee.enable = edata->eee_enabled; params.eee.tx_lpi_enable = edata->tx_lpi_enabled; diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c index adee5e712871..99c84af25746 100644 --- a/drivers/net/phy/phy-c45.c +++ b/drivers/net/phy/phy-c45.c @@ -1463,12 +1463,12 @@ int genphy_c45_ethtool_get_eee(struct phy_device *phydev, data->eee_enabled = is_enabled; data->eee_active = ret; - if (!ethtool_convert_link_mode_to_legacy_u32(&data->supported, + if (!ethtool_convert_link_mode_to_legacy_u32(&data->supported_u32, phydev->supported_eee)) overflow = true; - if (!ethtool_convert_link_mode_to_legacy_u32(&data->advertised, adv)) + if (!ethtool_convert_link_mode_to_legacy_u32(&data->advertised_u32, adv)) overflow = true; - if (!ethtool_convert_link_mode_to_legacy_u32(&data->lp_advertised, lp)) + if (!ethtool_convert_link_mode_to_legacy_u32(&data->lp_advertised_u32, lp)) overflow = true; if (overflow) @@ -1495,11 +1495,11 @@ int genphy_c45_ethtool_set_eee(struct phy_device *phydev, int ret; if (data->eee_enabled) { - if (data->advertised) { + if (data->advertised_u32) { __ETHTOOL_DECLARE_LINK_MODE_MASK(adv); ethtool_convert_legacy_u32_to_link_mode(adv, - data->advertised); + data->advertised_u32); linkmode_andnot(adv, adv, phydev->supported_eee); if (!linkmode_empty(adv)) { phydev_warn(phydev, "At least some EEE link modes are not supported.\n"); @@ -1507,7 +1507,7 @@ int genphy_c45_ethtool_set_eee(struct phy_device *phydev, } ethtool_convert_legacy_u32_to_link_mode(phydev->advertising_eee, - data->advertised); + data->advertised_u32); } else { linkmode_copy(phydev->advertising_eee, phydev->supported_eee); diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index 3922a9afdd1f..d6168eaa286f 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -676,21 +676,21 @@ ax88179_ethtool_get_eee(struct usbnet *dev, struct ethtool_keee *data) MDIO_MMD_PCS); if (val < 0) return val; - data->supported = mmd_eee_cap_to_ethtool_sup_t(val); + data->supported_u32 = mmd_eee_cap_to_ethtool_sup_t(val); /* Get advertisement EEE */ val = ax88179_phy_read_mmd_indirect(dev, MDIO_AN_EEE_ADV, MDIO_MMD_AN); if (val < 0) return val; - data->advertised = mmd_eee_adv_to_ethtool_adv_t(val); + data->advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(val); /* Get LP advertisement EEE */ val = ax88179_phy_read_mmd_indirect(dev, MDIO_AN_EEE_LPABLE, MDIO_MMD_AN); if (val < 0) return val; - data->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(val); + data->lp_advertised_u32 = mmd_eee_adv_to_ethtool_adv_t(val); return 0; } @@ -698,7 +698,7 @@ ax88179_ethtool_get_eee(struct usbnet *dev, struct ethtool_keee *data) static int ax88179_ethtool_set_eee(struct usbnet *dev, struct ethtool_keee *data) { - u16 tmp16 = ethtool_adv_to_mmd_eee_adv_t(data->advertised); + u16 tmp16 = ethtool_adv_to_mmd_eee_adv_t(data->advertised_u32); return ax88179_phy_write_mmd_indirect(dev, MDIO_AN_EEE_ADV, MDIO_MMD_AN, tmp16); @@ -1663,7 +1663,7 @@ static int ax88179_reset(struct usbnet *dev) ax88179_disable_eee(dev); ax88179_ethtool_get_eee(dev, &eee_data); - eee_data.advertised = 0; + eee_data.advertised_u32 = 0; ax88179_ethtool_set_eee(dev, &eee_data); /* Restart autoneg */ diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index dc163b766a89..3d806b3ff425 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -8938,16 +8938,16 @@ static int r8152_get_eee(struct r8152 *tp, struct ethtool_keee *eee) eee->eee_enabled = tp->eee_en; eee->eee_active = !!(supported & adv & lp); - eee->supported = supported; - eee->advertised = tp->eee_adv; - eee->lp_advertised = lp; + eee->supported_u32 = supported; + eee->advertised_u32 = tp->eee_adv; + eee->lp_advertised_u32 = lp; return 0; } static int r8152_set_eee(struct r8152 *tp, struct ethtool_keee *eee) { - u16 val = ethtool_adv_to_mmd_eee_adv_t(eee->advertised); + u16 val = ethtool_adv_to_mmd_eee_adv_t(eee->advertised_u32); tp->eee_en = eee->eee_enabled; tp->eee_adv = val; @@ -8973,9 +8973,9 @@ static int r8153_get_eee(struct r8152 *tp, struct ethtool_keee *eee) eee->eee_enabled = tp->eee_en; eee->eee_active = !!(supported & adv & lp); - eee->supported = supported; - eee->advertised = tp->eee_adv; - eee->lp_advertised = lp; + eee->supported_u32 = supported; + eee->advertised_u32 = tp->eee_adv; + eee->lp_advertised_u32 = lp; return 0; } diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 14549cb9e2b2..89807c30f5a7 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -223,9 +223,9 @@ __ethtool_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *link_ksettings); struct ethtool_keee { - u32 supported; - u32 advertised; - u32 lp_advertised; + u32 supported_u32; + u32 advertised_u32; + u32 lp_advertised_u32; u32 tx_lpi_timer; bool tx_lpi_enabled; bool eee_active; diff --git a/net/ethtool/eee.c b/net/ethtool/eee.c index ac9f694ffe20..ca56f28173d5 100644 --- a/net/ethtool/eee.c +++ b/net/ethtool/eee.c @@ -5,7 +5,7 @@ #include "bitset.h" #define EEE_MODES_COUNT \ - (sizeof_field(struct ethtool_keee, supported) * BITS_PER_BYTE) + (sizeof_field(struct ethtool_keee, supported_u32) * BITS_PER_BYTE) struct eee_req_info { struct ethnl_req_info base; @@ -52,19 +52,19 @@ static int eee_reply_size(const struct ethnl_req_info *req_base, int len = 0; int ret; - BUILD_BUG_ON(sizeof(eee->advertised) * BITS_PER_BYTE != + BUILD_BUG_ON(sizeof(eee->advertised_u32) * BITS_PER_BYTE != EEE_MODES_COUNT); - BUILD_BUG_ON(sizeof(eee->lp_advertised) * BITS_PER_BYTE != + BUILD_BUG_ON(sizeof(eee->lp_advertised_u32) * BITS_PER_BYTE != EEE_MODES_COUNT); /* MODES_OURS */ - ret = ethnl_bitset32_size(&eee->advertised, &eee->supported, + ret = ethnl_bitset32_size(&eee->advertised_u32, &eee->supported_u32, EEE_MODES_COUNT, link_mode_names, compact); if (ret < 0) return ret; len += ret; /* MODES_PEERS */ - ret = ethnl_bitset32_size(&eee->lp_advertised, NULL, + ret = ethnl_bitset32_size(&eee->lp_advertised_u32, NULL, EEE_MODES_COUNT, link_mode_names, compact); if (ret < 0) return ret; @@ -88,12 +88,12 @@ static int eee_fill_reply(struct sk_buff *skb, int ret; ret = ethnl_put_bitset32(skb, ETHTOOL_A_EEE_MODES_OURS, - &eee->advertised, &eee->supported, + &eee->advertised_u32, &eee->supported_u32, EEE_MODES_COUNT, link_mode_names, compact); if (ret < 0) return ret; ret = ethnl_put_bitset32(skb, ETHTOOL_A_EEE_MODES_PEER, - &eee->lp_advertised, NULL, EEE_MODES_COUNT, + &eee->lp_advertised_u32, NULL, EEE_MODES_COUNT, link_mode_names, compact); if (ret < 0) return ret; @@ -140,7 +140,7 @@ ethnl_set_eee(struct ethnl_req_info *req_info, struct genl_info *info) if (ret < 0) return ret; - ret = ethnl_update_bitset32(&eee.advertised, EEE_MODES_COUNT, + ret = ethnl_update_bitset32(&eee.advertised_u32, EEE_MODES_COUNT, tb[ETHTOOL_A_EEE_MODES_OURS], link_mode_names, info->extack, &mod); if (ret < 0) diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 46c29b369aeb..5b2ca72e3203 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1513,9 +1513,9 @@ static void eee_to_keee(struct ethtool_keee *keee, { memset(keee, 0, sizeof(*keee)); - keee->supported = eee->supported; - keee->advertised = eee->advertised; - keee->lp_advertised = eee->lp_advertised; + keee->supported_u32 = eee->supported; + keee->advertised_u32 = eee->advertised; + keee->lp_advertised_u32 = eee->lp_advertised; keee->eee_active = eee->eee_active; keee->eee_enabled = eee->eee_enabled; keee->tx_lpi_enabled = eee->tx_lpi_enabled; @@ -1527,9 +1527,9 @@ static void keee_to_eee(struct ethtool_eee *eee, { memset(eee, 0, sizeof(*eee)); - eee->supported = keee->supported; - eee->advertised = keee->advertised; - eee->lp_advertised = keee->lp_advertised; + eee->supported = keee->supported_u32; + eee->advertised = keee->advertised_u32; + eee->lp_advertised = keee->lp_advertised_u32; eee->eee_active = keee->eee_active; eee->eee_enabled = keee->eee_enabled; eee->tx_lpi_enabled = keee->tx_lpi_enabled; -- cgit v1.2.3 From 1f069de63602e8d39d7d9fd6195f65235316f79a Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 27 Jan 2024 14:29:33 +0100 Subject: ethtool: add linkmode bitmap support to struct ethtool_keee Add linkmode bitmap members to struct ethtool_keee, but keep the legacy u32 bitmaps for compatibility with existing drivers. Use linkmode "supported" not being empty as indicator that a user wants to use the linkmode bitmap members instead of the legacy bitmaps. Signed-off-by: Heiner Kallweit Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- include/linux/ethtool.h | 3 +++ net/ethtool/common.c | 5 +++++ net/ethtool/common.h | 1 + net/ethtool/eee.c | 49 +++++++++++++++++++++++++++++++++++-------------- net/ethtool/ioctl.c | 27 ++++++++++++++++++++++++--- 5 files changed, 68 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 89807c30f5a7..b90c33607594 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -223,6 +223,9 @@ __ethtool_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *link_ksettings); struct ethtool_keee { + __ETHTOOL_DECLARE_LINK_MODE_MASK(supported); + __ETHTOOL_DECLARE_LINK_MODE_MASK(advertised); + __ETHTOOL_DECLARE_LINK_MODE_MASK(lp_advertised); u32 supported_u32; u32 advertised_u32; u32 lp_advertised_u32; diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 6b2a360dcdf0..ce486cec346c 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -712,3 +712,8 @@ ethtool_forced_speed_maps_init(struct ethtool_forced_speed_map *maps, u32 size) } } EXPORT_SYMBOL_GPL(ethtool_forced_speed_maps_init); + +bool ethtool_eee_use_linkmodes(const struct ethtool_keee *eee) +{ + return !linkmode_empty(eee->supported); +} diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 28b8aaaf9bcb..0f2b5f7eacee 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -55,5 +55,6 @@ int ethtool_get_module_eeprom_call(struct net_device *dev, struct ethtool_eeprom *ee, u8 *data); bool __ethtool_dev_mm_supported(struct net_device *dev); +bool ethtool_eee_use_linkmodes(const struct ethtool_keee *eee); #endif /* _ETHTOOL_COMMON_H */ diff --git a/net/ethtool/eee.c b/net/ethtool/eee.c index ca56f28173d5..db6faa18fe41 100644 --- a/net/ethtool/eee.c +++ b/net/ethtool/eee.c @@ -30,6 +30,7 @@ static int eee_prepare_data(const struct ethnl_req_info *req_base, { struct eee_reply_data *data = EEE_REPDATA(reply_base); struct net_device *dev = reply_base->dev; + struct ethtool_keee *eee = &data->eee; int ret; if (!dev->ethtool_ops->get_eee) @@ -37,9 +38,18 @@ static int eee_prepare_data(const struct ethnl_req_info *req_base, ret = ethnl_ops_begin(dev); if (ret < 0) return ret; - ret = dev->ethtool_ops->get_eee(dev, &data->eee); + ret = dev->ethtool_ops->get_eee(dev, eee); ethnl_ops_complete(dev); + if (!ret && !ethtool_eee_use_linkmodes(eee)) { + ethtool_convert_legacy_u32_to_link_mode(eee->supported, + eee->supported_u32); + ethtool_convert_legacy_u32_to_link_mode(eee->advertised, + eee->advertised_u32); + ethtool_convert_legacy_u32_to_link_mode(eee->lp_advertised, + eee->lp_advertised_u32); + } + return ret; } @@ -58,14 +68,16 @@ static int eee_reply_size(const struct ethnl_req_info *req_base, EEE_MODES_COUNT); /* MODES_OURS */ - ret = ethnl_bitset32_size(&eee->advertised_u32, &eee->supported_u32, - EEE_MODES_COUNT, link_mode_names, compact); + ret = ethnl_bitset_size(eee->advertised, eee->supported, + __ETHTOOL_LINK_MODE_MASK_NBITS, + link_mode_names, compact); if (ret < 0) return ret; len += ret; /* MODES_PEERS */ - ret = ethnl_bitset32_size(&eee->lp_advertised_u32, NULL, - EEE_MODES_COUNT, link_mode_names, compact); + ret = ethnl_bitset_size(eee->lp_advertised, NULL, + __ETHTOOL_LINK_MODE_MASK_NBITS, + link_mode_names, compact); if (ret < 0) return ret; len += ret; @@ -87,14 +99,16 @@ static int eee_fill_reply(struct sk_buff *skb, const struct ethtool_keee *eee = &data->eee; int ret; - ret = ethnl_put_bitset32(skb, ETHTOOL_A_EEE_MODES_OURS, - &eee->advertised_u32, &eee->supported_u32, - EEE_MODES_COUNT, link_mode_names, compact); + ret = ethnl_put_bitset(skb, ETHTOOL_A_EEE_MODES_OURS, + eee->advertised, eee->supported, + __ETHTOOL_LINK_MODE_MASK_NBITS, + link_mode_names, compact); if (ret < 0) return ret; - ret = ethnl_put_bitset32(skb, ETHTOOL_A_EEE_MODES_PEER, - &eee->lp_advertised_u32, NULL, EEE_MODES_COUNT, - link_mode_names, compact); + ret = ethnl_put_bitset(skb, ETHTOOL_A_EEE_MODES_PEER, + eee->lp_advertised, NULL, + __ETHTOOL_LINK_MODE_MASK_NBITS, + link_mode_names, compact); if (ret < 0) return ret; @@ -140,9 +154,16 @@ ethnl_set_eee(struct ethnl_req_info *req_info, struct genl_info *info) if (ret < 0) return ret; - ret = ethnl_update_bitset32(&eee.advertised_u32, EEE_MODES_COUNT, - tb[ETHTOOL_A_EEE_MODES_OURS], - link_mode_names, info->extack, &mod); + if (ethtool_eee_use_linkmodes(&eee)) { + ret = ethnl_update_bitset(eee.advertised, + __ETHTOOL_LINK_MODE_MASK_NBITS, + tb[ETHTOOL_A_EEE_MODES_OURS], + link_mode_names, info->extack, &mod); + } else { + ret = ethnl_update_bitset32(&eee.advertised_u32, EEE_MODES_COUNT, + tb[ETHTOOL_A_EEE_MODES_OURS], + link_mode_names, info->extack, &mod); + } if (ret < 0) return ret; ethnl_update_bool(&eee.eee_enabled, tb[ETHTOOL_A_EEE_ENABLED], &mod); diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 5b2ca72e3203..1763e8b697e1 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1520,6 +1520,13 @@ static void eee_to_keee(struct ethtool_keee *keee, keee->eee_enabled = eee->eee_enabled; keee->tx_lpi_enabled = eee->tx_lpi_enabled; keee->tx_lpi_timer = eee->tx_lpi_timer; + + ethtool_convert_legacy_u32_to_link_mode(keee->supported, + eee->supported); + ethtool_convert_legacy_u32_to_link_mode(keee->advertised, + eee->advertised); + ethtool_convert_legacy_u32_to_link_mode(keee->lp_advertised, + eee->lp_advertised); } static void keee_to_eee(struct ethtool_eee *eee, @@ -1527,13 +1534,27 @@ static void keee_to_eee(struct ethtool_eee *eee, { memset(eee, 0, sizeof(*eee)); - eee->supported = keee->supported_u32; - eee->advertised = keee->advertised_u32; - eee->lp_advertised = keee->lp_advertised_u32; eee->eee_active = keee->eee_active; eee->eee_enabled = keee->eee_enabled; eee->tx_lpi_enabled = keee->tx_lpi_enabled; eee->tx_lpi_timer = keee->tx_lpi_timer; + + if (ethtool_eee_use_linkmodes(keee)) { + bool overflow; + + overflow = !ethtool_convert_link_mode_to_legacy_u32(&eee->supported, + keee->supported); + ethtool_convert_link_mode_to_legacy_u32(&eee->advertised, + keee->advertised); + ethtool_convert_link_mode_to_legacy_u32(&eee->lp_advertised, + keee->lp_advertised); + if (overflow) + pr_warn("Ethtool ioctl interface doesn't support passing EEE linkmodes beyond bit 32\n"); + } else { + eee->supported = keee->supported_u32; + eee->advertised = keee->advertised_u32; + eee->lp_advertised = keee->lp_advertised_u32; + } } static int ethtool_get_eee(struct net_device *dev, char __user *useraddr) -- cgit v1.2.3 From f3a052391822b772b4e27f2594526cf1eb103cab Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:04:58 +0800 Subject: cpufreq: amd-pstate: Enable amd-pstate preferred core support amd-pstate driver utilizes the functions and data structures provided by the ITMT architecture to enable the scheduler to favor scheduling on cores which can be get a higher frequency with lower voltage. We call it amd-pstate preferrred core. Here sched_set_itmt_core_prio() is called to set priorities and sched_set_itmt_support() is called to enable ITMT feature. amd-pstate driver uses the highest performance value to indicate the priority of CPU. The higher value has a higher priority. The initial core rankings are set up by amd-pstate when the system boots. Add a variable hw_prefcore in cpudata structure. It will check if the processor and power firmware support preferred core feature. Add one new early parameter `disable` to allow user to disable the preferred core. Only when hardware supports preferred core and user set `enabled` in early parameter, amd pstate driver supports preferred core featue. Tested-by: Oleksandr Natalenko Reviewed-by: Huang Rui Reviewed-by: Wyes Karny Reviewed-by: Mario Limonciello Co-developed-by: Perry Yuan Signed-off-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 131 ++++++++++++++++++++++++++++++++++++++++--- include/linux/amd-pstate.h | 4 ++ 2 files changed, 127 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 1791d37fbc53..7c533ad91f4f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ #define AMD_PSTATE_TRANSITION_LATENCY 20000 #define AMD_PSTATE_TRANSITION_DELAY 1000 +#define AMD_PSTATE_PREFCORE_THRESHOLD 166 /* * TODO: We need more time to fine tune processors with shared memory solution @@ -64,6 +66,7 @@ static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; static bool cppc_enabled; +static bool amd_pstate_prefcore = true; /* * AMD Energy Preference Performance (EPP) @@ -297,13 +300,14 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - /* - * TODO: Introduce AMD specific power feature. - * - * CPPC entry doesn't indicate the highest performance in some ASICs. + /* For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as + * the default max perf. */ - highest_perf = amd_get_highest_perf(); - if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) + if (cpudata->hw_prefcore) + highest_perf = AMD_PSTATE_PREFCORE_THRESHOLD; + else highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); WRITE_ONCE(cpudata->highest_perf, highest_perf); @@ -324,8 +328,9 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - highest_perf = amd_get_highest_perf(); - if (highest_perf > cppc_perf.highest_perf) + if (cpudata->hw_prefcore) + highest_perf = AMD_PSTATE_PREFCORE_THRESHOLD; + else highest_perf = cppc_perf.highest_perf; WRITE_ONCE(cpudata->highest_perf, highest_perf); @@ -706,6 +711,80 @@ static void amd_perf_ctl_reset(unsigned int cpu) wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); } +/* + * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks + * due to locking, so queue the work for later. + */ +static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) +{ + sched_set_itmt_support(); +} +static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); + +/* + * Get the highest performance register value. + * @cpu: CPU from which to get highest performance. + * @highest_perf: Return address. + * + * Return: 0 for success, -EIO otherwise. + */ +static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) +{ + int ret; + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + u64 cap1; + + ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); + if (ret) + return ret; + WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); + } else { + u64 cppc_highest_perf; + + ret = cppc_get_highest_perf(cpu, &cppc_highest_perf); + if (ret) + return ret; + WRITE_ONCE(*highest_perf, cppc_highest_perf); + } + + return (ret); +} + +#define CPPC_MAX_PERF U8_MAX + +static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) +{ + int ret, prio; + u32 highest_perf; + + ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf); + if (ret) + return; + + cpudata->hw_prefcore = true; + /* check if CPPC preferred core feature is enabled*/ + if (highest_perf < CPPC_MAX_PERF) + prio = (int)highest_perf; + else { + pr_debug("AMD CPPC preferred core is unsupported!\n"); + cpudata->hw_prefcore = false; + return; + } + + if (!amd_pstate_prefcore) + return; + + /* + * The priorities can be set regardless of whether or not + * sched_set_itmt_support(true) has been called and it is valid to + * update them at any time after it has been called. + */ + sched_set_itmt_core_prio(prio, cpudata->cpu); + + schedule_work(&sched_prefcore_work); +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -727,6 +806,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; @@ -877,6 +958,17 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sysfs_emit(buf, "%u\n", perf); } +static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, + char *buf) +{ + bool hw_prefcore; + struct amd_cpudata *cpudata = policy->driver_data; + + hw_prefcore = READ_ONCE(cpudata->hw_prefcore); + + return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); +} + static ssize_t show_energy_performance_available_preferences( struct cpufreq_policy *policy, char *buf) { @@ -1074,18 +1166,27 @@ static ssize_t status_store(struct device *a, struct device_attribute *b, return ret < 0 ? ret : count; } +static ssize_t prefcore_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); +} + cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); static DEVICE_ATTR_RW(status); +static DEVICE_ATTR_RO(prefcore); static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_hw_prefcore, NULL, }; @@ -1093,6 +1194,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_hw_prefcore, &energy_performance_preference, &energy_performance_available_preferences, NULL, @@ -1100,6 +1202,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { static struct attribute *pstate_global_attributes[] = { &dev_attr_status.attr, + &dev_attr_prefcore.attr, NULL }; @@ -1151,6 +1254,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; cpudata->epp_policy = 0; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; @@ -1567,7 +1672,17 @@ static int __init amd_pstate_param(char *str) return amd_pstate_set_driver(mode_idx); } + +static int __init amd_prefcore_param(char *str) +{ + if (!strcmp(str, "disable")) + amd_pstate_prefcore = false; + + return 0; +} + early_param("amd_pstate", amd_pstate_param); +early_param("amd_prefcore", amd_prefcore_param); MODULE_AUTHOR("Huang Rui "); MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 6ad02ad9c7b4..68fc1bd8d851 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -52,6 +52,9 @@ struct amd_aperf_mperf { * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value * @boost_supported: check whether the Processor or SBIOS supports boost mode + * @hw_prefcore: check whether HW supports preferred core featue. + * Only when hw_prefcore and early prefcore param are true, + * AMD P-State driver supports preferred core featue. * @epp_policy: Last saved policy used to set energy-performance preference * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value @@ -85,6 +88,7 @@ struct amd_cpudata { u64 freq; bool boost_supported; + bool hw_prefcore; /* EPP feature related attributes*/ s16 epp_policy; -- cgit v1.2.3 From 9c4a13a08a9b7afa4bc33f57675358f0195e302c Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:04:59 +0800 Subject: ACPI: cpufreq: Add highest perf change notification Platform firmware sends notify 0x85 to inform the OS that the highest performance of a CPU has changed. This will be used by the AMD P-state driver to update the ranking of preferred cores and set the priority of cores accordingly. Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Link: https://uefi.org/specs/ACPI/6.5/05_ACPI_Software_Programming_Model.html#processor-device-notification-values [ rjw: New subject, changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_driver.c | 6 ++++++ include/linux/cpufreq.h | 1 + 2 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 4bd16b3f0781..67db60eda370 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -27,6 +27,7 @@ #define ACPI_PROCESSOR_NOTIFY_PERFORMANCE 0x80 #define ACPI_PROCESSOR_NOTIFY_POWER 0x81 #define ACPI_PROCESSOR_NOTIFY_THROTTLING 0x82 +#define ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED 0x85 MODULE_AUTHOR("Paul Diefenbaugh"); MODULE_DESCRIPTION("ACPI Processor Driver"); @@ -83,6 +84,11 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data) acpi_bus_generate_netlink_event(device->pnp.device_class, dev_name(&device->dev), event, 0); break; + case ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED: + cpufreq_update_limits(pr->id); + acpi_bus_generate_netlink_event(device->pnp.device_class, + dev_name(&device->dev), event, 0); + break; default: acpi_handle_debug(handle, "Unsupported event [0x%x]\n", event); break; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index afda5f24d3dd..9bebeec24abb 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -263,6 +263,7 @@ static inline bool cpufreq_supports_freq_invariance(void) return false; } static inline void disable_cpufreq(void) { } +static inline void cpufreq_update_limits(unsigned int cpu) { } #endif #ifdef CONFIG_CPU_FREQ_STAT -- cgit v1.2.3 From e571a5e2068ef57945fcd5d0fb950f8f96da6dc8 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:00 +0800 Subject: cpufreq: amd-pstate: Update amd-pstate preferred core ranking dynamically Preferred core rankings can be changed dynamically by the platform based on the workload and platform conditions and accounting for thermals and aging. When this occurs, cpu priority need to be set. Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 52 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/amd-pstate.h | 6 +++++ 2 files changed, 58 insertions(+) (limited to 'include/linux') diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7c533ad91f4f..08e112444c27 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -315,6 +315,7 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); + WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); return 0; } @@ -339,6 +340,7 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->lowest_nonlinear_perf, cppc_perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); + WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); if (cppc_state == AMD_PSTATE_ACTIVE) @@ -785,6 +787,40 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) schedule_work(&sched_prefcore_work); } +static void amd_pstate_update_limits(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct amd_cpudata *cpudata = policy->driver_data; + u32 prev_high = 0, cur_high = 0; + int ret; + bool highest_perf_changed = false; + + mutex_lock(&amd_pstate_driver_lock); + if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) + goto free_cpufreq_put; + + ret = amd_pstate_get_highest_perf(cpu, &cur_high); + if (ret) + goto free_cpufreq_put; + + prev_high = READ_ONCE(cpudata->prefcore_ranking); + if (prev_high != cur_high) { + highest_perf_changed = true; + WRITE_ONCE(cpudata->prefcore_ranking, cur_high); + + if (cur_high < CPPC_MAX_PERF) + sched_set_itmt_core_prio((int)cur_high, cpu); + } + +free_cpufreq_put: + cpufreq_cpu_put(policy); + + if (!highest_perf_changed) + cpufreq_update_policy(cpu); + + mutex_unlock(&amd_pstate_driver_lock); +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -958,6 +994,17 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sysfs_emit(buf, "%u\n", perf); } +static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, + char *buf) +{ + u32 perf; + struct amd_cpudata *cpudata = policy->driver_data; + + perf = READ_ONCE(cpudata->prefcore_ranking); + + return sysfs_emit(buf, "%u\n", perf); +} + static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, char *buf) { @@ -1176,6 +1223,7 @@ cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); @@ -1186,6 +1234,7 @@ static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, &amd_pstate_hw_prefcore, NULL, }; @@ -1194,6 +1243,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, &amd_pstate_hw_prefcore, &energy_performance_preference, &energy_performance_available_preferences, @@ -1537,6 +1587,7 @@ static struct cpufreq_driver amd_pstate_driver = { .suspend = amd_pstate_cpu_suspend, .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate", .attr = amd_pstate_attr, }; @@ -1551,6 +1602,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .online = amd_pstate_epp_cpu_online, .suspend = amd_pstate_epp_suspend, .resume = amd_pstate_epp_resume, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate-epp", .attr = amd_pstate_epp_attr, }; diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 68fc1bd8d851..d21838835abd 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -39,11 +39,16 @@ struct amd_aperf_mperf { * @cppc_req_cached: cached performance request hints * @highest_perf: the maximum performance an individual processor may reach, * assuming ideal conditions + * For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the fixed value as the highest_perf. * @nominal_perf: the maximum sustained performance level of the processor, * assuming ideal operating conditions * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power * savings are achieved * @lowest_perf: the absolute lowest performance level of the processor + * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher + * priority. * @max_freq: the frequency that mapped to highest_perf * @min_freq: the frequency that mapped to lowest_perf * @nominal_freq: the frequency that mapped to nominal_perf @@ -73,6 +78,7 @@ struct amd_cpudata { u32 nominal_perf; u32 lowest_nonlinear_perf; u32 lowest_perf; + u32 prefcore_ranking; u32 min_limit_perf; u32 max_limit_perf; u32 min_limit_freq; -- cgit v1.2.3 From 2a71528427c635f0a8bff704b2e62ce81c641d6f Mon Sep 17 00:00:00 2001 From: Arend van Spriel Date: Sun, 28 Jan 2024 10:30:57 +0100 Subject: wifi: brcmfmac: fix copyright year mentioned in platform_data header The driver found its inception a little after the year 201. According git blame output it was added in 2016 so lets go with that. Fixes: 4d7928959832 ("brcmfmac: switch to new platform data") Reported-by: Dmitry Antipov Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo Link: https://msgid.link/20240128093057.164791-3-arend.vanspriel@broadcom.com --- include/linux/platform_data/brcmfmac.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/brcmfmac.h b/include/linux/platform_data/brcmfmac.h index f922a192fe58..ec99b7b73d1d 100644 --- a/include/linux/platform_data/brcmfmac.h +++ b/include/linux/platform_data/brcmfmac.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 201 Broadcom Corporation + * Copyright (c) 2016 Broadcom Corporation * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above -- cgit v1.2.3 From e6c5812dc4d0b3e890608cb9c98597d1bed7e937 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Wed, 31 Jan 2024 11:07:27 -0600 Subject: spi: reorder spi_message struct member doc comments The members of `struct spi_message` were reordered in commit ae2ade4ba581 ("spi: Reorder fields in 'struct spi_message'") but the documentation comments were not updated to match. This commit updates the comments to match the new order. Signed-off-by: David Lechner Link: https://msgid.link/r/20240131170732.1665105-1-dlechner@baylibre.com Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index f306aececeaf..29c3e4dd5d93 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1113,16 +1113,16 @@ struct spi_transfer { * @spi: SPI device to which the transaction is queued * @is_dma_mapped: if true, the caller provided both DMA and CPU virtual * addresses for each transfer buffer + * @prepared: spi_prepare_message was called for the this message + * @status: zero for success, else negative errno * @complete: called to report transaction completions * @context: the argument to complete() when it's called * @frame_length: the total number of bytes in the message * @actual_length: the total number of bytes that were transferred in all * successful segments - * @status: zero for success, else negative errno * @queue: for use by whichever driver currently owns the message * @state: for use by whichever driver currently owns the message * @resources: for resource management when the SPI message is processed - * @prepared: spi_prepare_message was called for the this message * * A @spi_message is used to execute an atomic sequence of data transfers, * each represented by a struct spi_transfer. The sequence is "atomic" -- cgit v1.2.3 From 79b47344bbc5a693a92ed6b2b09dac59254bfac8 Mon Sep 17 00:00:00 2001 From: Daniel Xu Date: Sun, 28 Jan 2024 18:24:06 -0700 Subject: bpf: btf: Support flags for BTF_SET8 sets This commit adds support for flags on BTF_SET8s. struct btf_id_set8 already supported 32 bits worth of flags, but was only used for alignment purposes before. We now use these bits to encode flags. The first use case is tagging kfunc sets with a flag so that pahole can recognize which BTF_ID_FLAGS(func, ..) are actual kfuncs. Signed-off-by: Daniel Xu Link: https://lore.kernel.org/r/7bb152ec76d6c2c930daec88e995bf18484a5ebb.1706491398.git.dxu@dxuuu.xyz Signed-off-by: Alexei Starovoitov --- include/linux/btf_ids.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index a9cb10b0e2e9..dca09b7f21dc 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -21,6 +21,7 @@ struct btf_id_set8 { #include /* for __PASTE */ #include /* for __maybe_unused */ +#include /* * Following macros help to define lists of BTF IDs placed @@ -183,17 +184,18 @@ extern struct btf_id_set name; * .word (1 << 3) | (1 << 1) | (1 << 2) * */ -#define __BTF_SET8_START(name, scope) \ +#define __BTF_SET8_START(name, scope, flags) \ +__BTF_ID_LIST(name, local) \ asm( \ ".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ "." #scope " __BTF_ID__set8__" #name "; \n" \ "__BTF_ID__set8__" #name ":; \n" \ -".zero 8 \n" \ +".zero 4 \n" \ +".long " __stringify(flags) "\n" \ ".popsection; \n"); #define BTF_SET8_START(name) \ -__BTF_ID_LIST(name, local) \ -__BTF_SET8_START(name, local) +__BTF_SET8_START(name, local, 0) #define BTF_SET8_END(name) \ asm( \ -- cgit v1.2.3 From a05e90427ef6706f59188b379ad6366b9d298bc5 Mon Sep 17 00:00:00 2001 From: Daniel Xu Date: Sun, 28 Jan 2024 18:24:07 -0700 Subject: bpf: btf: Add BTF_KFUNCS_START/END macro pair This macro pair is functionally equivalent to BTF_SET8_START/END, except with BTF_SET8_KFUNCS flag set in the btf_id_set8 flags field. The next commit will codemod all kfunc set8s to this new variant such that all kfuncs are tagged as such in .BTF_ids section. Signed-off-by: Daniel Xu Link: https://lore.kernel.org/r/d536c57c7c2af428686853cc7396b7a44faa53b7.1706491398.git.dxu@dxuuu.xyz Signed-off-by: Alexei Starovoitov --- include/linux/btf_ids.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include/linux') diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index dca09b7f21dc..e24aabfe8ecc 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -8,6 +8,9 @@ struct btf_id_set { u32 ids[]; }; +/* This flag implies BTF_SET8 holds kfunc(s) */ +#define BTF_SET8_KFUNCS (1 << 0) + struct btf_id_set8 { u32 cnt; u32 flags; @@ -204,6 +207,12 @@ asm( \ ".popsection; \n"); \ extern struct btf_id_set8 name; +#define BTF_KFUNCS_START(name) \ +__BTF_SET8_START(name, local, BTF_SET8_KFUNCS) + +#define BTF_KFUNCS_END(name) \ +BTF_SET8_END(name) + #else #define BTF_ID_LIST(name) static u32 __maybe_unused name[64]; @@ -218,6 +227,8 @@ extern struct btf_id_set8 name; #define BTF_SET_END(name) #define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; #define BTF_SET8_END(name) +#define BTF_KFUNCS_START(name) static struct btf_id_set8 __maybe_unused name = { .flags = BTF_SET8_KFUNCS }; +#define BTF_KFUNCS_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ -- cgit v1.2.3 From 3723b56d6f73f7c8c3b521a80556f129830f6fb9 Mon Sep 17 00:00:00 2001 From: Philippe Schenker Date: Tue, 30 Jan 2024 09:34:19 +0100 Subject: net: dsa: Add KSZ8567 switch support This commit introduces support for the KSZ8567, a robust 7-port Ethernet switch. The KSZ8567 features two RGMII/MII/RMII interfaces, each capable of gigabit speeds, complemented by five 10/100 Mbps MAC/PHYs. Signed-off-by: Philippe Schenker Acked-by: Arun Ramadoss Reviewed-by: Andrew Lunn Reviewed-by: Florian Fainelli Link: https://lore.kernel.org/r/20240130083419.135763-2-dev@pschenker.ch Signed-off-by: Paolo Abeni --- drivers/net/dsa/microchip/ksz9477_i2c.c | 4 +++ drivers/net/dsa/microchip/ksz_common.c | 43 ++++++++++++++++++++++++++++- drivers/net/dsa/microchip/ksz_common.h | 1 + drivers/net/dsa/microchip/ksz_spi.c | 5 ++++ include/linux/platform_data/microchip-ksz.h | 1 + 5 files changed, 53 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/microchip/ksz9477_i2c.c b/drivers/net/dsa/microchip/ksz9477_i2c.c index cac4a607e54a..82bebee4615c 100644 --- a/drivers/net/dsa/microchip/ksz9477_i2c.c +++ b/drivers/net/dsa/microchip/ksz9477_i2c.c @@ -103,6 +103,10 @@ static const struct of_device_id ksz9477_dt_ids[] = { .compatible = "microchip,ksz8563", .data = &ksz_switch_chips[KSZ8563] }, + { + .compatible = "microchip,ksz8567", + .data = &ksz_switch_chips[KSZ8567] + }, { .compatible = "microchip,ksz9567", .data = &ksz_switch_chips[KSZ9567] diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index a7b5ddb8656e..9ff5132f3ac6 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -1476,6 +1476,39 @@ const struct ksz_chip_data ksz_switch_chips[] = { .gbit_capable = {true, true, true}, }, + [KSZ8567] = { + .chip_id = KSZ8567_CHIP_ID, + .dev_name = "KSZ8567", + .num_vlans = 4096, + .num_alus = 4096, + .num_statics = 16, + .cpu_ports = 0x7F, /* can be configured as cpu port */ + .port_cnt = 7, /* total port count */ + .port_nirqs = 3, + .num_tx_queues = 4, + .tc_cbs_supported = true, + .tc_ets_supported = true, + .ops = &ksz9477_dev_ops, + .mib_names = ksz9477_mib_names, + .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), + .reg_mib_cnt = MIB_COUNTER_NUM, + .regs = ksz9477_regs, + .masks = ksz9477_masks, + .shifts = ksz9477_shifts, + .xmii_ctrl0 = ksz9477_xmii_ctrl0, + .xmii_ctrl1 = ksz9477_xmii_ctrl1, + .supports_mii = {false, false, false, false, + false, true, true}, + .supports_rmii = {false, false, false, false, + false, true, true}, + .supports_rgmii = {false, false, false, false, + false, true, true}, + .internal_phy = {true, true, true, true, + true, false, false}, + .gbit_capable = {false, false, false, false, false, + true, true}, + }, + [KSZ9567] = { .chip_id = KSZ9567_CHIP_ID, .dev_name = "KSZ9567", @@ -2649,6 +2682,7 @@ static void ksz_port_teardown(struct dsa_switch *ds, int port) switch (dev->chip_id) { case KSZ8563_CHIP_ID: + case KSZ8567_CHIP_ID: case KSZ9477_CHIP_ID: case KSZ9563_CHIP_ID: case KSZ9567_CHIP_ID: @@ -2705,7 +2739,8 @@ static enum dsa_tag_protocol ksz_get_tag_protocol(struct dsa_switch *ds, dev->chip_id == KSZ9563_CHIP_ID) proto = DSA_TAG_PROTO_KSZ9893; - if (dev->chip_id == KSZ9477_CHIP_ID || + if (dev->chip_id == KSZ8567_CHIP_ID || + dev->chip_id == KSZ9477_CHIP_ID || dev->chip_id == KSZ9896_CHIP_ID || dev->chip_id == KSZ9897_CHIP_ID || dev->chip_id == KSZ9567_CHIP_ID) @@ -2813,6 +2848,7 @@ static int ksz_max_mtu(struct dsa_switch *ds, int port) case KSZ8830_CHIP_ID: return KSZ8863_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN; case KSZ8563_CHIP_ID: + case KSZ8567_CHIP_ID: case KSZ9477_CHIP_ID: case KSZ9563_CHIP_ID: case KSZ9567_CHIP_ID: @@ -2839,6 +2875,7 @@ static int ksz_validate_eee(struct dsa_switch *ds, int port) switch (dev->chip_id) { case KSZ8563_CHIP_ID: + case KSZ8567_CHIP_ID: case KSZ9477_CHIP_ID: case KSZ9563_CHIP_ID: case KSZ9567_CHIP_ID: @@ -3183,6 +3220,7 @@ static int ksz_switch_detect(struct ksz_device *dev) case KSZ9896_CHIP_ID: case KSZ9897_CHIP_ID: case KSZ9567_CHIP_ID: + case KSZ8567_CHIP_ID: case LAN9370_CHIP_ID: case LAN9371_CHIP_ID: case LAN9372_CHIP_ID: @@ -3220,6 +3258,7 @@ static int ksz_cls_flower_add(struct dsa_switch *ds, int port, switch (dev->chip_id) { case KSZ8563_CHIP_ID: + case KSZ8567_CHIP_ID: case KSZ9477_CHIP_ID: case KSZ9563_CHIP_ID: case KSZ9567_CHIP_ID: @@ -3239,6 +3278,7 @@ static int ksz_cls_flower_del(struct dsa_switch *ds, int port, switch (dev->chip_id) { case KSZ8563_CHIP_ID: + case KSZ8567_CHIP_ID: case KSZ9477_CHIP_ID: case KSZ9563_CHIP_ID: case KSZ9567_CHIP_ID: @@ -4142,6 +4182,7 @@ static int ksz_parse_drive_strength(struct ksz_device *dev) case KSZ8794_CHIP_ID: case KSZ8765_CHIP_ID: case KSZ8563_CHIP_ID: + case KSZ8567_CHIP_ID: case KSZ9477_CHIP_ID: case KSZ9563_CHIP_ID: case KSZ9567_CHIP_ID: diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index 15612101a155..060c5de9aa05 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -187,6 +187,7 @@ struct ksz_device { /* List of supported models */ enum ksz_model { KSZ8563, + KSZ8567, KSZ8795, KSZ8794, KSZ8765, diff --git a/drivers/net/dsa/microchip/ksz_spi.c b/drivers/net/dsa/microchip/ksz_spi.c index 6f6d878e742c..c8166fb440ab 100644 --- a/drivers/net/dsa/microchip/ksz_spi.c +++ b/drivers/net/dsa/microchip/ksz_spi.c @@ -164,6 +164,10 @@ static const struct of_device_id ksz_dt_ids[] = { .compatible = "microchip,ksz8563", .data = &ksz_switch_chips[KSZ8563] }, + { + .compatible = "microchip,ksz8567", + .data = &ksz_switch_chips[KSZ8567] + }, { .compatible = "microchip,ksz9567", .data = &ksz_switch_chips[KSZ9567] @@ -204,6 +208,7 @@ static const struct spi_device_id ksz_spi_ids[] = { { "ksz9893" }, { "ksz9563" }, { "ksz8563" }, + { "ksz8567" }, { "ksz9567" }, { "lan9370" }, { "lan9371" }, diff --git a/include/linux/platform_data/microchip-ksz.h b/include/linux/platform_data/microchip-ksz.h index f177416635a2..8c659db4da6b 100644 --- a/include/linux/platform_data/microchip-ksz.h +++ b/include/linux/platform_data/microchip-ksz.h @@ -33,6 +33,7 @@ enum ksz_chip_id { KSZ9897_CHIP_ID = 0x00989700, KSZ9893_CHIP_ID = 0x00989300, KSZ9563_CHIP_ID = 0x00956300, + KSZ8567_CHIP_ID = 0x00856700, KSZ9567_CHIP_ID = 0x00956700, LAN9370_CHIP_ID = 0x00937000, LAN9371_CHIP_ID = 0x00937100, -- cgit v1.2.3 From 4d2ff655fb85a0bf1ecec6022ffdacb2a5f83fd2 Mon Sep 17 00:00:00 2001 From: Lukasz Majczak Date: Fri, 26 Jan 2024 09:57:19 +0000 Subject: platform/chrome: Update binary interface for EC-based watchdog Update structures and defines related to EC_CMD_HANG_DETECT to allow usage of new EC-based watchdog. Signed-off-by: Lukasz Majczak Reviewed-by: Guenter Roeck Acked-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20240126095721.782782-2-lma@chromium.org Signed-off-by: Lee Jones --- include/linux/platform_data/cros_ec_commands.h | 78 ++++++++++++-------------- 1 file changed, 35 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 7dae17b62a4d..ecc47d5fe239 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -3961,60 +3961,52 @@ struct ec_response_i2c_passthru { } __ec_align1; /*****************************************************************************/ -/* Power button hang detect */ - +/* AP hang detect */ #define EC_CMD_HANG_DETECT 0x009F -/* Reasons to start hang detection timer */ -/* Power button pressed */ -#define EC_HANG_START_ON_POWER_PRESS BIT(0) - -/* Lid closed */ -#define EC_HANG_START_ON_LID_CLOSE BIT(1) - - /* Lid opened */ -#define EC_HANG_START_ON_LID_OPEN BIT(2) - -/* Start of AP S3->S0 transition (booting or resuming from suspend) */ -#define EC_HANG_START_ON_RESUME BIT(3) - -/* Reasons to cancel hang detection */ +#define EC_HANG_DETECT_MIN_TIMEOUT 5 +#define EC_HANG_DETECT_MAX_TIMEOUT 65535 -/* Power button released */ -#define EC_HANG_STOP_ON_POWER_RELEASE BIT(8) +/* EC hang detect commands */ +enum ec_hang_detect_cmds { + /* Reload AP hang detect timer. */ + EC_HANG_DETECT_CMD_RELOAD = 0x0, -/* Any host command from AP received */ -#define EC_HANG_STOP_ON_HOST_COMMAND BIT(9) + /* Stop AP hang detect timer. */ + EC_HANG_DETECT_CMD_CANCEL = 0x1, -/* Stop on end of AP S0->S3 transition (suspending or shutting down) */ -#define EC_HANG_STOP_ON_SUSPEND BIT(10) + /* Configure watchdog with given reboot timeout and + * cancel currently running AP hang detect timer. + */ + EC_HANG_DETECT_CMD_SET_TIMEOUT = 0x2, -/* - * If this flag is set, all the other fields are ignored, and the hang detect - * timer is started. This provides the AP a way to start the hang timer - * without reconfiguring any of the other hang detect settings. Note that - * you must previously have configured the timeouts. - */ -#define EC_HANG_START_NOW BIT(30) + /* Get last hang status - whether the AP boot was clear or not */ + EC_HANG_DETECT_CMD_GET_STATUS = 0x3, -/* - * If this flag is set, all the other fields are ignored (including - * EC_HANG_START_NOW). This provides the AP a way to stop the hang timer - * without reconfiguring any of the other hang detect settings. - */ -#define EC_HANG_STOP_NOW BIT(31) + /* Clear last hang status. Called when AP is rebooting/shutting down + * gracefully. + */ + EC_HANG_DETECT_CMD_CLEAR_STATUS = 0x4 +}; struct ec_params_hang_detect { - /* Flags; see EC_HANG_* */ - uint32_t flags; - - /* Timeout in msec before generating host event, if enabled */ - uint16_t host_event_timeout_msec; + uint16_t command; /* enum ec_hang_detect_cmds */ + /* Timeout in seconds before generating reboot */ + uint16_t reboot_timeout_sec; +} __ec_align2; - /* Timeout in msec before generating warm reboot, if enabled */ - uint16_t warm_reboot_timeout_msec; -} __ec_align4; +/* Status codes that describe whether AP has boot normally or the hang has been + * detected and EC has reset AP + */ +enum ec_hang_detect_status { + EC_HANG_DETECT_AP_BOOT_NORMAL = 0x0, + EC_HANG_DETECT_AP_BOOT_EC_WDT = 0x1, + EC_HANG_DETECT_AP_BOOT_COUNT, +}; +struct ec_response_hang_detect { + uint8_t status; /* enum ec_hang_detect_status */ +} __ec_align1; /*****************************************************************************/ /* Commands for battery charging */ -- cgit v1.2.3 From c1f5204efcbcced83f67f12fa8f1a7f5f244fb87 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 28 Jan 2024 22:21:04 -0800 Subject: cpumask: add cpumask_weight_andnot() Similarly to cpumask_weight_and(), cpumask_weight_andnot() is a handy helper that may help to avoid creating an intermediate mask just to calculate number of bits that set in a 1st given mask, and clear in 2nd one. Signed-off-by: Yury Norov Reviewed-by: Jacob Keller Signed-off-by: Paolo Abeni --- include/linux/bitmap.h | 12 ++++++++++++ include/linux/cpumask.h | 13 +++++++++++++ lib/bitmap.c | 7 +++++++ 3 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 99451431e4d6..5814e9ee40ba 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -54,6 +54,7 @@ struct device; * bitmap_full(src, nbits) Are all bits set in *src? * bitmap_weight(src, nbits) Hamming Weight: number set bits * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap + * bitmap_weight_andnot(src1, src2, nbits) Hamming Weight of andnot'ed bitmap * bitmap_set(dst, pos, nbits) Set specified bit area * bitmap_clear(dst, pos, nbits) Clear specified bit area * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area @@ -169,6 +170,8 @@ bool __bitmap_subset(const unsigned long *bitmap1, unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); unsigned int __bitmap_weight_and(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int nbits); void __bitmap_set(unsigned long *map, unsigned int start, int len); void __bitmap_clear(unsigned long *map, unsigned int start, int len); @@ -425,6 +428,15 @@ unsigned long bitmap_weight_and(const unsigned long *src1, return __bitmap_weight_and(src1, src2, nbits); } +static __always_inline +unsigned long bitmap_weight_andnot(const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) +{ + if (small_const_nbits(nbits)) + return hweight_long(*src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)); + return __bitmap_weight_andnot(src1, src2, nbits); +} + static __always_inline void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits) { diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index cfb545841a2c..228c23eb36d2 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -719,6 +719,19 @@ static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); } +/** + * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2) + * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. + * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. + * + * Return: count of bits set in both *srcp1 and *srcp2 + */ +static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, + const struct cpumask *srcp2) +{ + return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); +} + /** * cpumask_shift_right - *dstp = *srcp >> n * @dstp: the cpumask result diff --git a/lib/bitmap.c b/lib/bitmap.c index 09522af227f1..b97692854966 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -348,6 +348,13 @@ unsigned int __bitmap_weight_and(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_weight_and); +unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(bitmap1[idx] & ~bitmap2[idx], bits); +} +EXPORT_SYMBOL(__bitmap_weight_andnot); + void __bitmap_set(unsigned long *map, unsigned int start, int len) { unsigned long *p = map + BIT_WORD(start); -- cgit v1.2.3 From dcee228078c34b63089c4b589d4bddf08019d0f6 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 28 Jan 2024 22:21:05 -0800 Subject: cpumask: define cleanup function for cpumasks Now we can simplify code that allocates cpumasks for local needs. Signed-off-by: Yury Norov Signed-off-by: Paolo Abeni --- include/linux/cpumask.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 228c23eb36d2..1c29947db848 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -7,6 +7,7 @@ * set of CPUs in a system, one bit position per CPU number. In general, * only nr_cpu_ids (<= NR_CPUS) bits are valid. */ +#include #include #include #include @@ -990,6 +991,8 @@ static inline bool cpumask_available(cpumask_var_t mask) } #endif /* CONFIG_CPUMASK_OFFSTACK */ +DEFINE_FREE(free_cpumask_var, struct cpumask *, if (_T) free_cpumask_var(_T)); + /* It's common to want to use cpu_all_mask in struct member initializers, * so it has to refer to an address rather than a pointer. */ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); -- cgit v1.2.3 From e2ca9e75849e63eab6544549b6888595997e8153 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Tue, 30 Jan 2024 13:08:30 +0100 Subject: dpll: extend lock_status_get() op by status error and expose to user Pass additional argunent status_error over lock_status_get() so drivers can fill it up. In case they do, expose the value over previously introduced attribute to user. Do it only in case the current lock_status is either "unlocked" or "holdover". Signed-off-by: Jiri Pirko Acked-by: Vadim Fedorenko Reviewed-by: Simon Horman Signed-off-by: Paolo Abeni --- drivers/dpll/dpll_netlink.c | 9 ++++++++- drivers/net/ethernet/intel/ice/ice_dpll.c | 2 ++ drivers/net/ethernet/mellanox/mlx5/core/dpll.c | 9 +++++---- drivers/ptp/ptp_ocp.c | 9 +++++---- include/linux/dpll.h | 1 + 5 files changed, 21 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index 314bb3775465..cf3313517ae1 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -121,14 +121,21 @@ dpll_msg_add_lock_status(struct sk_buff *msg, struct dpll_device *dpll, struct netlink_ext_ack *extack) { const struct dpll_device_ops *ops = dpll_device_ops(dpll); + enum dpll_lock_status_error status_error = 0; enum dpll_lock_status status; int ret; - ret = ops->lock_status_get(dpll, dpll_priv(dpll), &status, extack); + ret = ops->lock_status_get(dpll, dpll_priv(dpll), &status, + &status_error, extack); if (ret) return ret; if (nla_put_u32(msg, DPLL_A_LOCK_STATUS, status)) return -EMSGSIZE; + if (status_error && + (status == DPLL_LOCK_STATUS_UNLOCKED || + status == DPLL_LOCK_STATUS_HOLDOVER) && + nla_put_u32(msg, DPLL_A_LOCK_STATUS_ERROR, status_error)) + return -EMSGSIZE; return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c index b9c5eced6326..c0256564e998 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.c +++ b/drivers/net/ethernet/intel/ice/ice_dpll.c @@ -488,6 +488,7 @@ ice_dpll_hw_input_prio_set(struct ice_pf *pf, struct ice_dpll *dpll, * @dpll: registered dpll pointer * @dpll_priv: private data pointer passed on dpll registration * @status: on success holds dpll's lock status + * @status_error: status error value * @extack: error reporting * * Dpll subsystem callback, provides dpll's lock status. @@ -500,6 +501,7 @@ ice_dpll_hw_input_prio_set(struct ice_pf *pf, struct ice_dpll *dpll, static int ice_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv, enum dpll_lock_status *status, + enum dpll_lock_status_error *status_error, struct netlink_ext_ack *extack) { struct ice_dpll *d = dpll_priv; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c index 18fed2b34fb1..07f43d5c90c6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c @@ -118,10 +118,11 @@ mlx5_dpll_pin_ffo_get(struct mlx5_dpll_synce_status *synce_status, return 0; } -static int mlx5_dpll_device_lock_status_get(const struct dpll_device *dpll, - void *priv, - enum dpll_lock_status *status, - struct netlink_ext_ack *extack) +static int +mlx5_dpll_device_lock_status_get(const struct dpll_device *dpll, void *priv, + enum dpll_lock_status *status, + enum dpll_lock_status_error *status_error, + struct netlink_ext_ack *extack) { struct mlx5_dpll_synce_status synce_status; struct mlx5_dpll *mdpll = priv; diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index 5f858e426bbd..9507681e0d12 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -4209,10 +4209,11 @@ ptp_ocp_detach(struct ptp_ocp *bp) device_unregister(&bp->dev); } -static int ptp_ocp_dpll_lock_status_get(const struct dpll_device *dpll, - void *priv, - enum dpll_lock_status *status, - struct netlink_ext_ack *extack) +static int +ptp_ocp_dpll_lock_status_get(const struct dpll_device *dpll, void *priv, + enum dpll_lock_status *status, + enum dpll_lock_status_error *status_error, + struct netlink_ext_ack *extack) { struct ptp_ocp *bp = priv; diff --git a/include/linux/dpll.h b/include/linux/dpll.h index 9cf896ea1d41..9cb02ad73d51 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -19,6 +19,7 @@ struct dpll_device_ops { enum dpll_mode *mode, struct netlink_ext_ack *extack); int (*lock_status_get)(const struct dpll_device *dpll, void *dpll_priv, enum dpll_lock_status *status, + enum dpll_lock_status_error *status_error, struct netlink_ext_ack *extack); int (*temp_get)(const struct dpll_device *dpll, void *dpll_priv, s32 *temp, struct netlink_ext_ack *extack); -- cgit v1.2.3 From 2c54a4d71246379f4ffb9beb6a780f9a49fdfc24 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Tue, 30 Jan 2024 13:08:31 +0100 Subject: net/mlx5: DPLL, Implement lock status error value Fill-up the lock status error value properly. Signed-off-by: Jiri Pirko Acked-by: Vadim Fedorenko Reviewed-by: Simon Horman Signed-off-by: Paolo Abeni --- drivers/net/ethernet/mellanox/mlx5/core/dpll.c | 23 +++++++++++++++++++++++ include/linux/mlx5/mlx5_ifc.h | 8 ++++++++ 2 files changed, 31 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c index 07f43d5c90c6..4ad3d2d3d4c8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c @@ -41,6 +41,7 @@ struct mlx5_dpll_synce_status { enum mlx5_msees_oper_status oper_status; bool ho_acq; bool oper_freq_measure; + enum mlx5_msees_failure_reason failure_reason; s32 frequency_diff; }; @@ -60,6 +61,7 @@ mlx5_dpll_synce_status_get(struct mlx5_core_dev *mdev, synce_status->oper_status = MLX5_GET(msees_reg, out, oper_status); synce_status->ho_acq = MLX5_GET(msees_reg, out, ho_acq); synce_status->oper_freq_measure = MLX5_GET(msees_reg, out, oper_freq_measure); + synce_status->failure_reason = MLX5_GET(msees_reg, out, failure_reason); synce_status->frequency_diff = MLX5_GET(msees_reg, out, frequency_diff); return 0; } @@ -99,6 +101,26 @@ mlx5_dpll_lock_status_get(struct mlx5_dpll_synce_status *synce_status) } } +static enum dpll_lock_status_error +mlx5_dpll_lock_status_error_get(struct mlx5_dpll_synce_status *synce_status) +{ + switch (synce_status->oper_status) { + case MLX5_MSEES_OPER_STATUS_FAIL_HOLDOVER: + fallthrough; + case MLX5_MSEES_OPER_STATUS_FAIL_FREE_RUNNING: + switch (synce_status->failure_reason) { + case MLX5_MSEES_FAILURE_REASON_PORT_DOWN: + return DPLL_LOCK_STATUS_ERROR_MEDIA_DOWN; + case MLX5_MSEES_FAILURE_REASON_TOO_HIGH_FREQUENCY_DIFF: + return DPLL_LOCK_STATUS_ERROR_FRACTIONAL_FREQUENCY_OFFSET_TOO_HIGH; + default: + return DPLL_LOCK_STATUS_ERROR_UNDEFINED; + } + default: + return DPLL_LOCK_STATUS_ERROR_NONE; + } +} + static enum dpll_pin_state mlx5_dpll_pin_state_get(struct mlx5_dpll_synce_status *synce_status) { @@ -132,6 +154,7 @@ mlx5_dpll_device_lock_status_get(const struct dpll_device *dpll, void *priv, if (err) return err; *status = mlx5_dpll_lock_status_get(&synce_status); + *status_error = mlx5_dpll_lock_status_error_get(&synce_status); return 0; } diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index c726f90ab752..6c44f107b8ba 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -12705,6 +12705,14 @@ enum mlx5_msees_oper_status { MLX5_MSEES_OPER_STATUS_FAIL_FREE_RUNNING = 0x5, }; +enum mlx5_msees_failure_reason { + MLX5_MSEES_FAILURE_REASON_UNDEFINED_ERROR = 0x0, + MLX5_MSEES_FAILURE_REASON_PORT_DOWN = 0x1, + MLX5_MSEES_FAILURE_REASON_TOO_HIGH_FREQUENCY_DIFF = 0x2, + MLX5_MSEES_FAILURE_REASON_NET_SYNCHRONIZER_DEVICE_ERROR = 0x3, + MLX5_MSEES_FAILURE_REASON_LACK_OF_RESOURCES = 0x4, +}; + struct mlx5_ifc_msees_reg_bits { u8 reserved_at_0[0x8]; u8 local_port[0x8]; -- cgit v1.2.3 From 7092e9b3bed1252c7d3f5812b9fb9d82375b73a6 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 17 Oct 2023 10:53:15 +0530 Subject: mm/util: Introduce kmemdup_array() Introduce kmemdup_array() API to duplicate `n` number of elements from a given array. This internally uses kmemdup to allocate and duplicate the `src` array. Signed-off-by: Kartik Acked-by: Kees Cook Signed-off-by: Thierry Reding --- include/linux/string.h | 1 + mm/util.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/string.h b/include/linux/string.h index ab148d8dbfc1..4795ee5c50c6 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -217,6 +217,7 @@ extern char *kstrndup(const char *s, size_t len, gfp_t gfp); extern void *kmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp); +extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp); extern char **argv_split(gfp_t gfp, const char *str, int *argcp); extern void argv_free(char **argv); diff --git a/mm/util.c b/mm/util.c index 5a6a9802583b..5faf3adc6f43 100644 --- a/mm/util.c +++ b/mm/util.c @@ -135,6 +135,23 @@ void *kmemdup(const void *src, size_t len, gfp_t gfp) } EXPORT_SYMBOL(kmemdup); +/** + * kmemdup_array - duplicate a given array. + * + * @src: array to duplicate. + * @element_size: size of each element of array. + * @count: number of elements to duplicate from array. + * @gfp: GFP mask to use. + * + * Return: duplicated array of @src or %NULL in case of error, + * result is physically contiguous. Use kfree() to free. + */ +void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp) +{ + return kmemdup(src, size_mul(element_size, count), gfp); +} +EXPORT_SYMBOL(kmemdup_array); + /** * kvmemdup - duplicate region of memory * -- cgit v1.2.3 From 66a5c40f60f5d88ad8d47ba6a4ba05892853fa1f Mon Sep 17 00:00:00 2001 From: Tanzir Hasan Date: Tue, 26 Dec 2023 18:00:00 +0000 Subject: kernel.h: removed REPEAT_BYTE from kernel.h This patch creates wordpart.h and includes it in asm/word-at-a-time.h for all architectures. WORD_AT_A_TIME_CONSTANTS depends on kernel.h because of REPEAT_BYTE. Moving this to another header and including it where necessary allows us to not include the bloated kernel.h. Making this implicit dependency on REPEAT_BYTE explicit allows for later improvements in the lib/string.c inclusion list. Suggested-by: Al Viro Suggested-by: Andy Shevchenko Signed-off-by: Tanzir Hasan Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20231226-libstringheader-v6-1-80aa08c7652c@google.com Signed-off-by: Kees Cook --- arch/arm/include/asm/word-at-a-time.h | 3 ++- arch/arm64/include/asm/word-at-a-time.h | 3 ++- arch/powerpc/include/asm/word-at-a-time.h | 4 ++-- arch/riscv/include/asm/word-at-a-time.h | 3 ++- arch/s390/include/asm/word-at-a-time.h | 3 ++- arch/sh/include/asm/word-at-a-time.h | 2 ++ arch/x86/include/asm/word-at-a-time.h | 3 ++- arch/x86/kvm/mmu/mmu.c | 1 + fs/namei.c | 2 +- include/asm-generic/word-at-a-time.h | 3 ++- include/linux/kernel.h | 8 -------- include/linux/wordpart.h | 13 +++++++++++++ 12 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 include/linux/wordpart.h (limited to 'include/linux') diff --git a/arch/arm/include/asm/word-at-a-time.h b/arch/arm/include/asm/word-at-a-time.h index 352ab213520d..f9a3897b06e7 100644 --- a/arch/arm/include/asm/word-at-a-time.h +++ b/arch/arm/include/asm/word-at-a-time.h @@ -8,7 +8,8 @@ * Little-endian word-at-a-time zero byte handling. * Heavily based on the x86 algorithm. */ -#include +#include +#include struct word_at_a_time { const unsigned long one_bits, high_bits; diff --git a/arch/arm64/include/asm/word-at-a-time.h b/arch/arm64/include/asm/word-at-a-time.h index f3b151ed0d7a..14251abee23c 100644 --- a/arch/arm64/include/asm/word-at-a-time.h +++ b/arch/arm64/include/asm/word-at-a-time.h @@ -9,7 +9,8 @@ #ifndef __AARCH64EB__ -#include +#include +#include struct word_at_a_time { const unsigned long one_bits, high_bits; diff --git a/arch/powerpc/include/asm/word-at-a-time.h b/arch/powerpc/include/asm/word-at-a-time.h index 30a12d208687..54653a863414 100644 --- a/arch/powerpc/include/asm/word-at-a-time.h +++ b/arch/powerpc/include/asm/word-at-a-time.h @@ -4,8 +4,8 @@ /* * Word-at-a-time interfaces for PowerPC. */ - -#include +#include +#include #include #include diff --git a/arch/riscv/include/asm/word-at-a-time.h b/arch/riscv/include/asm/word-at-a-time.h index f3f031e34191..3802cda71ab7 100644 --- a/arch/riscv/include/asm/word-at-a-time.h +++ b/arch/riscv/include/asm/word-at-a-time.h @@ -10,7 +10,8 @@ #include -#include +#include +#include struct word_at_a_time { const unsigned long one_bits, high_bits; diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index 2579f1694b82..203acd6e431b 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -2,7 +2,8 @@ #ifndef _ASM_WORD_AT_A_TIME_H #define _ASM_WORD_AT_A_TIME_H -#include +#include +#include #include #include diff --git a/arch/sh/include/asm/word-at-a-time.h b/arch/sh/include/asm/word-at-a-time.h index 4aa398455b94..95100ce128d6 100644 --- a/arch/sh/include/asm/word-at-a-time.h +++ b/arch/sh/include/asm/word-at-a-time.h @@ -5,6 +5,8 @@ #ifdef CONFIG_CPU_BIG_ENDIAN # include #else +#include +#include /* * Little-endian version cribbed from x86. */ diff --git a/arch/x86/include/asm/word-at-a-time.h b/arch/x86/include/asm/word-at-a-time.h index 46b4f1f7f354..e8d7d4941c4c 100644 --- a/arch/x86/include/asm/word-at-a-time.h +++ b/arch/x86/include/asm/word-at-a-time.h @@ -2,7 +2,8 @@ #ifndef _ASM_WORD_AT_A_TIME_H #define _ASM_WORD_AT_A_TIME_H -#include +#include +#include /* * This is largely generic for little-endian machines, but the diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 2d6cdeab1f8a..6bb42b4fddb4 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include diff --git a/fs/namei.c b/fs/namei.c index 4e0de939fea1..6a548c2c40b9 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -17,8 +17,8 @@ #include #include -#include #include +#include #include #include #include diff --git a/include/asm-generic/word-at-a-time.h b/include/asm-generic/word-at-a-time.h index 95a1d214108a..ef3f841c6625 100644 --- a/include/asm-generic/word-at-a-time.h +++ b/include/asm-generic/word-at-a-time.h @@ -2,7 +2,8 @@ #ifndef _ASM_WORD_AT_A_TIME_H #define _ASM_WORD_AT_A_TIME_H -#include +#include +#include #include #ifdef __BIG_ENDIAN diff --git a/include/linux/kernel.h b/include/linux/kernel.h index d9ad21058eed..f4a1d582b79d 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -39,14 +39,6 @@ #define STACK_MAGIC 0xdeadbeef -/** - * REPEAT_BYTE - repeat the value @x multiple times as an unsigned long value - * @x: value to repeat - * - * NOTE: @x is not checked for > 0xff; larger values produce odd results. - */ -#define REPEAT_BYTE(x) ((~0ul / 0xff) * (x)) - /* generic data direction definitions */ #define READ 0 #define WRITE 1 diff --git a/include/linux/wordpart.h b/include/linux/wordpart.h new file mode 100644 index 000000000000..c9e6bd773ebd --- /dev/null +++ b/include/linux/wordpart.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LINUX_WORDPART_H +#define _LINUX_WORDPART_H +/** + * REPEAT_BYTE - repeat the value @x multiple times as an unsigned long value + * @x: value to repeat + * + * NOTE: @x is not checked for > 0xff; larger values produce odd results. + */ +#define REPEAT_BYTE(x) ((~0ul / 0xff) * (x)) + +#endif // _LINUX_WORDPART_H -- cgit v1.2.3 From 5b2dd77be1d85ac3a8be3749f5605bf0830e2998 Mon Sep 17 00:00:00 2001 From: Anjelique Melendez Date: Thu, 1 Feb 2024 12:44:24 -0800 Subject: soc: qcom: add QCOM PBS driver Add the Qualcomm PBS (Programmable Boot Sequencer) driver. The QCOM PBS driver supports configuring software PBS trigger events through PBS RAM on Qualcomm Technologies, Inc (QTI) PMICs. Signed-off-by: Anjelique Melendez Link: https://lore.kernel.org/r/20240201204421.16992-6-quic_amelende@quicinc.com Signed-off-by: Bjorn Andersson --- drivers/soc/qcom/Kconfig | 9 ++ drivers/soc/qcom/Makefile | 1 + drivers/soc/qcom/qcom-pbs.c | 236 ++++++++++++++++++++++++++++++++++++++ include/linux/soc/qcom/qcom-pbs.h | 30 +++++ 4 files changed, 276 insertions(+) create mode 100644 drivers/soc/qcom/qcom-pbs.c create mode 100644 include/linux/soc/qcom/qcom-pbs.h (limited to 'include/linux') diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index c6ca4de42586..5af33b0e3470 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -268,4 +268,13 @@ config QCOM_INLINE_CRYPTO_ENGINE tristate select QCOM_SCM +config QCOM_PBS + tristate "PBS trigger support for Qualcomm Technologies, Inc. PMICS" + depends on SPMI + help + This driver supports configuring software programmable boot sequencer (PBS) + trigger event through PBS RAM on Qualcomm Technologies, Inc. PMICs. + This module provides the APIs to the client drivers that wants to send the + PBS trigger event to the PBS RAM. + endmenu diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 05b3d54e8dc9..0a419b458fb2 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -34,3 +34,4 @@ obj-$(CONFIG_QCOM_KRYO_L2_ACCESSORS) += kryo-l2-accessors.o obj-$(CONFIG_QCOM_ICC_BWMON) += icc-bwmon.o qcom_ice-objs += ice.o obj-$(CONFIG_QCOM_INLINE_CRYPTO_ENGINE) += qcom_ice.o +obj-$(CONFIG_QCOM_PBS) += qcom-pbs.o diff --git a/drivers/soc/qcom/qcom-pbs.c b/drivers/soc/qcom/qcom-pbs.c new file mode 100644 index 000000000000..6af49b5060e5 --- /dev/null +++ b/drivers/soc/qcom/qcom-pbs.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PBS_CLIENT_TRIG_CTL 0x42 +#define PBS_CLIENT_SW_TRIG_BIT BIT(7) +#define PBS_CLIENT_SCRATCH1 0x50 +#define PBS_CLIENT_SCRATCH2 0x51 +#define PBS_CLIENT_SCRATCH2_ERROR 0xFF + +#define RETRIES 2000 +#define DELAY 1100 + +struct pbs_dev { + struct device *dev; + struct regmap *regmap; + struct mutex lock; + struct device_link *link; + + u32 base; +}; + +static int qcom_pbs_wait_for_ack(struct pbs_dev *pbs, u8 bit_pos) +{ + unsigned int val; + int ret; + + ret = regmap_read_poll_timeout(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH2, + val, val & BIT(bit_pos), DELAY, DELAY * RETRIES); + + if (ret < 0) { + dev_err(pbs->dev, "Timeout for PBS ACK/NACK for bit %u\n", bit_pos); + return -ETIMEDOUT; + } + + if (val == PBS_CLIENT_SCRATCH2_ERROR) { + ret = regmap_write(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH2, 0); + dev_err(pbs->dev, "NACK from PBS for bit %u\n", bit_pos); + return -EINVAL; + } + + dev_dbg(pbs->dev, "PBS sequence for bit %u executed!\n", bit_pos); + return 0; +} + +/** + * qcom_pbs_trigger_event() - Trigger the PBS RAM sequence + * @pbs: Pointer to PBS device + * @bitmap: bitmap + * + * This function is used to trigger the PBS RAM sequence to be + * executed by the client driver. + * + * The PBS trigger sequence involves + * 1. setting the PBS sequence bit in PBS_CLIENT_SCRATCH1 + * 2. Initiating the SW PBS trigger + * 3. Checking the equivalent bit in PBS_CLIENT_SCRATCH2 for the + * completion of the sequence. + * 4. If PBS_CLIENT_SCRATCH2 == 0xFF, the PBS sequence failed to execute + * + * Return: 0 on success, < 0 on failure + */ +int qcom_pbs_trigger_event(struct pbs_dev *pbs, u8 bitmap) +{ + unsigned int val; + u16 bit_pos; + int ret; + + if (WARN_ON(!bitmap)) + return -EINVAL; + + if (IS_ERR_OR_NULL(pbs)) + return -EINVAL; + + mutex_lock(&pbs->lock); + ret = regmap_read(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH2, &val); + if (ret < 0) + goto out; + + if (val == PBS_CLIENT_SCRATCH2_ERROR) { + /* PBS error - clear SCRATCH2 register */ + ret = regmap_write(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH2, 0); + if (ret < 0) + goto out; + } + + for (bit_pos = 0; bit_pos < 8; bit_pos++) { + if (!(bitmap & BIT(bit_pos))) + continue; + + /* Clear the PBS sequence bit position */ + ret = regmap_update_bits(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH2, + BIT(bit_pos), 0); + if (ret < 0) + goto out_clear_scratch1; + + /* Set the PBS sequence bit position */ + ret = regmap_update_bits(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH1, + BIT(bit_pos), BIT(bit_pos)); + if (ret < 0) + goto out_clear_scratch1; + + /* Initiate the SW trigger */ + ret = regmap_update_bits(pbs->regmap, pbs->base + PBS_CLIENT_TRIG_CTL, + PBS_CLIENT_SW_TRIG_BIT, PBS_CLIENT_SW_TRIG_BIT); + if (ret < 0) + goto out_clear_scratch1; + + ret = qcom_pbs_wait_for_ack(pbs, bit_pos); + if (ret < 0) + goto out_clear_scratch1; + + /* Clear the PBS sequence bit position */ + regmap_update_bits(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH1, BIT(bit_pos), 0); + regmap_update_bits(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH2, BIT(bit_pos), 0); + } + +out_clear_scratch1: + /* Clear all the requested bitmap */ + ret = regmap_update_bits(pbs->regmap, pbs->base + PBS_CLIENT_SCRATCH1, bitmap, 0); + +out: + mutex_unlock(&pbs->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(qcom_pbs_trigger_event); + +/** + * get_pbs_client_device() - Get the PBS device used by client + * @dev: Client device + * + * This function is used to get the PBS device that is being + * used by the client. + * + * Return: pbs_dev on success, ERR_PTR on failure + */ +struct pbs_dev *get_pbs_client_device(struct device *dev) +{ + struct device_node *pbs_dev_node; + struct platform_device *pdev; + struct pbs_dev *pbs; + + pbs_dev_node = of_parse_phandle(dev->of_node, "qcom,pbs", 0); + if (!pbs_dev_node) { + dev_err(dev, "Missing qcom,pbs property\n"); + return ERR_PTR(-ENODEV); + } + + pdev = of_find_device_by_node(pbs_dev_node); + if (!pdev) { + dev_err(dev, "Unable to find PBS dev_node\n"); + pbs = ERR_PTR(-EPROBE_DEFER); + goto out; + } + + pbs = platform_get_drvdata(pdev); + if (!pbs) { + dev_err(dev, "Cannot get pbs instance from %s\n", dev_name(&pdev->dev)); + platform_device_put(pdev); + pbs = ERR_PTR(-EPROBE_DEFER); + goto out; + } + + pbs->link = device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER); + if (!pbs->link) { + dev_err(&pdev->dev, "Failed to create device link to consumer %s\n", dev_name(dev)); + platform_device_put(pdev); + pbs = ERR_PTR(-EINVAL); + goto out; + } + +out: + of_node_put(pbs_dev_node); + return pbs; +} +EXPORT_SYMBOL_GPL(get_pbs_client_device); + +static int qcom_pbs_probe(struct platform_device *pdev) +{ + struct pbs_dev *pbs; + u32 val; + int ret; + + pbs = devm_kzalloc(&pdev->dev, sizeof(*pbs), GFP_KERNEL); + if (!pbs) + return -ENOMEM; + + pbs->dev = &pdev->dev; + pbs->regmap = dev_get_regmap(pbs->dev->parent, NULL); + if (!pbs->regmap) { + dev_err(pbs->dev, "Couldn't get parent's regmap\n"); + return -EINVAL; + } + + ret = device_property_read_u32(pbs->dev, "reg", &val); + if (ret < 0) { + dev_err(pbs->dev, "Couldn't find reg, ret = %d\n", ret); + return ret; + } + pbs->base = val; + mutex_init(&pbs->lock); + + platform_set_drvdata(pdev, pbs); + + return 0; +} + +static const struct of_device_id qcom_pbs_match_table[] = { + { .compatible = "qcom,pbs" }, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_pbs_match_table); + +static struct platform_driver qcom_pbs_driver = { + .driver = { + .name = "qcom-pbs", + .of_match_table = qcom_pbs_match_table, + }, + .probe = qcom_pbs_probe, +}; +module_platform_driver(qcom_pbs_driver) + +MODULE_DESCRIPTION("QCOM PBS DRIVER"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/soc/qcom/qcom-pbs.h b/include/linux/soc/qcom/qcom-pbs.h new file mode 100644 index 000000000000..8a46209ccf13 --- /dev/null +++ b/include/linux/soc/qcom/qcom-pbs.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _QCOM_PBS_H +#define _QCOM_PBS_H + +#include +#include + +struct device_node; +struct pbs_dev; + +#if IS_ENABLED(CONFIG_QCOM_PBS) +int qcom_pbs_trigger_event(struct pbs_dev *pbs, u8 bitmap); +struct pbs_dev *get_pbs_client_device(struct device *client_dev); +#else +static inline int qcom_pbs_trigger_event(struct pbs_dev *pbs, u8 bitmap) +{ + return -ENODEV; +} + +static inline struct pbs_dev *get_pbs_client_device(struct device *client_dev) +{ + return ERR_PTR(-ENODEV); +} +#endif + +#endif -- cgit v1.2.3 From b377252eeec91f347cd538011f956a4fe73794b3 Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Tue, 30 Jan 2024 12:12:33 +0100 Subject: thermal: core: Change governor name to const char pointer All users are already assigning a const char * to the `governor_name` member of struct thermal_zone_params and to the `name` member of struct thermal_governor. Even if users are technically wrong, it just makes more sense to change this member to be a const char pointer instead of doing the other way around. Signed-off-by: AngeloGioacchino Del Regno Signed-off-by: Rafael J. Wysocki --- include/linux/thermal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/thermal.h b/include/linux/thermal.h index b7a3deb372fd..65d8f92a9a0d 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -214,7 +214,7 @@ struct thermal_zone_device { * @governor_list: node in thermal_governor_list (in thermal_core.c) */ struct thermal_governor { - char name[THERMAL_NAME_LENGTH]; + const char *name; int (*bind_to_tz)(struct thermal_zone_device *tz); void (*unbind_from_tz)(struct thermal_zone_device *tz); int (*throttle)(struct thermal_zone_device *tz, @@ -226,7 +226,7 @@ struct thermal_governor { /* Structure to define Thermal Zone parameters */ struct thermal_zone_params { - char governor_name[THERMAL_NAME_LENGTH]; + const char *governor_name; /* * a boolean to indicate if the thermal to hwmon sysfs interface -- cgit v1.2.3 From 398ec3e925eb1c4d5850ec60f7075e0c20199003 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 21 Dec 2023 10:02:46 +0100 Subject: init: Declare rodata_enabled and mark_rodata_ro() at all time Declaring rodata_enabled and mark_rodata_ro() at all time helps removing related #ifdefery in C files. Signed-off-by: Christophe Leroy Signed-off-by: Luis Chamberlain --- include/linux/init.h | 4 ---- init/main.c | 21 +++++++-------------- 2 files changed, 7 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/init.h b/include/linux/init.h index 3fa3f6241350..58cef4c2e59a 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -168,12 +168,8 @@ extern initcall_entry_t __initcall_end[]; extern struct file_system_type rootfs_fs_type; -#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX) extern bool rodata_enabled; -#endif -#ifdef CONFIG_STRICT_KERNEL_RWX void mark_rodata_ro(void); -#endif extern void (*late_time_init)(void); diff --git a/init/main.c b/init/main.c index e24b0780fdff..807df08c501f 100644 --- a/init/main.c +++ b/init/main.c @@ -1396,10 +1396,9 @@ static int __init set_debug_rodata(char *str) early_param("rodata", set_debug_rodata); #endif -#ifdef CONFIG_STRICT_KERNEL_RWX static void mark_readonly(void) { - if (rodata_enabled) { + if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled) { /* * load_module() results in W+X mappings, which are cleaned * up with call_rcu(). Let's make sure that queued work is @@ -1409,20 +1408,14 @@ static void mark_readonly(void) rcu_barrier(); mark_rodata_ro(); rodata_test(); - } else + } else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) { pr_info("Kernel memory protection disabled.\n"); + } else if (IS_ENABLED(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)) { + pr_warn("Kernel memory protection not selected by kernel config.\n"); + } else { + pr_warn("This architecture does not have kernel memory protection.\n"); + } } -#elif defined(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX) -static inline void mark_readonly(void) -{ - pr_warn("Kernel memory protection not selected by kernel config.\n"); -} -#else -static inline void mark_readonly(void) -{ - pr_warn("This architecture does not have kernel memory protection.\n"); -} -#endif void __weak free_initmem(void) { -- cgit v1.2.3 From c1e6148cb4f83cec841db1f066e8db4a86c1f118 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Sat, 27 Jan 2024 19:52:34 +0200 Subject: bpf: Preserve boundaries and track scalars on narrowing fill When the width of a fill is smaller than the width of the preceding spill, the information about scalar boundaries can still be preserved, as long as it's coerced to the right width (done by coerce_reg_to_size). Even further, if the actual value fits into the fill width, the ID can be preserved as well for further tracking of equal scalars. Implement the above improvements, which makes narrowing fills behave the same as narrowing spills and MOVs between registers. Two tests are adjusted to accommodate for endianness differences and to take into account that it's now allowed to do a narrowing fill from the least significant bits. reg_bounds_sync is added to coerce_reg_to_size to correctly adjust umin/umax boundaries after the var_off truncation, for example, a 64-bit value 0xXXXXXXXX00000000, when read as a 32-bit, gets umin = 0, umax = 0xFFFFFFFF, var_off = (0x0; 0xffffffff00000000), which needs to be synced down to umax = 0, otherwise reg_bounds_sanity_check doesn't pass. Signed-off-by: Maxim Mikityanskiy Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20240127175237.526726-4-maxtram95@gmail.com --- include/linux/bpf_verifier.h | 9 ++++++++ kernel/bpf/verifier.c | 15 +++++++++---- .../selftests/bpf/progs/verifier_spill_fill.c | 26 ++++++++++++++++------ 3 files changed, 39 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 0dcde339dc7e..84365e6dd85d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -919,6 +919,15 @@ static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) env->scratched_stack_slots = ~0ULL; } +static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size) +{ +#ifdef __BIG_ENDIAN + off -= spill_size - fill_size; +#endif + + return !(off % BPF_REG_SIZE); +} + const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type); const char *dynptr_type_str(enum bpf_dynptr_type type); const char *iter_type_str(const struct btf *btf, u32 btf_id); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 28f62f24da7e..82af971926ac 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4778,7 +4778,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (dst_regno < 0) return 0; - if (!(off % BPF_REG_SIZE) && size == spill_size) { + if (size <= spill_size && + bpf_stack_narrow_access_ok(off, size, spill_size)) { /* The earlier check_reg_arg() has decided the * subreg_def for this insn. Save it first. */ @@ -4786,6 +4787,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, copy_register_state(&state->regs[dst_regno], reg); state->regs[dst_regno].subreg_def = subreg_def; + + /* Break the relation on a narrowing fill. + * coerce_reg_to_size will adjust the boundaries. + */ + if (get_reg_width(reg) > size * BITS_PER_BYTE) + state->regs[dst_regno].id = 0; } else { int spill_cnt = 0, zero_cnt = 0; @@ -6061,10 +6068,10 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) * values are also truncated so we push 64-bit bounds into * 32-bit bounds. Above were truncated < 32-bits already. */ - if (size < 4) { + if (size < 4) __mark_reg32_unbounded(reg); - reg_bounds_sync(reg); - } + + reg_bounds_sync(reg); } static void set_sext64_default_val(struct bpf_reg_state *reg, int size) diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index f9803005e1c0..3e5d063ea7e8 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -217,7 +217,7 @@ __naked void uninit_u32_from_the_stack(void) SEC("tc") __description("Spill a u32 const scalar. Refill as u16. Offset to skb->data") -__failure __msg("invalid access to packet") +__success __retval(0) __naked void u16_offset_to_skb_data(void) { asm volatile (" \ @@ -225,13 +225,19 @@ __naked void u16_offset_to_skb_data(void) r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ w4 = 20; \ *(u32*)(r10 - 8) = r4; \ - r4 = *(u16*)(r10 - 8); \ + " +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + "r4 = *(u16*)(r10 - 8);" +#else + "r4 = *(u16*)(r10 - 6);" +#endif + " \ r0 = r2; \ - /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=65535 */\ + /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=20 */\ r0 += r4; \ - /* if (r0 > r3) R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=umax=65535 */\ + /* if (r0 > r3) R0=pkt,off=20 R2=pkt R3=pkt_end R4=20 */\ if r0 > r3 goto l0_%=; \ - /* r0 = *(u32 *)r2 R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=20 */\ + /* r0 = *(u32 *)r2 R0=pkt,off=20 R2=pkt R3=pkt_end R4=20 */\ r0 = *(u32*)(r2 + 0); \ l0_%=: r0 = 0; \ exit; \ @@ -268,7 +274,7 @@ l0_%=: r0 = 0; \ } SEC("tc") -__description("Spill a u32 const scalar. Refill as u16 from fp-6. Offset to skb->data") +__description("Spill a u32 const scalar. Refill as u16 from MSB. Offset to skb->data") __failure __msg("invalid access to packet") __naked void _6_offset_to_skb_data(void) { @@ -277,7 +283,13 @@ __naked void _6_offset_to_skb_data(void) r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ w4 = 20; \ *(u32*)(r10 - 8) = r4; \ - r4 = *(u16*)(r10 - 6); \ + " +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + "r4 = *(u16*)(r10 - 6);" +#else + "r4 = *(u16*)(r10 - 8);" +#endif + " \ r0 = r2; \ /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=65535 */\ r0 += r4; \ -- cgit v1.2.3 From bd8a8d5ec5048ef74002d9f3db5cae971e68712c Mon Sep 17 00:00:00 2001 From: Yunjian Wang Date: Fri, 2 Feb 2024 15:25:55 +0800 Subject: tun: Fix code style issues in This fixes the following code style problem: - WARNING: please, no spaces at the start of a line - CHECK: Please use a blank line after function/struct/union/enum declarations Signed-off-by: Yunjian Wang Reviewed-by: Jiri Pirko Reviewed-by: Willem de Bruijn Acked-by: Jason Wang Signed-off-by: David S. Miller --- include/linux/if_tun.h | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/if_tun.h b/include/linux/if_tun.h index 2a7660843444..043d442994b0 100644 --- a/include/linux/if_tun.h +++ b/include/linux/if_tun.h @@ -27,44 +27,54 @@ struct tun_xdp_hdr { #if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE) struct socket *tun_get_socket(struct file *); struct ptr_ring *tun_get_tx_ring(struct file *file); + static inline bool tun_is_xdp_frame(void *ptr) { - return (unsigned long)ptr & TUN_XDP_FLAG; + return (unsigned long)ptr & TUN_XDP_FLAG; } + static inline void *tun_xdp_to_ptr(struct xdp_frame *xdp) { - return (void *)((unsigned long)xdp | TUN_XDP_FLAG); + return (void *)((unsigned long)xdp | TUN_XDP_FLAG); } + static inline struct xdp_frame *tun_ptr_to_xdp(void *ptr) { - return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG); + return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG); } + void tun_ptr_free(void *ptr); #else #include #include struct file; struct socket; + static inline struct socket *tun_get_socket(struct file *f) { return ERR_PTR(-EINVAL); } + static inline struct ptr_ring *tun_get_tx_ring(struct file *f) { return ERR_PTR(-EINVAL); } + static inline bool tun_is_xdp_frame(void *ptr) { return false; } + static inline void *tun_xdp_to_ptr(struct xdp_frame *xdp) { return NULL; } + static inline struct xdp_frame *tun_ptr_to_xdp(void *ptr) { return NULL; } + static inline void tun_ptr_free(void *ptr) { } -- cgit v1.2.3 From ffabe98cb576097b77d404d39e8b3df03caa986a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 2 Feb 2024 10:11:06 +0000 Subject: net: make dev_unreg_count global We can use a global dev_unreg_count counter instead of a per netns one. As a bonus we can factorize the changes done on it for bulk device removals. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/rtnetlink.h | 1 + include/net/net_namespace.h | 2 -- net/core/dev.c | 12 +++++++++--- net/core/rtnetlink.c | 11 +---------- 4 files changed, 11 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 410529fca18b..21780608cf47 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -47,6 +47,7 @@ extern int rtnl_lock_killable(void); extern bool refcount_dec_and_rtnl_lock(refcount_t *r); extern wait_queue_head_t netdev_unregistering_wq; +extern atomic_t dev_unreg_count; extern struct rw_semaphore pernet_ops_rwsem; extern struct rw_semaphore net_rwsem; diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index 13b3a4e29fdb..cd0c2eedbb5e 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -67,8 +67,6 @@ struct net { */ spinlock_t rules_mod_lock; - atomic_t dev_unreg_count; - unsigned int dev_base_seq; /* protected by rtnl_mutex */ u32 ifindex; diff --git a/net/core/dev.c b/net/core/dev.c index b53b9c94de40..27ba057d06c4 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -9698,11 +9698,11 @@ static void dev_index_release(struct net *net, int ifindex) /* Delayed registration/unregisteration */ LIST_HEAD(net_todo_list); DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); +atomic_t dev_unreg_count = ATOMIC_INIT(0); static void net_set_todo(struct net_device *dev) { list_add_tail(&dev->todo_list, &net_todo_list); - atomic_inc(&dev_net(dev)->dev_unreg_count); } static netdev_features_t netdev_sync_upper_features(struct net_device *lower, @@ -10529,6 +10529,7 @@ void netdev_run_todo(void) { struct net_device *dev, *tmp; struct list_head list; + int cnt; #ifdef CONFIG_LOCKDEP struct list_head unlink_list; @@ -10565,6 +10566,7 @@ void netdev_run_todo(void) linkwatch_sync_dev(dev); } + cnt = 0; while (!list_empty(&list)) { dev = netdev_wait_allrefs_any(&list); list_del(&dev->todo_list); @@ -10582,12 +10584,13 @@ void netdev_run_todo(void) if (dev->needs_free_netdev) free_netdev(dev); - if (atomic_dec_and_test(&dev_net(dev)->dev_unreg_count)) - wake_up(&netdev_unregistering_wq); + cnt++; /* Free network device */ kobject_put(&dev->dev.kobj); } + if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count)) + wake_up(&netdev_unregistering_wq); } /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has @@ -11034,6 +11037,7 @@ void unregister_netdevice_many_notify(struct list_head *head, { struct net_device *dev, *tmp; LIST_HEAD(close_head); + int cnt = 0; BUG_ON(dev_boot_phase); ASSERT_RTNL(); @@ -11130,7 +11134,9 @@ void unregister_netdevice_many_notify(struct list_head *head, list_for_each_entry(dev, head, unreg_list) { netdev_put(dev, &dev->dev_registered_tracker); net_set_todo(dev); + cnt++; } + atomic_add(cnt, &dev_unreg_count); list_del(head); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index f6f29eb03ec2..31f433950c8d 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -483,24 +483,15 @@ EXPORT_SYMBOL_GPL(__rtnl_link_unregister); */ static void rtnl_lock_unregistering_all(void) { - struct net *net; - bool unregistering; DEFINE_WAIT_FUNC(wait, woken_wake_function); add_wait_queue(&netdev_unregistering_wq, &wait); for (;;) { - unregistering = false; rtnl_lock(); /* We held write locked pernet_ops_rwsem, and parallel * setup_net() and cleanup_net() are not possible. */ - for_each_net(net) { - if (atomic_read(&net->dev_unreg_count) > 0) { - unregistering = true; - break; - } - } - if (!unregistering) + if (!atomic_read(&dev_unreg_count)) break; __rtnl_unlock(); -- cgit v1.2.3 From bc88528cda2eddc3e5ea304fc3f147f1b4186aa4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:09:44 +0100 Subject: PM: sleep: stats: Use array of suspend step names Replace suspend_step_name() in the suspend statistics code with an array of suspend step names which has fewer lines of code and less overhead. While at it, remove two unnecessary line breaks in suspend_stats_show() and adjust some white space in there to the kernel coding style for a more consistent code layout. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- include/linux/suspend.h | 3 ++- kernel/power/main.c | 50 ++++++++++++++++++------------------------------- 2 files changed, 20 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index ef503088942d..58f7352af205 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -41,7 +41,8 @@ typedef int __bitwise suspend_state_t; #define PM_SUSPEND_MAX ((__force suspend_state_t) 4) enum suspend_stat_step { - SUSPEND_FREEZE = 1, + SUSPEND_WORKING = 0, + SUSPEND_FREEZE, SUSPEND_PREPARE, SUSPEND_SUSPEND, SUSPEND_SUSPEND_LATE, diff --git a/kernel/power/main.c b/kernel/power/main.c index b1ae9b677d03..dca14543dfed 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -319,25 +319,17 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, power_attr(pm_test); #endif /* CONFIG_PM_SLEEP_DEBUG */ -static char *suspend_step_name(enum suspend_stat_step step) -{ - switch (step) { - case SUSPEND_FREEZE: - return "freeze"; - case SUSPEND_PREPARE: - return "prepare"; - case SUSPEND_SUSPEND: - return "suspend"; - case SUSPEND_SUSPEND_NOIRQ: - return "suspend_noirq"; - case SUSPEND_RESUME_NOIRQ: - return "resume_noirq"; - case SUSPEND_RESUME: - return "resume"; - default: - return ""; - } -} +static const char * const suspend_step_names[] = { + [SUSPEND_WORKING] = "", + [SUSPEND_FREEZE] = "freeze", + [SUSPEND_PREPARE] = "prepare", + [SUSPEND_SUSPEND] = "suspend", + [SUSPEND_SUSPEND_LATE] = "suspend_late", + [SUSPEND_SUSPEND_NOIRQ] = "suspend_noirq", + [SUSPEND_RESUME_NOIRQ] = "resume_noirq", + [SUSPEND_RESUME_EARLY] = "resume_early", + [SUSPEND_RESUME] = "resume", +}; #define suspend_attr(_name, format_str) \ static ssize_t _name##_show(struct kobject *kobj, \ @@ -392,16 +384,14 @@ static struct kobj_attribute last_failed_errno = __ATTR_RO(last_failed_errno); static ssize_t last_failed_step_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - int index; enum suspend_stat_step step; - char *last_failed_step = NULL; + int index; index = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; index %= REC_FAILED_NUM; step = suspend_stats.failed_steps[index]; - last_failed_step = suspend_step_name(step); - return sprintf(buf, "%s\n", last_failed_step); + return sprintf(buf, "%s\n", suspend_step_names[step]); } static struct kobj_attribute last_failed_step = __ATTR_RO(last_failed_step); @@ -473,30 +463,26 @@ static int suspend_stats_show(struct seq_file *s, void *unused) "failed_resume_noirq", suspend_stats.failed_resume_noirq); seq_printf(s, "failures:\n last_failed_dev:\t%-s\n", - suspend_stats.failed_devs[last_dev]); + suspend_stats.failed_devs[last_dev]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_dev + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; - seq_printf(s, "\t\t\t%-s\n", - suspend_stats.failed_devs[index]); + seq_printf(s, "\t\t\t%-s\n", suspend_stats.failed_devs[index]); } seq_printf(s, " last_failed_errno:\t%-d\n", suspend_stats.errno[last_errno]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_errno + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; - seq_printf(s, "\t\t\t%-d\n", - suspend_stats.errno[index]); + seq_printf(s, "\t\t\t%-d\n", suspend_stats.errno[index]); } seq_printf(s, " last_failed_step:\t%-s\n", - suspend_step_name( - suspend_stats.failed_steps[last_step])); + suspend_step_names[suspend_stats.failed_steps[last_step]]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_step + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; seq_printf(s, "\t\t\t%-s\n", - suspend_step_name( - suspend_stats.failed_steps[index])); + suspend_step_names[suspend_stats.failed_steps[index]]); } return 0; -- cgit v1.2.3 From 34a956739d295de6010cdaafeed698ccbba87ea4 Mon Sep 17 00:00:00 2001 From: Ezra Buehler Date: Thu, 25 Jan 2024 22:01:07 +0200 Subject: mtd: spinand: Add support for 5-byte IDs E.g. ESMT chips will return an identification code with a length of 5 bytes. In order to prevent ambiguity, flash chips would actually need to return IDs that are up to 17 or more bytes long due to JEDEC's continuation scheme. I understand that if a manufacturer ID is located in bank N of JEDEC's database (there are currently 16 banks), N - 1 continuation codes (7Fh) need to be added to the identification code (comprising of manufacturer ID and device ID). However, most flash chip manufacturers don't seem to implement this (correctly). Signed-off-by: Ezra Buehler Reviewed-by: Martin Kurbanov Tested-by: Martin Kurbanov Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20240125200108.24374-2-ezra@easyb.ch --- include/linux/mtd/spinand.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index badb4c1ac079..5c19ead60499 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -169,7 +169,7 @@ struct spinand_op; struct spinand_device; -#define SPINAND_MAX_ID_LEN 4 +#define SPINAND_MAX_ID_LEN 5 /* * For erase, write and read operation, we got the following timings : * tBERS (erase) 1ms to 4ms -- cgit v1.2.3 From b730bab0b9c4204d7dda3f5bc8adf4292497fc39 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:11:57 +0100 Subject: PM: sleep: stats: Use an array of step failure counters Instead of using a set of individual struct suspend_stats fields representing suspend step failure counters, use an array of counters indexed by enum suspend_stat_step for this purpose, which allows dpm_save_failed_step() to increment the appropriate counter automatically, so that its callers don't need to do that directly. It also allows suspend_stats_show() to carry out a loop over the counters array to print their values. Because the counters cannot become negative, use unsigned int for representing them. The only user-observable impact of this change is a different ordering of entries in the suspend_stats debugfs file which is not expected to matter. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 22 +++++++++----------- include/linux/suspend.h | 12 ++++------- kernel/power/main.c | 51 +++++++++++++++++++++++++---------------------- kernel/power/suspend.c | 1 - 4 files changed, 40 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index a2cdef95d8c4..896450503d0d 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -686,7 +686,6 @@ Out: TRACE_RESUME(error); if (error) { - suspend_stats.failed_resume_noirq++; dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); @@ -817,7 +816,6 @@ Out: complete_all(&dev->power.completion); if (error) { - suspend_stats.failed_resume_early++; dpm_save_failed_step(SUSPEND_RESUME_EARLY); dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async early" : " early", error); @@ -974,7 +972,6 @@ static void device_resume(struct device *dev, pm_message_t state, bool async) TRACE_RESUME(error); if (error) { - suspend_stats.failed_resume++; dpm_save_failed_step(SUSPEND_RESUME); dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async" : "", error); @@ -1323,10 +1320,9 @@ static int dpm_noirq_suspend_devices(pm_message_t state) if (!error) error = async_error; - if (error) { - suspend_stats.failed_suspend_noirq++; + if (error) dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ); - } + dpm_show_time(starttime, state, error, "noirq"); trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false); return error; @@ -1509,8 +1505,8 @@ int dpm_suspend_late(pm_message_t state) async_synchronize_full(); if (!error) error = async_error; + if (error) { - suspend_stats.failed_suspend_late++; dpm_save_failed_step(SUSPEND_SUSPEND_LATE); dpm_resume_early(resume_event(state)); } @@ -1789,10 +1785,10 @@ int dpm_suspend(pm_message_t state) async_synchronize_full(); if (!error) error = async_error; - if (error) { - suspend_stats.failed_suspend++; + + if (error) dpm_save_failed_step(SUSPEND_SUSPEND); - } + dpm_show_time(starttime, state, error, NULL); trace_suspend_resume(TPS("dpm_suspend"), state.event, false); return error; @@ -1943,11 +1939,11 @@ int dpm_suspend_start(pm_message_t state) int error; error = dpm_prepare(state); - if (error) { - suspend_stats.failed_prepare++; + if (error) dpm_save_failed_step(SUSPEND_PREPARE); - } else + else error = dpm_suspend(state); + dpm_show_time(starttime, state, error, "start"); return error; } diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 58f7352af205..5e4c4d4aed95 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -52,17 +52,12 @@ enum suspend_stat_step { SUSPEND_RESUME }; +#define SUSPEND_NR_STEPS SUSPEND_RESUME + struct suspend_stats { + unsigned int step_failures[SUSPEND_NR_STEPS]; int success; int fail; - int failed_freeze; - int failed_prepare; - int failed_suspend; - int failed_suspend_late; - int failed_suspend_noirq; - int failed_resume; - int failed_resume_early; - int failed_resume_noirq; #define REC_FAILED_NUM 2 int last_failed_dev; char failed_devs[REC_FAILED_NUM][40]; @@ -95,6 +90,7 @@ static inline void dpm_save_failed_errno(int err) static inline void dpm_save_failed_step(enum suspend_stat_step step) { + suspend_stats.step_failures[step-1]++; suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; suspend_stats.last_failed_step++; suspend_stats.last_failed_step %= REC_FAILED_NUM; diff --git a/kernel/power/main.c b/kernel/power/main.c index dca14543dfed..d7a02105b183 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -341,18 +341,28 @@ static struct kobj_attribute _name = __ATTR_RO(_name) suspend_attr(success, "%d\n"); suspend_attr(fail, "%d\n"); -suspend_attr(failed_freeze, "%d\n"); -suspend_attr(failed_prepare, "%d\n"); -suspend_attr(failed_suspend, "%d\n"); -suspend_attr(failed_suspend_late, "%d\n"); -suspend_attr(failed_suspend_noirq, "%d\n"); -suspend_attr(failed_resume, "%d\n"); -suspend_attr(failed_resume_early, "%d\n"); -suspend_attr(failed_resume_noirq, "%d\n"); suspend_attr(last_hw_sleep, "%llu\n"); suspend_attr(total_hw_sleep, "%llu\n"); suspend_attr(max_hw_sleep, "%llu\n"); +#define suspend_step_attr(_name, step) \ +static ssize_t _name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%u\n", \ + suspend_stats.step_failures[step-1]); \ +} \ +static struct kobj_attribute _name = __ATTR_RO(_name) + +suspend_step_attr(failed_freeze, SUSPEND_FREEZE); +suspend_step_attr(failed_prepare, SUSPEND_PREPARE); +suspend_step_attr(failed_suspend, SUSPEND_SUSPEND); +suspend_step_attr(failed_suspend_late, SUSPEND_SUSPEND_LATE); +suspend_step_attr(failed_suspend_noirq, SUSPEND_SUSPEND_NOIRQ); +suspend_step_attr(failed_resume, SUSPEND_RESUME); +suspend_step_attr(failed_resume_early, SUSPEND_RESUME_EARLY); +suspend_step_attr(failed_resume_noirq, SUSPEND_RESUME_NOIRQ); + static ssize_t last_failed_dev_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -439,6 +449,7 @@ static const struct attribute_group suspend_attr_group = { static int suspend_stats_show(struct seq_file *s, void *unused) { int i, index, last_dev, last_errno, last_step; + enum suspend_stat_step step; last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1; last_dev %= REC_FAILED_NUM; @@ -446,22 +457,14 @@ static int suspend_stats_show(struct seq_file *s, void *unused) last_errno %= REC_FAILED_NUM; last_step = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; last_step %= REC_FAILED_NUM; - seq_printf(s, "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n" - "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n", - "success", suspend_stats.success, - "fail", suspend_stats.fail, - "failed_freeze", suspend_stats.failed_freeze, - "failed_prepare", suspend_stats.failed_prepare, - "failed_suspend", suspend_stats.failed_suspend, - "failed_suspend_late", - suspend_stats.failed_suspend_late, - "failed_suspend_noirq", - suspend_stats.failed_suspend_noirq, - "failed_resume", suspend_stats.failed_resume, - "failed_resume_early", - suspend_stats.failed_resume_early, - "failed_resume_noirq", - suspend_stats.failed_resume_noirq); + + seq_printf(s, "success: %d\nfail: %d\n", + suspend_stats.success, suspend_stats.fail); + + for (step = SUSPEND_FREEZE; step <= SUSPEND_NR_STEPS; step++) + seq_printf(s, "failed_%s: %u\n", suspend_step_names[step], + suspend_stats.step_failures[step-1]); + seq_printf(s, "failures:\n last_failed_dev:\t%-s\n", suspend_stats.failed_devs[last_dev]); for (i = 1; i < REC_FAILED_NUM; i++) { diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index fa3bf161d13f..07bde5bba49e 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -367,7 +367,6 @@ static int suspend_prepare(suspend_state_t state) if (!error) return 0; - suspend_stats.failed_freeze++; dpm_save_failed_step(SUSPEND_FREEZE); pm_notifier_call_chain(PM_POST_SUSPEND); Restore: -- cgit v1.2.3 From 2231f78d3e15e45abe534db1997bc6a2153dc01c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:13:14 +0100 Subject: PM: sleep: stats: Use unsigned int for success and failure counters Change the type of the "success" and "fail" fields in struct suspend_stats to unsigned int, because they cannot be negative. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- include/linux/suspend.h | 4 ++-- kernel/power/main.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 5e4c4d4aed95..216bae989535 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -56,8 +56,8 @@ enum suspend_stat_step { struct suspend_stats { unsigned int step_failures[SUSPEND_NR_STEPS]; - int success; - int fail; + unsigned int success; + unsigned int fail; #define REC_FAILED_NUM 2 int last_failed_dev; char failed_devs[REC_FAILED_NUM][40]; diff --git a/kernel/power/main.c b/kernel/power/main.c index d7a02105b183..d6b4a9258288 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -339,8 +339,8 @@ static ssize_t _name##_show(struct kobject *kobj, \ } \ static struct kobj_attribute _name = __ATTR_RO(_name) -suspend_attr(success, "%d\n"); -suspend_attr(fail, "%d\n"); +suspend_attr(success, "%u\n"); +suspend_attr(fail, "%u\n"); suspend_attr(last_hw_sleep, "%llu\n"); suspend_attr(total_hw_sleep, "%llu\n"); suspend_attr(max_hw_sleep, "%llu\n"); @@ -458,7 +458,7 @@ static int suspend_stats_show(struct seq_file *s, void *unused) last_step = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; last_step %= REC_FAILED_NUM; - seq_printf(s, "success: %d\nfail: %d\n", + seq_printf(s, "success: %u\nfail: %u\n", suspend_stats.success, suspend_stats.fail); for (step = SUSPEND_FREEZE; step <= SUSPEND_NR_STEPS; step++) -- cgit v1.2.3 From 18af7e357033f1a1cee50db2663ef982b4a2226e Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Mon, 5 Feb 2024 12:09:55 +0200 Subject: mtd: flashchip: explicitly include MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While reviewing the hyperbus sfdp proposal the following problem was noticed: In file included from ./include/linux/mtd/gen_probe.h:10, from drivers/mtd/hyperbus/hyperbus-sfdp.c:6: ./include/linux/mtd/flashchip.h:77:9: error: unknown type name ‘wait_queue_head_t’ 77 | wait_queue_head_t wq; /* Wait on here when we're waiting for the chip | ^~~~~~~~~~~~~~~~~ It is good practice to directly include all headers used, it avoids implicit dependencies and spurious breakage if someone rearranges headers and causes the implicit include to vanish. Explicitly include in include/linux/mtd/flashchip. Signed-off-by: Tudor Ambarus Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20240205100955.149755-1-tudor.ambarus@linaro.org --- include/linux/mtd/flashchip.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mtd/flashchip.h b/include/linux/mtd/flashchip.h index c04f690871ca..9798c1a1d3b6 100644 --- a/include/linux/mtd/flashchip.h +++ b/include/linux/mtd/flashchip.h @@ -13,6 +13,7 @@ */ #include #include +#include typedef enum { FL_READY, -- cgit v1.2.3 From 9ff544fa5f94fe07f99a36d2138075b322067546 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:30:44 +0100 Subject: PM: sleep: stats: Define suspend_stats next to the code using it It is not necessary to define struct suspend_stats in a header file and the suspend_stats variable in the core device system-wide PM code. They both can be defined in kernel/power/main.c, next to the sysfs and debugfs code accessing suspend_stats, which can be static. Modify the code in question in accordance with the above observation and replace the static inline functions manipulating suspend_stats with regular ones defined in kernel/power/main.c. While at it, move the enum suspend_stat_step to the end of suspend.h which is a more suitable place for it. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 1 - include/linux/suspend.h | 71 ++++++++++--------------------------------- kernel/power/main.c | 76 +++++++++++++++++++++++++++++++++++++++-------- kernel/power/power.h | 2 ++ kernel/power/suspend.c | 7 +---- 5 files changed, 81 insertions(+), 76 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 896450503d0d..761b03b4edb1 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -60,7 +60,6 @@ static LIST_HEAD(dpm_suspended_list); static LIST_HEAD(dpm_late_early_list); static LIST_HEAD(dpm_noirq_list); -struct suspend_stats suspend_stats; static DEFINE_MUTEX(dpm_list_mtx); static pm_message_t pm_transition; diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 216bae989535..da6ebca3ff77 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -40,62 +40,6 @@ typedef int __bitwise suspend_state_t; #define PM_SUSPEND_MIN PM_SUSPEND_TO_IDLE #define PM_SUSPEND_MAX ((__force suspend_state_t) 4) -enum suspend_stat_step { - SUSPEND_WORKING = 0, - SUSPEND_FREEZE, - SUSPEND_PREPARE, - SUSPEND_SUSPEND, - SUSPEND_SUSPEND_LATE, - SUSPEND_SUSPEND_NOIRQ, - SUSPEND_RESUME_NOIRQ, - SUSPEND_RESUME_EARLY, - SUSPEND_RESUME -}; - -#define SUSPEND_NR_STEPS SUSPEND_RESUME - -struct suspend_stats { - unsigned int step_failures[SUSPEND_NR_STEPS]; - unsigned int success; - unsigned int fail; -#define REC_FAILED_NUM 2 - int last_failed_dev; - char failed_devs[REC_FAILED_NUM][40]; - int last_failed_errno; - int errno[REC_FAILED_NUM]; - int last_failed_step; - u64 last_hw_sleep; - u64 total_hw_sleep; - u64 max_hw_sleep; - enum suspend_stat_step failed_steps[REC_FAILED_NUM]; -}; - -extern struct suspend_stats suspend_stats; - -static inline void dpm_save_failed_dev(const char *name) -{ - strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], - name, - sizeof(suspend_stats.failed_devs[0])); - suspend_stats.last_failed_dev++; - suspend_stats.last_failed_dev %= REC_FAILED_NUM; -} - -static inline void dpm_save_failed_errno(int err) -{ - suspend_stats.errno[suspend_stats.last_failed_errno] = err; - suspend_stats.last_failed_errno++; - suspend_stats.last_failed_errno %= REC_FAILED_NUM; -} - -static inline void dpm_save_failed_step(enum suspend_stat_step step) -{ - suspend_stats.step_failures[step-1]++; - suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; - suspend_stats.last_failed_step++; - suspend_stats.last_failed_step %= REC_FAILED_NUM; -} - /** * struct platform_suspend_ops - Callbacks for managing platform dependent * system sleep states. @@ -623,4 +567,19 @@ static inline void queue_up_suspend_work(void) {} #endif /* !CONFIG_PM_AUTOSLEEP */ +enum suspend_stat_step { + SUSPEND_WORKING = 0, + SUSPEND_FREEZE, + SUSPEND_PREPARE, + SUSPEND_SUSPEND, + SUSPEND_SUSPEND_LATE, + SUSPEND_SUSPEND_NOIRQ, + SUSPEND_RESUME_NOIRQ, + SUSPEND_RESUME_EARLY, + SUSPEND_RESUME +}; + +void dpm_save_failed_dev(const char *name); +void dpm_save_failed_step(enum suspend_stat_step step); + #endif /* _LINUX_SUSPEND_H */ diff --git a/kernel/power/main.c b/kernel/power/main.c index d6b4a9258288..8c4bf5a54805 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -95,19 +95,6 @@ int unregister_pm_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(unregister_pm_notifier); -void pm_report_hw_sleep_time(u64 t) -{ - suspend_stats.last_hw_sleep = t; - suspend_stats.total_hw_sleep += t; -} -EXPORT_SYMBOL_GPL(pm_report_hw_sleep_time); - -void pm_report_max_hw_sleep(u64 t) -{ - suspend_stats.max_hw_sleep = t; -} -EXPORT_SYMBOL_GPL(pm_report_max_hw_sleep); - int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down) { int ret; @@ -319,6 +306,69 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, power_attr(pm_test); #endif /* CONFIG_PM_SLEEP_DEBUG */ +#define SUSPEND_NR_STEPS SUSPEND_RESUME +#define REC_FAILED_NUM 2 + +struct suspend_stats { + unsigned int step_failures[SUSPEND_NR_STEPS]; + unsigned int success; + unsigned int fail; + int last_failed_dev; + char failed_devs[REC_FAILED_NUM][40]; + int last_failed_errno; + int errno[REC_FAILED_NUM]; + int last_failed_step; + u64 last_hw_sleep; + u64 total_hw_sleep; + u64 max_hw_sleep; + enum suspend_stat_step failed_steps[REC_FAILED_NUM]; +}; + +static struct suspend_stats suspend_stats; + +void dpm_save_failed_dev(const char *name) +{ + strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], + name, sizeof(suspend_stats.failed_devs[0])); + suspend_stats.last_failed_dev++; + suspend_stats.last_failed_dev %= REC_FAILED_NUM; +} + +void dpm_save_failed_step(enum suspend_stat_step step) +{ + suspend_stats.step_failures[step-1]++; + suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; + suspend_stats.last_failed_step++; + suspend_stats.last_failed_step %= REC_FAILED_NUM; +} + +void dpm_save_errno(int err) +{ + if (!err) { + suspend_stats.success++; + return; + } + + suspend_stats.fail++; + + suspend_stats.errno[suspend_stats.last_failed_errno] = err; + suspend_stats.last_failed_errno++; + suspend_stats.last_failed_errno %= REC_FAILED_NUM; +} + +void pm_report_hw_sleep_time(u64 t) +{ + suspend_stats.last_hw_sleep = t; + suspend_stats.total_hw_sleep += t; +} +EXPORT_SYMBOL_GPL(pm_report_hw_sleep_time); + +void pm_report_max_hw_sleep(u64 t) +{ + suspend_stats.max_hw_sleep = t; +} +EXPORT_SYMBOL_GPL(pm_report_max_hw_sleep); + static const char * const suspend_step_names[] = { [SUSPEND_WORKING] = "", [SUSPEND_FREEZE] = "freeze", diff --git a/kernel/power/power.h b/kernel/power/power.h index 8499a39c62f4..4e03046b9c4d 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -327,3 +327,5 @@ static inline void pm_sleep_enable_secondary_cpus(void) suspend_enable_secondary_cpus(); cpuidle_resume(); } + +void dpm_save_errno(int err); diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 07bde5bba49e..742eb26618cc 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -616,12 +616,7 @@ int pm_suspend(suspend_state_t state) pr_info("suspend entry (%s)\n", mem_sleep_labels[state]); error = enter_state(state); - if (error) { - suspend_stats.fail++; - dpm_save_failed_errno(error); - } else { - suspend_stats.success++; - } + dpm_save_errno(error); pr_info("suspend exit\n"); return error; } -- cgit v1.2.3 From c5d74fe6a7f4240f6060dc51dd113b8a45f6cb56 Mon Sep 17 00:00:00 2001 From: R SUNDAR Date: Sun, 4 Feb 2024 21:15:06 +0530 Subject: spi: Remove the @multi_cs_cap to prevent kernel-doc warnings ./include/linux/spi/spi.h:778: warning: Excess struct member 'multi_cs_cap' description in 'spi_controller' Signed-off-by: R SUNDAR Link: https://lore.kernel.org/r/20240204154506.3561-1-prosunofficial@gmail.com Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 600fbd5daf68..0b0616b2a9f9 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -422,8 +422,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @bus_lock_spinlock: spinlock for SPI bus locking * @bus_lock_mutex: mutex for exclusion of multiple callers * @bus_lock_flag: indicates that the SPI bus is locked for exclusive use - * @multi_cs_cap: indicates that the SPI Controller can assert/de-assert - * more than one chip select at once. * @setup: updates the device mode and clocking records used by a * device's SPI controller; protocol code may call this. This * must fail if an unrecognized or unsupported mode is requested. -- cgit v1.2.3 From 0ec74ad3c157bd4bcbcc8b294777733687e8cd2a Mon Sep 17 00:00:00 2001 From: Jan Dakinevich Date: Fri, 26 Jan 2024 23:08:36 +0300 Subject: regmap: rework ->max_register handling When regmap consists of single register, 'regmap' subsystem is unable to understand whether ->max_register is set or not, because in both cases it is equal to zero. It leads to that the logic based on value of ->max_register doesn't work. For example using of REGCACHE_FLAT fails. This patch introduces an extra parameter to regmap config, indicating that zero value in ->max_register is authentic. Signed-off-by: Jan Dakinevich Link: https://lore.kernel.org/r/20240126200836.1829995-1-jan.dakinevich@salutedevices.com Signed-off-by: Mark Brown --- drivers/base/regmap/internal.h | 1 + drivers/base/regmap/regcache-flat.c | 2 +- drivers/base/regmap/regcache.c | 4 +++- drivers/base/regmap/regmap.c | 10 ++++++---- include/linux/regmap.h | 5 +++++ 5 files changed, 16 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h index 583dd5d7d46b..bcdb25bec77c 100644 --- a/drivers/base/regmap/internal.h +++ b/drivers/base/regmap/internal.h @@ -93,6 +93,7 @@ struct regmap { #endif unsigned int max_register; + bool max_register_is_set; bool (*writeable_reg)(struct device *dev, unsigned int reg); bool (*readable_reg)(struct device *dev, unsigned int reg); bool (*volatile_reg)(struct device *dev, unsigned int reg); diff --git a/drivers/base/regmap/regcache-flat.c b/drivers/base/regmap/regcache-flat.c index b7e4b2464102..9b17c77dec9d 100644 --- a/drivers/base/regmap/regcache-flat.c +++ b/drivers/base/regmap/regcache-flat.c @@ -23,7 +23,7 @@ static int regcache_flat_init(struct regmap *map) int i; unsigned int *cache; - if (!map || map->reg_stride_order < 0 || !map->max_register) + if (!map || map->reg_stride_order < 0 || !map->max_register_is_set) return -EINVAL; map->cache = kcalloc(regcache_flat_get_index(map, map->max_register) diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index ac63a73ccdaa..2e41cb12b8e2 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -187,8 +187,10 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) return 0; } - if (!map->max_register && map->num_reg_defaults_raw) + if (!map->max_register_is_set && map->num_reg_defaults_raw) { map->max_register = (map->num_reg_defaults_raw - 1) * map->reg_stride; + map->max_register_is_set = true; + } if (map->cache_ops->init) { dev_dbg(map->dev, "Initializing %s cache\n", diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 6db77d8e45f9..5cb425f6f02d 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -89,7 +89,7 @@ EXPORT_SYMBOL_GPL(regmap_check_range_table); bool regmap_writeable(struct regmap *map, unsigned int reg) { - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->writeable_reg) @@ -112,7 +112,7 @@ bool regmap_cached(struct regmap *map, unsigned int reg) if (!map->cache_ops) return false; - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; map->lock(map->lock_arg); @@ -129,7 +129,7 @@ bool regmap_readable(struct regmap *map, unsigned int reg) if (!map->reg_read) return false; - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->format.format_write) @@ -787,6 +787,7 @@ struct regmap *__regmap_init(struct device *dev, map->bus = bus; map->bus_context = bus_context; map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->wr_table = config->wr_table; map->rd_table = config->rd_table; map->volatile_table = config->volatile_table; @@ -1412,6 +1413,7 @@ int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config) regmap_debugfs_exit(map); map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->writeable_reg = config->writeable_reg; map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; @@ -3383,7 +3385,7 @@ EXPORT_SYMBOL_GPL(regmap_get_val_bytes); */ int regmap_get_max_register(struct regmap *map) { - return map->max_register ? map->max_register : -EINVAL; + return map->max_register_is_set ? map->max_register : -EINVAL; } EXPORT_SYMBOL_GPL(regmap_get_max_register); diff --git a/include/linux/regmap.h b/include/linux/regmap.h index c9182a47736e..b743241cfb7c 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -332,6 +332,10 @@ typedef void (*regmap_unlock)(void *); * @io_port: Support IO port accessors. Makes sense only when MMIO vs. IO port * access can be distinguished. * @max_register: Optional, specifies the maximum valid register address. + * @max_register_is_0: Optional, specifies that zero value in @max_register + * should be taken into account. This is a workaround to + * apply handling of @max_register for regmap that contains + * only one register. * @wr_table: Optional, points to a struct regmap_access_table specifying * valid ranges for write access. * @rd_table: As above, for read access. @@ -422,6 +426,7 @@ struct regmap_config { bool io_port; unsigned int max_register; + bool max_register_is_0; const struct regmap_access_table *wr_table; const struct regmap_access_table *rd_table; const struct regmap_access_table *volatile_table; -- cgit v1.2.3 From 2719a9e7156c4b3983b43db467c1ff96801bda99 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 31 Jan 2024 23:37:25 +0100 Subject: wifi: cw1200: Convert to GPIO descriptors The CW1200 uses two GPIOs to control the powerup and reset pins, get these from GPIO descriptors instead of being passed as platform data from boardfiles. The RESET line will need to be marked as active low as we will let gpiolib handle the polarity inversion. The SDIO case is a bit special since the "card" need to be powered up before it gets detected on the SDIO bus and properly probed. Fix this by using board-specific GPIOs assigned to device "NULL". There are currently no in-tree users. Signed-off-by: Linus Walleij Signed-off-by: Kalle Valo Link: https://msgid.link/20240131-descriptors-wireless-v1-6-e1c7c5d68746@linaro.org --- drivers/net/wireless/st/cw1200/cw1200_sdio.c | 42 +++++++++------- drivers/net/wireless/st/cw1200/cw1200_spi.c | 71 ++++++++++++++++------------ include/linux/platform_data/net-cw1200.h | 4 -- 3 files changed, 65 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wireless/st/cw1200/cw1200_sdio.c b/drivers/net/wireless/st/cw1200/cw1200_sdio.c index 4c30b5772ce0..00c4731d8f8e 100644 --- a/drivers/net/wireless/st/cw1200/cw1200_sdio.c +++ b/drivers/net/wireless/st/cw1200/cw1200_sdio.c @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include @@ -178,12 +178,15 @@ static int cw1200_sdio_irq_unsubscribe(struct hwbus_priv *self) return ret; } +/* Like the rest of the driver, this only supports one device per system */ +static struct gpio_desc *cw1200_reset; +static struct gpio_desc *cw1200_powerup; + static int cw1200_sdio_off(const struct cw1200_platform_data_sdio *pdata) { - if (pdata->reset) { - gpio_set_value(pdata->reset, 0); + if (cw1200_reset) { + gpiod_set_value(cw1200_reset, 0); msleep(30); /* Min is 2 * CLK32K cycles */ - gpio_free(pdata->reset); } if (pdata->power_ctrl) @@ -196,16 +199,21 @@ static int cw1200_sdio_off(const struct cw1200_platform_data_sdio *pdata) static int cw1200_sdio_on(const struct cw1200_platform_data_sdio *pdata) { - /* Ensure I/Os are pulled low */ - if (pdata->reset) { - gpio_request(pdata->reset, "cw1200_wlan_reset"); - gpio_direction_output(pdata->reset, 0); + /* Ensure I/Os are pulled low (reset is active low) */ + cw1200_reset = devm_gpiod_get_optional(NULL, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(cw1200_reset)) { + pr_err("could not get CW1200 SDIO reset GPIO\n"); + return PTR_ERR(cw1200_reset); } - if (pdata->powerup) { - gpio_request(pdata->powerup, "cw1200_wlan_powerup"); - gpio_direction_output(pdata->powerup, 0); + gpiod_set_consumer_name(cw1200_reset, "cw1200_wlan_reset"); + cw1200_powerup = devm_gpiod_get_optional(NULL, "powerup", GPIOD_OUT_LOW); + if (IS_ERR(cw1200_powerup)) { + pr_err("could not get CW1200 SDIO powerup GPIO\n"); + return PTR_ERR(cw1200_powerup); } - if (pdata->reset || pdata->powerup) + gpiod_set_consumer_name(cw1200_powerup, "cw1200_wlan_powerup"); + + if (cw1200_reset || cw1200_powerup) msleep(10); /* Settle time? */ /* Enable 3v3 and 1v8 to hardware */ @@ -226,13 +234,13 @@ static int cw1200_sdio_on(const struct cw1200_platform_data_sdio *pdata) } /* Enable POWERUP signal */ - if (pdata->powerup) { - gpio_set_value(pdata->powerup, 1); + if (cw1200_powerup) { + gpiod_set_value(cw1200_powerup, 1); msleep(250); /* or more..? */ } - /* Enable RSTn signal */ - if (pdata->reset) { - gpio_set_value(pdata->reset, 1); + /* Deassert RSTn signal, note active low */ + if (cw1200_reset) { + gpiod_set_value(cw1200_reset, 0); msleep(50); /* Or more..? */ } return 0; diff --git a/drivers/net/wireless/st/cw1200/cw1200_spi.c b/drivers/net/wireless/st/cw1200/cw1200_spi.c index b27b57fc25bc..fb3aafcafe18 100644 --- a/drivers/net/wireless/st/cw1200/cw1200_spi.c +++ b/drivers/net/wireless/st/cw1200/cw1200_spi.c @@ -11,7 +11,7 @@ */ #include -#include +#include #include #include #include @@ -38,6 +38,8 @@ struct hwbus_priv { const struct cw1200_platform_data_spi *pdata; spinlock_t lock; /* Serialize all bus operations */ wait_queue_head_t wq; + struct gpio_desc *reset; + struct gpio_desc *powerup; int claimed; }; @@ -269,12 +271,12 @@ static void cw1200_spi_irq_unsubscribe(struct hwbus_priv *self) free_irq(self->func->irq, self); } -static int cw1200_spi_off(const struct cw1200_platform_data_spi *pdata) +static int cw1200_spi_off(struct hwbus_priv *self, const struct cw1200_platform_data_spi *pdata) { - if (pdata->reset) { - gpio_set_value(pdata->reset, 0); + if (self->reset) { + /* Assert RESET, note active low */ + gpiod_set_value(self->reset, 1); msleep(30); /* Min is 2 * CLK32K cycles */ - gpio_free(pdata->reset); } if (pdata->power_ctrl) @@ -285,18 +287,12 @@ static int cw1200_spi_off(const struct cw1200_platform_data_spi *pdata) return 0; } -static int cw1200_spi_on(const struct cw1200_platform_data_spi *pdata) +static int cw1200_spi_on(struct hwbus_priv *self, const struct cw1200_platform_data_spi *pdata) { /* Ensure I/Os are pulled low */ - if (pdata->reset) { - gpio_request(pdata->reset, "cw1200_wlan_reset"); - gpio_direction_output(pdata->reset, 0); - } - if (pdata->powerup) { - gpio_request(pdata->powerup, "cw1200_wlan_powerup"); - gpio_direction_output(pdata->powerup, 0); - } - if (pdata->reset || pdata->powerup) + gpiod_direction_output(self->reset, 1); /* Active low */ + gpiod_direction_output(self->powerup, 0); + if (self->reset || self->powerup) msleep(10); /* Settle time? */ /* Enable 3v3 and 1v8 to hardware */ @@ -317,13 +313,13 @@ static int cw1200_spi_on(const struct cw1200_platform_data_spi *pdata) } /* Enable POWERUP signal */ - if (pdata->powerup) { - gpio_set_value(pdata->powerup, 1); + if (self->powerup) { + gpiod_set_value(self->powerup, 1); msleep(250); /* or more..? */ } - /* Enable RSTn signal */ - if (pdata->reset) { - gpio_set_value(pdata->reset, 1); + /* Assert RSTn signal, note active low */ + if (self->reset) { + gpiod_set_value(self->reset, 0); msleep(50); /* Or more..? */ } return 0; @@ -375,20 +371,33 @@ static int cw1200_spi_probe(struct spi_device *func) spi_get_chipselect(func, 0), func->mode, func->bits_per_word, func->max_speed_hz); - if (cw1200_spi_on(plat_data)) { + self = devm_kzalloc(&func->dev, sizeof(*self), GFP_KERNEL); + if (!self) { + pr_err("Can't allocate SPI hwbus_priv."); + return -ENOMEM; + } + + /* Request reset asserted */ + self->reset = devm_gpiod_get_optional(&func->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(self->reset)) + return dev_err_probe(&func->dev, PTR_ERR(self->reset), + "could not get reset GPIO\n"); + gpiod_set_consumer_name(self->reset, "cw1200_wlan_reset"); + + self->powerup = devm_gpiod_get_optional(&func->dev, "powerup", GPIOD_OUT_LOW); + if (IS_ERR(self->powerup)) + return dev_err_probe(&func->dev, PTR_ERR(self->powerup), + "could not get powerup GPIO\n"); + gpiod_set_consumer_name(self->reset, "cw1200_wlan_powerup"); + + if (cw1200_spi_on(self, plat_data)) { pr_err("spi_on() failed!\n"); - return -1; + return -ENODEV; } if (spi_setup(func)) { pr_err("spi_setup() failed!\n"); - return -1; - } - - self = devm_kzalloc(&func->dev, sizeof(*self), GFP_KERNEL); - if (!self) { - pr_err("Can't allocate SPI hwbus_priv."); - return -ENOMEM; + return -ENODEV; } self->pdata = plat_data; @@ -410,7 +419,7 @@ static int cw1200_spi_probe(struct spi_device *func) if (status) { cw1200_spi_irq_unsubscribe(self); - cw1200_spi_off(plat_data); + cw1200_spi_off(self, plat_data); } return status; @@ -428,7 +437,7 @@ static void cw1200_spi_disconnect(struct spi_device *func) self->core = NULL; } } - cw1200_spi_off(dev_get_platdata(&func->dev)); + cw1200_spi_off(self, dev_get_platdata(&func->dev)); } static int __maybe_unused cw1200_spi_suspend(struct device *dev) diff --git a/include/linux/platform_data/net-cw1200.h b/include/linux/platform_data/net-cw1200.h index c510734405bb..89d0ec6f7d46 100644 --- a/include/linux/platform_data/net-cw1200.h +++ b/include/linux/platform_data/net-cw1200.h @@ -14,8 +14,6 @@ struct cw1200_platform_data_spi { /* All others are optional */ bool have_5ghz; - int reset; /* GPIO to RSTn signal (0 disables) */ - int powerup; /* GPIO to POWERUP signal (0 disables) */ int (*power_ctrl)(const struct cw1200_platform_data_spi *pdata, bool enable); /* Control 3v3 / 1v8 supply */ int (*clk_ctrl)(const struct cw1200_platform_data_spi *pdata, @@ -30,8 +28,6 @@ struct cw1200_platform_data_sdio { /* All others are optional */ bool have_5ghz; bool no_nptb; /* SDIO hardware does not support non-power-of-2-blocksizes */ - int reset; /* GPIO to RSTn signal (0 disables) */ - int powerup; /* GPIO to POWERUP signal (0 disables) */ int irq; /* IRQ line or 0 to use SDIO IRQ */ int (*power_ctrl)(const struct cw1200_platform_data_sdio *pdata, bool enable); /* Control 3v3 / 1v8 supply */ -- cgit v1.2.3 From fd2bc4195d5107f88c1b90e1ec935888ccbfc5c0 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Oct 2023 20:57:20 +0300 Subject: xfrm: generalize xdo_dev_state_update_curlft to allow statistics update In order to allow drivers to fill all statistics, change the name of xdo_dev_state_update_curlft to be xdo_dev_state_update_stats. Acked-by: Steffen Klassert Signed-off-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- Documentation/networking/xfrm_device.rst | 4 ++-- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c | 7 ++++--- include/linux/netdevice.h | 2 +- include/net/xfrm.h | 11 ++++------- net/xfrm/xfrm_state.c | 4 ++-- net/xfrm/xfrm_user.c | 2 +- 6 files changed, 14 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/xfrm_device.rst b/Documentation/networking/xfrm_device.rst index 535077cbeb07..bfea9d8579ed 100644 --- a/Documentation/networking/xfrm_device.rst +++ b/Documentation/networking/xfrm_device.rst @@ -71,9 +71,9 @@ Callbacks to implement bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + void (*xdo_dev_state_update_stats) (struct xfrm_state *x); /* Solely packet offload callbacks */ - void (*xdo_dev_state_update_curlft) (struct xfrm_state *x); int (*xdo_dev_policy_add) (struct xfrm_policy *x, struct netlink_ext_ack *extack); void (*xdo_dev_policy_delete) (struct xfrm_policy *x); void (*xdo_dev_policy_free) (struct xfrm_policy *x); @@ -191,6 +191,6 @@ xdo_dev_policy_free() on any remaining offloaded states. Outcome of HW handling packets, the XFRM core can't count hard, soft limits. The HW/driver are responsible to perform it and provide accurate data when -xdo_dev_state_update_curlft() is called. In case of one of these limits +xdo_dev_state_update_stats() is called. In case of one of these limits occuried, the driver needs to call to xfrm_state_check_expire() to make sure that XFRM performs rekeying sequence. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 05612d9c6080..f160522fbe75 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -984,7 +984,7 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) queue_work(sa_entry->ipsec->wq, &work->work); } -static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) +static void mlx5e_xfrm_update_stats(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; @@ -993,7 +993,8 @@ static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) lockdep_assert(lockdep_is_held(&x->lock) || lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_cfg_mutex)); - if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) + if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ || + x->xso.type != XFRM_DEV_OFFLOAD_PACKET) return; mlx5_fc_query_cached(ipsec_rule->fc, &bytes, &packets, &lastuse); @@ -1156,7 +1157,7 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, - .xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft, + .xdo_dev_state_update_stats = mlx5e_xfrm_update_stats, .xdo_dev_policy_add = mlx5e_xfrm_add_policy, .xdo_dev_policy_delete = mlx5e_xfrm_del_policy, .xdo_dev_policy_free = mlx5e_xfrm_free_policy, diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 118c40258d07..9538576dbebc 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1062,7 +1062,7 @@ struct xfrmdev_ops { bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); - void (*xdo_dev_state_update_curlft) (struct xfrm_state *x); + void (*xdo_dev_state_update_stats) (struct xfrm_state *x); int (*xdo_dev_policy_add) (struct xfrm_policy *x, struct netlink_ext_ack *extack); void (*xdo_dev_policy_delete) (struct xfrm_policy *x); void (*xdo_dev_policy_free) (struct xfrm_policy *x); diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 1d107241b901..4ca2f3205190 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -1578,21 +1578,18 @@ struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, unsigned short family); int xfrm_state_check_expire(struct xfrm_state *x); #ifdef CONFIG_XFRM_OFFLOAD -static inline void xfrm_dev_state_update_curlft(struct xfrm_state *x) +static inline void xfrm_dev_state_update_stats(struct xfrm_state *x) { struct xfrm_dev_offload *xdo = &x->xso; struct net_device *dev = xdo->dev; - if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) - return; - if (dev && dev->xfrmdev_ops && - dev->xfrmdev_ops->xdo_dev_state_update_curlft) - dev->xfrmdev_ops->xdo_dev_state_update_curlft(x); + dev->xfrmdev_ops->xdo_dev_state_update_stats) + dev->xfrmdev_ops->xdo_dev_state_update_stats(x); } #else -static inline void xfrm_dev_state_update_curlft(struct xfrm_state *x) {} +static inline void xfrm_dev_state_update_stats(struct xfrm_state *x) {} #endif void xfrm_state_insert(struct xfrm_state *x); int xfrm_state_add(struct xfrm_state *x); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index bda5327bf34d..d8701b2d0d57 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -570,7 +570,7 @@ static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) int err = 0; spin_lock(&x->lock); - xfrm_dev_state_update_curlft(x); + xfrm_dev_state_update_stats(x); if (x->km.state == XFRM_STATE_DEAD) goto out; @@ -1935,7 +1935,7 @@ EXPORT_SYMBOL(xfrm_state_update); int xfrm_state_check_expire(struct xfrm_state *x) { - xfrm_dev_state_update_curlft(x); + xfrm_dev_state_update_stats(x); if (!READ_ONCE(x->curlft.use_time)) WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds()); diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index ad01997c3aa9..dc4f9b8d7cb0 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -902,7 +902,7 @@ static void copy_to_user_state(struct xfrm_state *x, struct xfrm_usersa_info *p) memcpy(&p->sel, &x->sel, sizeof(p->sel)); memcpy(&p->lft, &x->lft, sizeof(p->lft)); if (x->xso.dev) - xfrm_dev_state_update_curlft(x); + xfrm_dev_state_update_stats(x); memcpy(&p->curlft, &x->curlft, sizeof(p->curlft)); put_unaligned(x->stats.replay_window, &p->stats.replay_window); put_unaligned(x->stats.replay, &p->stats.replay); -- cgit v1.2.3 From 91a72ada66053b4dba95cf1a60a5a23fdbd6faf7 Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Tue, 26 Dec 2023 10:22:08 +0200 Subject: net/mlx5: Remove initial segmentation duplicate definitions Device definitions belong in mlx5_ifc, remove the duplicates in mlx5_core.h. Signed-off-by: Gal Pressman Reviewed-by: Jianbo Liu Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/fw.c | 6 +++--- drivers/net/ethernet/mellanox/mlx5/core/health.c | 20 ++++++++++---------- drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h | 7 ------- include/linux/mlx5/mlx5_ifc.h | 1 + 4 files changed, 14 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index 58f4c0d0fafa..e7faf7e73ca4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -366,18 +366,18 @@ int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev) return -EIO; } - mlx5_set_nic_state(dev, MLX5_NIC_IFC_DISABLED); + mlx5_set_nic_state(dev, MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED); /* Loop until device state turns to disable */ end = jiffies + msecs_to_jiffies(delay_ms); do { - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) break; cond_resched(); } while (!time_after(jiffies, end)); - if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { + if (mlx5_get_nic_state(dev) != MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) { dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", mlx5_get_nic_state(dev), delay_ms); return -EIO; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 5c2ac2d9dbd9..9463ede84d8d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -116,9 +116,9 @@ u32 mlx5_health_check_fatal_sensors(struct mlx5_core_dev *dev) return MLX5_SENSOR_PCI_COMM_ERR; if (pci_channel_offline(dev->pdev)) return MLX5_SENSOR_PCI_ERR; - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) return MLX5_SENSOR_NIC_DISABLED; - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET) return MLX5_SENSOR_NIC_SW_RESET; if (sensor_fw_synd_rfr(dev)) return MLX5_SENSOR_FW_SYND_RFR; @@ -185,7 +185,7 @@ static bool reset_fw_if_needed(struct mlx5_core_dev *dev) /* Write the NIC interface field to initiate the reset, the command * interface address also resides here, don't overwrite it. */ - mlx5_set_nic_state(dev, MLX5_NIC_IFC_SW_RESET); + mlx5_set_nic_state(dev, MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET); return true; } @@ -246,13 +246,13 @@ recover_from_sw_reset: /* Recover from SW reset */ end = jiffies + msecs_to_jiffies(delay_ms); do { - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) break; msleep(20); } while (!time_after(jiffies, end)); - if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { + if (mlx5_get_nic_state(dev) != MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) { dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", mlx5_get_nic_state(dev), delay_ms); } @@ -272,26 +272,26 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) u8 nic_interface = mlx5_get_nic_state(dev); switch (nic_interface) { - case MLX5_NIC_IFC_FULL: + case MLX5_INITIAL_SEG_NIC_INTERFACE_FULL_DRIVER: mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); break; - case MLX5_NIC_IFC_DISABLED: + case MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED: mlx5_core_warn(dev, "starting teardown\n"); break; - case MLX5_NIC_IFC_NO_DRAM_NIC: + case MLX5_INITIAL_SEG_NIC_INTERFACE_NO_DRAM_NIC: mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); break; - case MLX5_NIC_IFC_SW_RESET: + case MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET: /* The IFC mode field is 3 bits, so it will read 0x7 in 2 cases: * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded * and this is a VF), this is not recoverable by SW reset. * Logging of this is handled elsewhere. * 2. FW reset has been issued by another function, driver can * be reloaded to recover after the mode switches to - * MLX5_NIC_IFC_DISABLED. + * MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED. */ if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR) mlx5_core_warn(dev, "NIC SW reset in progress\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index a79b7959361b..58732f44940f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -312,13 +312,6 @@ static inline int mlx5_rescan_drivers(struct mlx5_core_dev *dev) return ret; } -enum { - MLX5_NIC_IFC_FULL = 0, - MLX5_NIC_IFC_DISABLED = 1, - MLX5_NIC_IFC_NO_DRAM_NIC = 2, - MLX5_NIC_IFC_SW_RESET = 7 -}; - u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 6c44f107b8ba..7f5e846eb46d 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -10661,6 +10661,7 @@ enum { MLX5_INITIAL_SEG_NIC_INTERFACE_FULL_DRIVER = 0x0, MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED = 0x1, MLX5_INITIAL_SEG_NIC_INTERFACE_NO_DRAM_NIC = 0x2, + MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET = 0x7, }; enum { -- cgit v1.2.3 From 0bd199fd9c19aa545f677fd0a99f2be101cb6309 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 2 Feb 2024 17:41:45 +0000 Subject: net: phy: constify phydev->drv Device driver structures are shared between all devices that they match, and thus nothing should never write to the device driver structure through the phydev->drv pointer. Let's make this pointer const to catch code that attempts to do so. Suggested-by: Christian Marangi Signed-off-by: Russell King (Oracle) Link: https://lore.kernel.org/r/E1rVxXt-002YqY-9G@rmk-PC.armlinux.org.uk Signed-off-by: Paolo Abeni --- drivers/net/phy/phy.c | 3 +-- drivers/net/phy/phy_device.c | 6 +++--- drivers/net/phy/xilinx_gmii2rgmii.c | 2 +- include/linux/phy.h | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 3b9531143be1..14224e06d69f 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -1290,7 +1290,6 @@ int phy_disable_interrupts(struct phy_device *phydev) static irqreturn_t phy_interrupt(int irq, void *phy_dat) { struct phy_device *phydev = phy_dat; - struct phy_driver *drv = phydev->drv; irqreturn_t ret; /* Wakeup interrupts may occur during a system sleep transition. @@ -1316,7 +1315,7 @@ static irqreturn_t phy_interrupt(int irq, void *phy_dat) } mutex_lock(&phydev->lock); - ret = drv->handle_interrupt(phydev); + ret = phydev->drv->handle_interrupt(phydev); mutex_unlock(&phydev->lock); return ret; diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 52828d1c64f7..2eed8f03621d 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1413,7 +1413,7 @@ int phy_sfp_probe(struct phy_device *phydev, } EXPORT_SYMBOL(phy_sfp_probe); -static bool phy_drv_supports_irq(struct phy_driver *phydrv) +static bool phy_drv_supports_irq(const struct phy_driver *phydrv) { return phydrv->config_intr && phydrv->handle_interrupt; } @@ -1867,7 +1867,7 @@ int phy_suspend(struct phy_device *phydev) { struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; struct net_device *netdev = phydev->attached_dev; - struct phy_driver *phydrv = phydev->drv; + const struct phy_driver *phydrv = phydev->drv; int ret; if (phydev->suspended) @@ -1892,7 +1892,7 @@ EXPORT_SYMBOL(phy_suspend); int __phy_resume(struct phy_device *phydev) { - struct phy_driver *phydrv = phydev->drv; + const struct phy_driver *phydrv = phydev->drv; int ret; lockdep_assert_held(&phydev->lock); diff --git a/drivers/net/phy/xilinx_gmii2rgmii.c b/drivers/net/phy/xilinx_gmii2rgmii.c index 7fd9fe6a602b..7b1bc5fcef9b 100644 --- a/drivers/net/phy/xilinx_gmii2rgmii.c +++ b/drivers/net/phy/xilinx_gmii2rgmii.c @@ -22,7 +22,7 @@ struct gmii2rgmii { struct phy_device *phy_dev; - struct phy_driver *phy_drv; + const struct phy_driver *phy_drv; struct phy_driver conv_phy_drv; struct mdio_device *mdio; }; diff --git a/include/linux/phy.h b/include/linux/phy.h index a66f07d3f5f4..ad93f8b1b128 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -638,7 +638,7 @@ struct phy_device { /* Information about the PHY type */ /* And management functions */ - struct phy_driver *drv; + const struct phy_driver *drv; struct device_link *devlink; -- cgit v1.2.3 From 8284765f03b7a0b18968cefeb5e78aca647b8f8b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 9 Jan 2024 17:15:32 -0800 Subject: KVM: Get reference to VM's address space in the async #PF worker Get a reference to the target VM's address space in async_pf_execute() instead of gifting a reference from kvm_setup_async_pf(). Keeping the address space alive just to service an async #PF is counter-productive, i.e. if the process is exiting and all vCPUs are dead, then NOT doing get_user_pages_remote() and freeing the address space asap is desirable. Handling the mm reference entirely within async_pf_execute() also simplifies the async #PF flows as a whole, e.g. it's not immediately obvious when the worker task vs. the vCPU task is responsible for putting the gifted mm reference. Reviewed-by: Vitaly Kuznetsov Reviewed-by: Xu Yilun Link: https://lore.kernel.org/r/20240110011533.503302-4-seanjc@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 1 - virt/kvm/async_pf.c | 32 ++++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 7e7fd25b09b3..bbfefd7e612f 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -238,7 +238,6 @@ struct kvm_async_pf { struct list_head link; struct list_head queue; struct kvm_vcpu *vcpu; - struct mm_struct *mm; gpa_t cr2_or_gpa; unsigned long addr; struct kvm_arch_async_pf arch; diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 85acc9e824b9..628f6df7609f 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -46,8 +46,8 @@ static void async_pf_execute(struct work_struct *work) { struct kvm_async_pf *apf = container_of(work, struct kvm_async_pf, work); - struct mm_struct *mm = apf->mm; struct kvm_vcpu *vcpu = apf->vcpu; + struct mm_struct *mm = vcpu->kvm->mm; unsigned long addr = apf->addr; gpa_t cr2_or_gpa = apf->cr2_or_gpa; int locked = 1; @@ -56,16 +56,24 @@ static void async_pf_execute(struct work_struct *work) might_sleep(); /* - * This work is run asynchronously to the task which owns - * mm and might be done in another context, so we must - * access remotely. + * Attempt to pin the VM's host address space, and simply skip gup() if + * acquiring a pin fail, i.e. if the process is exiting. Note, KVM + * holds a reference to its associated mm_struct until the very end of + * kvm_destroy_vm(), i.e. the struct itself won't be freed before this + * work item is fully processed. */ - mmap_read_lock(mm); - get_user_pages_remote(mm, addr, 1, FOLL_WRITE, NULL, &locked); - if (locked) - mmap_read_unlock(mm); - mmput(mm); + if (mmget_not_zero(mm)) { + mmap_read_lock(mm); + get_user_pages_remote(mm, addr, 1, FOLL_WRITE, NULL, &locked); + if (locked) + mmap_read_unlock(mm); + mmput(mm); + } + /* + * Notify and kick the vCPU even if faulting in the page failed, e.g. + * so that the vCPU can retry the fault synchronously. + */ if (IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC)) kvm_arch_async_page_present(vcpu, apf); @@ -131,10 +139,8 @@ void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu) #ifdef CONFIG_KVM_ASYNC_PF_SYNC flush_work(&work->work); #else - if (cancel_work_sync(&work->work)) { - mmput(work->mm); + if (cancel_work_sync(&work->work)) kmem_cache_free(async_pf_cache, work); - } #endif spin_lock(&vcpu->async_pf.lock); } @@ -205,8 +211,6 @@ bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, work->cr2_or_gpa = cr2_or_gpa; work->addr = hva; work->arch = *arch; - work->mm = current->mm; - mmget(work->mm); INIT_WORK(&work->work, async_pf_execute); -- cgit v1.2.3 From 574849054d97cee5be78d6c149d84685647fe774 Mon Sep 17 00:00:00 2001 From: Kuninori Morimoto Date: Mon, 5 Feb 2024 05:37:28 +0000 Subject: of: property: use unsigned int return on of_graph_get_endpoint_count() Because of of_graph_get_endpoint_count() doesn't report error, just return count of endpoint, the return type should be unsigned. Tidyup it. Signed-off-by: Kuninori Morimoto Reviewed-by: Laurent Pinchart Link: https://lore.kernel.org/r/87plxbcvzb.wl-kuninori.morimoto.gx@renesas.com Signed-off-by: Rob Herring --- drivers/of/property.c | 4 ++-- include/linux/of_graph.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/of/property.c b/drivers/of/property.c index 455dda225556..470d1eb5e56d 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -818,10 +818,10 @@ EXPORT_SYMBOL(of_graph_get_remote_port); * * Return: count of endpoint of this device node */ -int of_graph_get_endpoint_count(const struct device_node *np) +unsigned int of_graph_get_endpoint_count(const struct device_node *np) { struct device_node *endpoint; - int num = 0; + unsigned int num = 0; for_each_endpoint_of_node(np, endpoint) num++; diff --git a/include/linux/of_graph.h b/include/linux/of_graph.h index 4d7756087b6b..a4bea62bfa29 100644 --- a/include/linux/of_graph.h +++ b/include/linux/of_graph.h @@ -41,7 +41,7 @@ struct of_endpoint { bool of_graph_is_present(const struct device_node *node); int of_graph_parse_endpoint(const struct device_node *node, struct of_endpoint *endpoint); -int of_graph_get_endpoint_count(const struct device_node *np); +unsigned int of_graph_get_endpoint_count(const struct device_node *np); struct device_node *of_graph_get_port_by_id(struct device_node *node, u32 id); struct device_node *of_graph_get_next_endpoint(const struct device_node *parent, struct device_node *previous); @@ -68,7 +68,7 @@ static inline int of_graph_parse_endpoint(const struct device_node *node, return -ENOSYS; } -static inline int of_graph_get_endpoint_count(const struct device_node *np) +static inline unsigned int of_graph_get_endpoint_count(const struct device_node *np) { return 0; } -- cgit v1.2.3 From b64691274f5d33fc9d93af73483162967f7ec5bb Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 3 Feb 2024 20:53:15 +0100 Subject: net: phy: add helper phy_advertise_eee_all Per default phylib preserves the EEE advertising at the time of phy probing. The EEE advertising can be changed from user space, in addition this helper allows to set the EEE advertising to all supported modes from drivers in kernel space. Suggested-by: Andrew Lunn Signed-off-by: Heiner Kallweit Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20bfc471-aeeb-4ae4-ba09-7d6d4be6b86a@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy_device.c | 16 ++++++++++++++++ include/linux/phy.h | 1 + 2 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 2eed8f03621d..839e42f9e16a 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -2777,6 +2777,22 @@ void phy_advertise_supported(struct phy_device *phydev) } EXPORT_SYMBOL(phy_advertise_supported); +/** + * phy_advertise_eee_all - Advertise all supported EEE modes + * @phydev: target phy_device struct + * + * Description: Per default phylib preserves the EEE advertising at the time of + * phy probing, which might be a subset of the supported EEE modes. Use this + * function when all supported EEE modes should be advertised. This does not + * trigger auto-negotiation, so must be called before phy_start()/ + * phylink_start() which will start auto-negotiation. + */ +void phy_advertise_eee_all(struct phy_device *phydev) +{ + linkmode_copy(phydev->advertising_eee, phydev->supported_eee); +} +EXPORT_SYMBOL_GPL(phy_advertise_eee_all); + /** * phy_support_sym_pause - Enable support of symmetrical pause * @phydev: target phy_device struct diff --git a/include/linux/phy.h b/include/linux/phy.h index ad93f8b1b128..fd8dbea9b4d9 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1960,6 +1960,7 @@ int phy_get_rate_matching(struct phy_device *phydev, void phy_set_max_speed(struct phy_device *phydev, u32 max_speed); void phy_remove_link_mode(struct phy_device *phydev, u32 link_mode); void phy_advertise_supported(struct phy_device *phydev); +void phy_advertise_eee_all(struct phy_device *phydev); void phy_support_sym_pause(struct phy_device *phydev); void phy_support_asym_pause(struct phy_device *phydev); void phy_set_sym_pause(struct phy_device *phydev, bool rx, bool tx, -- cgit v1.2.3 From c0c0293cf7a0f21ef461956d44e4add718574f3f Mon Sep 17 00:00:00 2001 From: David Lechner Date: Tue, 6 Feb 2024 14:06:46 -0600 Subject: spi: drop gpf arg from __spi_split_transfer_maxsize() The __spi_split_transfer_maxsize() function has a gpf argument to allow callers to specify the type of memory allocation that needs to be used. However, this function only allocates struct spi_transfer and is not intended to be used from atomic contexts so this type should always be GFP_KERNEL, so we can just drop the argument. Some callers of these functions also passed GFP_DMA, but since only struct spi_transfer is allocated and not any tx/rx buffers, this is not actually necessary and is removed in this commit. Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20240206200648.1782234-1-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-stm32.c | 4 +--- drivers/spi/spi.c | 22 ++++++++-------------- include/linux/spi/spi.h | 6 ++---- 3 files changed, 11 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index e61302ef3c21..c32e57bb38bd 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1170,9 +1170,7 @@ static int stm32_spi_prepare_msg(struct spi_controller *ctrl, if (spi->cfg->set_number_of_data) { int ret; - ret = spi_split_transfers_maxwords(ctrl, msg, - spi->t_size_max, - GFP_KERNEL | GFP_DMA); + ret = spi_split_transfers_maxwords(ctrl, msg, spi->t_size_max); if (ret) return ret; } diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 517558e01559..3270a1337a5b 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1752,7 +1752,7 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, */ if ((msg->spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || spi_is_csgpiod(msg->spi))) { - ret = spi_split_transfers_maxwords(ctlr, msg, 1, GFP_KERNEL); + ret = spi_split_transfers_maxwords(ctlr, msg, 1); if (ret) { msg->status = ret; spi_finalize_current_message(ctlr); @@ -1767,8 +1767,7 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, } } else { ret = spi_split_transfers_maxsize(ctlr, msg, - spi_max_transfer_size(msg->spi), - GFP_KERNEL | GFP_DMA); + spi_max_transfer_size(msg->spi)); if (ret) { msg->status = ret; spi_finalize_current_message(ctlr); @@ -3707,8 +3706,7 @@ static struct spi_replaced_transfers *spi_replace_transfers( static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, struct spi_message *msg, struct spi_transfer **xferp, - size_t maxsize, - gfp_t gfp) + size_t maxsize) { struct spi_transfer *xfer = *xferp, *xfers; struct spi_replaced_transfers *srt; @@ -3719,7 +3717,7 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, count = DIV_ROUND_UP(xfer->len, maxsize); /* Create replacement */ - srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, gfp); + srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, GFP_KERNEL); if (IS_ERR(srt)) return PTR_ERR(srt); xfers = srt->inserted_transfers; @@ -3779,14 +3777,12 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, * @ctlr: the @spi_controller for this transfer * @msg: the @spi_message to transform * @maxsize: the maximum when to apply this - * @gfp: GFP allocation flags * * Return: status of transformation */ int spi_split_transfers_maxsize(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxsize, - gfp_t gfp) + size_t maxsize) { struct spi_transfer *xfer; int ret; @@ -3801,7 +3797,7 @@ int spi_split_transfers_maxsize(struct spi_controller *ctlr, list_for_each_entry(xfer, &msg->transfers, transfer_list) { if (xfer->len > maxsize) { ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer, - maxsize, gfp); + maxsize); if (ret) return ret; } @@ -3819,14 +3815,12 @@ EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize); * @ctlr: the @spi_controller for this transfer * @msg: the @spi_message to transform * @maxwords: the number of words to limit each transfer to - * @gfp: GFP allocation flags * * Return: status of transformation */ int spi_split_transfers_maxwords(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxwords, - gfp_t gfp) + size_t maxwords) { struct spi_transfer *xfer; @@ -3844,7 +3838,7 @@ int spi_split_transfers_maxwords(struct spi_controller *ctlr, maxsize = maxwords * roundup_pow_of_two(BITS_TO_BYTES(xfer->bits_per_word)); if (xfer->len > maxsize) { ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer, - maxsize, gfp); + maxsize); if (ret) return ret; } diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 29c3e4dd5d93..9339c8ed1f8f 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1365,12 +1365,10 @@ struct spi_replaced_transfers { extern int spi_split_transfers_maxsize(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxsize, - gfp_t gfp); + size_t maxsize); extern int spi_split_transfers_maxwords(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxwords, - gfp_t gfp); + size_t maxwords); /*---------------------------------------------------------------------------*/ -- cgit v1.2.3 From 7412dc6d55eed6b76180e40ac3601412ebde29bd Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:47:03 +0106 Subject: dump_stack: Do not get cpu_sync for panic CPU dump_stack() is called in panic(). If for some reason another CPU is holding the printk_cpu_sync and is unable to release it, the panic CPU will be unable to continue and print the stacktrace. Since non-panic CPUs are not allowed to store new printk messages anyway, there is no need to synchronize the stacktrace output in a panic situation. For the panic CPU, do not get the printk_cpu_sync because it is not needed and avoids a potential deadlock scenario in panic(). Link: https://lore.kernel.org/lkml/ZcIGKU8sxti38Kok@alley Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-15-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/printk.h | 2 ++ kernel/printk/internal.h | 1 - lib/dump_stack.c | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/printk.h b/include/linux/printk.h index 8ef499ab3c1e..955e31860095 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -273,6 +273,8 @@ static inline void printk_trigger_flush(void) } #endif +bool this_cpu_in_panic(void); + #ifdef CONFIG_SMP extern int __printk_cpu_sync_try_get(void); extern void __printk_cpu_sync_wait(void); diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index ac2d9750e5f8..6c2afee5ef62 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -130,7 +130,6 @@ struct printk_message { }; bool other_cpu_in_panic(void); -bool this_cpu_in_panic(void); bool printk_get_next_message(struct printk_message *pmsg, u64 seq, bool is_extended, bool may_supress); diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 83471e81501a..222c6d6c8281 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -96,15 +96,25 @@ static void __dump_stack(const char *log_lvl) */ asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { + bool in_panic = this_cpu_in_panic(); unsigned long flags; /* * Permit this cpu to perform nested stack dumps while serialising - * against other CPUs + * against other CPUs, unless this CPU is in panic. + * + * When in panic, non-panic CPUs are not permitted to store new + * printk messages so there is no need to synchronize the output. + * This avoids potential deadlock in panic() if another CPU is + * holding and unable to release the printk_cpu_sync. */ - printk_cpu_sync_get_irqsave(flags); + if (!in_panic) + printk_cpu_sync_get_irqsave(flags); + __dump_stack(log_lvl); - printk_cpu_sync_put_irqrestore(flags); + + if (!in_panic) + printk_cpu_sync_put_irqrestore(flags); } EXPORT_SYMBOL(dump_stack_lvl); -- cgit v1.2.3 From d160c66cda0ac8614adc53a5b5b0e6d6f1a05a5b Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Mon, 5 Feb 2024 12:30:22 +0200 Subject: net: Do not return value from init_dummy_netdev() init_dummy_netdev() always returns zero and all the callers do not check the returned value. Set the function to not return value, as it is not really used today. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Reviewed-by: Jiri Pirko Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20240205103022.440946-1-amcohen@nvidia.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 +- net/core/dev.c | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 118c40258d07..1845dd5043b4 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3198,7 +3198,7 @@ static inline void unregister_netdevice(struct net_device *dev) int netdev_refcnt_read(const struct net_device *dev); void free_netdev(struct net_device *dev); void netdev_freemem(struct net_device *dev); -int init_dummy_netdev(struct net_device *dev); +void init_dummy_netdev(struct net_device *dev); struct net_device *netdev_get_xmit_slave(struct net_device *dev, struct sk_buff *skb, diff --git a/net/core/dev.c b/net/core/dev.c index 27ba057d06c4..e52e2888cccd 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10345,7 +10345,7 @@ EXPORT_SYMBOL(register_netdevice); * that need to tie several hardware interfaces to a single NAPI * poll scheduler due to HW limitations. */ -int init_dummy_netdev(struct net_device *dev) +void init_dummy_netdev(struct net_device *dev) { /* Clear everything. Note we don't initialize spinlocks * are they aren't supposed to be taken by any of the @@ -10373,8 +10373,6 @@ int init_dummy_netdev(struct net_device *dev) * because users of this 'device' dont need to change * its refcount. */ - - return 0; } EXPORT_SYMBOL_GPL(init_dummy_netdev); -- cgit v1.2.3 From ccb49011bb2ebfd66164dbf68c5bff48917bb5ef Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 6 Feb 2024 15:08:19 +0100 Subject: quota: Properly annotate i_dquot arrays with __rcu Dquots pointed to from i_dquot arrays in inodes are protected by dquot_srcu. Annotate them as such and change .get_dquots callback to return properly annotated pointer to make sparse happy. Fixes: b9ba6f94b238 ("quota: remove dqptr_sem") Signed-off-by: Jan Kara --- fs/ext2/ext2.h | 2 +- fs/ext2/super.c | 2 +- fs/ext4/ext4.h | 2 +- fs/ext4/super.c | 2 +- fs/f2fs/f2fs.h | 2 +- fs/f2fs/super.c | 2 +- fs/jfs/jfs_incore.h | 2 +- fs/jfs/super.c | 2 +- fs/ocfs2/inode.h | 2 +- fs/ocfs2/super.c | 2 +- fs/quota/dquot.c | 3 +-- fs/reiserfs/reiserfs.h | 2 +- fs/reiserfs/super.c | 2 +- include/linux/fs.h | 2 +- include/linux/shmem_fs.h | 2 +- mm/shmem.c | 2 +- 16 files changed, 16 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 677a9ad45dcb..f38bdd46e4f7 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -674,7 +674,7 @@ struct ext2_inode_info { struct inode vfs_inode; struct list_head i_orphan; /* unlinked but open inodes */ #ifdef CONFIG_QUOTA - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif }; diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 01f9addc8b1f..6d8587505cea 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -320,7 +320,7 @@ static ssize_t ext2_quota_read(struct super_block *sb, int type, char *data, siz static ssize_t ext2_quota_write(struct super_block *sb, int type, const char *data, size_t len, loff_t off); static int ext2_quota_on(struct super_block *sb, int type, int format_id, const struct path *path); -static struct dquot **ext2_get_dquots(struct inode *inode) +static struct dquot __rcu **ext2_get_dquots(struct inode *inode) { return EXT2_I(inode)->i_dquot; } diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index a5d784872303..3205d46bc967 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1156,7 +1156,7 @@ struct ext4_inode_info { tid_t i_datasync_tid; #ifdef CONFIG_QUOTA - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif /* Precomputed uuid+inum+igen checksum for seeding inode checksums */ diff --git a/fs/ext4/super.c b/fs/ext4/super.c index dcba0f85dfe2..3f6ed580444b 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1600,7 +1600,7 @@ static ssize_t ext4_quota_write(struct super_block *sb, int type, static int ext4_quota_enable(struct super_block *sb, int type, int format_id, unsigned int flags); -static struct dquot **ext4_get_dquots(struct inode *inode) +static struct dquot __rcu **ext4_get_dquots(struct inode *inode) { return EXT4_I(inode)->i_dquot; } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 65294e3b0bef..31554e2a0a32 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -829,7 +829,7 @@ struct f2fs_inode_info { spinlock_t i_size_lock; /* protect last_disk_size */ #ifdef CONFIG_QUOTA - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; /* quota space reservation, managed internally by quota code */ qsize_t i_reserved_quota; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index d45ab0992ae5..18cc4829f7e8 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2768,7 +2768,7 @@ int f2fs_dquot_initialize(struct inode *inode) return dquot_initialize(inode); } -static struct dquot **f2fs_get_dquots(struct inode *inode) +static struct dquot __rcu **f2fs_get_dquots(struct inode *inode) { return F2FS_I(inode)->i_dquot; } diff --git a/fs/jfs/jfs_incore.h b/fs/jfs/jfs_incore.h index dd4264aa9bed..10934f9a11be 100644 --- a/fs/jfs/jfs_incore.h +++ b/fs/jfs/jfs_incore.h @@ -92,7 +92,7 @@ struct jfs_inode_info { } link; } u; #ifdef CONFIG_QUOTA - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif u32 dev; /* will die when we get wide dev_t */ struct inode vfs_inode; diff --git a/fs/jfs/super.c b/fs/jfs/super.c index 8d8e556bd610..ff135a43b5b7 100644 --- a/fs/jfs/super.c +++ b/fs/jfs/super.c @@ -824,7 +824,7 @@ out: return len - towrite; } -static struct dquot **jfs_get_dquots(struct inode *inode) +static struct dquot __rcu **jfs_get_dquots(struct inode *inode) { return JFS_IP(inode)->i_dquot; } diff --git a/fs/ocfs2/inode.h b/fs/ocfs2/inode.h index 82b28fdacc7e..accf03d4765e 100644 --- a/fs/ocfs2/inode.h +++ b/fs/ocfs2/inode.h @@ -65,7 +65,7 @@ struct ocfs2_inode_info tid_t i_sync_tid; tid_t i_datasync_tid; - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; }; /* diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index 6b906424902b..1259fe02cd53 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -122,7 +122,7 @@ static int ocfs2_susp_quotas(struct ocfs2_super *osb, int unsuspend); static int ocfs2_enable_quotas(struct ocfs2_super *osb); static void ocfs2_disable_quotas(struct ocfs2_super *osb); -static struct dquot **ocfs2_get_dquots(struct inode *inode) +static struct dquot __rcu **ocfs2_get_dquots(struct inode *inode) { return OCFS2_I(inode)->i_dquot; } diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 22754460fb6f..f73016c7bc39 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -998,8 +998,7 @@ EXPORT_SYMBOL(dqget); static inline struct dquot __rcu **i_dquot(struct inode *inode) { - /* Force __rcu for now until filesystems are fixed */ - return (struct dquot __rcu **)inode->i_sb->s_op->get_dquots(inode); + return inode->i_sb->s_op->get_dquots(inode); } static int dqinit_needed(struct inode *inode, int type) diff --git a/fs/reiserfs/reiserfs.h b/fs/reiserfs/reiserfs.h index 725667880e62..b65549164590 100644 --- a/fs/reiserfs/reiserfs.h +++ b/fs/reiserfs/reiserfs.h @@ -97,7 +97,7 @@ struct reiserfs_inode_info { struct rw_semaphore i_xattr_sem; #endif #ifdef CONFIG_QUOTA - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif struct inode vfs_inode; diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c index 67b5510beded..7b3d5aeb2a6f 100644 --- a/fs/reiserfs/super.c +++ b/fs/reiserfs/super.c @@ -802,7 +802,7 @@ static ssize_t reiserfs_quota_write(struct super_block *, int, const char *, static ssize_t reiserfs_quota_read(struct super_block *, int, char *, size_t, loff_t); -static struct dquot **reiserfs_get_dquots(struct inode *inode) +static struct dquot __rcu **reiserfs_get_dquots(struct inode *inode) { return REISERFS_I(inode)->i_dquot; } diff --git a/include/linux/fs.h b/include/linux/fs.h index ed5966a70495..d0b849e4f6cd 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2159,7 +2159,7 @@ struct super_operations { #ifdef CONFIG_QUOTA ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t); ssize_t (*quota_write)(struct super_block *, int, const char *, size_t, loff_t); - struct dquot **(*get_dquots)(struct inode *); + struct dquot __rcu **(*get_dquots)(struct inode *); #endif long (*nr_cached_objects)(struct super_block *, struct shrink_control *); diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 2caa6b86106a..66828dfc6e74 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -37,7 +37,7 @@ struct shmem_inode_info { unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ #ifdef CONFIG_TMPFS_QUOTA - struct dquot *i_dquot[MAXQUOTAS]; + struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif struct inode vfs_inode; }; diff --git a/mm/shmem.c b/mm/shmem.c index d7c84ff62186..791a6dc16324 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -311,7 +311,7 @@ static void shmem_disable_quotas(struct super_block *sb) dquot_quota_off(sb, type); } -static struct dquot **shmem_get_dquots(struct inode *inode) +static struct dquot __rcu **shmem_get_dquots(struct inode *inode) { return SHMEM_I(inode)->i_dquot; } -- cgit v1.2.3 From 2259233110d90059187c5ba75537eb93eba8417b Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 7 Feb 2024 19:40:30 +0100 Subject: spi: bitbang: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into the SPI bitbang controller drivers. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/f7f949feb803acb8bea75798f41371a13287f4e8.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-ath79.c | 4 +-- drivers/spi/spi-au1550.c | 2 +- drivers/spi/spi-bitbang.c | 64 ++++++++++++++++++++--------------------- drivers/spi/spi-butterfly.c | 6 ++-- drivers/spi/spi-davinci.c | 6 ++-- drivers/spi/spi-gpio.c | 2 +- drivers/spi/spi-lm70llp.c | 6 ++-- drivers/spi/spi-oc-tiny.c | 6 ++-- drivers/spi/spi-omap-uwire.c | 4 +-- drivers/spi/spi-sh-sci.c | 10 +++---- drivers/spi/spi-xilinx.c | 4 +-- drivers/spi/spi-xtensa-xtfpga.c | 2 +- include/linux/spi/spi_bitbang.h | 2 +- 13 files changed, 59 insertions(+), 59 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c index b7ada981464a..d78762d4db98 100644 --- a/drivers/spi/spi-ath79.c +++ b/drivers/spi/spi-ath79.c @@ -189,7 +189,7 @@ static int ath79_spi_probe(struct platform_device *pdev) host->num_chipselect = 3; host->mem_ops = &ath79_mem_ops; - sp->bitbang.master = host; + sp->bitbang.ctlr = host; sp->bitbang.chipselect = ath79_spi_chipselect; sp->bitbang.txrx_word[SPI_MODE_0] = ath79_spi_txrx_mode0; sp->bitbang.flags = SPI_CS_HIGH; @@ -237,7 +237,7 @@ static void ath79_spi_remove(struct platform_device *pdev) spi_bitbang_stop(&sp->bitbang); ath79_spi_disable(sp); - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); } static void ath79_spi_shutdown(struct platform_device *pdev) diff --git a/drivers/spi/spi-au1550.c b/drivers/spi/spi-au1550.c index 1011b1a8f241..825d2f1cdff8 100644 --- a/drivers/spi/spi-au1550.c +++ b/drivers/spi/spi-au1550.c @@ -800,7 +800,7 @@ static int au1550_spi_probe(struct platform_device *pdev) init_completion(&hw->host_done); - hw->bitbang.master = hw->host; + hw->bitbang.ctlr = hw->host; hw->bitbang.setup_transfer = au1550_spi_setupxfer; hw->bitbang.chipselect = au1550_spi_chipsel; hw->bitbang.txrx_bufs = au1550_spi_txrx_bufs; diff --git a/drivers/spi/spi-bitbang.c b/drivers/spi/spi-bitbang.c index ecd44016c197..a0e2204fc039 100644 --- a/drivers/spi/spi-bitbang.c +++ b/drivers/spi/spi-bitbang.c @@ -187,7 +187,7 @@ int spi_bitbang_setup(struct spi_device *spi) bool initial_setup = false; int retval; - bitbang = spi_master_get_devdata(spi->master); + bitbang = spi_controller_get_devdata(spi->controller); if (!cs) { cs = kzalloc(sizeof(*cs), GFP_KERNEL); @@ -236,7 +236,7 @@ static int spi_bitbang_bufs(struct spi_device *spi, struct spi_transfer *t) unsigned nsecs = cs->nsecs; struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi->master); + bitbang = spi_controller_get_devdata(spi->controller); if (bitbang->set_line_direction) { int err; @@ -268,11 +268,11 @@ static int spi_bitbang_bufs(struct spi_device *spi, struct spi_transfer *t) * transfer-at-a-time ones to leverage dma or fifo hardware. */ -static int spi_bitbang_prepare_hardware(struct spi_master *spi) +static int spi_bitbang_prepare_hardware(struct spi_controller *spi) { struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi); + bitbang = spi_controller_get_devdata(spi); mutex_lock(&bitbang->lock); bitbang->busy = 1; @@ -281,11 +281,11 @@ static int spi_bitbang_prepare_hardware(struct spi_master *spi) return 0; } -static int spi_bitbang_transfer_one(struct spi_master *master, +static int spi_bitbang_transfer_one(struct spi_controller *ctlr, struct spi_device *spi, struct spi_transfer *transfer) { - struct spi_bitbang *bitbang = spi_master_get_devdata(master); + struct spi_bitbang *bitbang = spi_controller_get_devdata(ctlr); int status = 0; if (bitbang->setup_transfer) { @@ -303,16 +303,16 @@ static int spi_bitbang_transfer_one(struct spi_master *master, status = -EREMOTEIO; out: - spi_finalize_current_transfer(master); + spi_finalize_current_transfer(ctlr); return status; } -static int spi_bitbang_unprepare_hardware(struct spi_master *spi) +static int spi_bitbang_unprepare_hardware(struct spi_controller *spi) { struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi); + bitbang = spi_controller_get_devdata(spi); mutex_lock(&bitbang->lock); bitbang->busy = 0; @@ -323,7 +323,7 @@ static int spi_bitbang_unprepare_hardware(struct spi_master *spi) static void spi_bitbang_set_cs(struct spi_device *spi, bool enable) { - struct spi_bitbang *bitbang = spi_master_get_devdata(spi->master); + struct spi_bitbang *bitbang = spi_controller_get_devdata(spi->controller); /* SPI core provides CS high / low, but bitbang driver * expects CS active @@ -341,10 +341,10 @@ static void spi_bitbang_set_cs(struct spi_device *spi, bool enable) int spi_bitbang_init(struct spi_bitbang *bitbang) { - struct spi_master *master = bitbang->master; + struct spi_controller *ctlr = bitbang->ctlr; bool custom_cs; - if (!master) + if (!ctlr) return -EINVAL; /* * We only need the chipselect callback if we are actually using it. @@ -352,39 +352,39 @@ int spi_bitbang_init(struct spi_bitbang *bitbang) * SPI_CONTROLLER_GPIO_SS flag is set, we always need to call the * driver-specific chipselect routine. */ - custom_cs = (!master->use_gpio_descriptors || - (master->flags & SPI_CONTROLLER_GPIO_SS)); + custom_cs = (!ctlr->use_gpio_descriptors || + (ctlr->flags & SPI_CONTROLLER_GPIO_SS)); if (custom_cs && !bitbang->chipselect) return -EINVAL; mutex_init(&bitbang->lock); - if (!master->mode_bits) - master->mode_bits = SPI_CPOL | SPI_CPHA | bitbang->flags; + if (!ctlr->mode_bits) + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | bitbang->flags; - if (master->transfer || master->transfer_one_message) + if (ctlr->transfer || ctlr->transfer_one_message) return -EINVAL; - master->prepare_transfer_hardware = spi_bitbang_prepare_hardware; - master->unprepare_transfer_hardware = spi_bitbang_unprepare_hardware; - master->transfer_one = spi_bitbang_transfer_one; + ctlr->prepare_transfer_hardware = spi_bitbang_prepare_hardware; + ctlr->unprepare_transfer_hardware = spi_bitbang_unprepare_hardware; + ctlr->transfer_one = spi_bitbang_transfer_one; /* * When using GPIO descriptors, the ->set_cs() callback doesn't even * get called unless SPI_CONTROLLER_GPIO_SS is set. */ if (custom_cs) - master->set_cs = spi_bitbang_set_cs; + ctlr->set_cs = spi_bitbang_set_cs; if (!bitbang->txrx_bufs) { bitbang->use_dma = 0; bitbang->txrx_bufs = spi_bitbang_bufs; - if (!master->setup) { + if (!ctlr->setup) { if (!bitbang->setup_transfer) bitbang->setup_transfer = spi_bitbang_setup_transfer; - master->setup = spi_bitbang_setup; - master->cleanup = spi_bitbang_cleanup; + ctlr->setup = spi_bitbang_setup; + ctlr->cleanup = spi_bitbang_cleanup; } } @@ -411,18 +411,18 @@ EXPORT_SYMBOL_GPL(spi_bitbang_init); * master methods. Those methods are the defaults if the bitbang->txrx_bufs * routine isn't initialized. * - * This routine registers the spi_master, which will process requests in a + * This routine registers the spi_controller, which will process requests in a * dedicated task, keeping IRQs unblocked most of the time. To stop * processing those requests, call spi_bitbang_stop(). * - * On success, this routine will take a reference to master. The caller is - * responsible for calling spi_bitbang_stop() to decrement the reference and - * spi_master_put() as counterpart of spi_alloc_master() to prevent a memory + * On success, this routine will take a reference to the controller. The caller + * is responsible for calling spi_bitbang_stop() to decrement the reference and + * spi_controller_put() as counterpart of spi_alloc_master() to prevent a memory * leak. */ int spi_bitbang_start(struct spi_bitbang *bitbang) { - struct spi_master *master = bitbang->master; + struct spi_controller *ctlr = bitbang->ctlr; int ret; ret = spi_bitbang_init(bitbang); @@ -432,9 +432,9 @@ int spi_bitbang_start(struct spi_bitbang *bitbang) /* driver may get busy before register() returns, especially * if someone registered boardinfo for devices */ - ret = spi_register_master(spi_master_get(master)); + ret = spi_register_controller(spi_controller_get(ctlr)); if (ret) - spi_master_put(master); + spi_controller_put(ctlr); return ret; } @@ -445,7 +445,7 @@ EXPORT_SYMBOL_GPL(spi_bitbang_start); */ void spi_bitbang_stop(struct spi_bitbang *bitbang) { - spi_unregister_master(bitbang->master); + spi_unregister_controller(bitbang->ctlr); } EXPORT_SYMBOL_GPL(spi_bitbang_stop); diff --git a/drivers/spi/spi-butterfly.c b/drivers/spi/spi-butterfly.c index 289b4454242a..1d267e6c22a4 100644 --- a/drivers/spi/spi-butterfly.c +++ b/drivers/spi/spi-butterfly.c @@ -205,7 +205,7 @@ static void butterfly_attach(struct parport *p) host->bus_num = 42; host->num_chipselect = 2; - pp->bitbang.master = host; + pp->bitbang.ctlr = host; pp->bitbang.chipselect = butterfly_chipselect; pp->bitbang.txrx_word[SPI_MODE_0] = butterfly_txrx_word_mode0; @@ -263,7 +263,7 @@ static void butterfly_attach(struct parport *p) pp->info[0].platform_data = &flash; pp->info[0].chip_select = 1; pp->info[0].controller_data = pp; - pp->dataflash = spi_new_device(pp->bitbang.master, &pp->info[0]); + pp->dataflash = spi_new_device(pp->bitbang.ctlr, &pp->info[0]); if (pp->dataflash) pr_debug("%s: dataflash at %s\n", p->name, dev_name(&pp->dataflash->dev)); @@ -308,7 +308,7 @@ static void butterfly_detach(struct parport *p) parport_release(pp->pd); parport_unregister_device(pp->pd); - spi_controller_put(pp->bitbang.master); + spi_controller_put(pp->bitbang.ctlr); } static struct parport_driver butterfly_driver = { diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index 5688be245c68..be3998104bfb 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -459,7 +459,7 @@ static bool davinci_spi_can_dma(struct spi_controller *host, static int davinci_spi_check_error(struct davinci_spi *dspi, int int_status) { - struct device *sdev = dspi->bitbang.master->dev.parent; + struct device *sdev = dspi->bitbang.ctlr->dev.parent; if (int_status & SPIFLG_TIMEOUT_MASK) { dev_err(sdev, "SPI Time-out Error\n"); @@ -742,7 +742,7 @@ static irqreturn_t davinci_spi_irq(s32 irq, void *data) static int davinci_spi_request_dma(struct davinci_spi *dspi) { - struct device *sdev = dspi->bitbang.master->dev.parent; + struct device *sdev = dspi->bitbang.ctlr->dev.parent; dspi->dma_rx = dma_request_chan(sdev, "rx"); if (IS_ERR(dspi->dma_rx)) @@ -913,7 +913,7 @@ static int davinci_spi_probe(struct platform_device *pdev) if (ret) goto free_host; - dspi->bitbang.master = host; + dspi->bitbang.ctlr = host; dspi->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(dspi->clk)) { diff --git a/drivers/spi/spi-gpio.c b/drivers/spi/spi-gpio.c index d8db4564b406..909cce109bba 100644 --- a/drivers/spi/spi-gpio.c +++ b/drivers/spi/spi-gpio.c @@ -427,7 +427,7 @@ static int spi_gpio_probe(struct platform_device *pdev) host->cleanup = spi_gpio_cleanup; bb = &spi_gpio->bitbang; - bb->master = host; + bb->ctlr = host; /* * There is some additional business, apart from driving the CS GPIO * line, that we need to do on selection. This makes the local diff --git a/drivers/spi/spi-lm70llp.c b/drivers/spi/spi-lm70llp.c index e6a65b5c8c31..f982bdebd028 100644 --- a/drivers/spi/spi-lm70llp.c +++ b/drivers/spi/spi-lm70llp.c @@ -212,7 +212,7 @@ static void spi_lm70llp_attach(struct parport *p) /* * SPI and bitbang hookup. */ - pp->bitbang.master = host; + pp->bitbang.ctlr = host; pp->bitbang.chipselect = lm70_chipselect; pp->bitbang.txrx_word[SPI_MODE_0] = lm70_txrx; pp->bitbang.flags = SPI_3WIRE; @@ -264,7 +264,7 @@ static void spi_lm70llp_attach(struct parport *p) * the board info's (void *)controller_data. */ pp->info.controller_data = pp; - pp->spidev_lm70 = spi_new_device(pp->bitbang.master, &pp->info); + pp->spidev_lm70 = spi_new_device(pp->bitbang.ctlr, &pp->info); if (pp->spidev_lm70) dev_dbg(&pp->spidev_lm70->dev, "spidev_lm70 at %s\n", dev_name(&pp->spidev_lm70->dev)); @@ -309,7 +309,7 @@ static void spi_lm70llp_detach(struct parport *p) parport_release(pp->pd); parport_unregister_device(pp->pd); - spi_controller_put(pp->bitbang.master); + spi_controller_put(pp->bitbang.ctlr); lm70llp = NULL; } diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c index cf7c111088a6..6ea38f5e7d64 100644 --- a/drivers/spi/spi-oc-tiny.c +++ b/drivers/spi/spi-oc-tiny.c @@ -194,7 +194,7 @@ static int tiny_spi_of_probe(struct platform_device *pdev) if (!np) return 0; - hw->bitbang.master->dev.of_node = pdev->dev.of_node; + hw->bitbang.ctlr->dev.of_node = pdev->dev.of_node; if (!of_property_read_u32(np, "clock-frequency", &val)) hw->freq = val; if (!of_property_read_u32(np, "baud-width", &val)) @@ -229,7 +229,7 @@ static int tiny_spi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, hw); /* setup the state for the bitbang driver */ - hw->bitbang.master = host; + hw->bitbang.ctlr = host; hw->bitbang.setup_transfer = tiny_spi_setup_transfer; hw->bitbang.txrx_bufs = tiny_spi_txrx_bufs; @@ -274,7 +274,7 @@ exit: static void tiny_spi_remove(struct platform_device *pdev) { struct tiny_spi *hw = platform_get_drvdata(pdev); - struct spi_controller *host = hw->bitbang.master; + struct spi_controller *host = hw->bitbang.ctlr; spi_bitbang_stop(&hw->bitbang); spi_controller_put(host); diff --git a/drivers/spi/spi-omap-uwire.c b/drivers/spi/spi-omap-uwire.c index 2dd1c1bcf4bf..210a98d903fa 100644 --- a/drivers/spi/spi-omap-uwire.c +++ b/drivers/spi/spi-omap-uwire.c @@ -448,7 +448,7 @@ static void uwire_off(struct uwire_spi *uwire) { uwire_write_reg(UWIRE_SR3, 0); clk_disable_unprepare(uwire->ck); - spi_controller_put(uwire->bitbang.master); + spi_controller_put(uwire->bitbang.ctlr); } static int uwire_probe(struct platform_device *pdev) @@ -493,7 +493,7 @@ static int uwire_probe(struct platform_device *pdev) host->setup = uwire_setup; host->cleanup = uwire_cleanup; - uwire->bitbang.master = host; + uwire->bitbang.ctlr = host; uwire->bitbang.chipselect = uwire_chipselect; uwire->bitbang.setup_transfer = uwire_setup_transfer; uwire->bitbang.txrx_bufs = uwire_txrx; diff --git a/drivers/spi/spi-sh-sci.c b/drivers/spi/spi-sh-sci.c index 148d615d2f38..3d560b154ad3 100644 --- a/drivers/spi/spi-sh-sci.c +++ b/drivers/spi/spi-sh-sci.c @@ -136,9 +136,9 @@ static int sh_sci_spi_probe(struct platform_device *dev) } /* setup spi bitbang adaptor */ - sp->bitbang.master = host; - sp->bitbang.master->bus_num = sp->info->bus_num; - sp->bitbang.master->num_chipselect = sp->info->num_chipselect; + sp->bitbang.ctlr = host; + sp->bitbang.ctlr->bus_num = sp->info->bus_num; + sp->bitbang.ctlr->num_chipselect = sp->info->num_chipselect; sp->bitbang.chipselect = sh_sci_spi_chipselect; sp->bitbang.txrx_word[SPI_MODE_0] = sh_sci_spi_txrx_mode0; @@ -166,7 +166,7 @@ static int sh_sci_spi_probe(struct platform_device *dev) setbits(sp, PIN_INIT, 0); iounmap(sp->membase); err1: - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); err0: return ret; } @@ -178,7 +178,7 @@ static void sh_sci_spi_remove(struct platform_device *dev) spi_bitbang_stop(&sp->bitbang); setbits(sp, PIN_INIT, 0); iounmap(sp->membase); - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); } static struct platform_driver sh_sci_spi_drv = { diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c index 12355957be97..7795328427a6 100644 --- a/drivers/spi/spi-xilinx.c +++ b/drivers/spi/spi-xilinx.c @@ -434,7 +434,7 @@ static int xilinx_spi_probe(struct platform_device *pdev) xspi = spi_controller_get_devdata(host); xspi->cs_inactive = 0xffffffff; - xspi->bitbang.master = host; + xspi->bitbang.ctlr = host; xspi->bitbang.chipselect = xilinx_spi_chipselect; xspi->bitbang.setup_transfer = xilinx_spi_setup_transfer; xspi->bitbang.txrx_bufs = xilinx_spi_txrx_bufs; @@ -516,7 +516,7 @@ static void xilinx_spi_remove(struct platform_device *pdev) /* Disable the global IPIF interrupt */ xspi->write_fn(0, regs_base + XIPIF_V123B_DGIER_OFFSET); - spi_controller_put(xspi->bitbang.master); + spi_controller_put(xspi->bitbang.ctlr); } /* work with hotplug and coldplug */ diff --git a/drivers/spi/spi-xtensa-xtfpga.c b/drivers/spi/spi-xtensa-xtfpga.c index 3c7721894376..3c2cda315397 100644 --- a/drivers/spi/spi-xtensa-xtfpga.c +++ b/drivers/spi/spi-xtensa-xtfpga.c @@ -93,7 +93,7 @@ static int xtfpga_spi_probe(struct platform_device *pdev) host->dev.of_node = pdev->dev.of_node; xspi = spi_controller_get_devdata(host); - xspi->bitbang.master = host; + xspi->bitbang.ctlr = host; xspi->bitbang.chipselect = xtfpga_spi_chipselect; xspi->bitbang.txrx_word[SPI_MODE_0] = xtfpga_spi_txrx_word; xspi->regs = devm_platform_ioremap_resource(pdev, 0); diff --git a/include/linux/spi/spi_bitbang.h b/include/linux/spi/spi_bitbang.h index 4444c2a992cb..b930eca2ef7b 100644 --- a/include/linux/spi/spi_bitbang.h +++ b/include/linux/spi/spi_bitbang.h @@ -10,7 +10,7 @@ struct spi_bitbang { u8 use_dma; u16 flags; /* extra spi->mode support */ - struct spi_master *master; + struct spi_controller *ctlr; /* setup_transfer() changes clock and/or wordsize to match settings * for this transfer; zeroes restore defaults from spi_device. -- cgit v1.2.3 From 620d269f29a569ba37419cc03cf1da2d55f6252a Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 7 Feb 2024 19:40:45 +0100 Subject: spi: Drop compat layer from renaming "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that all in-tree users followed the rename, the compat stuff can go away. This completes the renaming started with commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/ad1d949325b61a4682e8d6ecf9d05da751e6a99f.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- Documentation/driver-api/driver-model/devres.rst | 2 +- drivers/spi/spi.c | 2 +- include/linux/spi/spi.h | 20 +------------------- 3 files changed, 3 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index c5f99d834ec5..49c6fd82cd5d 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -462,7 +462,7 @@ SLAVE DMA ENGINE SPI devm_spi_alloc_master() devm_spi_alloc_slave() - devm_spi_register_master() + devm_spi_register_controller() WATCHDOG devm_watchdog_register_device() diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index f2170f4b5077..4b3e7a624de2 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -584,7 +584,7 @@ struct spi_device *spi_alloc_device(struct spi_controller *ctlr) return NULL; } - spi->master = spi->controller = ctlr; + spi->controller = ctlr; spi->dev.parent = &ctlr->dev; spi->dev.bus = &spi_bus_type; spi->dev.release = spidev_release; diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 600fbd5daf68..30ada46b51cd 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -131,7 +131,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * struct spi_device - Controller side proxy for an SPI slave device * @dev: Driver model representation of the device. * @controller: SPI controller used with the device. - * @master: Copy of controller, for backwards compatibility. * @max_speed_hz: Maximum clock rate to be used with this chip * (on this board); may be changed by the device's driver. * The spi_transfer.speed_hz can override this for each transfer. @@ -185,7 +184,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, struct spi_device { struct device dev; struct spi_controller *controller; - struct spi_controller *master; /* Compatibility layer */ u32 max_speed_hz; u8 chip_select[SPI_CS_CNT_MAX]; u8 bits_per_word; @@ -1298,7 +1296,7 @@ spi_max_transfer_size(struct spi_device *spi) */ static inline bool spi_is_bpw_supported(struct spi_device *spi, u32 bpw) { - u32 bpw_mask = spi->master->bits_per_word_mask; + u32 bpw_mask = spi->controller->bits_per_word_mask; if (bpw == 8 || (bpw <= 32 && bpw_mask & SPI_BPW_MASK(bpw))) return true; @@ -1670,20 +1668,4 @@ spi_transfer_is_last(struct spi_controller *ctlr, struct spi_transfer *xfer) return list_is_last(&xfer->transfer_list, &ctlr->cur_msg->transfers); } -/* Compatibility layer */ -#define spi_master spi_controller - -#define spi_master_get_devdata(_ctlr) spi_controller_get_devdata(_ctlr) -#define spi_master_set_devdata(_ctlr, _data) \ - spi_controller_set_devdata(_ctlr, _data) -#define spi_master_get(_ctlr) spi_controller_get(_ctlr) -#define spi_master_put(_ctlr) spi_controller_put(_ctlr) -#define spi_master_suspend(_ctlr) spi_controller_suspend(_ctlr) -#define spi_master_resume(_ctlr) spi_controller_resume(_ctlr) - -#define spi_register_master(_ctlr) spi_register_controller(_ctlr) -#define devm_spi_register_master(_dev, _ctlr) \ - devm_spi_register_controller(_dev, _ctlr) -#define spi_unregister_master(_ctlr) spi_unregister_controller(_ctlr) - #endif /* __LINUX_SPI_H */ -- cgit v1.2.3 From c478db84c8544156b80c5e5d3a8c7840d557707a Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 29 Jan 2024 19:34:45 +0100 Subject: wifi: mac80211: refactor puncturing bitmap extraction Add a new inline helper function to ieee80211.h to extract the disabled subchannels bitmap from an EHT operation element, and use that in mac80211 where we do that. Link: https://msgid.link/20240129194108.d9f50dcec8d0.I8b08cbc2490a734fafcce0fa0fc328211ba6f10b@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 16 +++++++++++++++ net/mac80211/mlme.c | 52 ++++++++++++++++------------------------------- 2 files changed, 33 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index a70388ae3a7b..d9d2c1253157 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -3189,6 +3189,22 @@ ieee80211_eht_oper_size_ok(const u8 *data, u8 len) return len >= needed; } +/* must validate ieee80211_eht_oper_size_ok() first */ +static inline u16 +ieee80211_eht_oper_dis_subchan_bitmap(const struct ieee80211_eht_operation *eht_oper) +{ + const struct ieee80211_eht_operation_info *info = + (const void *)eht_oper->optional; + + if (!(eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) + return 0; + + if (!(eht_oper->params & IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT)) + return 0; + + return get_unaligned_le16(info->optional); +} + #define IEEE80211_BW_IND_DIS_SUBCH_PRESENT BIT(1) struct ieee80211_bandwidth_indication { diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index da202103faf0..74a15f18e7ee 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -813,36 +813,27 @@ again: } if (conn->mode >= IEEE80211_CONN_MODE_EHT) { - const struct ieee80211_eht_operation *eht_oper; + u16 bitmap; - eht_oper = elems->eht_operation; - - if (WARN_ON_ONCE(!eht_oper)) { + if (WARN_ON_ONCE(!elems->eht_operation)) { ret = -EINVAL; goto free; } - if (eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT && - eht_oper->params & IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT) { - const struct ieee80211_eht_operation_info *info = - (void *)eht_oper->optional; - const u8 *disable_subchannel_bitmap = info->optional; - u16 bitmap; - - bitmap = get_unaligned_le16(disable_subchannel_bitmap); - if (!cfg80211_valid_disable_subchannel_bitmap(&bitmap, - &ap_chandef) || - (bitmap && - ieee80211_hw_check(&local->hw, DISALLOW_PUNCTURING))) { - conn->mode = IEEE80211_CONN_MODE_HE; - conn->bw_limit = min_t(enum ieee80211_conn_bw_limit, - conn->bw_limit, - IEEE80211_CONN_BW_LIMIT_160); - sdata_info(sdata, - "AP has invalid/unsupported puncturing, disabling EHT\n"); - } - /* FIXME: store puncturing bitmap */ + bitmap = ieee80211_eht_oper_dis_subchan_bitmap(elems->eht_operation); + + if (!cfg80211_valid_disable_subchannel_bitmap(&bitmap, + &ap_chandef) || + (bitmap && + ieee80211_hw_check(&local->hw, DISALLOW_PUNCTURING))) { + conn->mode = IEEE80211_CONN_MODE_HE; + conn->bw_limit = min_t(enum ieee80211_conn_bw_limit, + conn->bw_limit, + IEEE80211_CONN_BW_LIMIT_160); + sdata_info(sdata, + "AP has invalid/unsupported puncturing, disabling EHT\n"); } + /* FIXME: store puncturing bitmap */ } /* the mode can only decrease, so this must terminate */ @@ -5879,18 +5870,9 @@ static bool ieee80211_config_puncturing(struct ieee80211_link_data *link, u64 *changed) { struct ieee80211_local *local = link->sdata->local; - u16 bitmap = 0, extracted; - - if ((eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT) && - (eht_oper->params & - IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT)) { - const struct ieee80211_eht_operation_info *info = - (void *)eht_oper->optional; - const u8 *disable_subchannel_bitmap = info->optional; - - bitmap = get_unaligned_le16(disable_subchannel_bitmap); - } + u16 bitmap, extracted; + bitmap = ieee80211_eht_oper_dis_subchan_bitmap(eht_oper); extracted = ieee80211_extract_dis_subch_bmap(eht_oper, &link->conf->chanreq.oper, bitmap); -- cgit v1.2.3 From 3c7a8e190bc580813ddd9259f62971c8d2a6b5ad Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 12 Dec 2023 11:15:28 -0500 Subject: uapi: introduce uapi-friendly macros for GENMASK Move __GENMASK and __GENMASK_ULL from include/ to include/uapi/ so that they can be used to define masks in userspace API headers. Compared to what is already in include/linux/bits.h, the definitions need to use the uglified versions of UL(), ULL(), BITS_PER_LONG and BITS_PER_LONG_LONG (which did not even exist), but otherwise expand to the same content. Signed-off-by: Paolo Bonzini --- include/linux/bits.h | 8 +------- include/uapi/asm-generic/bitsperlong.h | 4 ++++ include/uapi/linux/bits.h | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 include/uapi/linux/bits.h (limited to 'include/linux') diff --git a/include/linux/bits.h b/include/linux/bits.h index 7c0cf5031abe..0eb24d21aac2 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -4,6 +4,7 @@ #include #include +#include #include #define BIT_MASK(nr) (UL(1) << ((nr) % BITS_PER_LONG)) @@ -30,15 +31,8 @@ #define GENMASK_INPUT_CHECK(h, l) 0 #endif -#define __GENMASK(h, l) \ - (((~UL(0)) - (UL(1) << (l)) + 1) & \ - (~UL(0) >> (BITS_PER_LONG - 1 - (h)))) #define GENMASK(h, l) \ (GENMASK_INPUT_CHECK(h, l) + __GENMASK(h, l)) - -#define __GENMASK_ULL(h, l) \ - (((~ULL(0)) - (ULL(1) << (l)) + 1) & \ - (~ULL(0) >> (BITS_PER_LONG_LONG - 1 - (h)))) #define GENMASK_ULL(h, l) \ (GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l)) diff --git a/include/uapi/asm-generic/bitsperlong.h b/include/uapi/asm-generic/bitsperlong.h index 352cb81947b8..fadb3f857f28 100644 --- a/include/uapi/asm-generic/bitsperlong.h +++ b/include/uapi/asm-generic/bitsperlong.h @@ -24,4 +24,8 @@ #endif #endif +#ifndef __BITS_PER_LONG_LONG +#define __BITS_PER_LONG_LONG 64 +#endif + #endif /* _UAPI__ASM_GENERIC_BITS_PER_LONG */ diff --git a/include/uapi/linux/bits.h b/include/uapi/linux/bits.h new file mode 100644 index 000000000000..3c2a101986a3 --- /dev/null +++ b/include/uapi/linux/bits.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* bits.h: Macros for dealing with bitmasks. */ + +#ifndef _UAPI_LINUX_BITS_H +#define _UAPI_LINUX_BITS_H + +#define __GENMASK(h, l) \ + (((~_UL(0)) - (_UL(1) << (l)) + 1) & \ + (~_UL(0) >> (__BITS_PER_LONG - 1 - (h)))) + +#define __GENMASK_ULL(h, l) \ + (((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \ + (~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h)))) + +#endif /* _UAPI_LINUX_BITS_H */ -- cgit v1.2.3 From a3c78778f50c4db6cc0bb6aa2986c0174b1267d0 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:38 +0000 Subject: PM: EM: Refactor em_pd_get_efficient_state() to be more flexible The Energy Model (EM) is going to support runtime modification. There are going to be 2 EM tables which store information. This patch aims to prepare the code to be generic and use one of the tables. The function will no longer get a pointer to 'struct em_perf_domain' (the EM) but instead a pointer to 'struct em_perf_state' (which is one of the EM's tables). Prepare em_pd_get_efficient_state() for the upcoming changes and make it possible to be re-used. Return an index for the best performance state for a given EM table. The function arguments that are introduced should allow to work on different performance state arrays. The caller of em_pd_get_efficient_state() should be able to use the index either on the default or the modifiable EM table. Reviewed-by: Daniel Lezcano Reviewed-by: Hongyan Xia Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 88d91e087471..1dcd1645dde7 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -175,33 +175,35 @@ void em_dev_unregister_perf_domain(struct device *dev); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM - * @pd : Performance domain for which we want an efficient frequency - * @freq : Frequency to map with the EM + * @table: List of performance states, in ascending order + * @nr_perf_states: Number of performance states + * @freq: Frequency to map with the EM + * @pd_flags: Performance Domain flags * * It is called from the scheduler code quite frequently and as a consequence * doesn't implement any check. * - * Return: An efficient performance state, high enough to meet @freq + * Return: An efficient performance state id, high enough to meet @freq * requirement. */ -static inline -struct em_perf_state *em_pd_get_efficient_state(struct em_perf_domain *pd, - unsigned long freq) +static inline int +em_pd_get_efficient_state(struct em_perf_state *table, int nr_perf_states, + unsigned long freq, unsigned long pd_flags) { struct em_perf_state *ps; int i; - for (i = 0; i < pd->nr_perf_states; i++) { - ps = &pd->table[i]; + for (i = 0; i < nr_perf_states; i++) { + ps = &table[i]; if (ps->frequency >= freq) { - if (pd->flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && + if (pd_flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && ps->flags & EM_PERF_STATE_INEFFICIENT) continue; - break; + return i; } } - return ps; + return nr_perf_states - 1; } /** @@ -226,7 +228,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, { unsigned long freq, ref_freq, scale_cpu; struct em_perf_state *ps; - int cpu; + int cpu, i; if (!sum_util) return 0; @@ -250,7 +252,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * Find the lowest performance state of the Energy Model above the * requested frequency. */ - ps = em_pd_get_efficient_state(pd, freq); + i = em_pd_get_efficient_state(pd->table, pd->nr_perf_states, freq, + pd->flags); + ps = &pd->table[i]; /* * The capacity of a CPU in the domain at the performance state (ps) -- cgit v1.2.3 From ca0fc871f16f4bef746b5ba814b67afb59119700 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:42 +0000 Subject: PM: EM: Introduce runtime modifiable table The new runtime table can be populated with a new power data to better reflect the actual efficiency of the device e.g. CPU. The power can vary over time e.g. due to the SoC temperature change. Higher temperature can increase power values. For longer running scenarios, such as game or camera, when also other devices are used (e.g. GPU, ISP) the CPU power can change. The new EM framework is able to addresses this issue and change the EM data at runtime safely. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 12 ++++++++++ kernel/power/energy_model.c | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 1dcd1645dde7..8ddf1d8a9581 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -36,9 +36,20 @@ struct em_perf_state { */ #define EM_PERF_STATE_INEFFICIENT BIT(0) +/** + * struct em_perf_table - Performance states table + * @rcu: RCU used for safe access and destruction + * @state: List of performance states, in ascending order + */ +struct em_perf_table { + struct rcu_head rcu; + struct em_perf_state state[]; +}; + /** * struct em_perf_domain - Performance domain * @table: List of performance states, in ascending order + * @em_table: Pointer to the runtime modifiable em_perf_table * @nr_perf_states: Number of performance states * @flags: See "em_perf_domain flags" * @cpus: Cpumask covering the CPUs of the domain. It's here @@ -54,6 +65,7 @@ struct em_perf_state { */ struct em_perf_domain { struct em_perf_state *table; + struct em_perf_table __rcu *em_table; int nr_perf_states; unsigned long flags; unsigned long cpus[]; diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 7468fa92134b..131ff1d0dc5b 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -23,6 +23,9 @@ */ static DEFINE_MUTEX(em_pd_mutex); +static void em_cpufreq_update_efficiencies(struct device *dev, + struct em_perf_state *table); + static bool _is_cpu_device(struct device *dev) { return (dev->bus == &cpu_subsys); @@ -103,6 +106,31 @@ static void em_debug_create_pd(struct device *dev) {} static void em_debug_remove_pd(struct device *dev) {} #endif +static void em_destroy_table_rcu(struct rcu_head *rp) +{ + struct em_perf_table __rcu *table; + + table = container_of(rp, struct em_perf_table, rcu); + kfree(table); +} + +static void em_free_table(struct em_perf_table __rcu *table) +{ + call_rcu(&table->rcu, em_destroy_table_rcu); +} + +static struct em_perf_table __rcu * +em_allocate_table(struct em_perf_domain *pd) +{ + struct em_perf_table __rcu *table; + int table_size; + + table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + + table = kzalloc(sizeof(*table) + table_size, GFP_KERNEL); + return table; +} + static int em_compute_costs(struct device *dev, struct em_perf_state *table, struct em_data_callback *cb, int nr_states, unsigned long flags) @@ -153,6 +181,24 @@ static int em_allocate_perf_table(struct em_perf_domain *pd, return 0; } +static int em_create_runtime_table(struct em_perf_domain *pd) +{ + struct em_perf_table __rcu *table; + int table_size; + + table = em_allocate_table(pd); + if (!table) + return -ENOMEM; + + /* Initialize runtime table with existing data */ + table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + memcpy(table->state, pd->table, table_size); + + rcu_assign_pointer(pd->em_table, table); + + return 0; +} + static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, struct em_perf_state *table, struct em_data_callback *cb, @@ -245,6 +291,10 @@ static int em_create_pd(struct device *dev, int nr_states, if (ret) goto free_pd_table; + ret = em_create_runtime_table(pd); + if (ret) + goto free_pd_table; + if (_is_cpu_device(dev)) for_each_cpu(cpu, cpus) { cpu_dev = get_cpu_device(cpu); @@ -461,6 +511,9 @@ void em_dev_unregister_perf_domain(struct device *dev) em_debug_remove_pd(dev); kfree(dev->em_pd->table); + + em_free_table(dev->em_pd->em_table); + kfree(dev->em_pd); dev->em_pd = NULL; mutex_unlock(&em_pd_mutex); -- cgit v1.2.3 From aa11a7ebfd5d698f541641922beede1cb474bf70 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:43 +0000 Subject: PM: EM: Use runtime modified EM for CPUs energy estimation in EAS The new Energy Model (EM) supports runtime modification of the performance state table to better model the power used by the SoC. Use this new feature to improve energy estimation and therefore task placement in Energy Aware Scheduler (EAS). Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 8ddf1d8a9581..5f842da3bb0c 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -239,9 +239,14 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, unsigned long allowed_cpu_cap) { unsigned long freq, ref_freq, scale_cpu; + struct em_perf_table *em_table; struct em_perf_state *ps; int cpu, i; +#ifdef CONFIG_SCHED_DEBUG + WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n"); +#endif + if (!sum_util) return 0; @@ -264,9 +269,10 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * Find the lowest performance state of the Energy Model above the * requested frequency. */ - i = em_pd_get_efficient_state(pd->table, pd->nr_perf_states, freq, - pd->flags); - ps = &pd->table[i]; + em_table = rcu_dereference(pd->em_table); + i = em_pd_get_efficient_state(em_table->state, pd->nr_perf_states, + freq, pd->flags); + ps = &em_table->state[i]; /* * The capacity of a CPU in the domain at the performance state (ps) -- cgit v1.2.3 From ffcf9bce7af02a21fb73738999de1e3d4fde5aca Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:44 +0000 Subject: PM: EM: Add functions for memory allocations for new EM tables The runtime modified EM table can be provided from drivers. Create mechanism which allows safely allocate and free the table for device drivers. The same table can be used by the EAS in task scheduler code paths, so make sure the memory is not freed when the device driver module is unloaded. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 11 +++++++++++ kernel/power/energy_model.c | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 5f842da3bb0c..27911dc1887e 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -39,10 +40,12 @@ struct em_perf_state { /** * struct em_perf_table - Performance states table * @rcu: RCU used for safe access and destruction + * @kref: Reference counter to track the users * @state: List of performance states, in ascending order */ struct em_perf_table { struct rcu_head rcu; + struct kref kref; struct em_perf_state state[]; }; @@ -184,6 +187,8 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, bool microwatts); void em_dev_unregister_perf_domain(struct device *dev); +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); +void em_table_free(struct em_perf_table __rcu *table); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM @@ -365,6 +370,12 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd) { return 0; } +static inline +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) +{ + return NULL; +} +static inline void em_table_free(struct em_perf_table __rcu *table) {} #endif #endif diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 131ff1d0dc5b..16795743f969 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -114,13 +114,36 @@ static void em_destroy_table_rcu(struct rcu_head *rp) kfree(table); } -static void em_free_table(struct em_perf_table __rcu *table) +static void em_release_table_kref(struct kref *kref) { + struct em_perf_table __rcu *table; + + /* It was the last owner of this table so we can free */ + table = container_of(kref, struct em_perf_table, kref); + call_rcu(&table->rcu, em_destroy_table_rcu); } -static struct em_perf_table __rcu * -em_allocate_table(struct em_perf_domain *pd) +/** + * em_table_free() - Handles safe free of the EM table when needed + * @table : EM table which is going to be freed + * + * No return values. + */ +void em_table_free(struct em_perf_table __rcu *table) +{ + kref_put(&table->kref, em_release_table_kref); +} + +/** + * em_table_alloc() - Allocate a new EM table + * @pd : EM performance domain for which this must be done + * + * Allocate a new EM table and initialize its kref to indicate that it + * has a user. + * Returns allocated table or NULL. + */ +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) { struct em_perf_table __rcu *table; int table_size; @@ -128,6 +151,11 @@ em_allocate_table(struct em_perf_domain *pd) table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; table = kzalloc(sizeof(*table) + table_size, GFP_KERNEL); + if (!table) + return NULL; + + kref_init(&table->kref); + return table; } @@ -186,7 +214,7 @@ static int em_create_runtime_table(struct em_perf_domain *pd) struct em_perf_table __rcu *table; int table_size; - table = em_allocate_table(pd); + table = em_table_alloc(pd); if (!table) return -ENOMEM; @@ -512,7 +540,7 @@ void em_dev_unregister_perf_domain(struct device *dev) kfree(dev->em_pd->table); - em_free_table(dev->em_pd->em_table); + em_table_free(dev->em_pd->em_table); kfree(dev->em_pd); dev->em_pd = NULL; -- cgit v1.2.3 From 977230d5d50314f9920d3ee6348773d8babbfb58 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:45 +0000 Subject: PM: EM: Introduce em_dev_update_perf_domain() for EM updates Add API function em_dev_update_perf_domain() which allows the EM to be changed safely. Concurrent updaters are serialized with a mutex and the removal of memory that will not be used any more is carried out with the help of RCU. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 8 ++++++++ kernel/power/energy_model.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 27911dc1887e..324a3a8e0a2d 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -183,6 +183,8 @@ struct em_data_callback { struct em_perf_domain *em_cpu_get(int cpu); struct em_perf_domain *em_pd_get(struct device *dev); +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table); int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, bool microwatts); @@ -376,6 +378,12 @@ struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) return NULL; } static inline void em_table_free(struct em_perf_table __rcu *table) {} +static inline +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table) +{ + return -EINVAL; +} #endif #endif diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 16795743f969..667619b70be7 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -209,6 +209,50 @@ static int em_allocate_perf_table(struct em_perf_domain *pd, return 0; } +/** + * em_dev_update_perf_domain() - Update runtime EM table for a device + * @dev : Device for which the EM is to be updated + * @new_table : The new EM table that is going to be used from now + * + * Update EM runtime modifiable table for the @dev using the provided @table. + * + * This function uses a mutex to serialize writers, so it must not be called + * from a non-sleeping context. + * + * Return 0 on success or an error code on failure. + */ +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table) +{ + struct em_perf_table __rcu *old_table; + struct em_perf_domain *pd; + + if (!dev) + return -EINVAL; + + /* Serialize update/unregister or concurrent updates */ + mutex_lock(&em_pd_mutex); + + if (!dev->em_pd) { + mutex_unlock(&em_pd_mutex); + return -EINVAL; + } + pd = dev->em_pd; + + kref_get(&new_table->kref); + + old_table = pd->em_table; + rcu_assign_pointer(pd->em_table, new_table); + + em_cpufreq_update_efficiencies(dev, new_table->state); + + em_table_free(old_table); + + mutex_unlock(&em_pd_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(em_dev_update_perf_domain); + static int em_create_runtime_table(struct em_perf_domain *pd) { struct em_perf_table __rcu *table; -- cgit v1.2.3 From ee1a19873ce1234a3c2e6f84af3624fc73bfbd9c Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:46 +0000 Subject: PM: EM: Add em_perf_state_from_pd() to get performance states table Introduce a wrapper to get the performance states table of the performance domain. The function should be called within the RCU read critical section. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 324a3a8e0a2d..158dad6ea313 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -338,6 +338,23 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd) return pd->nr_perf_states; } +/** + * em_perf_state_from_pd() - Get the performance states table of perf. + * domain + * @pd : performance domain for which this must be done + * + * To use this function the rcu_read_lock() should be hold. After the usage + * of the performance states table is finished, the rcu_read_unlock() should + * be called. + * + * Return: the pointer to performance states table of the performance domain + */ +static inline +struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) +{ + return rcu_dereference(pd->em_table)->state; +} + #else struct em_data_callback {}; #define EM_ADV_DATA_CB(_active_power_cb, _cost_cb) { } @@ -384,6 +401,11 @@ int em_dev_update_perf_domain(struct device *dev, { return -EINVAL; } +static inline +struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) +{ + return NULL; +} #endif #endif -- cgit v1.2.3 From 5a367f7b7014af86bd1ac0865a42db55187dbd3c Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:47 +0000 Subject: PM: EM: Add performance field to struct em_perf_state and optimize The performance doesn't scale linearly with the frequency. Also, it may be different in different workloads. Some CPUs are designed to be particularly good at some applications e.g. images or video processing and other CPUs in different. When those different types of CPUs are combined in one SoC they should be properly modeled to get max of the HW in Energy Aware Scheduler (EAS). The Energy Model (EM) provides the power vs. performance curves to the EAS, but assumes the CPUs capacity is fixed and scales linearly with the frequency. This patch allows to adjust the curve on the 'performance' axis as well. Code speed optimization: Removing map_util_freq() allows to avoid one division and one multiplication operations from the EAS hot code path. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 24 ++++++++++++------------ kernel/power/energy_model.c | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 158dad6ea313..ce24ea3fe41c 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -13,6 +13,7 @@ /** * struct em_perf_state - Performance state of a performance domain + * @performance: CPU performance (capacity) at a given frequency * @frequency: The frequency in KHz, for consistency with CPUFreq * @power: The power consumed at this level (by 1 CPU or by a registered * device). It can be a total power: static and dynamic. @@ -21,6 +22,7 @@ * @flags: see "em_perf_state flags" description below. */ struct em_perf_state { + unsigned long performance; unsigned long frequency; unsigned long power; unsigned long cost; @@ -196,25 +198,25 @@ void em_table_free(struct em_perf_table __rcu *table); * em_pd_get_efficient_state() - Get an efficient performance state from the EM * @table: List of performance states, in ascending order * @nr_perf_states: Number of performance states - * @freq: Frequency to map with the EM + * @max_util: Max utilization to map with the EM * @pd_flags: Performance Domain flags * * It is called from the scheduler code quite frequently and as a consequence * doesn't implement any check. * - * Return: An efficient performance state id, high enough to meet @freq + * Return: An efficient performance state id, high enough to meet @max_util * requirement. */ static inline int em_pd_get_efficient_state(struct em_perf_state *table, int nr_perf_states, - unsigned long freq, unsigned long pd_flags) + unsigned long max_util, unsigned long pd_flags) { struct em_perf_state *ps; int i; for (i = 0; i < nr_perf_states; i++) { ps = &table[i]; - if (ps->frequency >= freq) { + if (ps->performance >= max_util) { if (pd_flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && ps->flags & EM_PERF_STATE_INEFFICIENT) continue; @@ -245,9 +247,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, unsigned long max_util, unsigned long sum_util, unsigned long allowed_cpu_cap) { - unsigned long freq, ref_freq, scale_cpu; struct em_perf_table *em_table; struct em_perf_state *ps; + unsigned long scale_cpu; int cpu, i; #ifdef CONFIG_SCHED_DEBUG @@ -260,25 +262,23 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, /* * In order to predict the performance state, map the utilization of * the most utilized CPU of the performance domain to a requested - * frequency, like schedutil. Take also into account that the real - * frequency might be set lower (due to thermal capping). Thus, clamp + * performance, like schedutil. Take also into account that the real + * performance might be set lower (due to thermal capping). Thus, clamp * max utilization to the allowed CPU capacity before calculating - * effective frequency. + * effective performance. */ cpu = cpumask_first(to_cpumask(pd->cpus)); scale_cpu = arch_scale_cpu_capacity(cpu); - ref_freq = arch_scale_freq_ref(cpu); max_util = min(max_util, allowed_cpu_cap); - freq = map_util_freq(max_util, ref_freq, scale_cpu); /* * Find the lowest performance state of the Energy Model above the - * requested frequency. + * requested performance. */ em_table = rcu_dereference(pd->em_table); i = em_pd_get_efficient_state(em_table->state, pd->nr_perf_states, - freq, pd->flags); + max_util, pd->flags); ps = &em_table->state[i]; /* diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 667619b70be7..41418aa6daa6 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -46,6 +46,7 @@ static void em_debug_create_ps(struct em_perf_state *ps, struct dentry *pd) debugfs_create_ulong("frequency", 0444, d, &ps->frequency); debugfs_create_ulong("power", 0444, d, &ps->power); debugfs_create_ulong("cost", 0444, d, &ps->cost); + debugfs_create_ulong("performance", 0444, d, &ps->performance); debugfs_create_ulong("inefficient", 0444, d, &ps->flags); } @@ -159,6 +160,30 @@ struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) return table; } +static void em_init_performance(struct device *dev, struct em_perf_domain *pd, + struct em_perf_state *table, int nr_states) +{ + u64 fmax, max_cap; + int i, cpu; + + /* This is needed only for CPUs and EAS skip other devices */ + if (!_is_cpu_device(dev)) + return; + + cpu = cpumask_first(em_span_cpus(pd)); + + /* + * Calculate the performance value for each frequency with + * linear relationship. The final CPU capacity might not be ready at + * boot time, but the EM will be updated a bit later with correct one. + */ + fmax = (u64) table[nr_states - 1].frequency; + max_cap = (u64) arch_scale_cpu_capacity(cpu); + for (i = 0; i < nr_states; i++) + table[i].performance = div64_u64(max_cap * table[i].frequency, + fmax); +} + static int em_compute_costs(struct device *dev, struct em_perf_state *table, struct em_data_callback *cb, int nr_states, unsigned long flags) @@ -318,6 +343,8 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, table[i].frequency = prev_freq = freq; } + em_init_performance(dev, pd, table, nr_states); + ret = em_compute_costs(dev, table, cb, nr_states, flags); if (ret) return -EINVAL; -- cgit v1.2.3 From 1b600da510735a0f92c8b4140a7e2cb037a6a6c3 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:49 +0000 Subject: PM: EM: Optimize em_cpu_energy() and remove division The Energy Model (EM) can be modified at runtime which brings new possibilities. The em_cpu_energy() is called by the Energy Aware Scheduler (EAS) in its hot path. The energy calculation uses power value for a given performance state (ps) and the CPU busy time as percentage for that given frequency. It is possible to avoid the division by 'scale_cpu' at runtime, because EM is updated whenever new max capacity CPU is set in the system. Use that feature and do the needed division during the calculation of the coefficient 'ps->cost'. That enhanced 'ps->cost' value can be then just multiplied simply by utilization: pd_nrg = ps->cost * \Sum cpu_util to get the needed energy for whole Performance Domain (PD). With this optimization and earlier removal of map_util_freq(), the em_cpu_energy() should run faster on the Big CPU by 1.43x and on the Little CPU by 1.69x (RockPi 4B board). Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 55 ++++++++++++-------------------------------- kernel/power/energy_model.c | 7 +++--- 2 files changed, 18 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index ce24ea3fe41c..aabfc26fcd31 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -115,27 +115,6 @@ struct em_perf_domain { #define EM_MAX_NUM_CPUS 16 #endif -/* - * To avoid an overflow on 32bit machines while calculating the energy - * use a different order in the operation. First divide by the 'cpu_scale' - * which would reduce big value stored in the 'cost' field, then multiply by - * the 'sum_util'. This would allow to handle existing platforms, which have - * e.g. power ~1.3 Watt at max freq, so the 'cost' value > 1mln micro-Watts. - * In such scenario, where there are 4 CPUs in the Perf. Domain the 'sum_util' - * could be 4096, then multiplication: 'cost' * 'sum_util' would overflow. - * This reordering of operations has some limitations, we lose small - * precision in the estimation (comparing to 64bit platform w/o reordering). - * - * We are safe on 64bit machine. - */ -#ifdef CONFIG_64BIT -#define em_estimate_energy(cost, sum_util, scale_cpu) \ - (((cost) * (sum_util)) / (scale_cpu)) -#else -#define em_estimate_energy(cost, sum_util, scale_cpu) \ - (((cost) / (scale_cpu)) * (sum_util)) -#endif - struct em_data_callback { /** * active_power() - Provide power at the next performance state of @@ -249,8 +228,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, { struct em_perf_table *em_table; struct em_perf_state *ps; - unsigned long scale_cpu; - int cpu, i; + int i; #ifdef CONFIG_SCHED_DEBUG WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n"); @@ -267,9 +245,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * max utilization to the allowed CPU capacity before calculating * effective performance. */ - cpu = cpumask_first(to_cpumask(pd->cpus)); - scale_cpu = arch_scale_cpu_capacity(cpu); - + max_util = map_util_perf(max_util); max_util = min(max_util, allowed_cpu_cap); /* @@ -282,12 +258,12 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, ps = &em_table->state[i]; /* - * The capacity of a CPU in the domain at the performance state (ps) - * can be computed as: + * The performance (capacity) of a CPU in the domain at the performance + * state (ps) can be computed as: * - * ps->freq * scale_cpu - * ps->cap = -------------------- (1) - * cpu_max_freq + * ps->freq * scale_cpu + * ps->performance = -------------------- (1) + * cpu_max_freq * * So, ignoring the costs of idle states (which are not available in * the EM), the energy consumed by this CPU at that performance state @@ -295,9 +271,10 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * * ps->power * cpu_util * cpu_nrg = -------------------- (2) - * ps->cap + * ps->performance * - * since 'cpu_util / ps->cap' represents its percentage of busy time. + * since 'cpu_util / ps->performance' represents its percentage of busy + * time. * * NOTE: Although the result of this computation actually is in * units of power, it can be manipulated as an energy value @@ -307,9 +284,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * By injecting (1) in (2), 'cpu_nrg' can be re-expressed as a product * of two terms: * - * ps->power * cpu_max_freq cpu_util - * cpu_nrg = ------------------------ * --------- (3) - * ps->freq scale_cpu + * ps->power * cpu_max_freq + * cpu_nrg = ------------------------ * cpu_util (3) + * ps->freq * scale_cpu * * The first term is static, and is stored in the em_perf_state struct * as 'ps->cost'. @@ -319,11 +296,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * total energy of the domain (which is the simple sum of the energy of * all of its CPUs) can be factorized as: * - * ps->cost * \Sum cpu_util - * pd_nrg = ------------------------ (4) - * scale_cpu + * pd_nrg = ps->cost * \Sum cpu_util (4) */ - return em_estimate_energy(ps->cost, sum_util, scale_cpu); + return ps->cost * sum_util; } /** diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index b192b0ac8c6e..a631d7d52c40 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -192,11 +192,9 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, unsigned long flags) { unsigned long prev_cost = ULONG_MAX; - u64 fmax; int i, ret; /* Compute the cost of each performance state. */ - fmax = (u64) table[nr_states - 1].frequency; for (i = nr_states - 1; i >= 0; i--) { unsigned long power_res, cost; @@ -208,8 +206,9 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return -EINVAL; } } else { - power_res = table[i].power; - cost = div64_u64(fmax * power_res, table[i].frequency); + /* increase resolution of 'cost' precision */ + power_res = table[i].power * 10; + cost = power_res / table[i].performance; } table[i].cost = cost; -- cgit v1.2.3 From 24e9fb635df2790eccb0e95ff65c6dee7a97fcb7 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:55 +0000 Subject: PM: EM: Remove old table Remove the old EM table which wasn't able to modify the data. Clean the unneeded function and refactor the code a bit. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 2 -- kernel/power/energy_model.c | 46 +++++++------------------------------------- 2 files changed, 7 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index aabfc26fcd31..92866a81abe4 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -53,7 +53,6 @@ struct em_perf_table { /** * struct em_perf_domain - Performance domain - * @table: List of performance states, in ascending order * @em_table: Pointer to the runtime modifiable em_perf_table * @nr_perf_states: Number of performance states * @flags: See "em_perf_domain flags" @@ -69,7 +68,6 @@ struct em_perf_table { * field is unused. */ struct em_perf_domain { - struct em_perf_state *table; struct em_perf_table __rcu *em_table; int nr_perf_states; unsigned long flags; diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 548908e686ed..57838d28af85 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -276,17 +276,6 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return 0; } -static int em_allocate_perf_table(struct em_perf_domain *pd, - int nr_states) -{ - pd->table = kcalloc(nr_states, sizeof(struct em_perf_state), - GFP_KERNEL); - if (!pd->table) - return -ENOMEM; - - return 0; -} - /** * em_dev_update_perf_domain() - Update runtime EM table for a device * @dev : Device for which the EM is to be updated @@ -331,24 +320,6 @@ int em_dev_update_perf_domain(struct device *dev, } EXPORT_SYMBOL_GPL(em_dev_update_perf_domain); -static int em_create_runtime_table(struct em_perf_domain *pd) -{ - struct em_perf_table __rcu *table; - int table_size; - - table = em_table_alloc(pd); - if (!table) - return -ENOMEM; - - /* Initialize runtime table with existing data */ - table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; - memcpy(table->state, pd->table, table_size); - - rcu_assign_pointer(pd->em_table, table); - - return 0; -} - static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, struct em_perf_state *table, struct em_data_callback *cb, @@ -409,6 +380,7 @@ static int em_create_pd(struct device *dev, int nr_states, struct em_data_callback *cb, cpumask_t *cpus, unsigned long flags) { + struct em_perf_table __rcu *em_table; struct em_perf_domain *pd; struct device *cpu_dev; int cpu, ret, num_cpus; @@ -435,17 +407,15 @@ static int em_create_pd(struct device *dev, int nr_states, pd->nr_perf_states = nr_states; - ret = em_allocate_perf_table(pd, nr_states); - if (ret) + em_table = em_table_alloc(pd); + if (!em_table) goto free_pd; - ret = em_create_perf_table(dev, pd, pd->table, cb, flags); + ret = em_create_perf_table(dev, pd, em_table->state, cb, flags); if (ret) goto free_pd_table; - ret = em_create_runtime_table(pd); - if (ret) - goto free_pd_table; + rcu_assign_pointer(pd->em_table, em_table); if (_is_cpu_device(dev)) for_each_cpu(cpu, cpus) { @@ -458,7 +428,7 @@ static int em_create_pd(struct device *dev, int nr_states, return 0; free_pd_table: - kfree(pd->table); + kfree(em_table); free_pd: kfree(pd); return -EINVAL; @@ -629,7 +599,7 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, dev->em_pd->flags |= flags; - em_cpufreq_update_efficiencies(dev, dev->em_pd->table); + em_cpufreq_update_efficiencies(dev, dev->em_pd->em_table->state); em_debug_create_pd(dev); dev_info(dev, "EM: created perf domain\n"); @@ -666,8 +636,6 @@ void em_dev_unregister_perf_domain(struct device *dev) mutex_lock(&em_pd_mutex); em_debug_remove_pd(dev); - kfree(dev->em_pd->table); - em_table_free(dev->em_pd->em_table); kfree(dev->em_pd); -- cgit v1.2.3 From 22ea02848c07d1cbd15a5f442138ca429866300d Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:56 +0000 Subject: PM: EM: Add em_dev_compute_costs() The device drivers can modify EM at runtime by providing a new EM table. The EM is used by the EAS and the em_perf_state::cost stores pre-calculated value to avoid overhead. This patch provides the API for device drivers to calculate the cost values properly (and not duplicate the same code). Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 8 ++++++++ kernel/power/energy_model.c | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 92866a81abe4..770755df852f 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -170,6 +170,8 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, void em_dev_unregister_perf_domain(struct device *dev); struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); void em_table_free(struct em_perf_table __rcu *table); +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM @@ -379,6 +381,12 @@ struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) { return NULL; } +static inline +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states) +{ + return -EINVAL; +} #endif #endif diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 57838d28af85..7101fa3fa0c0 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -276,6 +276,24 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return 0; } +/** + * em_dev_compute_costs() - Calculate cost values for new runtime EM table + * @dev : Device for which the EM table is to be updated + * @table : The new EM table that is going to get the costs calculated + * + * Calculate the em_perf_state::cost values for new runtime EM table. The + * values are used for EAS during task placement. It also calculates and sets + * the efficiency flag for each performance state. When the function finish + * successfully the EM table is ready to be updated and used by EAS. + * + * Return 0 on success or a proper error in case of failure. + */ +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states) +{ + return em_compute_costs(dev, table, NULL, nr_states, 0); +} + /** * em_dev_update_perf_domain() - Update runtime EM table for a device * @dev : Device for which the EM is to be updated -- cgit v1.2.3 From b1344b1399daec9aca62bd0b2ea94874f5b8e126 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 29 Jan 2024 20:04:56 +0100 Subject: wifi: mac80211: add/use ieee80211_get_sn() This will also be useful for MLO duplicate multicast detection, but add it already here and use it in one place that trivially converts. Link: https://msgid.link/20240129200456.f0ff49c80006.I850d2785ab1640e56e262d3ad7343b87f6962552@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 7 ++++++- net/mac80211/rx.c | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index d9d2c1253157..b9367d5f04c4 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -9,7 +9,7 @@ * Copyright (c) 2006, Michael Wu * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH * Copyright (c) 2016 - 2017 Intel Deutschland GmbH - * Copyright (c) 2018 - 2023 Intel Corporation + * Copyright (c) 2018 - 2024 Intel Corporation */ #ifndef LINUX_IEEE80211_H @@ -808,6 +808,11 @@ static inline bool ieee80211_is_frag(struct ieee80211_hdr *hdr) hdr->seq_ctrl & cpu_to_le16(IEEE80211_SCTL_FRAG); } +static inline u16 ieee80211_get_sn(struct ieee80211_hdr *hdr) +{ + return le16_get_bits(hdr->seq_ctrl, IEEE80211_SCTL_SEQ); +} + struct ieee80211s_hdr { u8 flags; u8 ttl; diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 76798e8057f7..53c4764dc1ed 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -6,7 +6,7 @@ * Copyright 2007-2010 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright(c) 2015 - 2017 Intel Deutschland GmbH - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation */ #include @@ -1251,8 +1251,7 @@ static bool ieee80211_sta_manage_reorder_buf(struct ieee80211_sub_if_data *sdata { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); - u16 sc = le16_to_cpu(hdr->seq_ctrl); - u16 mpdu_seq_num = (sc & IEEE80211_SCTL_SEQ) >> 4; + u16 mpdu_seq_num = ieee80211_get_sn(hdr); u16 head_seq_num, buf_size; int index; bool ret = true; -- cgit v1.2.3 From 676259100cf3a81dd2d47918b36edb237986b9df Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 29 Jan 2024 20:04:57 +0100 Subject: wifi: mac80211: implement MLO multicast deduplication If the vif is an MLD then it may receive multicast from different links, and should drop those frames according to the SN. Implement that. Link: https://msgid.link/20240129200456.693b77d14b44.I491846f2bea0058c14eab6422962c10bfae9b675@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 5 +++++ net/mac80211/ieee80211_i.h | 4 +++- net/mac80211/mlme.c | 3 +++ net/mac80211/rx.c | 22 ++++++++++++++++++++-- 4 files changed, 31 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index b9367d5f04c4..e9078143b822 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -191,6 +191,11 @@ static inline bool ieee80211_sn_less(u16 sn1, u16 sn2) return ((sn1 - sn2) & IEEE80211_SN_MASK) > (IEEE80211_SN_MODULO >> 1); } +static inline bool ieee80211_sn_less_eq(u16 sn1, u16 sn2) +{ + return ((sn2 - sn1) & IEEE80211_SN_MASK) <= (IEEE80211_SN_MODULO >> 1); +} + static inline u16 ieee80211_sn_add(u16 sn1, u16 sn2) { return (sn1 + sn2) & IEEE80211_SN_MASK; diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 534cac3fc8df..46b517cf47ea 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -5,7 +5,7 @@ * Copyright 2006-2007 Jiri Benc * Copyright 2007-2010 Johannes Berg * Copyright 2013-2015 Intel Mobile Communications GmbH - * Copyright (C) 2018-2022 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation */ #ifndef IEEE80211_I_H @@ -523,6 +523,8 @@ struct ieee80211_if_managed { unsigned int flags; + u16 mcast_seq_last; + bool status_acked; bool status_received; __le16 status_fc; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 4eaf5c10efdb..35dda5982854 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -3351,6 +3351,8 @@ static void ieee80211_set_disassoc(struct ieee80211_sub_if_data *sdata, wiphy_delayed_work_cancel(sdata->local->hw.wiphy, &ifmgd->neg_ttlm_timeout_work); ieee80211_vif_set_links(sdata, 0, 0); + + ifmgd->mcast_seq_last = IEEE80211_SN_MODULO; } static void ieee80211_reset_ap_probe(struct ieee80211_sub_if_data *sdata) @@ -7512,6 +7514,7 @@ void ieee80211_sta_setup_sdata(struct ieee80211_sub_if_data *sdata) spin_lock_init(&ifmgd->teardown_lock); ifmgd->teardown_skb = NULL; ifmgd->orig_teardown_skb = NULL; + ifmgd->mcast_seq_last = IEEE80211_SN_MODULO; } static void ieee80211_recalc_smps_work(struct wiphy *wiphy, diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 53c4764dc1ed..9902ea69af0a 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -1434,13 +1434,31 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx) return RX_CONTINUE; if (ieee80211_is_ctl(hdr->frame_control) || - ieee80211_is_any_nullfunc(hdr->frame_control) || - is_multicast_ether_addr(hdr->addr1)) + ieee80211_is_any_nullfunc(hdr->frame_control)) return RX_CONTINUE; if (!rx->sta) return RX_CONTINUE; + if (unlikely(is_multicast_ether_addr(hdr->addr1))) { + struct ieee80211_sub_if_data *sdata = rx->sdata; + u16 sn = ieee80211_get_sn(hdr); + + if (!ieee80211_is_data_present(hdr->frame_control)) + return RX_CONTINUE; + + if (!ieee80211_vif_is_mld(&sdata->vif) || + sdata->vif.type != NL80211_IFTYPE_STATION) + return RX_CONTINUE; + + if (sdata->u.mgd.mcast_seq_last != IEEE80211_SN_MODULO && + ieee80211_sn_less_eq(sn, sdata->u.mgd.mcast_seq_last)) + return RX_DROP_U_DUP; + + sdata->u.mgd.mcast_seq_last = sn; + return RX_CONTINUE; + } + if (unlikely(ieee80211_has_retry(hdr->frame_control) && rx->sta->last_seq_ctrl[rx->seqno_idx] == hdr->seq_ctrl)) { I802_DEBUG_INC(rx->local->dot11FrameDuplicateCount); -- cgit v1.2.3 From 6239da18d2f947523a80fb1f85f8d8a13d1726c1 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 29 Jan 2024 20:19:28 +0100 Subject: wifi: mac80211: adjust EHT capa when lowering bandwidth If intending to associate with a lower bandwidth, remove capabilities related to 320 MHz from the EHT capabilities element. Also change the EHT MCS-NSS set accordingly: if just reducing 320->160 or similar the format doesn't change, just cut off the last bytes. If changing from higher bandwidth to 20 MHz only EHT STA, adjust the format. Note that this also requires adjusting the caller in mlme.c since the data written can now be shorter than it determined. We need to clean all that up. Since the other callers pass NULL for the conn limit, we don't need to change things there. Link: https://msgid.link/20240129202041.b5f6df108c77.I0d8ea04079c61cb3744cc88625eeaf0d4776dc2b@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 3 ++ net/mac80211/ieee80211_i.h | 3 +- net/mac80211/mesh.c | 3 +- net/mac80211/mlme.c | 15 ++++-- net/mac80211/tdls.c | 5 +- net/mac80211/util.c | 114 ++++++++++++++++++++++++++++++++++----------- 6 files changed, 106 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index e9078143b822..e4322238f273 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -3060,6 +3060,9 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) #define IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF 0x40 #define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK 0x07 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ 0x08 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ 0x30 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ 0x40 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK 0x78 #define IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP 0x80 diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 44400ce9a0b1..43c55ea6349c 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -2652,7 +2652,8 @@ void ieee80211_init_frag_cache(struct ieee80211_fragment_cache *cache); void ieee80211_destroy_frag_cache(struct ieee80211_fragment_cache *cache); u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata); -u8 *ieee80211_ie_build_eht_cap(u8 *pos, +u8 *ieee80211_ie_build_eht_cap(const struct ieee80211_conn_settings *conn, + u8 *pos, const struct ieee80211_sta_he_cap *he_cap, const struct ieee80211_sta_eht_cap *eht_cap, u8 *end, diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index 9fd209e4ca19..000fa9484b4e 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -668,7 +668,8 @@ int mesh_add_eht_cap_ie(struct ieee80211_sub_if_data *sdata, return -ENOMEM; pos = skb_put(skb, ie_len); - ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + ie_len, false); + ieee80211_ie_build_eht_cap(NULL, pos, he_cap, eht_cap, pos + ie_len, + false); return 0; } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 9a0331d914d3..bbc7894ccad0 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -1072,9 +1072,10 @@ static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata, static void ieee80211_add_eht_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb, - struct ieee80211_supported_band *sband) + struct ieee80211_supported_band *sband, + const struct ieee80211_conn_settings *conn) { - u8 *pos; + u8 *pos, *pre_eht_pos; const struct ieee80211_sta_he_cap *he_cap; const struct ieee80211_sta_eht_cap *eht_cap; u8 eht_cap_size; @@ -1097,8 +1098,11 @@ static void ieee80211_add_eht_ie(struct ieee80211_sub_if_data *sdata, ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0], eht_cap->eht_cap_elem.phy_cap_info); pos = skb_put(skb, eht_cap_size); - ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + eht_cap_size, - false); + pre_eht_pos = pos; + pos = ieee80211_ie_build_eht_cap(conn, pos, he_cap, eht_cap, + pos + eht_cap_size, false); + /* trim excess if any */ + skb_trim(skb, skb->len - (pre_eht_pos + eht_cap_size - pos)); } static void ieee80211_assoc_add_rates(struct sk_buff *skb, @@ -1453,7 +1457,8 @@ static size_t ieee80211_assoc_link_elems(struct ieee80211_sub_if_data *sdata, present_elems = NULL; if (assoc_data->link[link_id].conn.mode >= IEEE80211_CONN_MODE_EHT) - ieee80211_add_eht_ie(sdata, skb, sband); + ieee80211_add_eht_ie(sdata, skb, sband, + &assoc_data->link[link_id].conn); if (sband->band == NL80211_BAND_S1GHZ) { ieee80211_add_aid_request_ie(sdata, skb); diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c index 0f4aa42e070f..57673f27daf4 100644 --- a/net/mac80211/tdls.c +++ b/net/mac80211/tdls.c @@ -6,7 +6,7 @@ * Copyright 2014, Intel Corporation * Copyright 2014 Intel Mobile Communications GmbH * Copyright 2015 - 2016 Intel Deutschland GmbH - * Copyright (C) 2019, 2021-2023 Intel Corporation + * Copyright (C) 2019, 2021-2024 Intel Corporation */ #include @@ -604,7 +604,8 @@ ieee80211_tdls_add_setup_start_ies(struct ieee80211_link_data *link, ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0], eht_cap->eht_cap_elem.phy_cap_info); pos = skb_put(skb, cap_size); - ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + cap_size, false); + ieee80211_ie_build_eht_cap(NULL, pos, he_cap, eht_cap, + pos + cap_size, false); } /* add any remaining IEs */ diff --git a/net/mac80211/util.c b/net/mac80211/util.c index e9a7978d47d4..5224c22b1afc 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -2250,7 +2250,8 @@ static int ieee80211_build_preq_ies_band(struct ieee80211_sub_if_data *sdata, cfg80211_any_usable_channels(local->hw.wiphy, BIT(sband->band), IEEE80211_CHAN_NO_HE | IEEE80211_CHAN_NO_EHT)) { - pos = ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, end, + pos = ieee80211_ie_build_eht_cap(NULL, pos, he_cap, eht_cap, + end, sdata->vif.type == NL80211_IFTYPE_AP); if (!pos) goto out_err; @@ -3294,6 +3295,24 @@ u8 ieee80211_ie_len_he_cap(struct ieee80211_sub_if_data *sdata) he_cap->he_cap_elem.phy_cap_info); } +static void +ieee80211_get_adjusted_he_cap(const struct ieee80211_conn_settings *conn, + const struct ieee80211_sta_he_cap *he_cap, + struct ieee80211_he_cap_elem *elem) +{ + *elem = he_cap->he_cap_elem; + + if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_40) + elem->phy_cap_info[0] &= + ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G); + + if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_160) + elem->phy_cap_info[0] &= + ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G); +} + u8 *ieee80211_ie_build_he_cap(const struct ieee80211_conn_settings *conn, const struct ieee80211_sta_he_cap *he_cap, u8 *pos, u8 *end) @@ -3307,25 +3326,11 @@ u8 *ieee80211_ie_build_he_cap(const struct ieee80211_conn_settings *conn, conn = &ieee80211_conn_settings_unlimited; /* Make sure we have place for the IE */ - /* - * TODO: the 1 added is because this temporarily is under the EXTENSION - * IE. Get rid of it when it moves. - */ if (!he_cap) return orig_pos; /* modify on stack first to calculate 'n' and 'ie_len' correctly */ - elem = he_cap->he_cap_elem; - - if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_40) - elem.phy_cap_info[0] &= - ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G); - - if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_160) - elem.phy_cap_info[0] &= - ~(IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | - IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G); + ieee80211_get_adjusted_he_cap(conn, he_cap, &elem); n = ieee80211_he_mcs_nss_size(&elem); ie_len = 2 + 1 + @@ -5096,25 +5101,65 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata) return 0; } -u8 *ieee80211_ie_build_eht_cap(u8 *pos, +u8 *ieee80211_ie_build_eht_cap(const struct ieee80211_conn_settings *conn, + u8 *pos, const struct ieee80211_sta_he_cap *he_cap, const struct ieee80211_sta_eht_cap *eht_cap, - u8 *end, - bool for_ap) + u8 *end, bool for_ap) { + struct ieee80211_eht_cap_elem_fixed fixed, *out; + struct ieee80211_he_cap_elem he; u8 mcs_nss_len, ppet_len; + u8 orig_mcs_nss_len; u8 ie_len; u8 *orig_pos = pos; + if (!conn) + conn = &ieee80211_conn_settings_unlimited; + /* Make sure we have place for the IE */ if (!he_cap || !eht_cap) return orig_pos; - mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem, - &eht_cap->eht_cap_elem, - for_ap); + orig_mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem, + &eht_cap->eht_cap_elem, + for_ap); + + ieee80211_get_adjusted_he_cap(conn, he_cap, &he); + + fixed = eht_cap->eht_cap_elem; + + if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_80) + fixed.phy_cap_info[6] &= + ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ; + + if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_160) { + fixed.phy_cap_info[1] &= + ~IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK; + fixed.phy_cap_info[2] &= + ~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK; + fixed.phy_cap_info[6] &= + ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ; + } + + if (conn->bw_limit < IEEE80211_CONN_BW_LIMIT_320) { + fixed.phy_cap_info[0] &= + ~IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + fixed.phy_cap_info[1] &= + ~IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK; + fixed.phy_cap_info[2] &= + ~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK; + fixed.phy_cap_info[6] &= + ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ; + } + + if (conn->bw_limit == IEEE80211_CONN_BW_LIMIT_20) + fixed.phy_cap_info[0] &= + ~IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ; + + mcs_nss_len = ieee80211_eht_mcs_nss_size(&he, &fixed, for_ap); ppet_len = ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0], - eht_cap->eht_cap_elem.phy_cap_info); + fixed.phy_cap_info); ie_len = 2 + 1 + sizeof(eht_cap->eht_cap_elem) + mcs_nss_len + ppet_len; if ((end - pos) < ie_len) @@ -5124,12 +5169,25 @@ u8 *ieee80211_ie_build_eht_cap(u8 *pos, *pos++ = ie_len - 2; *pos++ = WLAN_EID_EXT_EHT_CAPABILITY; - /* Fixed data */ - memcpy(pos, &eht_cap->eht_cap_elem, sizeof(eht_cap->eht_cap_elem)); - pos += sizeof(eht_cap->eht_cap_elem); + out = (void *)pos; + *out = fixed; + pos += sizeof(*out); - memcpy(pos, &eht_cap->eht_mcs_nss_supp, mcs_nss_len); - pos += mcs_nss_len; + if (mcs_nss_len == 4 && orig_mcs_nss_len != 4) { + /* + * If the (non-AP) STA became 20 MHz only, then convert from + * <=80 to 20-MHz-only format, where MCSes are indicated in + * the groups 0-7, 8-9, 10-11, 12-13 rather than just 0-9, + * 10-11, 12-13. Thus, use 0-9 for 0-7 and 8-9. + */ + *pos++ = eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs9_max_nss; + *pos++ = eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs9_max_nss; + *pos++ = eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs11_max_nss; + *pos++ = eht_cap->eht_mcs_nss_supp.bw._80.rx_tx_mcs13_max_nss; + } else { + memcpy(pos, &eht_cap->eht_mcs_nss_supp, mcs_nss_len); + pos += mcs_nss_len; + } if (ppet_len) { memcpy(pos, &eht_cap->eht_ppe_thres, ppet_len); -- cgit v1.2.3 From d1ff85fdf0b8f63a6e042ae7559c630f9b1c50e2 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:52 +0100 Subject: spi: pl022: Use typedef for dma_filter_fn Use existing typedef for dma_filter_fn to avoid duplicating type definition. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202154.630336-1-krzysztof.kozlowski@linaro.org Signed-off-by: Mark Brown --- include/linux/amba/pl022.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/amba/pl022.h b/include/linux/amba/pl022.h index 9bf58aac0df2..e08488df6d28 100644 --- a/include/linux/amba/pl022.h +++ b/include/linux/amba/pl022.h @@ -16,6 +16,7 @@ #ifndef _SSP_PL022_H #define _SSP_PL022_H +#include #include /** @@ -235,7 +236,7 @@ struct dma_chan; struct pl022_ssp_controller { u16 bus_id; u8 enable_dma:1; - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); + dma_filter_fn dma_filter; void *dma_rx_param; void *dma_tx_param; int autosuspend_delay; -- cgit v1.2.3 From c42d9bead493854507e1a180942ebe33c9180598 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:53 +0100 Subject: spi: pl022: Add missing dma_filter field kerneldoc Add kerneldoc for dma_filter field in struct pl022_ssp_controller. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202154.630336-2-krzysztof.kozlowski@linaro.org Signed-off-by: Mark Brown --- include/linux/amba/pl022.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/amba/pl022.h b/include/linux/amba/pl022.h index e08488df6d28..d7b07d0311e1 100644 --- a/include/linux/amba/pl022.h +++ b/include/linux/amba/pl022.h @@ -225,6 +225,7 @@ struct dma_chan; * struct pl022_ssp_master - device.platform_data for SPI controller devices. * @bus_id: identifier for this bus * @enable_dma: if true enables DMA driven transfers. + * @dma_filter: callback filter for dma_request_channel. * @dma_rx_param: parameter to locate an RX DMA channel. * @dma_tx_param: parameter to locate a TX DMA channel. * @autosuspend_delay: delay in ms following transfer completion before the -- cgit v1.2.3 From 3d4dd10b376e1b8b6d0409f7e7b752f9baa51c24 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:54 +0100 Subject: spi: pxa2xx: Use typedef for dma_filter_fn Use existing typedef for dma_filter_fn to avoid duplicating type definition. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202154.630336-3-krzysztof.kozlowski@linaro.org Signed-off-by: Mark Brown --- include/linux/spi/pxa2xx_spi.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/spi/pxa2xx_spi.h b/include/linux/spi/pxa2xx_spi.h index 0916cb9bcb0a..ca2cd4e30ead 100644 --- a/include/linux/spi/pxa2xx_spi.h +++ b/include/linux/spi/pxa2xx_spi.h @@ -5,6 +5,7 @@ #ifndef __LINUX_SPI_PXA2XX_SPI_H #define __LINUX_SPI_PXA2XX_SPI_H +#include #include #include @@ -22,7 +23,7 @@ struct pxa2xx_spi_controller { bool is_target; /* DMA engine specific config */ - bool (*dma_filter)(struct dma_chan *chan, void *param); + dma_filter_fn dma_filter; void *tx_param; void *rx_param; -- cgit v1.2.3 From e891becdccaa9048b1ab91c08ad5722edd571806 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Wed, 7 Feb 2024 22:39:14 +0100 Subject: PCI: endpoint: Refactor pci_epf_alloc_space() API Refactor pci_epf_alloc_space() API to accept "epc_features" as a parameter. This is a preparatory work to make the API more robust. Reviewed-by: Frank Li Reviewed-by: Manivannan Sadhasivam Signed-off-by: Niklas Cassel Link: https://lore.kernel.org/r/20240207213922.1796533-2-cassel@kernel.org [mani: reworded commit message] Signed-off-by: Manivannan Sadhasivam --- drivers/pci/endpoint/functions/pci-epf-ntb.c | 2 +- drivers/pci/endpoint/functions/pci-epf-test.c | 5 ++--- drivers/pci/endpoint/functions/pci-epf-vntb.c | 4 ++-- drivers/pci/endpoint/pci-epf-core.c | 6 ++++-- include/linux/pci-epf.h | 4 +++- 5 files changed, 12 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index 0553946005c4..43cd309ce22f 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -1067,7 +1067,7 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, else if (size < ctrl_size + spad_size) return -EINVAL; - base = pci_epf_alloc_space(epf, size, barno, align, type); + base = pci_epf_alloc_space(epf, size, barno, epc_features, type); if (!base) { dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n", pci_epc_interface_string(type)); diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 18c80002d3bd..15bfa7d83489 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -848,7 +848,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) } base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar, - epc_features->align, PRIMARY_INTERFACE); + epc_features, PRIMARY_INTERFACE); if (!base) { dev_err(dev, "Failed to allocated register space\n"); return -ENOMEM; @@ -866,8 +866,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) continue; base = pci_epf_alloc_space(epf, bar_size[bar], bar, - epc_features->align, - PRIMARY_INTERFACE); + epc_features, PRIMARY_INTERFACE); if (!base) dev_err(dev, "Failed to allocate space for BAR%d\n", bar); diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index e75a2af77328..ba509d67188b 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -446,7 +446,7 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb) else if (size < ctrl_size + spad_size) return -EINVAL; - base = pci_epf_alloc_space(epf, size, barno, align, 0); + base = pci_epf_alloc_space(epf, size, barno, epc_features, 0); if (!base) { dev_err(dev, "Config/Status/SPAD alloc region fail\n"); return -ENOMEM; @@ -550,7 +550,7 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb) barno = ntb->epf_ntb_bar[BAR_DB]; - mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0); + mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0); if (!mw_addr) { dev_err(dev, "Failed to allocate OB address\n"); return -ENOMEM; diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index bf655383e5ed..9da9a1e9c086 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -251,14 +251,16 @@ EXPORT_SYMBOL_GPL(pci_epf_free_space); * @epf: the EPF device to whom allocate the memory * @size: the size of the memory that has to be allocated * @bar: the BAR number corresponding to the allocated register space - * @align: alignment size for the allocation region + * @epc_features: the features provided by the EPC specific to this EPF * @type: Identifies if the allocation is for primary EPC or secondary EPC * * Invoke to allocate memory for the PCI EPF register space. */ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, - size_t align, enum pci_epc_interface_type type) + const struct pci_epc_features *epc_features, + enum pci_epc_interface_type type) { + size_t align = epc_features->align; struct pci_epf_bar *epf_bar; dma_addr_t phys_addr; struct pci_epc *epc; diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 77b146e0f672..adee6a1b35db 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -15,6 +15,7 @@ #include struct pci_epf; +struct pci_epc_features; enum pci_epc_interface_type; enum pci_barno { @@ -216,7 +217,8 @@ int __pci_epf_register_driver(struct pci_epf_driver *driver, struct module *owner); void pci_epf_unregister_driver(struct pci_epf_driver *driver); void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, - size_t align, enum pci_epc_interface_type type); + const struct pci_epc_features *epc_features, + enum pci_epc_interface_type type); void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, enum pci_epc_interface_type type); int pci_epf_bind(struct pci_epf *epf); -- cgit v1.2.3 From e3caf184107a4e2e196528b98b218ddc41e4cb8c Mon Sep 17 00:00:00 2001 From: Jinjian Song Date: Mon, 5 Feb 2024 18:22:27 +0800 Subject: wwan: core: Add WWAN fastboot port type Add a new WWAN port that connects to the device fastboot protocol interface. Signed-off-by: Jinjian Song Signed-off-by: David S. Miller --- drivers/net/wwan/wwan_core.c | 4 ++++ include/linux/wwan.h | 2 ++ 2 files changed, 6 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index 72e01e550a16..2ed20b20e7fc 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -328,6 +328,10 @@ static const struct { .name = "XMMRPC", .devsuf = "xmmrpc", }, + [WWAN_PORT_FASTBOOT] = { + .name = "FASTBOOT", + .devsuf = "fastboot", + }, }; static ssize_t type_show(struct device *dev, struct device_attribute *attr, diff --git a/include/linux/wwan.h b/include/linux/wwan.h index 01fa15506286..170fdee6339c 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -16,6 +16,7 @@ * @WWAN_PORT_QCDM: Qcom Modem diagnostic interface * @WWAN_PORT_FIREHOSE: XML based command protocol * @WWAN_PORT_XMMRPC: Control protocol for Intel XMM modems + * @WWAN_PORT_FASTBOOT: Fastboot protocol control * * @WWAN_PORT_MAX: Highest supported port types * @WWAN_PORT_UNKNOWN: Special value to indicate an unknown port type @@ -28,6 +29,7 @@ enum wwan_port_type { WWAN_PORT_QCDM, WWAN_PORT_FIREHOSE, WWAN_PORT_XMMRPC, + WWAN_PORT_FASTBOOT, /* Add new port types above this line */ -- cgit v1.2.3 From 6f35fe5d8a0ad0125c52fb20f5d67000e369eb3a Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:01 +0000 Subject: iommu/amd: Introduce get_amd_iommu_from_dev() Introduce get_amd_iommu_from_dev() and get_amd_iommu_from_dev_data(). And replace rlookup_amd_iommu() with the new helper function where applicable to avoid unnecessary loop to look up struct amd_iommu from struct device. Suggested-by: Jason Gunthorpe Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-4-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 15 ++++++++++++ drivers/iommu/amd/iommu.c | 54 +++++++++++-------------------------------- include/linux/iommu.h | 16 +++++++++++++ 3 files changed, 44 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 7464222ad8af..4e1f9a444a1a 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -138,6 +138,21 @@ static inline void *alloc_pgtable_page(int nid, gfp_t gfp) return page ? page_address(page) : NULL; } +/* + * This must be called after device probe completes. During probe + * use rlookup_amd_iommu() get the iommu. + */ +static inline struct amd_iommu *get_amd_iommu_from_dev(struct device *dev) +{ + return iommu_get_iommu_dev(dev, struct amd_iommu, iommu); +} + +/* This must be called after device probe completes. */ +static inline struct amd_iommu *get_amd_iommu_from_dev_data(struct iommu_dev_data *dev_data) +{ + return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu); +} + bool translation_pre_enabled(struct amd_iommu *iommu); bool amd_iommu_is_attach_deferred(struct device *dev); int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 743465cb677f..2a5ae7a841cc 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1384,14 +1384,9 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu) static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, size_t size, ioasid_t pasid, bool gn) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct iommu_cmd cmd; - int qdep; - - qdep = dev_data->ats_qdep; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return -EINVAL; + int qdep = dev_data->ats_qdep; build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size, pasid, gn); @@ -1411,16 +1406,12 @@ static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) */ static int device_flush_dte(struct iommu_dev_data *dev_data) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct pci_dev *pdev = NULL; struct amd_iommu_pci_seg *pci_seg; u16 alias; int ret; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return -EINVAL; - if (dev_is_pci(dev_data->dev)) pdev = to_pci_dev(dev_data->dev); @@ -1805,11 +1796,7 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) static void do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct amd_iommu *iommu; - - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); /* Update data structures */ dev_data->domain = domain; @@ -1833,11 +1820,7 @@ static void do_attach(struct iommu_dev_data *dev_data, static void do_detach(struct iommu_dev_data *dev_data) { struct protection_domain *domain = dev_data->domain; - struct amd_iommu *iommu; - - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); /* Update data structures */ dev_data->domain = NULL; @@ -2003,10 +1986,8 @@ static void update_device_table(struct protection_domain *domain) struct iommu_dev_data *dev_data; list_for_each_entry(dev_data, &domain->dev_list, list) { - struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); - if (!iommu) - continue; set_dte_entry(iommu, dev_data); clone_aliases(iommu, dev_data->dev); } @@ -2187,11 +2168,8 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type, struct protection_domain *domain; struct amd_iommu *iommu = NULL; - if (dev) { - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return ERR_PTR(-ENODEV); - } + if (dev) + iommu = get_amd_iommu_from_dev(dev); /* * Since DTE[Mode]=0 is prohibited on SNP-enabled system, @@ -2272,7 +2250,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); struct protection_domain *domain = to_pdomain(dom); - struct amd_iommu *iommu = rlookup_amd_iommu(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; /* @@ -2411,7 +2389,7 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) case IOMMU_CAP_DEFERRED_FLUSH: return true; case IOMMU_CAP_DIRTY_TRACKING: { - struct amd_iommu *iommu = rlookup_amd_iommu(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); return amd_iommu_hd_support(iommu); } @@ -2440,9 +2418,7 @@ static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, } list_for_each_entry(dev_data, &pdomain->dev_list, list) { - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - continue; + iommu = get_amd_iommu_from_dev_data(dev_data); dev_table = get_dev_table(iommu); pte_root = dev_table[dev_data->devid].data[0]; @@ -2502,9 +2478,7 @@ static void amd_iommu_get_resv_regions(struct device *dev, return; devid = PCI_SBDF_TO_DEVID(sbdf); - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return; + iommu = get_amd_iommu_from_dev(dev); pci_seg = iommu->pci_seg; list_for_each_entry(entry, &pci_seg->unity_map, list) { @@ -2838,9 +2812,7 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, struct iommu_cmd cmd; dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = rlookup_amd_iommu(&pdev->dev); - if (!iommu) - return -ENODEV; + iommu = get_amd_iommu_from_dev(&pdev->dev); build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, dev_data->pri_tlp); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1ea2a820e1eb..32bb121e8032 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -654,6 +654,22 @@ static inline struct iommu_device *dev_to_iommu_device(struct device *dev) return (struct iommu_device *)dev_get_drvdata(dev); } +/** + * iommu_get_iommu_dev - Get iommu_device for a device + * @dev: an end-point device + * + * Note that this function must be called from the iommu_ops + * to retrieve the iommu_device for a device, which the core code + * guarentees it will not invoke the op without an attached iommu. + */ +static inline struct iommu_device *__iommu_get_iommu_dev(struct device *dev) +{ + return dev->iommu->iommu_dev; +} + +#define iommu_get_iommu_dev(dev, type, member) \ + container_of(__iommu_get_iommu_dev(dev), type, member) + static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) { *gather = (struct iommu_iotlb_gather) { -- cgit v1.2.3 From bf8aff2945ba4091f503df673b9df33002546e6a Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:07 +0000 Subject: iommu: Introduce iommu_group_mutex_assert() Add function to check iommu group mutex lock. So that device drivers can rely on group mutex lock instead of adding another driver level lock before modifying driver specific device data structure. Suggested-by: Jason Gunthorpe Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-10-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 19 +++++++++++++++++++ include/linux/iommu.h | 8 ++++++++ 2 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..8309e634977e 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1248,6 +1248,25 @@ void iommu_group_remove_device(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_group_remove_device); +#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API) +/** + * iommu_group_mutex_assert - Check device group mutex lock + * @dev: the device that has group param set + * + * This function is called by an iommu driver to check whether it holds + * group mutex lock for the given device or not. + * + * Note that this function must be called after device group param is set. + */ +void iommu_group_mutex_assert(struct device *dev) +{ + struct iommu_group *group = dev->iommu_group; + + lockdep_assert_held(&group->mutex); +} +EXPORT_SYMBOL_GPL(iommu_group_mutex_assert); +#endif + static struct device *iommu_group_first_dev(struct iommu_group *group) { lockdep_assert_held(&group->mutex); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 32bb121e8032..8141a37556d5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1356,6 +1356,14 @@ static inline ioasid_t iommu_alloc_global_pasid(struct device *dev) static inline void iommu_free_global_pasid(ioasid_t pasid) {} #endif /* CONFIG_IOMMU_API */ +#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API) +void iommu_group_mutex_assert(struct device *dev); +#else +static inline void iommu_group_mutex_assert(struct device *dev) +{ +} +#endif + /** * iommu_map_sgtable - Map the given buffer to the IOMMU domain * @domain: The IOMMU domain to perform the mapping -- cgit v1.2.3 From fd7eea27a3aed79b63b1726c00bde0d50cf207e2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 5 Feb 2024 16:48:43 +0100 Subject: Compiler Attributes: Add __uninitialized macro With INIT_STACK_ALL_PATTERN or INIT_STACK_ALL_ZERO enabled the kernel will be compiled with -ftrivial-auto-var-init=<...> which causes initialization of stack variables at function entry time. In order to avoid the performance impact that comes with this users can use the "uninitialized" attribute to prevent such initialization. Therefore provide the __uninitialized macro which can be used for cases where INIT_STACK_ALL_PATTERN or INIT_STACK_ALL_ZERO is enabled, but only selected variables should not be initialized. Acked-by: Kees Cook Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240205154844.3757121-2-hca@linux.ibm.com Signed-off-by: Heiko Carstens --- include/linux/compiler_attributes.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 28566624f008..f5859b8c68b4 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -333,6 +333,18 @@ */ #define __section(section) __attribute__((__section__(section))) +/* + * Optional: only supported since gcc >= 12 + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-uninitialized-variable-attribute + * clang: https://clang.llvm.org/docs/AttributeReference.html#uninitialized + */ +#if __has_attribute(__uninitialized__) +# define __uninitialized __attribute__((__uninitialized__)) +#else +# define __uninitialized +#endif + /* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-unused-function-attribute * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Type-Attributes.html#index-unused-type-attribute -- cgit v1.2.3 From fc5b764bef24d0cf722deb5c1a44948cd17d4afe Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 9 Feb 2024 18:54:23 +0200 Subject: spi: gpio: Follow renaming of SPI "master" to "controller" In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. Recent work by Uwe completes this renaming. However, there are plenty of leftovers in the comments and in-code documentation. Update them as well. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240209165423.2305493-1-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- include/linux/spi/spi_gpio.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/spi/spi_gpio.h b/include/linux/spi/spi_gpio.h index 9e7e83d8645b..5f0e1407917a 100644 --- a/include/linux/spi/spi_gpio.h +++ b/include/linux/spi/spi_gpio.h @@ -15,8 +15,8 @@ */ /** - * struct spi_gpio_platform_data - parameter for bitbanged SPI master - * @num_chipselect: how many slaves to allow + * struct spi_gpio_platform_data - parameter for bitbanged SPI host controller + * @num_chipselect: how many target devices to allow */ struct spi_gpio_platform_data { u16 num_chipselect; -- cgit v1.2.3 From 471e8fd3afcef5a9f9089f0bd21965ad9ba35c91 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Tue, 6 Feb 2024 18:31:06 +0100 Subject: net: phy: add devm/of_phy_package_join helper Add devm/of_phy_package_join helper to join PHYs in a PHY package. These are variant of the manual phy_package_join with the difference that these will use DT nodes to derive the base_addr instead of manually passing an hardcoded value. An additional value is added in phy_package_shared, "np" to reference the PHY package node pointer in specific PHY driver probe_once and config_init_once functions to make use of additional specific properties defined in the PHY package node in DT. The np value is filled only with of_phy_package_join if a valid PHY package node is found. A valid PHY package node must have the node name set to "ethernet-phy-package". Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/phy/phy_device.c | 96 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/phy.h | 6 +++ 2 files changed, 102 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 2e7d5bfb338e..71d48152e8d5 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1712,6 +1712,7 @@ int phy_package_join(struct phy_device *phydev, int base_addr, size_t priv_size) shared->priv_size = priv_size; } shared->base_addr = base_addr; + shared->np = NULL; refcount_set(&shared->refcnt, 1); bus->shared[base_addr] = shared; } else { @@ -1734,6 +1735,63 @@ err_unlock: } EXPORT_SYMBOL_GPL(phy_package_join); +/** + * of_phy_package_join - join a common PHY group in PHY package + * @phydev: target phy_device struct + * @priv_size: if non-zero allocate this amount of bytes for private data + * + * This is a variant of phy_package_join for PHY package defined in DT. + * + * The parent node of the @phydev is checked as a valid PHY package node + * structure (by matching the node name "ethernet-phy-package") and the + * base_addr for the PHY package is passed to phy_package_join. + * + * With this configuration the shared struct will also have the np value + * filled to use additional DT defined properties in PHY specific + * probe_once and config_init_once PHY package OPs. + * + * Returns < 0 on error, 0 on success. Esp. calling phy_package_join() + * with the same cookie but a different priv_size is an error. Or a parent + * node is not detected or is not valid or doesn't match the expected node + * name for PHY package. + */ +int of_phy_package_join(struct phy_device *phydev, size_t priv_size) +{ + struct device_node *node = phydev->mdio.dev.of_node; + struct device_node *package_node; + u32 base_addr; + int ret; + + if (!node) + return -EINVAL; + + package_node = of_get_parent(node); + if (!package_node) + return -EINVAL; + + if (!of_node_name_eq(package_node, "ethernet-phy-package")) { + ret = -EINVAL; + goto exit; + } + + if (of_property_read_u32(package_node, "reg", &base_addr)) { + ret = -EINVAL; + goto exit; + } + + ret = phy_package_join(phydev, base_addr, priv_size); + if (ret) + goto exit; + + phydev->shared->np = package_node; + + return 0; +exit: + of_node_put(package_node); + return ret; +} +EXPORT_SYMBOL_GPL(of_phy_package_join); + /** * phy_package_leave - leave a common PHY group * @phydev: target phy_device struct @@ -1750,6 +1808,10 @@ void phy_package_leave(struct phy_device *phydev) if (!shared) return; + /* Decrease the node refcount on leave if present */ + if (shared->np) + of_node_put(shared->np); + if (refcount_dec_and_mutex_lock(&shared->refcnt, &bus->shared_lock)) { bus->shared[shared->base_addr] = NULL; mutex_unlock(&bus->shared_lock); @@ -1802,6 +1864,40 @@ int devm_phy_package_join(struct device *dev, struct phy_device *phydev, } EXPORT_SYMBOL_GPL(devm_phy_package_join); +/** + * devm_of_phy_package_join - resource managed of_phy_package_join() + * @dev: device that is registering this PHY package + * @phydev: target phy_device struct + * @priv_size: if non-zero allocate this amount of bytes for private data + * + * Managed of_phy_package_join(). Shared storage fetched by this function, + * phy_package_leave() is automatically called on driver detach. See + * of_phy_package_join() for more information. + */ +int devm_of_phy_package_join(struct device *dev, struct phy_device *phydev, + size_t priv_size) +{ + struct phy_device **ptr; + int ret; + + ptr = devres_alloc(devm_phy_package_leave, sizeof(*ptr), + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ret = of_phy_package_join(phydev, priv_size); + + if (!ret) { + *ptr = phydev; + devres_add(dev, ptr); + } else { + devres_free(ptr); + } + + return ret; +} +EXPORT_SYMBOL_GPL(devm_of_phy_package_join); + /** * phy_detach - detach a PHY device from its network device * @phydev: target phy_device struct diff --git a/include/linux/phy.h b/include/linux/phy.h index fd8dbea9b4d9..cbd49418b819 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -329,6 +329,7 @@ struct mdio_bus_stats { * struct phy_package_shared - Shared information in PHY packages * @base_addr: Base PHY address of PHY package used to combine PHYs * in one package and for offset calculation of phy_package_read/write + * @np: Pointer to the Device Node if PHY package defined in DT * @refcnt: Number of PHYs connected to this shared data * @flags: Initialization of PHY package * @priv_size: Size of the shared private data @priv @@ -340,6 +341,8 @@ struct mdio_bus_stats { */ struct phy_package_shared { u8 base_addr; + /* With PHY package defined in DT this points to the PHY package node */ + struct device_node *np; refcount_t refcnt; unsigned long flags; size_t priv_size; @@ -2000,9 +2003,12 @@ int phy_ethtool_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *cmd); int phy_ethtool_nway_reset(struct net_device *ndev); int phy_package_join(struct phy_device *phydev, int base_addr, size_t priv_size); +int of_phy_package_join(struct phy_device *phydev, size_t priv_size); void phy_package_leave(struct phy_device *phydev); int devm_phy_package_join(struct device *dev, struct phy_device *phydev, int base_addr, size_t priv_size); +int devm_of_phy_package_join(struct device *dev, struct phy_device *phydev, + size_t priv_size); int __init mdio_bus_init(void); void mdio_bus_exit(void); -- cgit v1.2.3 From 9b1d5e055508393561e26bd1720f4c2639b03b1a Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Tue, 6 Feb 2024 18:31:09 +0100 Subject: net: phy: provide whether link has changed in c37_read_status Some PHY driver might require additional regs call after genphy_c37_read_status() is called. Expand genphy_c37_read_status to provide a bool wheather the link has changed or not to permit PHY driver to skip additional regs call if nothing has changed. Every user of genphy_c37_read_status() is updated with the new additional bool. Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/phy/broadcom.c | 3 ++- drivers/net/phy/phy_device.c | 11 +++++++++-- drivers/net/phy/qcom/at803x.c | 3 ++- include/linux/phy.h | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 312a8bb35d78..370e4ed45098 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -665,10 +665,11 @@ static int bcm54616s_config_aneg(struct phy_device *phydev) static int bcm54616s_read_status(struct phy_device *phydev) { struct bcm54616s_phy_priv *priv = phydev->priv; + bool changed; int err; if (priv->mode_1000bx_en) - err = genphy_c37_read_status(phydev); + err = genphy_c37_read_status(phydev, &changed); else err = genphy_read_status(phydev); diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 71d48152e8d5..9f37c0bfbf8d 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -2621,12 +2621,15 @@ EXPORT_SYMBOL(genphy_read_status); /** * genphy_c37_read_status - check the link status and update current link state * @phydev: target phy_device struct + * @changed: pointer where to store if link changed * * Description: Check the link, then figure out the current state * by comparing what we advertise with what the link partner * advertises. This function is for Clause 37 1000Base-X mode. + * + * If link has changed, @changed is set to true, false otherwise. */ -int genphy_c37_read_status(struct phy_device *phydev) +int genphy_c37_read_status(struct phy_device *phydev, bool *changed) { int lpa, err, old_link = phydev->link; @@ -2636,9 +2639,13 @@ int genphy_c37_read_status(struct phy_device *phydev) return err; /* why bother the PHY if nothing can have changed */ - if (phydev->autoneg == AUTONEG_ENABLE && old_link && phydev->link) + if (phydev->autoneg == AUTONEG_ENABLE && old_link && phydev->link) { + *changed = false; return 0; + } + /* Signal link has changed */ + *changed = true; phydev->duplex = DUPLEX_UNKNOWN; phydev->pause = 0; phydev->asym_pause = 0; diff --git a/drivers/net/phy/qcom/at803x.c b/drivers/net/phy/qcom/at803x.c index 3e3ee4c1d4bc..4717c59d51d0 100644 --- a/drivers/net/phy/qcom/at803x.c +++ b/drivers/net/phy/qcom/at803x.c @@ -912,9 +912,10 @@ static int at8031_config_intr(struct phy_device *phydev) static int at8031_read_status(struct phy_device *phydev) { struct at803x_priv *priv = phydev->priv; + bool changed; if (priv->is_1000basex) - return genphy_c37_read_status(phydev); + return genphy_c37_read_status(phydev, &changed); return at803x_read_status(phydev); } diff --git a/include/linux/phy.h b/include/linux/phy.h index cbd49418b819..2249cdb5957a 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1876,7 +1876,7 @@ int genphy_write_mmd_unsupported(struct phy_device *phdev, int devnum, /* Clause 37 */ int genphy_c37_config_aneg(struct phy_device *phydev); -int genphy_c37_read_status(struct phy_device *phydev); +int genphy_c37_read_status(struct phy_device *phydev, bool *changed); /* Clause 45 PHY */ int genphy_c45_restart_aneg(struct phy_device *phydev); -- cgit v1.2.3 From b9f55084aa0962af6247fa971f0b1e9c7aa676ef Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 17 Jan 2024 13:44:05 +0200 Subject: hwmon: put HWMON_CHANNEL_INFO() initializers in rodata HWMON_CHANNEL_INFO() is supposed to be used as initializer for arrays of const struct hwmon_channel_info *. However, without explicit const, HWMON_CHANNEL_INFO() creates mutable compound literals, and the const pointers point at the mutable data. Add const to place the data in rodata. Cc: Jean Delvare Cc: Guenter Roeck Signed-off-by: Jani Nikula Link: https://lore.kernel.org/r/20240117114405.1506775-1-jani.nikula@intel.com Signed-off-by: Guenter Roeck --- include/linux/hwmon.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 8cd6a6b33593..c2c0da18dfa3 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -425,12 +425,12 @@ struct hwmon_channel_info { const u32 *config; }; -#define HWMON_CHANNEL_INFO(stype, ...) \ - (&(struct hwmon_channel_info) { \ - .type = hwmon_##stype, \ - .config = (u32 []) { \ - __VA_ARGS__, 0 \ - } \ +#define HWMON_CHANNEL_INFO(stype, ...) \ + (&(const struct hwmon_channel_info) { \ + .type = hwmon_##stype, \ + .config = (const u32 []) { \ + __VA_ARGS__, 0 \ + } \ }) /** -- cgit v1.2.3 From 35c1bfb99fef9c71f9df5c1325db99a79300bc97 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Mon, 29 Jan 2024 17:13:24 +0100 Subject: hwmon: add fault attribute for voltage channels Sometimes a voltage channel might have an hard failure (eg: a shorted MOSFET). Hence, add a fault attribute to report such failures. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240129-b4-ltc4282-support-v4-2-fe75798164cc@analog.com Signed-off-by: Guenter Roeck --- Documentation/ABI/testing/sysfs-class-hwmon | 9 +++++++++ drivers/hwmon/hwmon.c | 1 + include/linux/hwmon.h | 2 ++ 3 files changed, 12 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon index 3dac923c9b0e..6c4e68ad4a83 100644 --- a/Documentation/ABI/testing/sysfs-class-hwmon +++ b/Documentation/ABI/testing/sysfs-class-hwmon @@ -149,6 +149,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/inY_fault +Description: + Reports a voltage hard failure (eg: shorted component) + + - 1: Failed + - 0: Ok + + RO + What: /sys/class/hwmon/hwmonX/cpuY_vid Description: CPU core reference voltage. diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index c7dd3f5b2bd5..18705049ad61 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -510,6 +510,7 @@ static const char * const hwmon_in_attr_templates[] = { [hwmon_in_rated_min] = "in%d_rated_min", [hwmon_in_rated_max] = "in%d_rated_max", [hwmon_in_beep] = "in%d_beep", + [hwmon_in_fault] = "in%d_fault", }; static const char * const hwmon_curr_attr_templates[] = { diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index c2c0da18dfa3..c7885fdce88f 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -141,6 +141,7 @@ enum hwmon_in_attributes { hwmon_in_rated_min, hwmon_in_rated_max, hwmon_in_beep, + hwmon_in_fault, }; #define HWMON_I_ENABLE BIT(hwmon_in_enable) @@ -162,6 +163,7 @@ enum hwmon_in_attributes { #define HWMON_I_RATED_MIN BIT(hwmon_in_rated_min) #define HWMON_I_RATED_MAX BIT(hwmon_in_rated_max) #define HWMON_I_BEEP BIT(hwmon_in_beep) +#define HWMON_I_FAULT BIT(hwmon_in_fault) enum hwmon_curr_attributes { hwmon_curr_enable, -- cgit v1.2.3 From 68bc61c26cacf152baf905786b5949769700f40d Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Wed, 7 Feb 2024 13:26:17 +0100 Subject: bpf: Allow compiler to inline most of bpf_local_storage_lookup() In various performance profiles of kernels with BPF programs attached, bpf_local_storage_lookup() appears as a significant portion of CPU cycles spent. To enable the compiler generate more optimal code, turn bpf_local_storage_lookup() into a static inline function, where only the cache insertion code path is outlined Notably, outlining cache insertion helps avoid bloating callers by duplicating setting up calls to raw_spin_{lock,unlock}_irqsave() (on architectures which do not inline spin_lock/unlock, such as x86), which would cause the compiler produce worse code by deciding to outline otherwise inlinable functions. The call overhead is neutral, because we make 2 calls either way: either calling raw_spin_lock_irqsave() and raw_spin_unlock_irqsave(); or call __bpf_local_storage_insert_cache(), which calls raw_spin_lock_irqsave(), followed by a tail-call to raw_spin_unlock_irqsave() where the compiler can perform TCO and (in optimized uninstrumented builds) turns it into a plain jump. The call to __bpf_local_storage_insert_cache() can be elided entirely if cacheit_lockit is a false constant expression. Based on results from './benchs/run_bench_local_storage.sh' (21 trials, reboot between each trial; x86 defconfig + BPF, clang 16) this produces improvements in throughput and latency in the majority of cases, with an average (geomean) improvement of 8%: +---- Hashmap Control -------------------- | | + num keys: 10 | : | | +-+ hashmap (control) sequential get +----------------------+---------------------- | +- hits throughput | 14.789 M ops/s | 14.745 M ops/s ( ~ ) | +- hits latency | 67.679 ns/op | 67.879 ns/op ( ~ ) | +- important_hits throughput | 14.789 M ops/s | 14.745 M ops/s ( ~ ) | | + num keys: 1000 | : | | +-+ hashmap (control) sequential get +----------------------+---------------------- | +- hits throughput | 12.233 M ops/s | 12.170 M ops/s ( ~ ) | +- hits latency | 81.754 ns/op | 82.185 ns/op ( ~ ) | +- important_hits throughput | 12.233 M ops/s | 12.170 M ops/s ( ~ ) | | + num keys: 10000 | : | | +-+ hashmap (control) sequential get +----------------------+---------------------- | +- hits throughput | 7.220 M ops/s | 7.204 M ops/s ( ~ ) | +- hits latency | 138.522 ns/op | 138.842 ns/op ( ~ ) | +- important_hits throughput | 7.220 M ops/s | 7.204 M ops/s ( ~ ) | | + num keys: 100000 | : | | +-+ hashmap (control) sequential get +----------------------+---------------------- | +- hits throughput | 5.061 M ops/s | 5.165 M ops/s (+2.1%) | +- hits latency | 198.483 ns/op | 194.270 ns/op (-2.1%) | +- important_hits throughput | 5.061 M ops/s | 5.165 M ops/s (+2.1%) | | + num keys: 4194304 | : | | +-+ hashmap (control) sequential get +----------------------+---------------------- | +- hits throughput | 2.864 M ops/s | 2.882 M ops/s ( ~ ) | +- hits latency | 365.220 ns/op | 361.418 ns/op (-1.0%) | +- important_hits throughput | 2.864 M ops/s | 2.882 M ops/s ( ~ ) | +---- Local Storage ---------------------- | | + num_maps: 1 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 33.005 M ops/s | 39.068 M ops/s (+18.4%) | +- hits latency | 30.300 ns/op | 25.598 ns/op (-15.5%) | +- important_hits throughput | 33.005 M ops/s | 39.068 M ops/s (+18.4%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 37.151 M ops/s | 44.926 M ops/s (+20.9%) | +- hits latency | 26.919 ns/op | 22.259 ns/op (-17.3%) | +- important_hits throughput | 37.151 M ops/s | 44.926 M ops/s (+20.9%) | | + num_maps: 10 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 32.288 M ops/s | 38.099 M ops/s (+18.0%) | +- hits latency | 30.972 ns/op | 26.248 ns/op (-15.3%) | +- important_hits throughput | 3.229 M ops/s | 3.810 M ops/s (+18.0%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 34.473 M ops/s | 41.145 M ops/s (+19.4%) | +- hits latency | 29.010 ns/op | 24.307 ns/op (-16.2%) | +- important_hits throughput | 12.312 M ops/s | 14.695 M ops/s (+19.4%) | | + num_maps: 16 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 32.524 M ops/s | 38.341 M ops/s (+17.9%) | +- hits latency | 30.748 ns/op | 26.083 ns/op (-15.2%) | +- important_hits throughput | 2.033 M ops/s | 2.396 M ops/s (+17.9%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 34.575 M ops/s | 41.338 M ops/s (+19.6%) | +- hits latency | 28.925 ns/op | 24.193 ns/op (-16.4%) | +- important_hits throughput | 11.001 M ops/s | 13.153 M ops/s (+19.6%) | | + num_maps: 17 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 28.861 M ops/s | 32.756 M ops/s (+13.5%) | +- hits latency | 34.649 ns/op | 30.530 ns/op (-11.9%) | +- important_hits throughput | 1.700 M ops/s | 1.929 M ops/s (+13.5%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 31.529 M ops/s | 36.110 M ops/s (+14.5%) | +- hits latency | 31.719 ns/op | 27.697 ns/op (-12.7%) | +- important_hits throughput | 9.598 M ops/s | 10.993 M ops/s (+14.5%) | | + num_maps: 24 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 18.602 M ops/s | 19.937 M ops/s (+7.2%) | +- hits latency | 53.767 ns/op | 50.166 ns/op (-6.7%) | +- important_hits throughput | 0.776 M ops/s | 0.831 M ops/s (+7.2%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 21.718 M ops/s | 23.332 M ops/s (+7.4%) | +- hits latency | 46.047 ns/op | 42.865 ns/op (-6.9%) | +- important_hits throughput | 6.110 M ops/s | 6.564 M ops/s (+7.4%) | | + num_maps: 32 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 14.118 M ops/s | 14.626 M ops/s (+3.6%) | +- hits latency | 70.856 ns/op | 68.381 ns/op (-3.5%) | +- important_hits throughput | 0.442 M ops/s | 0.458 M ops/s (+3.6%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 17.111 M ops/s | 17.906 M ops/s (+4.6%) | +- hits latency | 58.451 ns/op | 55.865 ns/op (-4.4%) | +- important_hits throughput | 4.776 M ops/s | 4.998 M ops/s (+4.6%) | | + num_maps: 100 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 5.281 M ops/s | 5.528 M ops/s (+4.7%) | +- hits latency | 192.398 ns/op | 183.059 ns/op (-4.9%) | +- important_hits throughput | 0.053 M ops/s | 0.055 M ops/s (+4.9%) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 6.265 M ops/s | 6.498 M ops/s (+3.7%) | +- hits latency | 161.436 ns/op | 152.877 ns/op (-5.3%) | +- important_hits throughput | 1.636 M ops/s | 1.697 M ops/s (+3.7%) | | + num_maps: 1000 | : | | +-+ local_storage cache sequential get +----------------------+---------------------- | +- hits throughput | 0.355 M ops/s | 0.354 M ops/s ( ~ ) | +- hits latency | 2826.538 ns/op | 2827.139 ns/op ( ~ ) | +- important_hits throughput | 0.000 M ops/s | 0.000 M ops/s ( ~ ) | : | : | | +-+ local_storage cache interleaved get +----------------------+---------------------- | +- hits throughput | 0.404 M ops/s | 0.403 M ops/s ( ~ ) | +- hits latency | 2481.190 ns/op | 2487.555 ns/op ( ~ ) | +- important_hits throughput | 0.102 M ops/s | 0.101 M ops/s ( ~ ) The on_lookup test in {cgrp,task}_ls_recursion.c is removed because the bpf_local_storage_lookup is no longer traceable and adding tracepoint will make the compiler generate worse code: https://lore.kernel.org/bpf/ZcJmok64Xqv6l4ZS@elver.google.com/ Signed-off-by: Marco Elver Cc: Martin KaFai Lau Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20240207122626.3508658-1-elver@google.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf_local_storage.h | 30 ++++++++++++- kernel/bpf/bpf_local_storage.c | 52 ++++++---------------- .../selftests/bpf/prog_tests/task_local_storage.c | 6 --- .../selftests/bpf/progs/cgrp_ls_recursion.c | 26 ----------- .../selftests/bpf/progs/task_ls_recursion.c | 17 ------- 5 files changed, 41 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 173ec7f43ed1..dcddb0aef7d8 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -129,10 +129,36 @@ bpf_local_storage_map_alloc(union bpf_attr *attr, struct bpf_local_storage_cache *cache, bool bpf_ma); -struct bpf_local_storage_data * +void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, + struct bpf_local_storage_map *smap, + struct bpf_local_storage_elem *selem); +/* If cacheit_lockit is false, this lookup function is lockless */ +static inline struct bpf_local_storage_data * bpf_local_storage_lookup(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap, - bool cacheit_lockit); + bool cacheit_lockit) +{ + struct bpf_local_storage_data *sdata; + struct bpf_local_storage_elem *selem; + + /* Fast path (cache hit) */ + sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx], + bpf_rcu_lock_held()); + if (sdata && rcu_access_pointer(sdata->smap) == smap) + return sdata; + + /* Slow path (cache miss) */ + hlist_for_each_entry_rcu(selem, &local_storage->list, snode, + rcu_read_lock_trace_held()) + if (rcu_access_pointer(SDATA(selem)->smap) == smap) + break; + + if (!selem) + return NULL; + if (cacheit_lockit) + __bpf_local_storage_insert_cache(local_storage, smap, selem); + return SDATA(selem); +} void bpf_local_storage_destroy(struct bpf_local_storage *local_storage); diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 146824cc9689..bdea1a459153 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -414,47 +414,21 @@ void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now) bpf_selem_unlink_storage(selem, reuse_now); } -/* If cacheit_lockit is false, this lookup function is lockless */ -struct bpf_local_storage_data * -bpf_local_storage_lookup(struct bpf_local_storage *local_storage, - struct bpf_local_storage_map *smap, - bool cacheit_lockit) +void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, + struct bpf_local_storage_map *smap, + struct bpf_local_storage_elem *selem) { - struct bpf_local_storage_data *sdata; - struct bpf_local_storage_elem *selem; - - /* Fast path (cache hit) */ - sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx], - bpf_rcu_lock_held()); - if (sdata && rcu_access_pointer(sdata->smap) == smap) - return sdata; - - /* Slow path (cache miss) */ - hlist_for_each_entry_rcu(selem, &local_storage->list, snode, - rcu_read_lock_trace_held()) - if (rcu_access_pointer(SDATA(selem)->smap) == smap) - break; - - if (!selem) - return NULL; - - sdata = SDATA(selem); - if (cacheit_lockit) { - unsigned long flags; - - /* spinlock is needed to avoid racing with the - * parallel delete. Otherwise, publishing an already - * deleted sdata to the cache will become a use-after-free - * problem in the next bpf_local_storage_lookup(). - */ - raw_spin_lock_irqsave(&local_storage->lock, flags); - if (selem_linked_to_storage(selem)) - rcu_assign_pointer(local_storage->cache[smap->cache_idx], - sdata); - raw_spin_unlock_irqrestore(&local_storage->lock, flags); - } + unsigned long flags; - return sdata; + /* spinlock is needed to avoid racing with the + * parallel delete. Otherwise, publishing an already + * deleted sdata to the cache will become a use-after-free + * problem in the next bpf_local_storage_lookup(). + */ + raw_spin_lock_irqsave(&local_storage->lock, flags); + if (selem_linked_to_storage(selem)) + rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem)); + raw_spin_unlock_irqrestore(&local_storage->lock, flags); } static int check_flags(const struct bpf_local_storage_data *old_sdata, diff --git a/tools/testing/selftests/bpf/prog_tests/task_local_storage.c b/tools/testing/selftests/bpf/prog_tests/task_local_storage.c index ea8537c54413..c33c05161a9e 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/task_local_storage.c @@ -117,12 +117,6 @@ static void test_recursion(void) ASSERT_OK(err, "lookup map_b"); ASSERT_EQ(value, 100, "map_b value"); - prog_fd = bpf_program__fd(skel->progs.on_lookup); - memset(&info, 0, sizeof(info)); - err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); - ASSERT_OK(err, "get prog info"); - ASSERT_GT(info.recursion_misses, 0, "on_lookup prog recursion"); - prog_fd = bpf_program__fd(skel->progs.on_update); memset(&info, 0, sizeof(info)); err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); diff --git a/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c b/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c index 610c2427fd93..3500e4b69ebe 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c +++ b/tools/testing/selftests/bpf/progs/cgrp_ls_recursion.c @@ -27,32 +27,6 @@ bool is_cgroup1 = 0; struct cgroup *bpf_task_get_cgroup1(struct task_struct *task, int hierarchy_id) __ksym; void bpf_cgroup_release(struct cgroup *cgrp) __ksym; -static void __on_lookup(struct cgroup *cgrp) -{ - bpf_cgrp_storage_delete(&map_a, cgrp); - bpf_cgrp_storage_delete(&map_b, cgrp); -} - -SEC("fentry/bpf_local_storage_lookup") -int BPF_PROG(on_lookup) -{ - struct task_struct *task = bpf_get_current_task_btf(); - struct cgroup *cgrp; - - if (is_cgroup1) { - cgrp = bpf_task_get_cgroup1(task, target_hid); - if (!cgrp) - return 0; - - __on_lookup(cgrp); - bpf_cgroup_release(cgrp); - return 0; - } - - __on_lookup(task->cgroups->dfl_cgrp); - return 0; -} - static void __on_update(struct cgroup *cgrp) { long *ptr; diff --git a/tools/testing/selftests/bpf/progs/task_ls_recursion.c b/tools/testing/selftests/bpf/progs/task_ls_recursion.c index 4542dc683b44..f1853c38aada 100644 --- a/tools/testing/selftests/bpf/progs/task_ls_recursion.c +++ b/tools/testing/selftests/bpf/progs/task_ls_recursion.c @@ -27,23 +27,6 @@ struct { __type(value, long); } map_b SEC(".maps"); -SEC("fentry/bpf_local_storage_lookup") -int BPF_PROG(on_lookup) -{ - struct task_struct *task = bpf_get_current_task_btf(); - - if (!test_pid || task->pid != test_pid) - return 0; - - /* The bpf_task_storage_delete will call - * bpf_local_storage_lookup. The prog->active will - * stop the recursion. - */ - bpf_task_storage_delete(&map_a, task); - bpf_task_storage_delete(&map_b, task); - return 0; -} - SEC("fentry/bpf_local_storage_update") int BPF_PROG(on_update) { -- cgit v1.2.3 From faf6efd2e5e23d3319501132d9671c8606ef21bd Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:27:04 +0100 Subject: gpio: constify opaque pointer in gpio_device_find() match function The match function used in gpio_device_find() should not modify the contents of passed opaque pointer, because such modification would not be necessary for actual matching and it could lead to quite unreadable, spaghetti code. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Mika Westerberg Reviewed-by: Linus Walleij Reviewed-by: Andy Shevchenko [Bartosz: fix coding style in header] Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-acpi.c | 2 +- drivers/gpio/gpiolib-of.c | 7 ++++--- drivers/gpio/gpiolib.c | 6 +++--- include/linux/gpio/driver.h | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index cd3e9657cc36..899cd505073e 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -126,7 +126,7 @@ static DEFINE_MUTEX(acpi_gpio_deferred_req_irqs_lock); static LIST_HEAD(acpi_gpio_deferred_req_irqs_list); static bool acpi_gpio_deferred_req_irqs_done; -static int acpi_gpiochip_find(struct gpio_chip *gc, void *data) +static int acpi_gpiochip_find(struct gpio_chip *gc, const void *data) { return device_match_acpi_handle(&gc->gpiodev->dev, data); } diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 77509aa19900..35d717fd393f 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -118,9 +118,10 @@ int of_gpio_get_count(struct device *dev, const char *con_id) return ret ? ret : -ENOENT; } -static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, + const void *data) { - struct of_phandle_args *gpiospec = data; + const struct of_phandle_args *gpiospec = data; return device_match_of_node(&chip->gpiodev->dev, gpiospec->np) && chip->of_xlate && @@ -852,7 +853,7 @@ static void of_gpiochip_remove_hog(struct gpio_chip *chip, gpiochip_free_own_desc(desc); } -static int of_gpiochip_match_node(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node(struct gpio_chip *chip, const void *data) { return device_match_of_node(&chip->gpiodev->dev, data); } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index d068788adb10..288aed5a9f6b 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1119,7 +1119,7 @@ EXPORT_SYMBOL_GPL(gpiochip_remove); */ struct gpio_device *gpio_device_find(void *data, int (*match)(struct gpio_chip *gc, - void *data)) + const void *data)) { struct gpio_device *gdev; @@ -1141,7 +1141,7 @@ struct gpio_device *gpio_device_find(void *data, } EXPORT_SYMBOL_GPL(gpio_device_find); -static int gpio_chip_match_by_label(struct gpio_chip *gc, void *label) +static int gpio_chip_match_by_label(struct gpio_chip *gc, const void *label) { return gc->label && !strcmp(gc->label, label); } @@ -1161,7 +1161,7 @@ struct gpio_device *gpio_device_find_by_label(const char *label) } EXPORT_SYMBOL_GPL(gpio_device_find_by_label); -static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, void *fwnode) +static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, const void *fwnode) { return device_match_fwnode(&gc->gpiodev->dev, fwnode); } diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 3a37d058cfcf..9d0023f83a57 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -629,7 +629,8 @@ int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, struct lock_class_key *request_key); struct gpio_device *gpio_device_find(void *data, - int (*match)(struct gpio_chip *gc, void *data)); + int (*match)(struct gpio_chip *gc, + const void *data)); struct gpio_device *gpio_device_find_by_label(const char *label); struct gpio_device *gpio_device_find_by_fwnode(const struct fwnode_handle *fwnode); -- cgit v1.2.3 From dd95255d44c05c9977f962bf0f2afe5e11f8ab3e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 5 Jan 2024 13:33:32 +0100 Subject: coresight: make coresight_bustype const Now that the driver core can properly handle constant struct bus_type, move the coresight_bustype variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Suzuki K Poulose Cc: Mike Leach Cc: James Clark Cc: Leo Yan Cc: Alexander Shishkin Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Signed-off-by: Greg Kroah-Hartman Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/2024010531-tinfoil-avert-4a57@gregkh --- drivers/hwtracing/coresight/coresight-core.c | 2 +- include/linux/coresight.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index d7f0e231feb9..5dde597403b3 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1799,7 +1799,7 @@ done: } EXPORT_SYMBOL_GPL(coresight_alloc_device_name); -struct bus_type coresight_bustype = { +const struct bus_type coresight_bustype = { .name = "coresight", }; diff --git a/include/linux/coresight.h b/include/linux/coresight.h index a4cb7dd6ca23..e8b6e388218c 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -35,7 +35,7 @@ #define CORESIGHT_UNLOCK 0xc5acce55 -extern struct bus_type coresight_bustype; +extern const struct bus_type coresight_bustype; enum coresight_dev_type { CORESIGHT_DEV_TYPE_SINK, -- cgit v1.2.3 From a0fef3f05cf36338d471e8f35a9ced88a054d583 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:33 +0000 Subject: coresight: Make language around "activated" sinks consistent Activated has the specific meaning of a sink that's selected for use by the user via sysfs. But comments in some code that's shared by Perf use the same word, so in those cases change them to just say "selected" instead. With selected implying either via Perf or "activated" via sysfs. coresight_get_enabled_sink() doesn't actually get an enabled sink, it only gets an activated one, so change that too. And change the activated variable name to include "sysfs" so it can't be confused as a general status. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-3-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-core.c | 51 +++++++++++----------------- drivers/hwtracing/coresight/coresight-priv.h | 2 -- include/linux/coresight.h | 14 ++++---- 3 files changed, 27 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 1dd3cd46fda4..37e59052e877 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -500,7 +500,7 @@ static void coresight_disable_path_from(struct list_head *path, /* * ETF devices are tricky... They can be a link or a sink, * depending on how they are configured. If an ETF has been - * "activated" it will be configured as a sink, otherwise + * selected as a sink it will be configured as a sink, otherwise * go ahead with the link configuration. */ if (type == CORESIGHT_DEV_TYPE_LINKSINK) @@ -578,7 +578,7 @@ int coresight_enable_path(struct list_head *path, enum cs_mode mode, /* * ETF devices are tricky... They can be a link or a sink, * depending on how they are configured. If an ETF has been - * "activated" it will be configured as a sink, otherwise + * selected as a sink it will be configured as a sink, otherwise * go ahead with the link configuration. */ if (type == CORESIGHT_DEV_TYPE_LINKSINK) @@ -635,15 +635,21 @@ struct coresight_device *coresight_get_sink(struct list_head *path) return csdev; } +/** + * coresight_find_activated_sysfs_sink - returns the first sink activated via + * sysfs using connection based search starting from the source reference. + * + * @csdev: Coresight source device reference + */ static struct coresight_device * -coresight_find_enabled_sink(struct coresight_device *csdev) +coresight_find_activated_sysfs_sink(struct coresight_device *csdev) { int i; struct coresight_device *sink = NULL; if ((csdev->type == CORESIGHT_DEV_TYPE_SINK || csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) && - csdev->activated) + csdev->sysfs_sink_activated) return csdev; /* @@ -654,7 +660,7 @@ coresight_find_enabled_sink(struct coresight_device *csdev) child_dev = csdev->pdata->out_conns[i]->dest_dev; if (child_dev) - sink = coresight_find_enabled_sink(child_dev); + sink = coresight_find_activated_sysfs_sink(child_dev); if (sink) return sink; } @@ -662,21 +668,6 @@ coresight_find_enabled_sink(struct coresight_device *csdev) return NULL; } -/** - * coresight_get_enabled_sink - returns the first enabled sink using - * connection based search starting from the source reference - * - * @source: Coresight source device reference - */ -struct coresight_device * -coresight_get_enabled_sink(struct coresight_device *source) -{ - if (!source) - return NULL; - - return coresight_find_enabled_sink(source); -} - static int coresight_sink_by_id(struct device *dev, const void *data) { struct coresight_device *csdev = to_coresight_device(dev); @@ -810,11 +801,10 @@ static void coresight_drop_device(struct coresight_device *csdev) * @sink: The final sink we want in this path. * @path: The list to add devices to. * - * The tree of Coresight device is traversed until an activated sink is - * found. From there the sink is added to the list along with all the - * devices that led to that point - the end result is a list from source - * to sink. In that list the source is the first device and the sink the - * last one. + * The tree of Coresight device is traversed until @sink is found. + * From there the sink is added to the list along with all the devices that led + * to that point - the end result is a list from source to sink. In that list + * the source is the first device and the sink the last one. */ static int _coresight_build_path(struct coresight_device *csdev, struct coresight_device *sink, @@ -824,7 +814,7 @@ static int _coresight_build_path(struct coresight_device *csdev, bool found = false; struct coresight_node *node; - /* An activated sink has been found. Enqueue the element */ + /* The sink has been found. Enqueue the element */ if (csdev == sink) goto out; @@ -1145,7 +1135,7 @@ int coresight_enable(struct coresight_device *csdev) goto out; } - sink = coresight_get_enabled_sink(csdev); + sink = coresight_find_activated_sysfs_sink(csdev); if (!sink) { ret = -EINVAL; goto out; @@ -1259,7 +1249,7 @@ static ssize_t enable_sink_show(struct device *dev, { struct coresight_device *csdev = to_coresight_device(dev); - return scnprintf(buf, PAGE_SIZE, "%u\n", csdev->activated); + return scnprintf(buf, PAGE_SIZE, "%u\n", csdev->sysfs_sink_activated); } static ssize_t enable_sink_store(struct device *dev, @@ -1274,10 +1264,7 @@ static ssize_t enable_sink_store(struct device *dev, if (ret) return ret; - if (val) - csdev->activated = true; - else - csdev->activated = false; + csdev->sysfs_sink_activated = !!val; return size; diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 30c051055e54..ced5be05a527 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -130,8 +130,6 @@ void coresight_disable_path(struct list_head *path); int coresight_enable_path(struct list_head *path, enum cs_mode mode, void *sink_data); struct coresight_device *coresight_get_sink(struct list_head *path); -struct coresight_device * -coresight_get_enabled_sink(struct coresight_device *source); struct coresight_device *coresight_get_sink_by_id(u32 id); struct coresight_device * coresight_find_default_sink(struct coresight_device *csdev); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index e8b6e388218c..516ab45ff3c2 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -229,10 +229,12 @@ struct coresight_sysfs_link { * @refcnt: keep track of what is in use. * @orphan: true if the component has connections that haven't been linked. * @enable: 'true' if component is currently part of an active path. - * @activated: 'true' only if a _sink_ has been activated. A sink can be - * activated but not yet enabled. Enabling for a _sink_ - * happens when a source has been selected and a path is enabled - * from source to that sink. + * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs + * by writing a 1 to the 'enable_sink' file. A sink can be + * activated but not yet enabled. Enabling for a _sink_ happens + * when a source has been selected and a path is enabled from + * source to that sink. A sink can also become enabled but not + * activated if it's used via Perf. * @ea: Device attribute for sink representation under PMU directory. * @def_sink: cached reference to default sink found for this device. * @nr_links: number of sysfs links created to other components from this @@ -252,9 +254,9 @@ struct coresight_device { struct device dev; atomic_t refcnt; bool orphan; - bool enable; /* true only if configured as part of a path */ + bool enable; /* sink specific fields */ - bool activated; /* true only if a sink is part of a path */ + bool sysfs_sink_activated; struct dev_ext_attribute *ea; struct coresight_device *def_sink; /* sysfs links between components */ -- cgit v1.2.3 From 9cae77cf23e317f31de036ced7ad2c261317dc76 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:35 +0000 Subject: coresight: Move mode to struct coresight_device Most devices use mode, so move the mode definition out of the individual devices and up to the Coresight device. This will allow the core code to also know the mode which will be useful in a later commit. This also fixes the inconsistency of the documentation of the mode field on the individual device types. For example ETB10 had "this ETB is being used". Two devices didn't require an atomic mode type, so these usages have been converted to atomic_get() and atomic_set() only to make it compile, but the documentation of the field in struct coresight_device explains this type of usage. In the future, manipulation of the mode could be completely moved out of the individual devices and into the core code because it's almost all duplicate code, and this change is a step towards that. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-5-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-etb10.c | 18 +++++++-------- drivers/hwtracing/coresight/coresight-etm.h | 2 -- drivers/hwtracing/coresight/coresight-etm3x-core.c | 13 +++++------ .../hwtracing/coresight/coresight-etm3x-sysfs.c | 4 ++-- drivers/hwtracing/coresight/coresight-etm4x-core.c | 16 ++++++------- drivers/hwtracing/coresight/coresight-etm4x.h | 1 - drivers/hwtracing/coresight/coresight-stm.c | 20 ++++++++--------- drivers/hwtracing/coresight/coresight-tmc-core.c | 2 +- drivers/hwtracing/coresight/coresight-tmc-etf.c | 26 +++++++++++----------- drivers/hwtracing/coresight/coresight-tmc-etr.c | 20 ++++++++--------- drivers/hwtracing/coresight/coresight-tmc.h | 2 -- drivers/hwtracing/coresight/ultrasoc-smb.c | 13 ++++++----- drivers/hwtracing/coresight/ultrasoc-smb.h | 2 -- include/linux/coresight.h | 6 +++++ 14 files changed, 69 insertions(+), 76 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 05e21cffd33b..145fb815ec1c 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -76,7 +76,6 @@ DEFINE_CORESIGHT_DEVLIST(etb_devs, "etb"); * @pid: Process ID of the process being monitored by the session * that is using this component. * @buf: area of memory where ETB buffer content gets sent. - * @mode: this ETB is being used. * @buffer_depth: size of @buf. * @trigger_cntr: amount of words to store after a trigger. */ @@ -89,7 +88,6 @@ struct etb_drvdata { local_t reading; pid_t pid; u8 *buf; - u32 mode; u32 buffer_depth; u32 trigger_cntr; }; @@ -150,17 +148,17 @@ static int etb_enable_sysfs(struct coresight_device *csdev) spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't messup with perf sessions. */ - if (drvdata->mode == CS_MODE_PERF) { + if (local_read(&csdev->mode) == CS_MODE_PERF) { ret = -EBUSY; goto out; } - if (drvdata->mode == CS_MODE_DISABLED) { + if (local_read(&csdev->mode) == CS_MODE_DISABLED) { ret = etb_enable_hw(drvdata); if (ret) goto out; - drvdata->mode = CS_MODE_SYSFS; + local_set(&csdev->mode, CS_MODE_SYSFS); } atomic_inc(&csdev->refcnt); @@ -181,7 +179,7 @@ static int etb_enable_perf(struct coresight_device *csdev, void *data) spin_lock_irqsave(&drvdata->spinlock, flags); /* No need to continue if the component is already in used by sysFS. */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { ret = -EBUSY; goto out; } @@ -216,7 +214,7 @@ static int etb_enable_perf(struct coresight_device *csdev, void *data) if (!ret) { /* Associate with monitored process. */ drvdata->pid = pid; - drvdata->mode = CS_MODE_PERF; + local_set(&drvdata->csdev->mode, CS_MODE_PERF); atomic_inc(&csdev->refcnt); } @@ -362,11 +360,11 @@ static int etb_disable(struct coresight_device *csdev) } /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); + WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); etb_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; - drvdata->mode = CS_MODE_DISABLED; + local_set(&csdev->mode, CS_MODE_DISABLED); spin_unlock_irqrestore(&drvdata->spinlock, flags); dev_dbg(&csdev->dev, "ETB disabled\n"); @@ -589,7 +587,7 @@ static void etb_dump(struct etb_drvdata *drvdata) unsigned long flags; spin_lock_irqsave(&drvdata->spinlock, flags); - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { __etb_disable_hw(drvdata); etb_dump_hw(drvdata); __etb_enable_hw(drvdata); diff --git a/drivers/hwtracing/coresight/coresight-etm.h b/drivers/hwtracing/coresight/coresight-etm.h index 9a0d08b092ae..e02c3ea972c9 100644 --- a/drivers/hwtracing/coresight/coresight-etm.h +++ b/drivers/hwtracing/coresight/coresight-etm.h @@ -215,7 +215,6 @@ struct etm_config { * @port_size: port size as reported by ETMCR bit 4-6 and 21. * @arch: ETM/PTM version number. * @use_cpu14: true if management registers need to be accessed via CP14. - * @mode: this tracer's mode, i.e sysFS, Perf or disabled. * @sticky_enable: true if ETM base configuration has been done. * @boot_enable:true if we should start tracing at boot time. * @os_unlock: true if access to management registers is allowed. @@ -238,7 +237,6 @@ struct etm_drvdata { int port_size; u8 arch; bool use_cp14; - local_t mode; bool sticky_enable; bool boot_enable; bool os_unlock; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 74ae26911b8a..333d0a32c95c 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -559,7 +559,7 @@ static int etm_enable(struct coresight_device *csdev, struct perf_event *event, u32 val; struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode); + val = local_cmpxchg(&drvdata->csdev->mode, CS_MODE_DISABLED, mode); /* Someone is already using the tracer */ if (val) @@ -578,7 +578,7 @@ static int etm_enable(struct coresight_device *csdev, struct perf_event *event, /* The tracer didn't start */ if (ret) - local_set(&drvdata->mode, CS_MODE_DISABLED); + local_set(&drvdata->csdev->mode, CS_MODE_DISABLED); return ret; } @@ -672,14 +672,13 @@ static void etm_disable(struct coresight_device *csdev, struct perf_event *event) { enum cs_mode mode; - struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); /* * For as long as the tracer isn't disabled another entity can't * change its status. As such we can read the status here without * fearing it will change under us. */ - mode = local_read(&drvdata->mode); + mode = local_read(&csdev->mode); switch (mode) { case CS_MODE_DISABLED: @@ -696,7 +695,7 @@ static void etm_disable(struct coresight_device *csdev, } if (mode) - local_set(&drvdata->mode, CS_MODE_DISABLED); + local_set(&csdev->mode, CS_MODE_DISABLED); } static const struct coresight_ops_source etm_source_ops = { @@ -730,7 +729,7 @@ static int etm_starting_cpu(unsigned int cpu) etmdrvdata[cpu]->os_unlock = true; } - if (local_read(&etmdrvdata[cpu]->mode)) + if (local_read(&etmdrvdata[cpu]->csdev->mode)) etm_enable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; @@ -742,7 +741,7 @@ static int etm_dying_cpu(unsigned int cpu) return 0; spin_lock(&etmdrvdata[cpu]->spinlock); - if (local_read(&etmdrvdata[cpu]->mode)) + if (local_read(&etmdrvdata[cpu]->csdev->mode)) etm_disable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c index 2f271b7fb048..6c8429c980b1 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c @@ -722,7 +722,7 @@ static ssize_t cntr_val_show(struct device *dev, struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etm_config *config = &drvdata->config; - if (!local_read(&drvdata->mode)) { + if (!local_read(&drvdata->csdev->mode)) { spin_lock(&drvdata->spinlock); for (i = 0; i < drvdata->nr_cntr; i++) ret += sprintf(buf, "counter %d: %x\n", @@ -941,7 +941,7 @@ static ssize_t seq_curr_state_show(struct device *dev, struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etm_config *config = &drvdata->config; - if (!local_read(&drvdata->mode)) { + if (!local_read(&drvdata->csdev->mode)) { val = config->seq_curr_state; goto out; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c5ea808ea662..7ceff627cd95 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -841,9 +841,8 @@ static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, { int ret; u32 val; - struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode); + val = local_cmpxchg(&csdev->mode, CS_MODE_DISABLED, mode); /* Someone is already using the tracer */ if (val) @@ -862,7 +861,7 @@ static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, /* The tracer didn't start */ if (ret) - local_set(&drvdata->mode, CS_MODE_DISABLED); + local_set(&csdev->mode, CS_MODE_DISABLED); return ret; } @@ -1004,14 +1003,13 @@ static void etm4_disable(struct coresight_device *csdev, struct perf_event *event) { enum cs_mode mode; - struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); /* * For as long as the tracer isn't disabled another entity can't * change its status. As such we can read the status here without * fearing it will change under us. */ - mode = local_read(&drvdata->mode); + mode = local_read(&csdev->mode); switch (mode) { case CS_MODE_DISABLED: @@ -1025,7 +1023,7 @@ static void etm4_disable(struct coresight_device *csdev, } if (mode) - local_set(&drvdata->mode, CS_MODE_DISABLED); + local_set(&csdev->mode, CS_MODE_DISABLED); } static const struct coresight_ops_source etm4_source_ops = { @@ -1663,7 +1661,7 @@ static int etm4_starting_cpu(unsigned int cpu) if (!etmdrvdata[cpu]->os_unlock) etm4_os_unlock(etmdrvdata[cpu]); - if (local_read(&etmdrvdata[cpu]->mode)) + if (local_read(&etmdrvdata[cpu]->csdev->mode)) etm4_enable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; @@ -1675,7 +1673,7 @@ static int etm4_dying_cpu(unsigned int cpu) return 0; spin_lock(&etmdrvdata[cpu]->spinlock); - if (local_read(&etmdrvdata[cpu]->mode)) + if (local_read(&etmdrvdata[cpu]->csdev->mode)) etm4_disable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; @@ -1833,7 +1831,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) * Save and restore the ETM Trace registers only if * the ETM is active. */ - if (local_read(&drvdata->mode) && drvdata->save_state) + if (local_read(&drvdata->csdev->mode) && drvdata->save_state) ret = __etm4_cpu_save(drvdata); return ret; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index da17b6c49b0f..9ea678bc2e8e 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -1016,7 +1016,6 @@ struct etmv4_drvdata { void __iomem *base; struct coresight_device *csdev; spinlock_t spinlock; - local_t mode; int cpu; u8 arch; u8 nr_pe; diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 891ee64efed8..f7fc645ea7a4 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -119,7 +119,6 @@ DEFINE_CORESIGHT_DEVLIST(stm_devs, "stm"); * @spinlock: only one at a time pls. * @chs: the channels accociated to this STM. * @stm: structure associated to the generic STM interface. - * @mode: this tracer's mode (enum cs_mode), i.e sysFS, or disabled. * @traceid: value of the current ID for this component. * @write_bytes: Maximus bytes this STM can write at a time. * @stmsper: settings for register STMSPER. @@ -136,7 +135,6 @@ struct stm_drvdata { spinlock_t spinlock; struct channel_space chs; struct stm_data stm; - local_t mode; u8 traceid; u32 write_bytes; u32 stmsper; @@ -201,7 +199,7 @@ static int stm_enable(struct coresight_device *csdev, struct perf_event *event, if (mode != CS_MODE_SYSFS) return -EINVAL; - val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode); + val = local_cmpxchg(&csdev->mode, CS_MODE_DISABLED, mode); /* Someone is already using the tracer */ if (val) @@ -266,7 +264,7 @@ static void stm_disable(struct coresight_device *csdev, * change its status. As such we can read the status here without * fearing it will change under us. */ - if (local_read(&drvdata->mode) == CS_MODE_SYSFS) { + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { spin_lock(&drvdata->spinlock); stm_disable_hw(drvdata); spin_unlock(&drvdata->spinlock); @@ -276,7 +274,7 @@ static void stm_disable(struct coresight_device *csdev, pm_runtime_put(csdev->dev.parent); - local_set(&drvdata->mode, CS_MODE_DISABLED); + local_set(&csdev->mode, CS_MODE_DISABLED); dev_dbg(&csdev->dev, "STM tracing disabled\n"); } } @@ -373,7 +371,7 @@ static long stm_generic_set_options(struct stm_data *stm_data, { struct stm_drvdata *drvdata = container_of(stm_data, struct stm_drvdata, stm); - if (!(drvdata && local_read(&drvdata->mode))) + if (!(drvdata && local_read(&drvdata->csdev->mode))) return -EINVAL; if (channel >= drvdata->numsp) @@ -408,7 +406,7 @@ static ssize_t notrace stm_generic_packet(struct stm_data *stm_data, struct stm_drvdata, stm); unsigned int stm_flags; - if (!(drvdata && local_read(&drvdata->mode))) + if (!(drvdata && local_read(&drvdata->csdev->mode))) return -EACCES; if (channel >= drvdata->numsp) @@ -515,7 +513,7 @@ static ssize_t port_select_show(struct device *dev, struct stm_drvdata *drvdata = dev_get_drvdata(dev->parent); unsigned long val; - if (!local_read(&drvdata->mode)) { + if (!local_read(&drvdata->csdev->mode)) { val = drvdata->stmspscr; } else { spin_lock(&drvdata->spinlock); @@ -541,7 +539,7 @@ static ssize_t port_select_store(struct device *dev, spin_lock(&drvdata->spinlock); drvdata->stmspscr = val; - if (local_read(&drvdata->mode)) { + if (local_read(&drvdata->csdev->mode)) { CS_UNLOCK(drvdata->base); /* Process as per ARM's TRM recommendation */ stmsper = readl_relaxed(drvdata->base + STMSPER); @@ -562,7 +560,7 @@ static ssize_t port_enable_show(struct device *dev, struct stm_drvdata *drvdata = dev_get_drvdata(dev->parent); unsigned long val; - if (!local_read(&drvdata->mode)) { + if (!local_read(&drvdata->csdev->mode)) { val = drvdata->stmsper; } else { spin_lock(&drvdata->spinlock); @@ -588,7 +586,7 @@ static ssize_t port_enable_store(struct device *dev, spin_lock(&drvdata->spinlock); drvdata->stmsper = val; - if (local_read(&drvdata->mode)) { + if (local_read(&drvdata->csdev->mode)) { CS_UNLOCK(drvdata->base); writel_relaxed(drvdata->stmsper, drvdata->base + STMSPER); CS_LOCK(drvdata->base); diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index 39bae35d4ffd..e8b2bbed047f 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -558,7 +558,7 @@ static void tmc_shutdown(struct amba_device *adev) spin_lock_irqsave(&drvdata->spinlock, flags); - if (drvdata->mode == CS_MODE_DISABLED) + if (local_read(&drvdata->csdev->mode) == CS_MODE_DISABLED) goto out; if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 7406b65e2cdd..2a7e516052a2 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -89,7 +89,7 @@ static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) * When operating in sysFS mode the content of the buffer needs to be * read before the TMC is disabled. */ - if (drvdata->mode == CS_MODE_SYSFS) + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) tmc_etb_dump_hw(drvdata); tmc_disable_hw(drvdata); @@ -205,7 +205,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) * sink is already enabled no memory is needed and the HW need not be * touched. */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { atomic_inc(&csdev->refcnt); goto out; } @@ -228,7 +228,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) ret = tmc_etb_enable_hw(drvdata); if (!ret) { - drvdata->mode = CS_MODE_SYSFS; + local_set(&csdev->mode, CS_MODE_SYSFS); atomic_inc(&csdev->refcnt); } else { /* Free up the buffer if we failed to enable */ @@ -262,7 +262,7 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) * No need to continue if the ETB/ETF is already operated * from sysFS. */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { ret = -EBUSY; break; } @@ -292,7 +292,7 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) if (!ret) { /* Associate with monitored process. */ drvdata->pid = pid; - drvdata->mode = CS_MODE_PERF; + local_set(&csdev->mode, CS_MODE_PERF); atomic_inc(&csdev->refcnt); } } while (0); @@ -344,11 +344,11 @@ static int tmc_disable_etf_sink(struct coresight_device *csdev) } /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); + WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); tmc_etb_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; - drvdata->mode = CS_MODE_DISABLED; + local_set(&csdev->mode, CS_MODE_DISABLED); spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -374,7 +374,7 @@ static int tmc_enable_etf_link(struct coresight_device *csdev, if (atomic_read(&csdev->refcnt) == 0) { ret = tmc_etf_enable_hw(drvdata); if (!ret) { - drvdata->mode = CS_MODE_SYSFS; + local_set(&csdev->mode, CS_MODE_SYSFS); first_enable = true; } } @@ -403,7 +403,7 @@ static void tmc_disable_etf_link(struct coresight_device *csdev, if (atomic_dec_return(&csdev->refcnt) == 0) { tmc_etf_disable_hw(drvdata); - drvdata->mode = CS_MODE_DISABLED; + local_set(&csdev->mode, CS_MODE_DISABLED); last_disable = true; } spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -483,7 +483,7 @@ static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, return 0; /* This shouldn't happen */ - if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF)) + if (WARN_ON_ONCE(local_read(&csdev->mode) != CS_MODE_PERF)) return 0; spin_lock_irqsave(&drvdata->spinlock, flags); @@ -629,7 +629,7 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) } /* Don't interfere if operated from Perf */ - if (drvdata->mode == CS_MODE_PERF) { + if (local_read(&drvdata->csdev->mode) == CS_MODE_PERF) { ret = -EINVAL; goto out; } @@ -641,7 +641,7 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) } /* Disable the TMC if need be */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { /* There is no point in reading a TMC in HW FIFO mode */ mode = readl_relaxed(drvdata->base + TMC_MODE); if (mode != TMC_MODE_CIRCULAR_BUFFER) { @@ -673,7 +673,7 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) spin_lock_irqsave(&drvdata->spinlock, flags); /* Re-enable the TMC if need be */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { /* There is no point in reading a TMC in HW FIFO mode */ mode = readl_relaxed(drvdata->base + TMC_MODE); if (mode != TMC_MODE_CIRCULAR_BUFFER) { diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index af02ba5d5f15..3dc989d4fcab 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1143,7 +1143,7 @@ static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata) * When operating in sysFS mode the content of the buffer needs to be * read before the TMC is disabled. */ - if (drvdata->mode == CS_MODE_SYSFS) + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) tmc_etr_sync_sysfs_buf(drvdata); tmc_disable_hw(drvdata); @@ -1189,7 +1189,7 @@ static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev) spin_lock_irqsave(&drvdata->spinlock, flags); } - if (drvdata->reading || drvdata->mode == CS_MODE_PERF) { + if (drvdata->reading || local_read(&csdev->mode) == CS_MODE_PERF) { ret = -EBUSY; goto out; } @@ -1230,14 +1230,14 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) * sink is already enabled no memory is needed and the HW need not be * touched, even if the buffer size has changed. */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { atomic_inc(&csdev->refcnt); goto out; } ret = tmc_etr_enable_hw(drvdata, sysfs_buf); if (!ret) { - drvdata->mode = CS_MODE_SYSFS; + local_set(&csdev->mode, CS_MODE_SYSFS); atomic_inc(&csdev->refcnt); } @@ -1652,7 +1652,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't use this sink if it is already claimed by sysFS */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { rc = -EBUSY; goto unlock_out; } @@ -1684,7 +1684,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) if (!rc) { /* Associate with monitored process. */ drvdata->pid = pid; - drvdata->mode = CS_MODE_PERF; + local_set(&csdev->mode, CS_MODE_PERF); drvdata->perf_buf = etr_perf->etr_buf; atomic_inc(&csdev->refcnt); } @@ -1725,11 +1725,11 @@ static int tmc_disable_etr_sink(struct coresight_device *csdev) } /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); + WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); tmc_etr_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; - drvdata->mode = CS_MODE_DISABLED; + local_set(&csdev->mode, CS_MODE_DISABLED); /* Reset perf specific data */ drvdata->perf_buf = NULL; @@ -1777,7 +1777,7 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata) } /* Disable the TMC if we are trying to read from a running session. */ - if (drvdata->mode == CS_MODE_SYSFS) + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) __tmc_etr_disable_hw(drvdata); drvdata->reading = true; @@ -1799,7 +1799,7 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata) spin_lock_irqsave(&drvdata->spinlock, flags); /* RE-enable the TMC if need be */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { /* * The trace run will continue with the same allocated trace * buffer. Since the tracer is still enabled drvdata::buf can't diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h index 8dcb426ac3e7..cef979c897e6 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.h +++ b/drivers/hwtracing/coresight/coresight-tmc.h @@ -178,7 +178,6 @@ struct etr_buf { * @size: trace buffer size for this TMC (common for all modes). * @max_burst_size: The maximum burst size that can be initiated by * TMC-ETR on AXI bus. - * @mode: how this TMC is being used. * @config_type: TMC variant, must be of type @tmc_config_type. * @memwidth: width of the memory interface databus, in bytes. * @trigger_cntr: amount of words to store after a trigger. @@ -203,7 +202,6 @@ struct tmc_drvdata { u32 len; u32 size; u32 max_burst_size; - u32 mode; enum tmc_config_type config_type; enum tmc_mem_intf_width memwidth; u32 trigger_cntr; diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c index f0b6806202d1..bef633960173 100644 --- a/drivers/hwtracing/coresight/ultrasoc-smb.c +++ b/drivers/hwtracing/coresight/ultrasoc-smb.c @@ -207,11 +207,11 @@ static void smb_enable_sysfs(struct coresight_device *csdev) { struct smb_drv_data *drvdata = dev_get_drvdata(csdev->dev.parent); - if (drvdata->mode != CS_MODE_DISABLED) + if (local_read(&csdev->mode) != CS_MODE_DISABLED) return; smb_enable_hw(drvdata); - drvdata->mode = CS_MODE_SYSFS; + local_set(&csdev->mode, CS_MODE_SYSFS); } static int smb_enable_perf(struct coresight_device *csdev, void *data) @@ -234,7 +234,7 @@ static int smb_enable_perf(struct coresight_device *csdev, void *data) if (drvdata->pid == -1) { smb_enable_hw(drvdata); drvdata->pid = pid; - drvdata->mode = CS_MODE_PERF; + local_set(&csdev->mode, CS_MODE_PERF); } return 0; @@ -253,7 +253,8 @@ static int smb_enable(struct coresight_device *csdev, enum cs_mode mode, return -EBUSY; /* Do nothing, the SMB is already enabled as other mode */ - if (drvdata->mode != CS_MODE_DISABLED && drvdata->mode != mode) + if (local_read(&csdev->mode) != CS_MODE_DISABLED && + local_read(&csdev->mode) != mode) return -EBUSY; switch (mode) { @@ -289,13 +290,13 @@ static int smb_disable(struct coresight_device *csdev) return -EBUSY; /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); + WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); smb_disable_hw(drvdata); /* Dissociate from the target process. */ drvdata->pid = -1; - drvdata->mode = CS_MODE_DISABLED; + local_set(&csdev->mode, CS_MODE_DISABLED); dev_dbg(&csdev->dev, "Ultrasoc SMB disabled\n"); return 0; diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.h b/drivers/hwtracing/coresight/ultrasoc-smb.h index 82a44c14a882..a91d39cfccb8 100644 --- a/drivers/hwtracing/coresight/ultrasoc-smb.h +++ b/drivers/hwtracing/coresight/ultrasoc-smb.h @@ -109,7 +109,6 @@ struct smb_data_buffer { * @reading: Synchronise user space access to SMB buffer. * @pid: Process ID of the process being monitored by the * session that is using this component. - * @mode: How this SMB is being used, perf mode or sysfs mode. */ struct smb_drv_data { void __iomem *base; @@ -119,7 +118,6 @@ struct smb_drv_data { spinlock_t spinlock; bool reading; pid_t pid; - enum cs_mode mode; }; #endif diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 516ab45ff3c2..01f67862ea2f 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -226,6 +226,11 @@ struct coresight_sysfs_link { * by @coresight_ops. * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. + * @mode: This tracer's mode, i.e sysFS, Perf or disabled. This is + * actually an 'enum cs_mode', but is stored in an atomic type. + * This is always accessed through local_read() and local_set(), + * but wherever it's done from within the Coresight device's lock, + * a non-atomic read would also work. * @refcnt: keep track of what is in use. * @orphan: true if the component has connections that haven't been linked. * @enable: 'true' if component is currently part of an active path. @@ -252,6 +257,7 @@ struct coresight_device { const struct coresight_ops *ops; struct csdev_access access; struct device dev; + local_t mode; atomic_t refcnt; bool orphan; bool enable; -- cgit v1.2.3 From d5e83f97eb5669bfdd894ec980083f65517df2fb Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:36 +0000 Subject: coresight: Remove the 'enable' field. 'enable', which probably should have been 'enabled', is only ever read in the core code in relation to controlling sources, and specifically only sources in sysfs mode. Confusingly it's not labelled as such and relying on it can be a source of bugs like the one fixed by commit 078dbba3f0c9 ("coresight: Fix crash when Perf and sysfs modes are used concurrently"). Most importantly, it can only be used when the coresight_mutex is held which is only done when enabling and disabling paths in sysfs mode, and not Perf mode. So to prevent its usage spreading and leaking out to other devices, remove it. It's use is equivalent to checking if the mode is currently sysfs, as due to the coresight_mutex lock, mode == CS_MODE_SYSFS can only become true or untrue when that lock is held, and when mode == CS_MODE_SYSFS the device is both enabled and in sysfs mode. The one place it was used outside of the core code is in TPDA, but that pattern is more appropriately represented using refcounts inside the device's own spinlock. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-6-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-core.c | 86 ++++++++++------------------ drivers/hwtracing/coresight/coresight-tpda.c | 12 +++- include/linux/coresight.h | 2 - 3 files changed, 38 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 5e93b52df105..b8d6520d47c9 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -279,29 +279,18 @@ EXPORT_SYMBOL_GPL(coresight_add_helper); static int coresight_enable_sink(struct coresight_device *csdev, enum cs_mode mode, void *data) { - int ret = sink_ops(csdev)->enable(csdev, mode, data); - - if (ret) - return ret; - - csdev->enable = true; - - return 0; + return sink_ops(csdev)->enable(csdev, mode, data); } static void coresight_disable_sink(struct coresight_device *csdev) { - int ret = sink_ops(csdev)->disable(csdev); - if (ret) - return; - csdev->enable = false; + sink_ops(csdev)->disable(csdev); } static int coresight_enable_link(struct coresight_device *csdev, struct coresight_device *parent, struct coresight_device *child) { - int ret = 0; int link_subtype; struct coresight_connection *inconn, *outconn; @@ -317,19 +306,13 @@ static int coresight_enable_link(struct coresight_device *csdev, if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_SPLIT && IS_ERR(outconn)) return PTR_ERR(outconn); - ret = link_ops(csdev)->enable(csdev, inconn, outconn); - if (!ret) - csdev->enable = true; - - return ret; + return link_ops(csdev)->enable(csdev, inconn, outconn); } static void coresight_disable_link(struct coresight_device *csdev, struct coresight_device *parent, struct coresight_device *child) { - int i; - int link_subtype; struct coresight_connection *inconn, *outconn; if (!parent || !child) @@ -337,26 +320,8 @@ static void coresight_disable_link(struct coresight_device *csdev, inconn = coresight_find_out_connection(parent, csdev); outconn = coresight_find_out_connection(csdev, child); - link_subtype = csdev->subtype.link_subtype; link_ops(csdev)->disable(csdev, inconn, outconn); - - if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_MERG) { - for (i = 0; i < csdev->pdata->nr_inconns; i++) - if (atomic_read(&csdev->pdata->in_conns[i]->dest_refcnt) != - 0) - return; - } else if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_SPLIT) { - for (i = 0; i < csdev->pdata->nr_outconns; i++) - if (atomic_read(&csdev->pdata->out_conns[i]->src_refcnt) != - 0) - return; - } else { - if (atomic_read(&csdev->refcnt) != 0) - return; - } - - csdev->enable = false; } int coresight_enable_source(struct coresight_device *csdev, enum cs_mode mode, @@ -364,11 +329,16 @@ int coresight_enable_source(struct coresight_device *csdev, enum cs_mode mode, { int ret; - if (!csdev->enable) { + /* + * Comparison with CS_MODE_SYSFS works without taking any device + * specific spinlock because the truthyness of that comparison can only + * change with coresight_mutex held, which we already have here. + */ + lockdep_assert_held(&coresight_mutex); + if (local_read(&csdev->mode) != CS_MODE_SYSFS) { ret = source_ops(csdev)->enable(csdev, data, mode); if (ret) return ret; - csdev->enable = true; } atomic_inc(&csdev->refcnt); @@ -385,22 +355,12 @@ static bool coresight_is_helper(struct coresight_device *csdev) static int coresight_enable_helper(struct coresight_device *csdev, enum cs_mode mode, void *data) { - int ret = helper_ops(csdev)->enable(csdev, mode, data); - - if (ret) - return ret; - - csdev->enable = true; - return 0; + return helper_ops(csdev)->enable(csdev, mode, data); } static void coresight_disable_helper(struct coresight_device *csdev) { - int ret = helper_ops(csdev)->disable(csdev, NULL); - - if (ret) - return; - csdev->enable = false; + helper_ops(csdev)->disable(csdev, NULL); } static void coresight_disable_helpers(struct coresight_device *csdev) @@ -445,11 +405,15 @@ EXPORT_SYMBOL_GPL(coresight_disable_source); static bool coresight_disable_source_sysfs(struct coresight_device *csdev, void *data) { + lockdep_assert_held(&coresight_mutex); + if (local_read(&csdev->mode) != CS_MODE_SYSFS) + return false; + if (atomic_dec_return(&csdev->refcnt) == 0) { coresight_disable_source(csdev, data); - csdev->enable = false; + return true; } - return !csdev->enable; + return false; } /* @@ -1097,7 +1061,13 @@ int coresight_enable(struct coresight_device *csdev) if (ret) goto out; - if (csdev->enable) { + /* + * mode == SYSFS implies that it's already enabled. Don't look at the + * refcount to determine this because we don't claim the source until + * coresight_enable_source() so can still race with Perf mode which + * doesn't hold coresight_mutex. + */ + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { /* * There could be multiple applications driving the software * source. So keep the refcount for each such user when the @@ -1183,7 +1153,7 @@ void coresight_disable(struct coresight_device *csdev) if (ret) goto out; - if (!csdev->enable || !coresight_disable_source_sysfs(csdev, NULL)) + if (!coresight_disable_source_sysfs(csdev, NULL)) goto out; switch (csdev->subtype.source_subtype) { @@ -1249,7 +1219,9 @@ static ssize_t enable_source_show(struct device *dev, { struct coresight_device *csdev = to_coresight_device(dev); - return scnprintf(buf, PAGE_SIZE, "%u\n", csdev->enable); + guard(mutex)(&coresight_mutex); + return scnprintf(buf, PAGE_SIZE, "%u\n", + local_read(&csdev->mode) == CS_MODE_SYSFS); } static ssize_t enable_source_store(struct device *dev, diff --git a/drivers/hwtracing/coresight/coresight-tpda.c b/drivers/hwtracing/coresight/coresight-tpda.c index 4ac954f4bc13..c813ec427b8d 100644 --- a/drivers/hwtracing/coresight/coresight-tpda.c +++ b/drivers/hwtracing/coresight/coresight-tpda.c @@ -148,7 +148,11 @@ static int __tpda_enable(struct tpda_drvdata *drvdata, int port) CS_UNLOCK(drvdata->base); - if (!drvdata->csdev->enable) + /* + * Only do pre-port enable for first port that calls enable when the + * device's main refcount is still 0 + */ + if (!atomic_read(&drvdata->csdev->refcnt)) tpda_enable_pre_port(drvdata); ret = tpda_enable_port(drvdata, port); @@ -169,6 +173,7 @@ static int tpda_enable(struct coresight_device *csdev, ret = __tpda_enable(drvdata, in->dest_port); if (!ret) { atomic_inc(&in->dest_refcnt); + atomic_inc(&csdev->refcnt); dev_dbg(drvdata->dev, "TPDA inport %d enabled.\n", in->dest_port); } } @@ -197,9 +202,10 @@ static void tpda_disable(struct coresight_device *csdev, struct tpda_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); spin_lock(&drvdata->spinlock); - if (atomic_dec_return(&in->dest_refcnt) == 0) + if (atomic_dec_return(&in->dest_refcnt) == 0) { __tpda_disable(drvdata, in->dest_port); - + atomic_dec(&csdev->refcnt); + } spin_unlock(&drvdata->spinlock); dev_dbg(drvdata->dev, "TPDA inport %d disabled\n", in->dest_port); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 01f67862ea2f..d1fd7070099c 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -233,7 +233,6 @@ struct coresight_sysfs_link { * a non-atomic read would also work. * @refcnt: keep track of what is in use. * @orphan: true if the component has connections that haven't been linked. - * @enable: 'true' if component is currently part of an active path. * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be * activated but not yet enabled. Enabling for a _sink_ happens @@ -260,7 +259,6 @@ struct coresight_device { local_t mode; atomic_t refcnt; bool orphan; - bool enable; /* sink specific fields */ bool sysfs_sink_activated; struct dev_ext_attribute *ea; -- cgit v1.2.3 From 1f5149c7751c50aba1a871143ffa6cb36af3fb49 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:37 +0000 Subject: coresight: Move all sysfs code to sysfs file At the moment the core file contains both sysfs functionality and core functionality, while the Perf mode is in a separate file in coresight-etm-perf.c Many of the functions have ambiguous names like coresight_enable_source() which actually only work in relation to the sysfs mode. To avoid further confusion, move everything that isn't core functionality into the sysfs file and append _sysfs to the ambiguous functions. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-7-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-core.c | 394 +-------------------- drivers/hwtracing/coresight/coresight-etm3x-core.c | 4 +- drivers/hwtracing/coresight/coresight-etm4x-core.c | 4 +- drivers/hwtracing/coresight/coresight-priv.h | 5 +- drivers/hwtracing/coresight/coresight-stm.c | 4 +- drivers/hwtracing/coresight/coresight-sysfs.c | 390 ++++++++++++++++++++ include/linux/coresight.h | 8 +- 7 files changed, 407 insertions(+), 402 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index b8d6520d47c9..b83613e34289 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -25,15 +24,12 @@ #include "coresight-priv.h" #include "coresight-syscfg.h" -static DEFINE_MUTEX(coresight_mutex); -static DEFINE_PER_CPU(struct coresight_device *, csdev_sink); - /* - * Use IDR to map the hash of the source's device name - * to the pointer of path for the source. The idr is for - * the sources which aren't associated with CPU. + * Mutex used to lock all sysfs enable and disable actions and loading and + * unloading devices by the Coresight core. */ -static DEFINE_IDR(path_idr); +DEFINE_MUTEX(coresight_mutex); +static DEFINE_PER_CPU(struct coresight_device *, csdev_sink); /** * struct coresight_node - elements of a path, from source to sink @@ -45,12 +41,6 @@ struct coresight_node { struct list_head link; }; -/* - * When operating Coresight drivers from the sysFS interface, only a single - * path can exist from a tracer (associated to a CPU) to a sink. - */ -static DEFINE_PER_CPU(struct list_head *, tracer_path); - /* * When losing synchronisation a new barrier packet needs to be inserted at the * beginning of the data collected in a buffer. That way the decoder knows that @@ -61,34 +51,6 @@ EXPORT_SYMBOL_GPL(coresight_barrier_pkt); static const struct cti_assoc_op *cti_assoc_ops; -ssize_t coresight_simple_show_pair(struct device *_dev, - struct device_attribute *attr, char *buf) -{ - struct coresight_device *csdev = container_of(_dev, struct coresight_device, dev); - struct cs_pair_attribute *cs_attr = container_of(attr, struct cs_pair_attribute, attr); - u64 val; - - pm_runtime_get_sync(_dev->parent); - val = csdev_access_relaxed_read_pair(&csdev->access, cs_attr->lo_off, cs_attr->hi_off); - pm_runtime_put_sync(_dev->parent); - return sysfs_emit(buf, "0x%llx\n", val); -} -EXPORT_SYMBOL_GPL(coresight_simple_show_pair); - -ssize_t coresight_simple_show32(struct device *_dev, - struct device_attribute *attr, char *buf) -{ - struct coresight_device *csdev = container_of(_dev, struct coresight_device, dev); - struct cs_off_attribute *cs_attr = container_of(attr, struct cs_off_attribute, attr); - u64 val; - - pm_runtime_get_sync(_dev->parent); - val = csdev_access_relaxed_read32(&csdev->access, cs_attr->off); - pm_runtime_put_sync(_dev->parent); - return sysfs_emit(buf, "0x%llx\n", val); -} -EXPORT_SYMBOL_GPL(coresight_simple_show32); - void coresight_set_cti_ops(const struct cti_assoc_op *cti_op) { cti_assoc_ops = cti_op; @@ -324,29 +286,6 @@ static void coresight_disable_link(struct coresight_device *csdev, link_ops(csdev)->disable(csdev, inconn, outconn); } -int coresight_enable_source(struct coresight_device *csdev, enum cs_mode mode, - void *data) -{ - int ret; - - /* - * Comparison with CS_MODE_SYSFS works without taking any device - * specific spinlock because the truthyness of that comparison can only - * change with coresight_mutex held, which we already have here. - */ - lockdep_assert_held(&coresight_mutex); - if (local_read(&csdev->mode) != CS_MODE_SYSFS) { - ret = source_ops(csdev)->enable(csdev, data, mode); - if (ret) - return ret; - } - - atomic_inc(&csdev->refcnt); - - return 0; -} -EXPORT_SYMBOL_GPL(coresight_enable_source); - static bool coresight_is_helper(struct coresight_device *csdev) { return csdev->type == CORESIGHT_DEV_TYPE_HELPER; @@ -392,30 +331,6 @@ void coresight_disable_source(struct coresight_device *csdev, void *data) } EXPORT_SYMBOL_GPL(coresight_disable_source); -/** - * coresight_disable_source_sysfs - Drop the reference count by 1 and disable - * the device if there are no users left. - * - * @csdev: The coresight device to disable - * @data: Opaque data to pass on to the disable function of the source device. - * For example in perf mode this is a pointer to the struct perf_event. - * - * Returns true if the device has been disabled. - */ -static bool coresight_disable_source_sysfs(struct coresight_device *csdev, - void *data) -{ - lockdep_assert_held(&coresight_mutex); - if (local_read(&csdev->mode) != CS_MODE_SYSFS) - return false; - - if (atomic_dec_return(&csdev->refcnt) == 0) { - coresight_disable_source(csdev, data); - return true; - } - return false; -} - /* * coresight_disable_path_from : Disable components in the given path beyond * @nd in the list. If @nd is NULL, all the components, except the SOURCE are @@ -572,39 +487,6 @@ struct coresight_device *coresight_get_sink(struct list_head *path) return csdev; } -/** - * coresight_find_activated_sysfs_sink - returns the first sink activated via - * sysfs using connection based search starting from the source reference. - * - * @csdev: Coresight source device reference - */ -static struct coresight_device * -coresight_find_activated_sysfs_sink(struct coresight_device *csdev) -{ - int i; - struct coresight_device *sink = NULL; - - if ((csdev->type == CORESIGHT_DEV_TYPE_SINK || - csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) && - csdev->sysfs_sink_activated) - return csdev; - - /* - * Recursively explore each port found on this element. - */ - for (i = 0; i < csdev->pdata->nr_outconns; i++) { - struct coresight_device *child_dev; - - child_dev = csdev->pdata->out_conns[i]->dest_dev; - if (child_dev) - sink = coresight_find_activated_sysfs_sink(child_dev); - if (sink) - return sink; - } - - return NULL; -} - static int coresight_sink_by_id(struct device *dev, const void *data) { struct coresight_device *csdev = to_coresight_device(dev); @@ -1015,274 +897,6 @@ static void coresight_clear_default_sink(struct coresight_device *csdev) } } -/** coresight_validate_source - make sure a source has the right credentials - * @csdev: the device structure for a source. - * @function: the function this was called from. - * - * Assumes the coresight_mutex is held. - */ -static int coresight_validate_source(struct coresight_device *csdev, - const char *function) -{ - u32 type, subtype; - - type = csdev->type; - subtype = csdev->subtype.source_subtype; - - if (type != CORESIGHT_DEV_TYPE_SOURCE) { - dev_err(&csdev->dev, "wrong device type in %s\n", function); - return -EINVAL; - } - - if (subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_PROC && - subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE && - subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM && - subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS) { - dev_err(&csdev->dev, "wrong device subtype in %s\n", function); - return -EINVAL; - } - - return 0; -} - -int coresight_enable(struct coresight_device *csdev) -{ - int cpu, ret = 0; - struct coresight_device *sink; - struct list_head *path; - enum coresight_dev_subtype_source subtype; - u32 hash; - - subtype = csdev->subtype.source_subtype; - - mutex_lock(&coresight_mutex); - - ret = coresight_validate_source(csdev, __func__); - if (ret) - goto out; - - /* - * mode == SYSFS implies that it's already enabled. Don't look at the - * refcount to determine this because we don't claim the source until - * coresight_enable_source() so can still race with Perf mode which - * doesn't hold coresight_mutex. - */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { - /* - * There could be multiple applications driving the software - * source. So keep the refcount for each such user when the - * source is already enabled. - */ - if (subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE) - atomic_inc(&csdev->refcnt); - goto out; - } - - sink = coresight_find_activated_sysfs_sink(csdev); - if (!sink) { - ret = -EINVAL; - goto out; - } - - path = coresight_build_path(csdev, sink); - if (IS_ERR(path)) { - pr_err("building path(s) failed\n"); - ret = PTR_ERR(path); - goto out; - } - - ret = coresight_enable_path(path, CS_MODE_SYSFS, NULL); - if (ret) - goto err_path; - - ret = coresight_enable_source(csdev, CS_MODE_SYSFS, NULL); - if (ret) - goto err_source; - - switch (subtype) { - case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC: - /* - * When working from sysFS it is important to keep track - * of the paths that were created so that they can be - * undone in 'coresight_disable()'. Since there can only - * be a single session per tracer (when working from sysFS) - * a per-cpu variable will do just fine. - */ - cpu = source_ops(csdev)->cpu_id(csdev); - per_cpu(tracer_path, cpu) = path; - break; - case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE: - case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM: - case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS: - /* - * Use the hash of source's device name as ID - * and map the ID to the pointer of the path. - */ - hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); - ret = idr_alloc_u32(&path_idr, path, &hash, hash, GFP_KERNEL); - if (ret) - goto err_source; - break; - default: - /* We can't be here */ - break; - } - -out: - mutex_unlock(&coresight_mutex); - return ret; - -err_source: - coresight_disable_path(path); - -err_path: - coresight_release_path(path); - goto out; -} -EXPORT_SYMBOL_GPL(coresight_enable); - -void coresight_disable(struct coresight_device *csdev) -{ - int cpu, ret; - struct list_head *path = NULL; - u32 hash; - - mutex_lock(&coresight_mutex); - - ret = coresight_validate_source(csdev, __func__); - if (ret) - goto out; - - if (!coresight_disable_source_sysfs(csdev, NULL)) - goto out; - - switch (csdev->subtype.source_subtype) { - case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC: - cpu = source_ops(csdev)->cpu_id(csdev); - path = per_cpu(tracer_path, cpu); - per_cpu(tracer_path, cpu) = NULL; - break; - case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE: - case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM: - case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS: - hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); - /* Find the path by the hash. */ - path = idr_find(&path_idr, hash); - if (path == NULL) { - pr_err("Path is not found for %s\n", dev_name(&csdev->dev)); - goto out; - } - idr_remove(&path_idr, hash); - break; - default: - /* We can't be here */ - break; - } - - coresight_disable_path(path); - coresight_release_path(path); - -out: - mutex_unlock(&coresight_mutex); -} -EXPORT_SYMBOL_GPL(coresight_disable); - -static ssize_t enable_sink_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct coresight_device *csdev = to_coresight_device(dev); - - return scnprintf(buf, PAGE_SIZE, "%u\n", csdev->sysfs_sink_activated); -} - -static ssize_t enable_sink_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) -{ - int ret; - unsigned long val; - struct coresight_device *csdev = to_coresight_device(dev); - - ret = kstrtoul(buf, 10, &val); - if (ret) - return ret; - - csdev->sysfs_sink_activated = !!val; - - return size; - -} -static DEVICE_ATTR_RW(enable_sink); - -static ssize_t enable_source_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct coresight_device *csdev = to_coresight_device(dev); - - guard(mutex)(&coresight_mutex); - return scnprintf(buf, PAGE_SIZE, "%u\n", - local_read(&csdev->mode) == CS_MODE_SYSFS); -} - -static ssize_t enable_source_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) -{ - int ret = 0; - unsigned long val; - struct coresight_device *csdev = to_coresight_device(dev); - - ret = kstrtoul(buf, 10, &val); - if (ret) - return ret; - - if (val) { - ret = coresight_enable(csdev); - if (ret) - return ret; - } else { - coresight_disable(csdev); - } - - return size; -} -static DEVICE_ATTR_RW(enable_source); - -static struct attribute *coresight_sink_attrs[] = { - &dev_attr_enable_sink.attr, - NULL, -}; -ATTRIBUTE_GROUPS(coresight_sink); - -static struct attribute *coresight_source_attrs[] = { - &dev_attr_enable_source.attr, - NULL, -}; -ATTRIBUTE_GROUPS(coresight_source); - -static struct device_type coresight_dev_type[] = { - { - .name = "sink", - .groups = coresight_sink_groups, - }, - { - .name = "link", - }, - { - .name = "linksink", - .groups = coresight_sink_groups, - }, - { - .name = "source", - .groups = coresight_source_groups, - }, - { - .name = "helper", - } -}; -/* Ensure the enum matches the names and groups */ -static_assert(ARRAY_SIZE(coresight_dev_type) == CORESIGHT_DEV_TYPE_MAX); - static void coresight_device_release(struct device *dev) { struct coresight_device *csdev = to_coresight_device(dev); diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 333d0a32c95c..de53c3e8db29 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -714,7 +714,7 @@ static int etm_online_cpu(unsigned int cpu) return 0; if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable) - coresight_enable(etmdrvdata[cpu]->csdev); + coresight_enable_sysfs(etmdrvdata[cpu]->csdev); return 0; } @@ -924,7 +924,7 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) dev_info(&drvdata->csdev->dev, "%s initialized\n", (char *)coresight_get_uci_data(id)); if (boot_enable) { - coresight_enable(drvdata->csdev); + coresight_enable_sysfs(drvdata->csdev); drvdata->boot_enable = true; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 7ceff627cd95..1c64b54459d7 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1648,7 +1648,7 @@ static int etm4_online_cpu(unsigned int cpu) return etm4_probe_cpu(cpu); if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable) - coresight_enable(etmdrvdata[cpu]->csdev); + coresight_enable_sysfs(etmdrvdata[cpu]->csdev); return 0; } @@ -2096,7 +2096,7 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) drvdata->cpu, type_name, major, minor); if (boot_enable) { - coresight_enable(drvdata->csdev); + coresight_enable_sysfs(drvdata->csdev); drvdata->boot_enable = true; } diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index ced5be05a527..eb365236f9a9 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -12,6 +12,9 @@ #include #include +extern struct mutex coresight_mutex; +extern struct device_type coresight_dev_type[]; + /* * Coresight management registers (0xf00-0xfcc) * 0xfa0 - 0xfa4: Management registers in PFTv1.0 @@ -229,8 +232,6 @@ void coresight_add_helper(struct coresight_device *csdev, void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev); struct coresight_device *coresight_get_percpu_sink(int cpu); -int coresight_enable_source(struct coresight_device *csdev, enum cs_mode mode, - void *data); void coresight_disable_source(struct coresight_device *csdev, void *data); #endif diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index f7fc645ea7a4..b2ccbe989ff8 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -332,7 +332,7 @@ static int stm_generic_link(struct stm_data *stm_data, if (!drvdata || !drvdata->csdev) return -EINVAL; - return coresight_enable(drvdata->csdev); + return coresight_enable_sysfs(drvdata->csdev); } static void stm_generic_unlink(struct stm_data *stm_data, @@ -343,7 +343,7 @@ static void stm_generic_unlink(struct stm_data *stm_data, if (!drvdata || !drvdata->csdev) return; - coresight_disable(drvdata->csdev); + coresight_disable_sysfs(drvdata->csdev); } static phys_addr_t diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index dd78e9fcfc4d..92cdf8139f23 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -5,10 +5,400 @@ */ #include +#include #include #include "coresight-priv.h" +/* + * Use IDR to map the hash of the source's device name + * to the pointer of path for the source. The idr is for + * the sources which aren't associated with CPU. + */ +static DEFINE_IDR(path_idr); + +/* + * When operating Coresight drivers from the sysFS interface, only a single + * path can exist from a tracer (associated to a CPU) to a sink. + */ +static DEFINE_PER_CPU(struct list_head *, tracer_path); + +ssize_t coresight_simple_show_pair(struct device *_dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = container_of(_dev, struct coresight_device, dev); + struct cs_pair_attribute *cs_attr = container_of(attr, struct cs_pair_attribute, attr); + u64 val; + + pm_runtime_get_sync(_dev->parent); + val = csdev_access_relaxed_read_pair(&csdev->access, cs_attr->lo_off, cs_attr->hi_off); + pm_runtime_put_sync(_dev->parent); + return sysfs_emit(buf, "0x%llx\n", val); +} +EXPORT_SYMBOL_GPL(coresight_simple_show_pair); + +ssize_t coresight_simple_show32(struct device *_dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = container_of(_dev, struct coresight_device, dev); + struct cs_off_attribute *cs_attr = container_of(attr, struct cs_off_attribute, attr); + u64 val; + + pm_runtime_get_sync(_dev->parent); + val = csdev_access_relaxed_read32(&csdev->access, cs_attr->off); + pm_runtime_put_sync(_dev->parent); + return sysfs_emit(buf, "0x%llx\n", val); +} +EXPORT_SYMBOL_GPL(coresight_simple_show32); + +static int coresight_enable_source_sysfs(struct coresight_device *csdev, + enum cs_mode mode, void *data) +{ + int ret; + + /* + * Comparison with CS_MODE_SYSFS works without taking any device + * specific spinlock because the truthyness of that comparison can only + * change with coresight_mutex held, which we already have here. + */ + lockdep_assert_held(&coresight_mutex); + if (local_read(&csdev->mode) != CS_MODE_SYSFS) { + ret = source_ops(csdev)->enable(csdev, data, mode); + if (ret) + return ret; + } + + atomic_inc(&csdev->refcnt); + + return 0; +} + +/** + * coresight_disable_source_sysfs - Drop the reference count by 1 and disable + * the device if there are no users left. + * + * @csdev: The coresight device to disable + * @data: Opaque data to pass on to the disable function of the source device. + * For example in perf mode this is a pointer to the struct perf_event. + * + * Returns true if the device has been disabled. + */ +static bool coresight_disable_source_sysfs(struct coresight_device *csdev, + void *data) +{ + lockdep_assert_held(&coresight_mutex); + if (local_read(&csdev->mode) != CS_MODE_SYSFS) + return false; + + if (atomic_dec_return(&csdev->refcnt) == 0) { + coresight_disable_source(csdev, data); + return true; + } + return false; +} + +/** + * coresight_find_activated_sysfs_sink - returns the first sink activated via + * sysfs using connection based search starting from the source reference. + * + * @csdev: Coresight source device reference + */ +static struct coresight_device * +coresight_find_activated_sysfs_sink(struct coresight_device *csdev) +{ + int i; + struct coresight_device *sink = NULL; + + if ((csdev->type == CORESIGHT_DEV_TYPE_SINK || + csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) && + csdev->sysfs_sink_activated) + return csdev; + + /* + * Recursively explore each port found on this element. + */ + for (i = 0; i < csdev->pdata->nr_outconns; i++) { + struct coresight_device *child_dev; + + child_dev = csdev->pdata->out_conns[i]->dest_dev; + if (child_dev) + sink = coresight_find_activated_sysfs_sink(child_dev); + if (sink) + return sink; + } + + return NULL; +} + +/** coresight_validate_source - make sure a source has the right credentials to + * be used via sysfs. + * @csdev: the device structure for a source. + * @function: the function this was called from. + * + * Assumes the coresight_mutex is held. + */ +static int coresight_validate_source_sysfs(struct coresight_device *csdev, + const char *function) +{ + u32 type, subtype; + + type = csdev->type; + subtype = csdev->subtype.source_subtype; + + if (type != CORESIGHT_DEV_TYPE_SOURCE) { + dev_err(&csdev->dev, "wrong device type in %s\n", function); + return -EINVAL; + } + + if (subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_PROC && + subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE && + subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM && + subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS) { + dev_err(&csdev->dev, "wrong device subtype in %s\n", function); + return -EINVAL; + } + + return 0; +} + +int coresight_enable_sysfs(struct coresight_device *csdev) +{ + int cpu, ret = 0; + struct coresight_device *sink; + struct list_head *path; + enum coresight_dev_subtype_source subtype; + u32 hash; + + subtype = csdev->subtype.source_subtype; + + mutex_lock(&coresight_mutex); + + ret = coresight_validate_source_sysfs(csdev, __func__); + if (ret) + goto out; + + /* + * mode == SYSFS implies that it's already enabled. Don't look at the + * refcount to determine this because we don't claim the source until + * coresight_enable_source() so can still race with Perf mode which + * doesn't hold coresight_mutex. + */ + if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + /* + * There could be multiple applications driving the software + * source. So keep the refcount for each such user when the + * source is already enabled. + */ + if (subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE) + atomic_inc(&csdev->refcnt); + goto out; + } + + sink = coresight_find_activated_sysfs_sink(csdev); + if (!sink) { + ret = -EINVAL; + goto out; + } + + path = coresight_build_path(csdev, sink); + if (IS_ERR(path)) { + pr_err("building path(s) failed\n"); + ret = PTR_ERR(path); + goto out; + } + + ret = coresight_enable_path(path, CS_MODE_SYSFS, NULL); + if (ret) + goto err_path; + + ret = coresight_enable_source_sysfs(csdev, CS_MODE_SYSFS, NULL); + if (ret) + goto err_source; + + switch (subtype) { + case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC: + /* + * When working from sysFS it is important to keep track + * of the paths that were created so that they can be + * undone in 'coresight_disable()'. Since there can only + * be a single session per tracer (when working from sysFS) + * a per-cpu variable will do just fine. + */ + cpu = source_ops(csdev)->cpu_id(csdev); + per_cpu(tracer_path, cpu) = path; + break; + case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE: + case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM: + case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS: + /* + * Use the hash of source's device name as ID + * and map the ID to the pointer of the path. + */ + hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); + ret = idr_alloc_u32(&path_idr, path, &hash, hash, GFP_KERNEL); + if (ret) + goto err_source; + break; + default: + /* We can't be here */ + break; + } + +out: + mutex_unlock(&coresight_mutex); + return ret; + +err_source: + coresight_disable_path(path); + +err_path: + coresight_release_path(path); + goto out; +} +EXPORT_SYMBOL_GPL(coresight_enable_sysfs); + +void coresight_disable_sysfs(struct coresight_device *csdev) +{ + int cpu, ret; + struct list_head *path = NULL; + u32 hash; + + mutex_lock(&coresight_mutex); + + ret = coresight_validate_source_sysfs(csdev, __func__); + if (ret) + goto out; + + if (!coresight_disable_source_sysfs(csdev, NULL)) + goto out; + + switch (csdev->subtype.source_subtype) { + case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC: + cpu = source_ops(csdev)->cpu_id(csdev); + path = per_cpu(tracer_path, cpu); + per_cpu(tracer_path, cpu) = NULL; + break; + case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE: + case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM: + case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS: + hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev))); + /* Find the path by the hash. */ + path = idr_find(&path_idr, hash); + if (path == NULL) { + pr_err("Path is not found for %s\n", dev_name(&csdev->dev)); + goto out; + } + idr_remove(&path_idr, hash); + break; + default: + /* We can't be here */ + break; + } + + coresight_disable_path(path); + coresight_release_path(path); + +out: + mutex_unlock(&coresight_mutex); +} +EXPORT_SYMBOL_GPL(coresight_disable_sysfs); + +static ssize_t enable_sink_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = to_coresight_device(dev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", csdev->sysfs_sink_activated); +} + +static ssize_t enable_sink_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct coresight_device *csdev = to_coresight_device(dev); + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + csdev->sysfs_sink_activated = !!val; + + return size; + +} +static DEVICE_ATTR_RW(enable_sink); + +static ssize_t enable_source_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = to_coresight_device(dev); + + guard(mutex)(&coresight_mutex); + return scnprintf(buf, PAGE_SIZE, "%u\n", + local_read(&csdev->mode) == CS_MODE_SYSFS); +} + +static ssize_t enable_source_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret = 0; + unsigned long val; + struct coresight_device *csdev = to_coresight_device(dev); + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + if (val) { + ret = coresight_enable_sysfs(csdev); + if (ret) + return ret; + } else { + coresight_disable_sysfs(csdev); + } + + return size; +} +static DEVICE_ATTR_RW(enable_source); + +static struct attribute *coresight_sink_attrs[] = { + &dev_attr_enable_sink.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_sink); + +static struct attribute *coresight_source_attrs[] = { + &dev_attr_enable_source.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_source); + +struct device_type coresight_dev_type[] = { + { + .name = "sink", + .groups = coresight_sink_groups, + }, + { + .name = "link", + }, + { + .name = "linksink", + .groups = coresight_sink_groups, + }, + { + .name = "source", + .groups = coresight_source_groups, + }, + { + .name = "helper", + } +}; +/* Ensure the enum matches the names and groups */ +static_assert(ARRAY_SIZE(coresight_dev_type) == CORESIGHT_DEV_TYPE_MAX); + /* * Connections group - links attribute. * Count of created links between coresight components in the group. diff --git a/include/linux/coresight.h b/include/linux/coresight.h index d1fd7070099c..365b28022c5b 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -578,8 +578,8 @@ static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) extern struct coresight_device * coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); -extern int coresight_enable(struct coresight_device *csdev); -extern void coresight_disable(struct coresight_device *csdev); +extern int coresight_enable_sysfs(struct coresight_device *csdev); +extern void coresight_disable_sysfs(struct coresight_device *csdev); extern int coresight_timeout(struct csdev_access *csa, u32 offset, int position, int value); @@ -609,8 +609,8 @@ static inline struct coresight_device * coresight_register(struct coresight_desc *desc) { return NULL; } static inline void coresight_unregister(struct coresight_device *csdev) {} static inline int -coresight_enable(struct coresight_device *csdev) { return -ENOSYS; } -static inline void coresight_disable(struct coresight_device *csdev) {} +coresight_enable_sysfs(struct coresight_device *csdev) { return -ENOSYS; } +static inline void coresight_disable_sysfs(struct coresight_device *csdev) {} static inline int coresight_timeout(struct csdev_access *csa, u32 offset, int position, int value) -- cgit v1.2.3 From 4545b38ef004a586295750ea49a505b6396a7c90 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:38 +0000 Subject: coresight: Remove atomic type from refcnt Refcnt is only ever accessed from either inside the coresight_mutex, or the device's spinlock, making the atomic type and atomic_dec_return() calls confusing and unnecessary. The only point of synchronisation outside of these two types of locks is already done with a compare and swap on 'mode', which a comment has been added for. There was one instance of refcnt being used outside of a lock in TPIU, but that can easily be fixed by making it the same as all the other devices and adding a spinlock. Potentially in the future all the refcounting and locking can be moved up into the core code, and all the mostly duplicate code from the individual devices can be removed. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-8-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-etb10.c | 11 ++++++----- drivers/hwtracing/coresight/coresight-sysfs.c | 7 ++++--- drivers/hwtracing/coresight/coresight-tmc-etf.c | 20 +++++++++++--------- drivers/hwtracing/coresight/coresight-tmc-etr.c | 13 +++++++------ drivers/hwtracing/coresight/coresight-tpda.c | 7 ++++--- drivers/hwtracing/coresight/coresight-tpiu.c | 14 ++++++++++++-- drivers/hwtracing/coresight/ultrasoc-smb.c | 9 +++++---- include/linux/coresight.h | 13 ++++++++++--- 8 files changed, 59 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 145fb815ec1c..306d2af47d63 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -161,7 +161,7 @@ static int etb_enable_sysfs(struct coresight_device *csdev) local_set(&csdev->mode, CS_MODE_SYSFS); } - atomic_inc(&csdev->refcnt); + csdev->refcnt++; out: spin_unlock_irqrestore(&drvdata->spinlock, flags); return ret; @@ -197,7 +197,7 @@ static int etb_enable_perf(struct coresight_device *csdev, void *data) * use for this session. */ if (drvdata->pid == pid) { - atomic_inc(&csdev->refcnt); + csdev->refcnt++; goto out; } @@ -215,7 +215,7 @@ static int etb_enable_perf(struct coresight_device *csdev, void *data) /* Associate with monitored process. */ drvdata->pid = pid; local_set(&drvdata->csdev->mode, CS_MODE_PERF); - atomic_inc(&csdev->refcnt); + csdev->refcnt++; } out: @@ -354,7 +354,8 @@ static int etb_disable(struct coresight_device *csdev) spin_lock_irqsave(&drvdata->spinlock, flags); - if (atomic_dec_return(&csdev->refcnt)) { + csdev->refcnt--; + if (csdev->refcnt) { spin_unlock_irqrestore(&drvdata->spinlock, flags); return -EBUSY; } @@ -445,7 +446,7 @@ static unsigned long etb_update_buffer(struct coresight_device *csdev, spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't do anything if another tracer is using this sink */ - if (atomic_read(&csdev->refcnt) != 1) + if (csdev->refcnt != 1) goto out; __etb_disable_hw(drvdata); diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index 92cdf8139f23..5992f2c2200a 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -68,7 +68,7 @@ static int coresight_enable_source_sysfs(struct coresight_device *csdev, return ret; } - atomic_inc(&csdev->refcnt); + csdev->refcnt++; return 0; } @@ -90,7 +90,8 @@ static bool coresight_disable_source_sysfs(struct coresight_device *csdev, if (local_read(&csdev->mode) != CS_MODE_SYSFS) return false; - if (atomic_dec_return(&csdev->refcnt) == 0) { + csdev->refcnt--; + if (csdev->refcnt == 0) { coresight_disable_source(csdev, data); return true; } @@ -190,7 +191,7 @@ int coresight_enable_sysfs(struct coresight_device *csdev) * source is already enabled. */ if (subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE) - atomic_inc(&csdev->refcnt); + csdev->refcnt++; goto out; } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 2a7e516052a2..f3281c958a57 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -206,7 +206,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) * touched. */ if (local_read(&csdev->mode) == CS_MODE_SYSFS) { - atomic_inc(&csdev->refcnt); + csdev->refcnt++; goto out; } @@ -229,7 +229,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) ret = tmc_etb_enable_hw(drvdata); if (!ret) { local_set(&csdev->mode, CS_MODE_SYSFS); - atomic_inc(&csdev->refcnt); + csdev->refcnt++; } else { /* Free up the buffer if we failed to enable */ used = false; @@ -284,7 +284,7 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) * use for this session. */ if (drvdata->pid == pid) { - atomic_inc(&csdev->refcnt); + csdev->refcnt++; break; } @@ -293,7 +293,7 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) /* Associate with monitored process. */ drvdata->pid = pid; local_set(&csdev->mode, CS_MODE_PERF); - atomic_inc(&csdev->refcnt); + csdev->refcnt++; } } while (0); spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -338,7 +338,8 @@ static int tmc_disable_etf_sink(struct coresight_device *csdev) return -EBUSY; } - if (atomic_dec_return(&csdev->refcnt)) { + csdev->refcnt--; + if (csdev->refcnt) { spin_unlock_irqrestore(&drvdata->spinlock, flags); return -EBUSY; } @@ -371,7 +372,7 @@ static int tmc_enable_etf_link(struct coresight_device *csdev, return -EBUSY; } - if (atomic_read(&csdev->refcnt) == 0) { + if (csdev->refcnt == 0) { ret = tmc_etf_enable_hw(drvdata); if (!ret) { local_set(&csdev->mode, CS_MODE_SYSFS); @@ -379,7 +380,7 @@ static int tmc_enable_etf_link(struct coresight_device *csdev, } } if (!ret) - atomic_inc(&csdev->refcnt); + csdev->refcnt++; spin_unlock_irqrestore(&drvdata->spinlock, flags); if (first_enable) @@ -401,7 +402,8 @@ static void tmc_disable_etf_link(struct coresight_device *csdev, return; } - if (atomic_dec_return(&csdev->refcnt) == 0) { + csdev->refcnt--; + if (csdev->refcnt == 0) { tmc_etf_disable_hw(drvdata); local_set(&csdev->mode, CS_MODE_DISABLED); last_disable = true; @@ -489,7 +491,7 @@ static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't do anything if another tracer is using this sink */ - if (atomic_read(&csdev->refcnt) != 1) + if (csdev->refcnt != 1) goto out; CS_UNLOCK(drvdata->base); diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 3dc989d4fcab..88a0fc375b4d 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1231,14 +1231,14 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) * touched, even if the buffer size has changed. */ if (local_read(&csdev->mode) == CS_MODE_SYSFS) { - atomic_inc(&csdev->refcnt); + csdev->refcnt++; goto out; } ret = tmc_etr_enable_hw(drvdata, sysfs_buf); if (!ret) { local_set(&csdev->mode, CS_MODE_SYSFS); - atomic_inc(&csdev->refcnt); + csdev->refcnt++; } out: @@ -1564,7 +1564,7 @@ tmc_update_etr_buffer(struct coresight_device *csdev, spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't do anything if another tracer is using this sink */ - if (atomic_read(&csdev->refcnt) != 1) { + if (csdev->refcnt != 1) { spin_unlock_irqrestore(&drvdata->spinlock, flags); goto out; } @@ -1676,7 +1676,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) * use for this session. */ if (drvdata->pid == pid) { - atomic_inc(&csdev->refcnt); + csdev->refcnt++; goto unlock_out; } @@ -1686,7 +1686,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) drvdata->pid = pid; local_set(&csdev->mode, CS_MODE_PERF); drvdata->perf_buf = etr_perf->etr_buf; - atomic_inc(&csdev->refcnt); + csdev->refcnt++; } unlock_out: @@ -1719,7 +1719,8 @@ static int tmc_disable_etr_sink(struct coresight_device *csdev) return -EBUSY; } - if (atomic_dec_return(&csdev->refcnt)) { + csdev->refcnt--; + if (csdev->refcnt) { spin_unlock_irqrestore(&drvdata->spinlock, flags); return -EBUSY; } diff --git a/drivers/hwtracing/coresight/coresight-tpda.c b/drivers/hwtracing/coresight/coresight-tpda.c index c813ec427b8d..f8f6413cc711 100644 --- a/drivers/hwtracing/coresight/coresight-tpda.c +++ b/drivers/hwtracing/coresight/coresight-tpda.c @@ -152,7 +152,8 @@ static int __tpda_enable(struct tpda_drvdata *drvdata, int port) * Only do pre-port enable for first port that calls enable when the * device's main refcount is still 0 */ - if (!atomic_read(&drvdata->csdev->refcnt)) + lockdep_assert_held(&drvdata->spinlock); + if (!drvdata->csdev->refcnt) tpda_enable_pre_port(drvdata); ret = tpda_enable_port(drvdata, port); @@ -173,7 +174,7 @@ static int tpda_enable(struct coresight_device *csdev, ret = __tpda_enable(drvdata, in->dest_port); if (!ret) { atomic_inc(&in->dest_refcnt); - atomic_inc(&csdev->refcnt); + csdev->refcnt++; dev_dbg(drvdata->dev, "TPDA inport %d enabled.\n", in->dest_port); } } @@ -204,7 +205,7 @@ static void tpda_disable(struct coresight_device *csdev, spin_lock(&drvdata->spinlock); if (atomic_dec_return(&in->dest_refcnt) == 0) { __tpda_disable(drvdata, in->dest_port); - atomic_dec(&csdev->refcnt); + csdev->refcnt--; } spin_unlock(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index 7e69048ac944..29024f880fda 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -58,6 +58,7 @@ struct tpiu_drvdata { void __iomem *base; struct clk *atclk; struct coresight_device *csdev; + spinlock_t spinlock; }; static void tpiu_enable_hw(struct csdev_access *csa) @@ -72,8 +73,11 @@ static void tpiu_enable_hw(struct csdev_access *csa) static int tpiu_enable(struct coresight_device *csdev, enum cs_mode mode, void *__unused) { + struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + guard(spinlock)(&drvdata->spinlock); tpiu_enable_hw(&csdev->access); - atomic_inc(&csdev->refcnt); + csdev->refcnt++; dev_dbg(&csdev->dev, "TPIU enabled\n"); return 0; } @@ -96,7 +100,11 @@ static void tpiu_disable_hw(struct csdev_access *csa) static int tpiu_disable(struct coresight_device *csdev) { - if (atomic_dec_return(&csdev->refcnt)) + struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + guard(spinlock)(&drvdata->spinlock); + csdev->refcnt--; + if (csdev->refcnt) return -EBUSY; tpiu_disable_hw(&csdev->access); @@ -132,6 +140,8 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id) if (!drvdata) return -ENOMEM; + spin_lock_init(&drvdata->spinlock); + drvdata->atclk = devm_clk_get(&adev->dev, "atclk"); /* optional */ if (!IS_ERR(drvdata->atclk)) { ret = clk_prepare_enable(drvdata->atclk); diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c index bef633960173..a92c5b750f81 100644 --- a/drivers/hwtracing/coresight/ultrasoc-smb.c +++ b/drivers/hwtracing/coresight/ultrasoc-smb.c @@ -103,7 +103,7 @@ static int smb_open(struct inode *inode, struct file *file) if (drvdata->reading) return -EBUSY; - if (atomic_read(&drvdata->csdev->refcnt)) + if (drvdata->csdev->refcnt) return -EBUSY; smb_update_data_size(drvdata); @@ -271,7 +271,7 @@ static int smb_enable(struct coresight_device *csdev, enum cs_mode mode, if (ret) return ret; - atomic_inc(&csdev->refcnt); + csdev->refcnt++; dev_dbg(&csdev->dev, "Ultrasoc SMB enabled\n"); return ret; @@ -286,7 +286,8 @@ static int smb_disable(struct coresight_device *csdev) if (drvdata->reading) return -EBUSY; - if (atomic_dec_return(&csdev->refcnt)) + csdev->refcnt--; + if (csdev->refcnt) return -EBUSY; /* Complain if we (somehow) got out of sync */ @@ -381,7 +382,7 @@ static unsigned long smb_update_buffer(struct coresight_device *csdev, guard(spinlock)(&drvdata->spinlock); /* Don't do anything if another tracer is using this sink. */ - if (atomic_read(&csdev->refcnt) != 1) + if (csdev->refcnt != 1) return 0; smb_disable_hw(drvdata); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 365b28022c5b..74bcec526aa9 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -230,8 +230,15 @@ struct coresight_sysfs_link { * actually an 'enum cs_mode', but is stored in an atomic type. * This is always accessed through local_read() and local_set(), * but wherever it's done from within the Coresight device's lock, - * a non-atomic read would also work. - * @refcnt: keep track of what is in use. + * a non-atomic read would also work. This is the main point of + * synchronisation between code happening inside the sysfs mode's + * coresight_mutex and outside when running in Perf mode. A compare + * and exchange swap is done to atomically claim one mode or the + * other. + * @refcnt: keep track of what is in use. Only access this outside of the + * device's spinlock when the coresight_mutex held and mode == + * CS_MODE_SYSFS. Otherwise it must be accessed from inside the + * spinlock. * @orphan: true if the component has connections that haven't been linked. * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be @@ -257,7 +264,7 @@ struct coresight_device { struct csdev_access access; struct device dev; local_t mode; - atomic_t refcnt; + int refcnt; bool orphan; /* sink specific fields */ bool sysfs_sink_activated; -- cgit v1.2.3 From 053ad9ad1d13f253605d7644de3aa20d958569ef Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:39 +0000 Subject: coresight: Remove unused stubs These are a bit annoying to keep up to date when the function signatures change. But if CONFIG_CORESIGHT isn't enabled, then they're not used anyway so just delete them. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-9-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- include/linux/coresight.h | 79 ----------------------------------------------- 1 file changed, 79 deletions(-) (limited to 'include/linux') diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 74bcec526aa9..ecf4b8aecca8 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -391,8 +391,6 @@ struct coresight_ops { const struct coresight_ops_helper *helper_ops; }; -#if IS_ENABLED(CONFIG_CORESIGHT) - static inline u32 csdev_access_relaxed_read32(struct csdev_access *csa, u32 offset) { @@ -611,83 +609,6 @@ void coresight_relaxed_write64(struct coresight_device *csdev, u64 val, u32 offset); void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset); -#else -static inline struct coresight_device * -coresight_register(struct coresight_desc *desc) { return NULL; } -static inline void coresight_unregister(struct coresight_device *csdev) {} -static inline int -coresight_enable_sysfs(struct coresight_device *csdev) { return -ENOSYS; } -static inline void coresight_disable_sysfs(struct coresight_device *csdev) {} - -static inline int coresight_timeout(struct csdev_access *csa, u32 offset, - int position, int value) -{ - return 1; -} - -static inline int coresight_claim_device_unlocked(struct coresight_device *csdev) -{ - return -EINVAL; -} - -static inline int coresight_claim_device(struct coresight_device *csdev) -{ - return -EINVAL; -} - -static inline void coresight_disclaim_device(struct coresight_device *csdev) {} -static inline void coresight_disclaim_device_unlocked(struct coresight_device *csdev) {} - -static inline bool coresight_loses_context_with_cpu(struct device *dev) -{ - return false; -} - -static inline u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset) -{ - WARN_ON_ONCE(1); - return 0; -} - -static inline u32 coresight_read32(struct coresight_device *csdev, u32 offset) -{ - WARN_ON_ONCE(1); - return 0; -} - -static inline void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset) -{ -} - -static inline void coresight_relaxed_write32(struct coresight_device *csdev, - u32 val, u32 offset) -{ -} - -static inline u64 coresight_relaxed_read64(struct coresight_device *csdev, - u32 offset) -{ - WARN_ON_ONCE(1); - return 0; -} - -static inline u64 coresight_read64(struct coresight_device *csdev, u32 offset) -{ - WARN_ON_ONCE(1); - return 0; -} - -static inline void coresight_relaxed_write64(struct coresight_device *csdev, - u64 val, u32 offset) -{ -} - -static inline void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset) -{ -} - -#endif /* IS_ENABLED(CONFIG_CORESIGHT) */ - extern int coresight_get_cpu(struct device *dev); struct coresight_platform_data *coresight_get_platform_data(struct device *dev); -- cgit v1.2.3 From d724f65218b994da234081df5dfe417c23802a65 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:41 +0000 Subject: coresight: Add helper for atomically taking the device Now that mode is in struct coresight_device, this pattern can be wrapped in a helper. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-11-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-etm3x-core.c | 8 +++----- drivers/hwtracing/coresight/coresight-etm4x-core.c | 8 +++----- drivers/hwtracing/coresight/coresight-stm.c | 8 +++----- include/linux/coresight.h | 11 +++++++++++ 4 files changed, 20 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index de53c3e8db29..c301f6a5f065 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -556,14 +556,12 @@ static int etm_enable(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode) { int ret; - u32 val; struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - val = local_cmpxchg(&drvdata->csdev->mode, CS_MODE_DISABLED, mode); - - /* Someone is already using the tracer */ - if (val) + if (!coresight_take_mode(csdev, mode)) { + /* Someone is already using the tracer */ return -EBUSY; + } switch (mode) { case CS_MODE_SYSFS: diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 1c64b54459d7..572ac4a5e4ff 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -840,13 +840,11 @@ static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode) { int ret; - u32 val; - val = local_cmpxchg(&csdev->mode, CS_MODE_DISABLED, mode); - - /* Someone is already using the tracer */ - if (val) + if (!coresight_take_mode(csdev, mode)) { + /* Someone is already using the tracer */ return -EBUSY; + } switch (mode) { case CS_MODE_SYSFS: diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index b2ccbe989ff8..ff578762f6bf 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -193,17 +193,15 @@ static void stm_enable_hw(struct stm_drvdata *drvdata) static int stm_enable(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode) { - u32 val; struct stm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); if (mode != CS_MODE_SYSFS) return -EINVAL; - val = local_cmpxchg(&csdev->mode, CS_MODE_DISABLED, mode); - - /* Someone is already using the tracer */ - if (val) + if (!coresight_take_mode(csdev, mode)) { + /* Someone is already using the tracer */ return -EBUSY; + } pm_runtime_get_sync(csdev->dev.parent); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index ecf4b8aecca8..414bcbbdaf62 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -580,6 +580,17 @@ static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) (csdev->subtype.sink_subtype == CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM); } +/* + * Atomically try to take the device and set a new mode. Returns true on + * success, false if the device is already taken by someone else. + */ +static inline bool coresight_take_mode(struct coresight_device *csdev, + enum cs_mode new_mode) +{ + return local_cmpxchg(&csdev->mode, CS_MODE_DISABLED, new_mode) == + CS_MODE_DISABLED; +} + extern struct coresight_device * coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); -- cgit v1.2.3 From c95c2733e5feb1f6848923f166849b2d1c7bf682 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:42 +0000 Subject: coresight: Add a helper for getting csdev->mode Now that mode is in struct coresight_device accesses can be wrapped. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-12-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-etb10.c | 10 +++++----- drivers/hwtracing/coresight/coresight-etm3x-core.c | 6 +++--- drivers/hwtracing/coresight/coresight-etm3x-sysfs.c | 4 ++-- drivers/hwtracing/coresight/coresight-etm4x-core.c | 8 ++++---- drivers/hwtracing/coresight/coresight-stm.c | 14 +++++++------- drivers/hwtracing/coresight/coresight-sysfs.c | 8 ++++---- drivers/hwtracing/coresight/coresight-tmc-core.c | 2 +- drivers/hwtracing/coresight/coresight-tmc-etf.c | 16 ++++++++-------- drivers/hwtracing/coresight/coresight-tmc-etr.c | 14 +++++++------- drivers/hwtracing/coresight/ultrasoc-smb.c | 8 ++++---- include/linux/coresight.h | 5 +++++ 11 files changed, 50 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 306d2af47d63..e3a54f7038ad 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -148,12 +148,12 @@ static int etb_enable_sysfs(struct coresight_device *csdev) spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't messup with perf sessions. */ - if (local_read(&csdev->mode) == CS_MODE_PERF) { + if (coresight_get_mode(csdev) == CS_MODE_PERF) { ret = -EBUSY; goto out; } - if (local_read(&csdev->mode) == CS_MODE_DISABLED) { + if (coresight_get_mode(csdev) == CS_MODE_DISABLED) { ret = etb_enable_hw(drvdata); if (ret) goto out; @@ -179,7 +179,7 @@ static int etb_enable_perf(struct coresight_device *csdev, void *data) spin_lock_irqsave(&drvdata->spinlock, flags); /* No need to continue if the component is already in used by sysFS. */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { ret = -EBUSY; goto out; } @@ -361,7 +361,7 @@ static int etb_disable(struct coresight_device *csdev) } /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); + WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED); etb_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; @@ -588,7 +588,7 @@ static void etb_dump(struct etb_drvdata *drvdata) unsigned long flags; spin_lock_irqsave(&drvdata->spinlock, flags); - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { __etb_disable_hw(drvdata); etb_dump_hw(drvdata); __etb_enable_hw(drvdata); diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index c301f6a5f065..64ff53887ef5 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -676,7 +676,7 @@ static void etm_disable(struct coresight_device *csdev, * change its status. As such we can read the status here without * fearing it will change under us. */ - mode = local_read(&csdev->mode); + mode = coresight_get_mode(csdev); switch (mode) { case CS_MODE_DISABLED: @@ -727,7 +727,7 @@ static int etm_starting_cpu(unsigned int cpu) etmdrvdata[cpu]->os_unlock = true; } - if (local_read(&etmdrvdata[cpu]->csdev->mode)) + if (coresight_get_mode(etmdrvdata[cpu]->csdev)) etm_enable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; @@ -739,7 +739,7 @@ static int etm_dying_cpu(unsigned int cpu) return 0; spin_lock(&etmdrvdata[cpu]->spinlock); - if (local_read(&etmdrvdata[cpu]->csdev->mode)) + if (coresight_get_mode(etmdrvdata[cpu]->csdev)) etm_disable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c index 6c8429c980b1..68c644be9813 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c @@ -722,7 +722,7 @@ static ssize_t cntr_val_show(struct device *dev, struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etm_config *config = &drvdata->config; - if (!local_read(&drvdata->csdev->mode)) { + if (!coresight_get_mode(drvdata->csdev)) { spin_lock(&drvdata->spinlock); for (i = 0; i < drvdata->nr_cntr; i++) ret += sprintf(buf, "counter %d: %x\n", @@ -941,7 +941,7 @@ static ssize_t seq_curr_state_show(struct device *dev, struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etm_config *config = &drvdata->config; - if (!local_read(&drvdata->csdev->mode)) { + if (!coresight_get_mode(drvdata->csdev)) { val = config->seq_curr_state; goto out; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 572ac4a5e4ff..e65232e982a7 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1007,7 +1007,7 @@ static void etm4_disable(struct coresight_device *csdev, * change its status. As such we can read the status here without * fearing it will change under us. */ - mode = local_read(&csdev->mode); + mode = coresight_get_mode(csdev); switch (mode) { case CS_MODE_DISABLED: @@ -1659,7 +1659,7 @@ static int etm4_starting_cpu(unsigned int cpu) if (!etmdrvdata[cpu]->os_unlock) etm4_os_unlock(etmdrvdata[cpu]); - if (local_read(&etmdrvdata[cpu]->csdev->mode)) + if (coresight_get_mode(etmdrvdata[cpu]->csdev)) etm4_enable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; @@ -1671,7 +1671,7 @@ static int etm4_dying_cpu(unsigned int cpu) return 0; spin_lock(&etmdrvdata[cpu]->spinlock); - if (local_read(&etmdrvdata[cpu]->csdev->mode)) + if (coresight_get_mode(etmdrvdata[cpu]->csdev)) etm4_disable_hw(etmdrvdata[cpu]); spin_unlock(&etmdrvdata[cpu]->spinlock); return 0; @@ -1829,7 +1829,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) * Save and restore the ETM Trace registers only if * the ETM is active. */ - if (local_read(&drvdata->csdev->mode) && drvdata->save_state) + if (coresight_get_mode(drvdata->csdev) && drvdata->save_state) ret = __etm4_cpu_save(drvdata); return ret; } diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index ff578762f6bf..1d02d42bea83 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -262,7 +262,7 @@ static void stm_disable(struct coresight_device *csdev, * change its status. As such we can read the status here without * fearing it will change under us. */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { spin_lock(&drvdata->spinlock); stm_disable_hw(drvdata); spin_unlock(&drvdata->spinlock); @@ -369,7 +369,7 @@ static long stm_generic_set_options(struct stm_data *stm_data, { struct stm_drvdata *drvdata = container_of(stm_data, struct stm_drvdata, stm); - if (!(drvdata && local_read(&drvdata->csdev->mode))) + if (!(drvdata && coresight_get_mode(drvdata->csdev))) return -EINVAL; if (channel >= drvdata->numsp) @@ -404,7 +404,7 @@ static ssize_t notrace stm_generic_packet(struct stm_data *stm_data, struct stm_drvdata, stm); unsigned int stm_flags; - if (!(drvdata && local_read(&drvdata->csdev->mode))) + if (!(drvdata && coresight_get_mode(drvdata->csdev))) return -EACCES; if (channel >= drvdata->numsp) @@ -511,7 +511,7 @@ static ssize_t port_select_show(struct device *dev, struct stm_drvdata *drvdata = dev_get_drvdata(dev->parent); unsigned long val; - if (!local_read(&drvdata->csdev->mode)) { + if (!coresight_get_mode(drvdata->csdev)) { val = drvdata->stmspscr; } else { spin_lock(&drvdata->spinlock); @@ -537,7 +537,7 @@ static ssize_t port_select_store(struct device *dev, spin_lock(&drvdata->spinlock); drvdata->stmspscr = val; - if (local_read(&drvdata->csdev->mode)) { + if (coresight_get_mode(drvdata->csdev)) { CS_UNLOCK(drvdata->base); /* Process as per ARM's TRM recommendation */ stmsper = readl_relaxed(drvdata->base + STMSPER); @@ -558,7 +558,7 @@ static ssize_t port_enable_show(struct device *dev, struct stm_drvdata *drvdata = dev_get_drvdata(dev->parent); unsigned long val; - if (!local_read(&drvdata->csdev->mode)) { + if (!coresight_get_mode(drvdata->csdev)) { val = drvdata->stmsper; } else { spin_lock(&drvdata->spinlock); @@ -584,7 +584,7 @@ static ssize_t port_enable_store(struct device *dev, spin_lock(&drvdata->spinlock); drvdata->stmsper = val; - if (local_read(&drvdata->csdev->mode)) { + if (coresight_get_mode(drvdata->csdev)) { CS_UNLOCK(drvdata->base); writel_relaxed(drvdata->stmsper, drvdata->base + STMSPER); CS_LOCK(drvdata->base); diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index fa52297c73d2..f9444e2cb1d9 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -62,7 +62,7 @@ static int coresight_enable_source_sysfs(struct coresight_device *csdev, * change with coresight_mutex held, which we already have here. */ lockdep_assert_held(&coresight_mutex); - if (local_read(&csdev->mode) != CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) != CS_MODE_SYSFS) { ret = source_ops(csdev)->enable(csdev, data, mode); if (ret) return ret; @@ -87,7 +87,7 @@ static bool coresight_disable_source_sysfs(struct coresight_device *csdev, void *data) { lockdep_assert_held(&coresight_mutex); - if (local_read(&csdev->mode) != CS_MODE_SYSFS) + if (coresight_get_mode(csdev) != CS_MODE_SYSFS) return false; csdev->refcnt--; @@ -184,7 +184,7 @@ int coresight_enable_sysfs(struct coresight_device *csdev) * coresight_enable_source() so can still race with Perf mode which * doesn't hold coresight_mutex. */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { /* * There could be multiple applications driving the software * source. So keep the refcount for each such user when the @@ -338,7 +338,7 @@ static ssize_t enable_source_show(struct device *dev, guard(mutex)(&coresight_mutex); return scnprintf(buf, PAGE_SIZE, "%u\n", - local_read(&csdev->mode) == CS_MODE_SYSFS); + coresight_get_mode(csdev) == CS_MODE_SYSFS); } static ssize_t enable_source_store(struct device *dev, diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index e8b2bbed047f..72005b0c633e 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -558,7 +558,7 @@ static void tmc_shutdown(struct amba_device *adev) spin_lock_irqsave(&drvdata->spinlock, flags); - if (local_read(&drvdata->csdev->mode) == CS_MODE_DISABLED) + if (coresight_get_mode(drvdata->csdev) == CS_MODE_DISABLED) goto out; if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index f3281c958a57..77ef67c976e9 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -89,7 +89,7 @@ static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) * When operating in sysFS mode the content of the buffer needs to be * read before the TMC is disabled. */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) tmc_etb_dump_hw(drvdata); tmc_disable_hw(drvdata); @@ -205,7 +205,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) * sink is already enabled no memory is needed and the HW need not be * touched. */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { csdev->refcnt++; goto out; } @@ -262,7 +262,7 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) * No need to continue if the ETB/ETF is already operated * from sysFS. */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { ret = -EBUSY; break; } @@ -345,7 +345,7 @@ static int tmc_disable_etf_sink(struct coresight_device *csdev) } /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); + WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED); tmc_etb_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; @@ -485,7 +485,7 @@ static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, return 0; /* This shouldn't happen */ - if (WARN_ON_ONCE(local_read(&csdev->mode) != CS_MODE_PERF)) + if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF)) return 0; spin_lock_irqsave(&drvdata->spinlock, flags); @@ -631,7 +631,7 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) } /* Don't interfere if operated from Perf */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_PERF) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) { ret = -EINVAL; goto out; } @@ -643,7 +643,7 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) } /* Disable the TMC if need be */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { /* There is no point in reading a TMC in HW FIFO mode */ mode = readl_relaxed(drvdata->base + TMC_MODE); if (mode != TMC_MODE_CIRCULAR_BUFFER) { @@ -675,7 +675,7 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) spin_lock_irqsave(&drvdata->spinlock, flags); /* Re-enable the TMC if need be */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { /* There is no point in reading a TMC in HW FIFO mode */ mode = readl_relaxed(drvdata->base + TMC_MODE); if (mode != TMC_MODE_CIRCULAR_BUFFER) { diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 88a0fc375b4d..383cb8647589 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1143,7 +1143,7 @@ static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata) * When operating in sysFS mode the content of the buffer needs to be * read before the TMC is disabled. */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) tmc_etr_sync_sysfs_buf(drvdata); tmc_disable_hw(drvdata); @@ -1189,7 +1189,7 @@ static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev) spin_lock_irqsave(&drvdata->spinlock, flags); } - if (drvdata->reading || local_read(&csdev->mode) == CS_MODE_PERF) { + if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) { ret = -EBUSY; goto out; } @@ -1230,7 +1230,7 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) * sink is already enabled no memory is needed and the HW need not be * touched, even if the buffer size has changed. */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { csdev->refcnt++; goto out; } @@ -1652,7 +1652,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) spin_lock_irqsave(&drvdata->spinlock, flags); /* Don't use this sink if it is already claimed by sysFS */ - if (local_read(&csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { rc = -EBUSY; goto unlock_out; } @@ -1726,7 +1726,7 @@ static int tmc_disable_etr_sink(struct coresight_device *csdev) } /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); + WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED); tmc_etr_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; @@ -1778,7 +1778,7 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata) } /* Disable the TMC if we are trying to read from a running session. */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) __tmc_etr_disable_hw(drvdata); drvdata->reading = true; @@ -1800,7 +1800,7 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata) spin_lock_irqsave(&drvdata->spinlock, flags); /* RE-enable the TMC if need be */ - if (local_read(&drvdata->csdev->mode) == CS_MODE_SYSFS) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { /* * The trace run will continue with the same allocated trace * buffer. Since the tracer is still enabled drvdata::buf can't diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c index a92c5b750f81..903cfffe0bcd 100644 --- a/drivers/hwtracing/coresight/ultrasoc-smb.c +++ b/drivers/hwtracing/coresight/ultrasoc-smb.c @@ -207,7 +207,7 @@ static void smb_enable_sysfs(struct coresight_device *csdev) { struct smb_drv_data *drvdata = dev_get_drvdata(csdev->dev.parent); - if (local_read(&csdev->mode) != CS_MODE_DISABLED) + if (coresight_get_mode(csdev) != CS_MODE_DISABLED) return; smb_enable_hw(drvdata); @@ -253,8 +253,8 @@ static int smb_enable(struct coresight_device *csdev, enum cs_mode mode, return -EBUSY; /* Do nothing, the SMB is already enabled as other mode */ - if (local_read(&csdev->mode) != CS_MODE_DISABLED && - local_read(&csdev->mode) != mode) + if (coresight_get_mode(csdev) != CS_MODE_DISABLED && + coresight_get_mode(csdev) != mode) return -EBUSY; switch (mode) { @@ -291,7 +291,7 @@ static int smb_disable(struct coresight_device *csdev) return -EBUSY; /* Complain if we (somehow) got out of sync */ - WARN_ON_ONCE(local_read(&csdev->mode) == CS_MODE_DISABLED); + WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED); smb_disable_hw(drvdata); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 414bcbbdaf62..a49e4e20e899 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -591,6 +591,11 @@ static inline bool coresight_take_mode(struct coresight_device *csdev, CS_MODE_DISABLED; } +static inline enum cs_mode coresight_get_mode(struct coresight_device *csdev) +{ + return local_read(&csdev->mode); +} + extern struct coresight_device * coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); -- cgit v1.2.3 From bcaabb95f0c9883fb8e1112bd13eaba9cfd62c15 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 29 Jan 2024 15:40:43 +0000 Subject: coresight: Add helper for setting csdev->mode Now that mode is in struct coresight_device, sets can be wrapped. This also allows us to add a sanity check that there have been no concurrent modifications of mode. Currently all usages of local_set() were inside the device's spin locks so this new warning shouldn't be triggered. coresight_take_mode() could maybe have been used in place of adding the warning, but there may be use cases which set the mode to the same mode which are valid but would fail in coresight_take_mode() because it requires the device to only be in the disabled state. Signed-off-by: James Clark Link: https://lore.kernel.org/r/20240129154050.569566-13-james.clark@arm.com Signed-off-by: Suzuki K Poulose --- drivers/hwtracing/coresight/coresight-etb10.c | 6 +++--- drivers/hwtracing/coresight/coresight-etm3x-core.c | 4 ++-- drivers/hwtracing/coresight/coresight-etm4x-core.c | 4 ++-- drivers/hwtracing/coresight/coresight-stm.c | 2 +- drivers/hwtracing/coresight/coresight-tmc-etf.c | 10 +++++----- drivers/hwtracing/coresight/coresight-tmc-etr.c | 6 +++--- drivers/hwtracing/coresight/ultrasoc-smb.c | 6 +++--- include/linux/coresight.h | 16 ++++++++++++++++ 8 files changed, 35 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index e3a54f7038ad..3aab182b562f 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -158,7 +158,7 @@ static int etb_enable_sysfs(struct coresight_device *csdev) if (ret) goto out; - local_set(&csdev->mode, CS_MODE_SYSFS); + coresight_set_mode(csdev, CS_MODE_SYSFS); } csdev->refcnt++; @@ -214,7 +214,7 @@ static int etb_enable_perf(struct coresight_device *csdev, void *data) if (!ret) { /* Associate with monitored process. */ drvdata->pid = pid; - local_set(&drvdata->csdev->mode, CS_MODE_PERF); + coresight_set_mode(drvdata->csdev, CS_MODE_PERF); csdev->refcnt++; } @@ -365,7 +365,7 @@ static int etb_disable(struct coresight_device *csdev) etb_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); spin_unlock_irqrestore(&drvdata->spinlock, flags); dev_dbg(&csdev->dev, "ETB disabled\n"); diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 64ff53887ef5..9d5c1391ffb1 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -576,7 +576,7 @@ static int etm_enable(struct coresight_device *csdev, struct perf_event *event, /* The tracer didn't start */ if (ret) - local_set(&drvdata->csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); return ret; } @@ -693,7 +693,7 @@ static void etm_disable(struct coresight_device *csdev, } if (mode) - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); } static const struct coresight_ops_source etm_source_ops = { diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index e65232e982a7..f087bc8d1da6 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -859,7 +859,7 @@ static int etm4_enable(struct coresight_device *csdev, struct perf_event *event, /* The tracer didn't start */ if (ret) - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); return ret; } @@ -1021,7 +1021,7 @@ static void etm4_disable(struct coresight_device *csdev, } if (mode) - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); } static const struct coresight_ops_source etm4_source_ops = { diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 1d02d42bea83..974d37e5f94c 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -272,7 +272,7 @@ static void stm_disable(struct coresight_device *csdev, pm_runtime_put(csdev->dev.parent); - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); dev_dbg(&csdev->dev, "STM tracing disabled\n"); } } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 77ef67c976e9..d4f641cd9de6 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -228,7 +228,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) ret = tmc_etb_enable_hw(drvdata); if (!ret) { - local_set(&csdev->mode, CS_MODE_SYSFS); + coresight_set_mode(csdev, CS_MODE_SYSFS); csdev->refcnt++; } else { /* Free up the buffer if we failed to enable */ @@ -292,7 +292,7 @@ static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) if (!ret) { /* Associate with monitored process. */ drvdata->pid = pid; - local_set(&csdev->mode, CS_MODE_PERF); + coresight_set_mode(csdev, CS_MODE_PERF); csdev->refcnt++; } } while (0); @@ -349,7 +349,7 @@ static int tmc_disable_etf_sink(struct coresight_device *csdev) tmc_etb_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -375,7 +375,7 @@ static int tmc_enable_etf_link(struct coresight_device *csdev, if (csdev->refcnt == 0) { ret = tmc_etf_enable_hw(drvdata); if (!ret) { - local_set(&csdev->mode, CS_MODE_SYSFS); + coresight_set_mode(csdev, CS_MODE_SYSFS); first_enable = true; } } @@ -405,7 +405,7 @@ static void tmc_disable_etf_link(struct coresight_device *csdev, csdev->refcnt--; if (csdev->refcnt == 0) { tmc_etf_disable_hw(drvdata); - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); last_disable = true; } spin_unlock_irqrestore(&drvdata->spinlock, flags); diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 383cb8647589..e75428fa1592 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1237,7 +1237,7 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) ret = tmc_etr_enable_hw(drvdata, sysfs_buf); if (!ret) { - local_set(&csdev->mode, CS_MODE_SYSFS); + coresight_set_mode(csdev, CS_MODE_SYSFS); csdev->refcnt++; } @@ -1684,7 +1684,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) if (!rc) { /* Associate with monitored process. */ drvdata->pid = pid; - local_set(&csdev->mode, CS_MODE_PERF); + coresight_set_mode(csdev, CS_MODE_PERF); drvdata->perf_buf = etr_perf->etr_buf; csdev->refcnt++; } @@ -1730,7 +1730,7 @@ static int tmc_disable_etr_sink(struct coresight_device *csdev) tmc_etr_disable_hw(drvdata); /* Dissociate from monitored process. */ drvdata->pid = -1; - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); /* Reset perf specific data */ drvdata->perf_buf = NULL; diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c index 903cfffe0bcd..f9ebf20c91e6 100644 --- a/drivers/hwtracing/coresight/ultrasoc-smb.c +++ b/drivers/hwtracing/coresight/ultrasoc-smb.c @@ -211,7 +211,7 @@ static void smb_enable_sysfs(struct coresight_device *csdev) return; smb_enable_hw(drvdata); - local_set(&csdev->mode, CS_MODE_SYSFS); + coresight_set_mode(csdev, CS_MODE_SYSFS); } static int smb_enable_perf(struct coresight_device *csdev, void *data) @@ -234,7 +234,7 @@ static int smb_enable_perf(struct coresight_device *csdev, void *data) if (drvdata->pid == -1) { smb_enable_hw(drvdata); drvdata->pid = pid; - local_set(&csdev->mode, CS_MODE_PERF); + coresight_set_mode(csdev, CS_MODE_PERF); } return 0; @@ -297,7 +297,7 @@ static int smb_disable(struct coresight_device *csdev) /* Dissociate from the target process. */ drvdata->pid = -1; - local_set(&csdev->mode, CS_MODE_DISABLED); + coresight_set_mode(csdev, CS_MODE_DISABLED); dev_dbg(&csdev->dev, "Ultrasoc SMB disabled\n"); return 0; diff --git a/include/linux/coresight.h b/include/linux/coresight.h index a49e4e20e899..5f288d475490 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -596,6 +596,22 @@ static inline enum cs_mode coresight_get_mode(struct coresight_device *csdev) return local_read(&csdev->mode); } +static inline void coresight_set_mode(struct coresight_device *csdev, + enum cs_mode new_mode) +{ + enum cs_mode current_mode = coresight_get_mode(csdev); + + /* + * Changing to a new mode must be done from an already disabled state + * unless it's synchronized with coresight_take_mode(). Otherwise the + * device is already in use and signifies a locking issue. + */ + WARN(new_mode != CS_MODE_DISABLED && current_mode != CS_MODE_DISABLED && + current_mode != new_mode, "Device already in use\n"); + + local_set(&csdev->mode, new_mode); +} + extern struct coresight_device * coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); -- cgit v1.2.3 From c0ef3df8dbaef51ee4cfd58a471adf2eaee6f6b3 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 30 Jan 2024 13:28:05 +0200 Subject: PM: runtime: Simplify pm_runtime_get_if_active() usage There are two ways to opportunistically increment a device's runtime PM usage count, calling either pm_runtime_get_if_active() or pm_runtime_get_if_in_use(). The former has an argument to tell whether to ignore the usage count or not, and the latter simply calls the former with ign_usage_count set to false. The other users that want to ignore the usage_count will have to explicitly set that argument to true which is a bit cumbersome. To make this function more practical to use, remove the ign_usage_count argument from the function. The main implementation is in a static function called pm_runtime_get_conditional() and implementations of pm_runtime_get_if_active() and pm_runtime_get_if_in_use() are moved to runtime.c. Signed-off-by: Sakari Ailus Reviewed-by: Alex Elder Reviewed-by: Laurent Pinchart Acked-by: Takashi Iwai # sound/ Reviewed-by: Jacek Lawrynowicz # drivers/accel/ivpu/ Acked-by: Rodrigo Vivi # drivers/gpu/drm/i915/ Reviewed-by: Rodrigo Vivi Acked-by: Bjorn Helgaas # drivers/pci/ Signed-off-by: Rafael J. Wysocki --- Documentation/power/runtime_pm.rst | 5 ++--- drivers/accel/ivpu/ivpu_pm.c | 2 +- drivers/base/power/runtime.c | 35 +++++++++++++++++++++++++++++++-- drivers/gpu/drm/i915/intel_runtime_pm.c | 5 ++++- drivers/gpu/drm/xe/xe_pm.c | 2 +- drivers/media/i2c/ccs/ccs-core.c | 2 +- drivers/media/i2c/ov64a40.c | 2 +- drivers/media/i2c/thp7312.c | 2 +- drivers/net/ipa/ipa_smp2p.c | 2 +- drivers/pci/pci.c | 2 +- include/linux/pm_runtime.h | 18 +++-------------- sound/hda/hdac_device.c | 2 +- 12 files changed, 50 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index 65b86e487afe..da99379071a4 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -396,10 +396,9 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: nonzero, increment the counter and return 1; otherwise return 0 without changing the counter - `int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count);` + `int pm_runtime_get_if_active(struct device *dev);` - return -EINVAL if 'power.disable_depth' is nonzero; otherwise, if the - runtime PM status is RPM_ACTIVE, and either ign_usage_count is true - or the device's usage_count is non-zero, increment the counter and + runtime PM status is RPM_ACTIVE, increment the counter and return 1; otherwise return 0 without changing the counter `void pm_runtime_put_noidle(struct device *dev);` diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c index f501f27ebafd..981777c93cf5 100644 --- a/drivers/accel/ivpu/ivpu_pm.c +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -304,7 +304,7 @@ int ivpu_rpm_get_if_active(struct ivpu_device *vdev) { int ret; - ret = pm_runtime_get_if_active(vdev->drm.dev, false); + ret = pm_runtime_get_if_in_use(vdev->drm.dev); drm_WARN_ON(&vdev->drm, ret < 0); return ret; diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 05793c9fbb84..5275a6b2e980 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1176,7 +1176,7 @@ int __pm_runtime_resume(struct device *dev, int rpmflags) EXPORT_SYMBOL_GPL(__pm_runtime_resume); /** - * pm_runtime_get_if_active - Conditionally bump up device usage counter. + * pm_runtime_get_conditional - Conditionally bump up device usage counter. * @dev: Device to handle. * @ign_usage_count: Whether or not to look at the current usage counter value. * @@ -1197,7 +1197,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_resume); * The caller is responsible for decrementing the runtime PM usage counter of * @dev after this function has returned a positive value for it. */ -int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) +static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count) { unsigned long flags; int retval; @@ -1218,8 +1218,39 @@ int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) return retval; } + +/** + * pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is + * in active state + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE, in which case it returns 1. If the device is in a different + * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the + * device, in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_active(struct device *dev) +{ + return pm_runtime_get_conditional(dev, true); +} EXPORT_SYMBOL_GPL(pm_runtime_get_if_active); +/** + * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE and its runtime PM usage counter is greater than 0, in which case + * it returns 1. If the device is in a different state or its usage_count is 0, + * 0 is returned. -EINVAL is returned if runtime PM is disabled for the device, + * in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_in_use(struct device *dev) +{ + return pm_runtime_get_conditional(dev, false); +} +EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use); + /** * __pm_runtime_set_status - Set runtime PM status of a device. * @dev: Device to handle. diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 860b51b56a92..d4e844128826 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -246,7 +246,10 @@ static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm * function, since the power state is undefined. This applies * atm to the late/early system suspend/resume handlers. */ - if (pm_runtime_get_if_active(rpm->kdev, ignore_usecount) <= 0) + if ((ignore_usecount && + pm_runtime_get_if_active(rpm->kdev) <= 0) || + (!ignore_usecount && + pm_runtime_get_if_in_use(rpm->kdev) <= 0)) return 0; } diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index b429c2876a76..dd110058bf74 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -330,7 +330,7 @@ int xe_pm_runtime_put(struct xe_device *xe) int xe_pm_runtime_get_if_active(struct xe_device *xe) { - return pm_runtime_get_if_active(xe->drm.dev, true); + return pm_runtime_get_if_active(xe->drm.dev); } void xe_pm_assert_unbounded_bridge(struct xe_device *xe) diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index e21287d50c15..e1ae0f9fad43 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -674,7 +674,7 @@ static int ccs_set_ctrl(struct v4l2_ctrl *ctrl) break; } - pm_status = pm_runtime_get_if_active(&client->dev, true); + pm_status = pm_runtime_get_if_active(&client->dev); if (!pm_status) return 0; diff --git a/drivers/media/i2c/ov64a40.c b/drivers/media/i2c/ov64a40.c index 4fba4c2cb064..541bf74581d2 100644 --- a/drivers/media/i2c/ov64a40.c +++ b/drivers/media/i2c/ov64a40.c @@ -3287,7 +3287,7 @@ static int ov64a40_set_ctrl(struct v4l2_ctrl *ctrl) exp_max, 1, exp_val); } - pm_status = pm_runtime_get_if_active(ov64a40->dev, true); + pm_status = pm_runtime_get_if_active(ov64a40->dev); if (!pm_status) return 0; diff --git a/drivers/media/i2c/thp7312.c b/drivers/media/i2c/thp7312.c index 2806887514dc..19bd923a7315 100644 --- a/drivers/media/i2c/thp7312.c +++ b/drivers/media/i2c/thp7312.c @@ -1052,7 +1052,7 @@ static int thp7312_s_ctrl(struct v4l2_ctrl *ctrl) if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE) return -EINVAL; - if (!pm_runtime_get_if_active(thp7312->dev, true)) + if (!pm_runtime_get_if_active(thp7312->dev)) return 0; switch (ctrl->id) { diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c index 5620dc271fac..cbf3d4761ce3 100644 --- a/drivers/net/ipa/ipa_smp2p.c +++ b/drivers/net/ipa/ipa_smp2p.c @@ -92,7 +92,7 @@ static void ipa_smp2p_notify(struct ipa_smp2p *smp2p) return; dev = &smp2p->ipa->pdev->dev; - smp2p->power_on = pm_runtime_get_if_active(dev, true) > 0; + smp2p->power_on = pm_runtime_get_if_active(dev) > 0; /* Signal whether the IPA power is enabled */ mask = BIT(smp2p->enabled_bit); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 9ab9b1008d8b..cb51c4079013 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2536,7 +2536,7 @@ static void pci_pme_list_scan(struct work_struct *work) * If the device is in a low power state it * should not be polled either. */ - pm_status = pm_runtime_get_if_active(dev, true); + pm_status = pm_runtime_get_if_active(dev); if (!pm_status) continue; diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 7c9b35448563..436baa167498 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -72,7 +72,8 @@ extern int pm_runtime_force_resume(struct device *dev); extern int __pm_runtime_idle(struct device *dev, int rpmflags); extern int __pm_runtime_suspend(struct device *dev, int rpmflags); extern int __pm_runtime_resume(struct device *dev, int rpmflags); -extern int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count); +extern int pm_runtime_get_if_active(struct device *dev); +extern int pm_runtime_get_if_in_use(struct device *dev); extern int pm_schedule_suspend(struct device *dev, unsigned int delay); extern int __pm_runtime_set_status(struct device *dev, unsigned int status); extern int pm_runtime_barrier(struct device *dev); @@ -94,18 +95,6 @@ extern void pm_runtime_release_supplier(struct device_link *link); extern int devm_pm_runtime_enable(struct device *dev); -/** - * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. - * @dev: Target device. - * - * Increment the runtime PM usage counter of @dev if its runtime PM status is - * %RPM_ACTIVE and its runtime PM usage counter is greater than 0. - */ -static inline int pm_runtime_get_if_in_use(struct device *dev) -{ - return pm_runtime_get_if_active(dev, false); -} - /** * pm_suspend_ignore_children - Set runtime PM behavior regarding children. * @dev: Target device. @@ -275,8 +264,7 @@ static inline int pm_runtime_get_if_in_use(struct device *dev) { return -EINVAL; } -static inline int pm_runtime_get_if_active(struct device *dev, - bool ign_usage_count) +static inline int pm_runtime_get_if_active(struct device *dev) { return -EINVAL; } diff --git a/sound/hda/hdac_device.c b/sound/hda/hdac_device.c index 7f7b67fe1b65..068c16e52dff 100644 --- a/sound/hda/hdac_device.c +++ b/sound/hda/hdac_device.c @@ -612,7 +612,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_power_up_pm); int snd_hdac_keep_power_up(struct hdac_device *codec) { if (!atomic_inc_not_zero(&codec->in_pm)) { - int ret = pm_runtime_get_if_active(&codec->dev, true); + int ret = pm_runtime_get_if_active(&codec->dev); if (!ret) return -1; if (ret < 0) -- cgit v1.2.3 From b7d46644e554ed017dfabc0841acf418d0584bc9 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 30 Jan 2024 13:28:32 +0200 Subject: PM: runtime: Add pm_runtime_put_autosuspend() replacement Add __pm_runtime_put_autosuspend() that replaces pm_runtime_put_autosuspend() for new users. The intent is to later re-purpose pm_runtime_put_autosuspend() to also mark the device's last busy stamp---which is what the vast majority of users actually need. This is also described in pm_runtime_put_autosuspend() documentation. Signed-off-by: Sakari Ailus Reviewed-by: Laurent Pinchart Signed-off-by: Rafael J. Wysocki --- Documentation/power/runtime_pm.rst | 17 +++++++++++------ include/linux/pm_runtime.h | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index da99379071a4..6fa50e4f87ce 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -154,7 +154,7 @@ suspending the device are satisfied) and to queue up a suspend request for the device in that case. If there is no idle callback, or if the callback returns 0, then the PM core will attempt to carry out a runtime suspend of the device, also respecting devices configured for autosuspend. In essence this means a -call to pm_runtime_autosuspend() (do note that drivers needs to update the +call to __pm_runtime_autosuspend() (do note that drivers needs to update the device last busy mark, pm_runtime_mark_last_busy(), to control the delay under this circumstance). To prevent this (for example, if the callback routine has started a delayed suspend), the routine must return a non-zero value. Negative @@ -409,6 +409,10 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: pm_request_idle(dev) and return its result `int pm_runtime_put_autosuspend(struct device *dev);` + - does the same as __pm_runtime_put_autosuspend() for now, but in the + future, will also call pm_runtime_mark_last_busy() as well, DO NOT USE! + + `int __pm_runtime_put_autosuspend(struct device *dev);` - decrement the device's usage counter; if the result is 0 then run pm_request_autosuspend(dev) and return its result @@ -539,6 +543,7 @@ It is safe to execute the following helper functions from interrupt context: - pm_runtime_put_noidle() - pm_runtime_put() - pm_runtime_put_autosuspend() +- __pm_runtime_put_autosuspend() - pm_runtime_enable() - pm_suspend_ignore_children() - pm_runtime_set_active() @@ -864,9 +869,9 @@ automatically be delayed until the desired period of inactivity has elapsed. Inactivity is determined based on the power.last_busy field. Drivers should call pm_runtime_mark_last_busy() to update this field after carrying out I/O, -typically just before calling pm_runtime_put_autosuspend(). The desired length -of the inactivity period is a matter of policy. Subsystems can set this length -initially by calling pm_runtime_set_autosuspend_delay(), but after device +typically just before calling __pm_runtime_put_autosuspend(). The desired +length of the inactivity period is a matter of policy. Subsystems can set this +length initially by calling pm_runtime_set_autosuspend_delay(), but after device registration the length should be controlled by user space, using the /sys/devices/.../power/autosuspend_delay_ms attribute. @@ -877,7 +882,7 @@ instead of the non-autosuspend counterparts:: Instead of: pm_runtime_suspend use: pm_runtime_autosuspend; Instead of: pm_schedule_suspend use: pm_request_autosuspend; - Instead of: pm_runtime_put use: pm_runtime_put_autosuspend; + Instead of: pm_runtime_put use: __pm_runtime_put_autosuspend; Instead of: pm_runtime_put_sync use: pm_runtime_put_sync_autosuspend. Drivers may also continue to use the non-autosuspend helper functions; they @@ -916,7 +921,7 @@ Here is a schematic pseudo-code example:: lock(&foo->private_lock); if (--foo->num_pending_requests == 0) { pm_runtime_mark_last_busy(&foo->dev); - pm_runtime_put_autosuspend(&foo->dev); + __pm_runtime_put_autosuspend(&foo->dev); } else { foo_process_next_request(foo); } diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 436baa167498..d39dc863f612 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -448,6 +448,18 @@ static inline int pm_runtime_put(struct device *dev) return __pm_runtime_idle(dev, RPM_GET_PUT | RPM_ASYNC); } +/** + * __pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0. + * @dev: Target device. + * + * Decrement the runtime PM usage counter of @dev and if it turns out to be + * equal to 0, queue up a work item for @dev like in pm_request_autosuspend(). + */ +static inline int __pm_runtime_put_autosuspend(struct device *dev) +{ + return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_ASYNC | RPM_AUTO); +} + /** * pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0. * @dev: Target device. -- cgit v1.2.3 From 7251b9e8a007ddd834aa81f8c7ea338884629fec Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Tue, 6 Feb 2024 09:54:09 +0800 Subject: thermal/intel: Fix intel_tcc_get_temp() to support negative CPU temperature CPU temperature can be negative in some cases. Thus the negative CPU temperature should not be considered as a failure. Fix intel_tcc_get_temp() and its users to support negative CPU temperature. Fixes: a3c1f066e1c5 ("thermal/intel: Introduce Intel TCC library") Signed-off-by: Zhang Rui Reviewed-by: Stanislaw Gruszka Cc: 6.3+ # 6.3+ Signed-off-by: Rafael J. Wysocki --- .../thermal/intel/int340x_thermal/processor_thermal_device.c | 8 ++++---- drivers/thermal/intel/intel_tcc.c | 12 ++++++------ drivers/thermal/intel/x86_pkg_temp_thermal.c | 8 ++++---- include/linux/intel_tcc.h | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index 649f67fdf345..d75fae7b7ed2 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -176,14 +176,14 @@ static int proc_thermal_get_zone_temp(struct thermal_zone_device *zone, int *temp) { int cpu; - int curr_temp; + int curr_temp, ret; *temp = 0; for_each_online_cpu(cpu) { - curr_temp = intel_tcc_get_temp(cpu, false); - if (curr_temp < 0) - return curr_temp; + ret = intel_tcc_get_temp(cpu, &curr_temp, false); + if (ret < 0) + return ret; if (!*temp || curr_temp > *temp) *temp = curr_temp; } diff --git a/drivers/thermal/intel/intel_tcc.c b/drivers/thermal/intel/intel_tcc.c index 2e5c741c41ca..5e8b7f34b395 100644 --- a/drivers/thermal/intel/intel_tcc.c +++ b/drivers/thermal/intel/intel_tcc.c @@ -103,18 +103,19 @@ EXPORT_SYMBOL_NS_GPL(intel_tcc_set_offset, INTEL_TCC); /** * intel_tcc_get_temp() - returns the current temperature * @cpu: cpu that the MSR should be run on, nagative value means any cpu. + * @temp: pointer to the memory for saving cpu temperature. * @pkg: true: Package Thermal Sensor. false: Core Thermal Sensor. * * Get the current temperature returned by the CPU core/package level * thermal sensor, in degrees C. * - * Return: Temperature in degrees C on success, negative error code otherwise. + * Return: 0 on success, negative error code otherwise. */ -int intel_tcc_get_temp(int cpu, bool pkg) +int intel_tcc_get_temp(int cpu, int *temp, bool pkg) { u32 low, high; u32 msr = pkg ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; - int tjmax, temp, err; + int tjmax, err; tjmax = intel_tcc_get_tjmax(cpu); if (tjmax < 0) @@ -131,9 +132,8 @@ int intel_tcc_get_temp(int cpu, bool pkg) if (!(low & BIT(31))) return -ENODATA; - temp = tjmax - ((low >> 16) & 0x7f); + *temp = tjmax - ((low >> 16) & 0x7f); - /* Do not allow negative CPU temperature */ - return temp >= 0 ? temp : -ENODATA; + return 0; } EXPORT_SYMBOL_NS_GPL(intel_tcc_get_temp, INTEL_TCC); diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 11a7f8108bbb..61c3d450ee60 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -108,11 +108,11 @@ static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu) static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp) { struct zone_device *zonedev = thermal_zone_device_priv(tzd); - int val; + int val, ret; - val = intel_tcc_get_temp(zonedev->cpu, true); - if (val < 0) - return val; + ret = intel_tcc_get_temp(zonedev->cpu, &val, true); + if (ret < 0) + return ret; *temp = val * 1000; pr_debug("sys_get_curr_temp %d\n", *temp); diff --git a/include/linux/intel_tcc.h b/include/linux/intel_tcc.h index f422612c28d6..8ff8eabb4a98 100644 --- a/include/linux/intel_tcc.h +++ b/include/linux/intel_tcc.h @@ -13,6 +13,6 @@ int intel_tcc_get_tjmax(int cpu); int intel_tcc_get_offset(int cpu); int intel_tcc_set_offset(int cpu, int offset); -int intel_tcc_get_temp(int cpu, bool pkg); +int intel_tcc_get_temp(int cpu, int *temp, bool pkg); #endif /* __INTEL_TCC_H__ */ -- cgit v1.2.3 From ed94a8f8ca75ea0f607c919edf2ed5a5e707ef44 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Thu, 1 Feb 2024 12:53:16 +0100 Subject: HID: protect hid_device::bpf by CONFIG_HID_BPF And not by CONFIG_BPF. BPF can be selected while HID_BPF does not have to. It actually cannot be on some platforms due to Kconfig dependences. This saves quite some bytes on those setups. Found by https://github.com/jirislaby/clang-struct. Signed-off-by: Jiri Slaby (SUSE) Cc: Jiri Kosina Cc: Benjamin Tissoires Signed-off-by: Jiri Kosina --- include/linux/hid.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hid.h b/include/linux/hid.h index 7c26db874ff0..b12cb1c8e682 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -683,9 +683,9 @@ struct hid_device { /* device report descriptor */ unsigned int id; /* system unique id */ -#ifdef CONFIG_BPF +#ifdef CONFIG_HID_BPF struct hid_bpf bpf; /* hid-bpf data */ -#endif /* CONFIG_BPF */ +#endif /* CONFIG_HID_BPF */ }; void hiddev_free(struct kref *ref); -- cgit v1.2.3 From 1aa09b9379a7a644cd2f75ae0bac82b8783df600 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 31 Jan 2024 19:37:09 +0800 Subject: powercap: intel_rapl: Fix locking in TPMI RAPL The RAPL framework uses CPU hotplug locking to protect the rapl_packages list and rp->lead_cpu to guarantee that 1. the RAPL package device is not unprobed and freed 2. the cached rp->lead_cpu is always valid for operations like powercap sysfs accesses. Current RAPL APIs assume being called from CPU hotplug callbacks which hold the CPU hotplug lock, but TPMI RAPL driver invokes the APIs in the driver's .probe() function without acquiring the CPU hotplug lock. Fix the problem by providing both locked and lockless versions of RAPL APIs. Fixes: 9eef7f9da928 ("powercap: intel_rapl: Introduce RAPL TPMI interface driver") Signed-off-by: Zhang Rui Cc: 6.5+ # 6.5+ Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 29 +++++++++++++++++++--- drivers/powercap/intel_rapl_msr.c | 8 +++--- .../intel/int340x_thermal/processor_thermal_rapl.c | 8 +++--- include/linux/intel_rapl.h | 6 +++++ 4 files changed, 40 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 1a739afd47d9..9d3e102f1a76 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -5,6 +5,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -1504,7 +1505,7 @@ static int rapl_detect_domains(struct rapl_package *rp) } /* called from CPU hotplug notifier, hotplug lock held */ -void rapl_remove_package(struct rapl_package *rp) +void rapl_remove_package_cpuslocked(struct rapl_package *rp) { struct rapl_domain *rd, *rd_package = NULL; @@ -1533,10 +1534,18 @@ void rapl_remove_package(struct rapl_package *rp) list_del(&rp->plist); kfree(rp); } +EXPORT_SYMBOL_GPL(rapl_remove_package_cpuslocked); + +void rapl_remove_package(struct rapl_package *rp) +{ + guard(cpus_read_lock)(); + rapl_remove_package_cpuslocked(rp); +} EXPORT_SYMBOL_GPL(rapl_remove_package); /* caller to ensure CPU hotplug lock is held */ -struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) +struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu) { struct rapl_package *rp; int uid; @@ -1554,10 +1563,17 @@ struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, return NULL; } +EXPORT_SYMBOL_GPL(rapl_find_package_domain_cpuslocked); + +struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) +{ + guard(cpus_read_lock)(); + return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu); +} EXPORT_SYMBOL_GPL(rapl_find_package_domain); /* called from CPU hotplug notifier, hotplug lock held */ -struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) +struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu) { struct rapl_package *rp; int ret; @@ -1603,6 +1619,13 @@ err_free_package: kfree(rp); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(rapl_add_package_cpuslocked); + +struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) +{ + guard(cpus_read_lock)(); + return rapl_add_package_cpuslocked(id, priv, id_is_cpu); +} EXPORT_SYMBOL_GPL(rapl_add_package); static void power_limit_state_save(void) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 250bd41a588c..b4b6930cacb0 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -73,9 +73,9 @@ static int rapl_cpu_online(unsigned int cpu) { struct rapl_package *rp; - rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, rapl_msr_priv, true); if (!rp) { - rp = rapl_add_package(cpu, rapl_msr_priv, true); + rp = rapl_add_package_cpuslocked(cpu, rapl_msr_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -88,14 +88,14 @@ static int rapl_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, rapl_msr_priv, true); if (!rp) return 0; cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); + rapl_remove_package_cpuslocked(rp); else if (rp->lead_cpu == cpu) rp->lead_cpu = lead_cpu; return 0; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index 2f00fc3bf274..e964a9375722 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -27,9 +27,9 @@ static int rapl_mmio_cpu_online(unsigned int cpu) if (topology_physical_package_id(cpu)) return 0; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, &rapl_mmio_priv, true); if (!rp) { - rp = rapl_add_package(cpu, &rapl_mmio_priv, true); + rp = rapl_add_package_cpuslocked(cpu, &rapl_mmio_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -42,14 +42,14 @@ static int rapl_mmio_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, &rapl_mmio_priv, true); if (!rp) return 0; cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); + rapl_remove_package_cpuslocked(rp); else if (rp->lead_cpu == cpu) rp->lead_cpu = lead_cpu; return 0; diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 33f21bd85dbf..f3196f82fd8a 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -178,6 +178,12 @@ struct rapl_package { struct rapl_if_priv *priv; }; +struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu); +struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu); +void rapl_remove_package_cpuslocked(struct rapl_package *rp); + struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu); struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu); void rapl_remove_package(struct rapl_package *rp); -- cgit v1.2.3 From 178c54666f9c4d2f49f2ea661d0c11b52f0ed190 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 6 Feb 2024 23:01:02 -0800 Subject: bpf: Mark bpf_spin_{lock,unlock}() helpers with notrace correctly Currently tracing is supposed not to allow for bpf_spin_{lock,unlock}() helper calls. This is to prevent deadlock for the following cases: - there is a prog (prog-A) calling bpf_spin_{lock,unlock}(). - there is a tracing program (prog-B), e.g., fentry, attached to bpf_spin_lock() and/or bpf_spin_unlock(). - prog-B calls bpf_spin_{lock,unlock}(). For such a case, when prog-A calls bpf_spin_{lock,unlock}(), a deadlock will happen. The related source codes are below in kernel/bpf/helpers.c: notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) notrace is supposed to prevent fentry prog from attaching to bpf_spin_{lock,unlock}(). But actually this is not the case and fentry prog can successfully attached to bpf_spin_lock(). Siddharth Chintamaneni reported the issue in [1]. The following is the macro definition for above BPF_CALL_1: #define BPF_CALL_x(x, name, ...) \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ { \ return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ } \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) #define BPF_CALL_1(name, ...) BPF_CALL_x(1, name, __VA_ARGS__) The notrace attribute is actually applied to the static always_inline function ____bpf_spin_{lock,unlock}(). The actual callback function bpf_spin_{lock,unlock}() is not marked with notrace, hence allowing fentry prog to attach to two helpers, and this may cause the above mentioned deadlock. Siddharth Chintamaneni actually has a reproducer in [2]. To fix the issue, a new macro NOTRACE_BPF_CALL_1 is introduced which will add notrace attribute to the original function instead of the hidden always_inline function and this fixed the problem. [1] https://lore.kernel.org/bpf/CAE5sdEigPnoGrzN8WU7Tx-h-iFuMZgW06qp0KHWtpvoXxf1OAQ@mail.gmail.com/ [2] https://lore.kernel.org/bpf/CAE5sdEg6yUc_Jz50AnUXEEUh6O73yQ1Z6NV2srJnef0ZrQkZew@mail.gmail.com/ Fixes: d83525ca62cf ("bpf: introduce bpf_spin_lock") Signed-off-by: Yonghong Song Signed-off-by: Andrii Nakryiko Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20240207070102.335167-1-yonghong.song@linux.dev --- include/linux/filter.h | 21 ++++++++++++--------- kernel/bpf/helpers.c | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index fee070b9826e..36cc29a2934c 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -547,24 +547,27 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) __BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2, \ u64, __ur_3, u64, __ur_4, u64, __ur_5) -#define BPF_CALL_x(x, name, ...) \ +#define BPF_CALL_x(x, attr, name, ...) \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ - u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ - u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ + attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ + attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ { \ return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ } \ static __always_inline \ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) -#define BPF_CALL_0(name, ...) BPF_CALL_x(0, name, __VA_ARGS__) -#define BPF_CALL_1(name, ...) BPF_CALL_x(1, name, __VA_ARGS__) -#define BPF_CALL_2(name, ...) BPF_CALL_x(2, name, __VA_ARGS__) -#define BPF_CALL_3(name, ...) BPF_CALL_x(3, name, __VA_ARGS__) -#define BPF_CALL_4(name, ...) BPF_CALL_x(4, name, __VA_ARGS__) -#define BPF_CALL_5(name, ...) BPF_CALL_x(5, name, __VA_ARGS__) +#define __NOATTR +#define BPF_CALL_0(name, ...) BPF_CALL_x(0, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_1(name, ...) BPF_CALL_x(1, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_2(name, ...) BPF_CALL_x(2, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_3(name, ...) BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_4(name, ...) BPF_CALL_x(4, __NOATTR, name, __VA_ARGS__) +#define BPF_CALL_5(name, ...) BPF_CALL_x(5, __NOATTR, name, __VA_ARGS__) + +#define NOTRACE_BPF_CALL_1(name, ...) BPF_CALL_x(1, notrace, name, __VA_ARGS__) #define bpf_ctx_range(TYPE, MEMBER) \ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 3503949b4c1b..93edf730d288 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -334,7 +334,7 @@ static inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock) __this_cpu_write(irqsave_flags, flags); } -notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) +NOTRACE_BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) { __bpf_spin_lock_irqsave(lock); return 0; @@ -357,7 +357,7 @@ static inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock) local_irq_restore(flags); } -notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) +NOTRACE_BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) { __bpf_spin_unlock_irqrestore(lock); return 0; -- cgit v1.2.3 From 77c0208e199ccb0986fb3612f2409c8cdcb036ad Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Thu, 8 Feb 2024 18:37:47 -0800 Subject: bpf: add btf pointer to struct bpf_ctx_arg_aux. Enable the providers to use types defined in a module instead of in the kernel (btf_vmlinux). Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240209023750.1153905-2-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 1 + kernel/bpf/btf.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1ebbee1d648e..3b7836f0a83e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1415,6 +1415,7 @@ struct bpf_jit_poke_descriptor { struct bpf_ctx_arg_aux { u32 offset; enum bpf_reg_type reg_type; + struct btf *btf; u32 btf_id; }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 8e06d29961f1..cf100b5573ca 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6266,7 +6266,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } info->reg_type = ctx_arg_info->reg_type; - info->btf = btf_vmlinux; + info->btf = ctx_arg_info->btf ? : btf_vmlinux; info->btf_id = ctx_arg_info->btf_id; return true; } -- cgit v1.2.3 From 6115a0aeef01aef152ad7738393aad11422bfb82 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Thu, 8 Feb 2024 18:37:48 -0800 Subject: bpf: Move __kfunc_param_match_suffix() to btf.c. Move __kfunc_param_match_suffix() to btf.c and rename it as btf_param_match_suffix(). It can be reused by bpf_struct_ops later. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240209023750.1153905-3-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/btf.h | 4 ++++ kernel/bpf/btf.c | 18 ++++++++++++++++++ kernel/bpf/verifier.c | 38 ++++++++++---------------------------- 3 files changed, 32 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 1ee8977b8c95..df76a14c64f6 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -495,6 +495,10 @@ static inline void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id) return bsearch(&id, set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func); } +bool btf_param_match_suffix(const struct btf *btf, + const struct btf_param *arg, + const char *suffix); + struct bpf_verifier_log; #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index cf100b5573ca..447da964f217 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8904,3 +8904,21 @@ errout: } EXPORT_SYMBOL_GPL(__register_bpf_struct_ops); #endif + +bool btf_param_match_suffix(const struct btf *btf, + const struct btf_param *arg, + const char *suffix) +{ + int suffix_len = strlen(suffix), len; + const char *param_name; + + /* In the future, this can be ported to use BTF tagging */ + param_name = btf_name_by_offset(btf, arg->name_off); + if (str_is_empty(param_name)) + return false; + len = strlen(param_name); + if (len <= suffix_len) + return false; + param_name += len - suffix_len; + return !strncmp(param_name, suffix, suffix_len); +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ddaf09db1175..c92d6af7d975 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10682,24 +10682,6 @@ static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta) return meta->kfunc_flags & KF_RCU_PROTECTED; } -static bool __kfunc_param_match_suffix(const struct btf *btf, - const struct btf_param *arg, - const char *suffix) -{ - int suffix_len = strlen(suffix), len; - const char *param_name; - - /* In the future, this can be ported to use BTF tagging */ - param_name = btf_name_by_offset(btf, arg->name_off); - if (str_is_empty(param_name)) - return false; - len = strlen(param_name); - if (len < suffix_len) - return false; - param_name += len - suffix_len; - return !strncmp(param_name, suffix, suffix_len); -} - static bool is_kfunc_arg_mem_size(const struct btf *btf, const struct btf_param *arg, const struct bpf_reg_state *reg) @@ -10710,7 +10692,7 @@ static bool is_kfunc_arg_mem_size(const struct btf *btf, if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) return false; - return __kfunc_param_match_suffix(btf, arg, "__sz"); + return btf_param_match_suffix(btf, arg, "__sz"); } static bool is_kfunc_arg_const_mem_size(const struct btf *btf, @@ -10723,47 +10705,47 @@ static bool is_kfunc_arg_const_mem_size(const struct btf *btf, if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) return false; - return __kfunc_param_match_suffix(btf, arg, "__szk"); + return btf_param_match_suffix(btf, arg, "__szk"); } static bool is_kfunc_arg_optional(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__opt"); + return btf_param_match_suffix(btf, arg, "__opt"); } static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__k"); + return btf_param_match_suffix(btf, arg, "__k"); } static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__ign"); + return btf_param_match_suffix(btf, arg, "__ign"); } static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__alloc"); + return btf_param_match_suffix(btf, arg, "__alloc"); } static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__uninit"); + return btf_param_match_suffix(btf, arg, "__uninit"); } static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__refcounted_kptr"); + return btf_param_match_suffix(btf, arg, "__refcounted_kptr"); } static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__nullable"); + return btf_param_match_suffix(btf, arg, "__nullable"); } static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg) { - return __kfunc_param_match_suffix(btf, arg, "__str"); + return btf_param_match_suffix(btf, arg, "__str"); } static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, -- cgit v1.2.3 From 1611603537a4b88cec7993f32b70c03113801a46 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Thu, 8 Feb 2024 18:37:49 -0800 Subject: bpf: Create argument information for nullable arguments. Collect argument information from the type information of stub functions to mark arguments of BPF struct_ops programs with PTR_MAYBE_NULL if they are nullable. A nullable argument is annotated by suffixing "__nullable" at the argument name of stub function. For nullable arguments, this patch sets a struct bpf_ctx_arg_aux to label their reg_type with PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL. This makes the verifier to check programs and ensure that they properly check the pointer. The programs should check if the pointer is null before accessing the pointed memory. The implementer of a struct_ops type should annotate the arguments that can be null. The implementer should define a stub function (empty) as a placeholder for each defined operator. The name of a stub function should be in the pattern "__". For example, for test_maybe_null of struct bpf_testmod_ops, it's stub function name should be "bpf_testmod_ops__test_maybe_null". You mark an argument nullable by suffixing the argument name with "__nullable" at the stub function. Since we already has stub functions for kCFI, we just reuse these stub functions with the naming convention mentioned earlier. These stub functions with the naming convention is only required if there are nullable arguments to annotate. For functions having not nullable arguments, stub functions are not necessary for the purpose of this patch. This patch will prepare a list of struct bpf_ctx_arg_aux, aka arg_info, for each member field of a struct_ops type. "arg_info" will be assigned to "prog->aux->ctx_arg_info" of BPF struct_ops programs in check_struct_ops_btf_id() so that it can be used by btf_ctx_access() later to set reg_type properly for the verifier. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240209023750.1153905-4-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 21 +++++ include/linux/btf.h | 2 + kernel/bpf/bpf_struct_ops.c | 213 +++++++++++++++++++++++++++++++++++++++++--- kernel/bpf/btf.c | 27 ++++++ kernel/bpf/verifier.c | 6 ++ 5 files changed, 257 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 3b7836f0a83e..c7aa99b44dbd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1709,6 +1709,19 @@ struct bpf_struct_ops { struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; }; +/* Every member of a struct_ops type has an instance even a member is not + * an operator (function pointer). The "info" field will be assigned to + * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the + * argument information required by the verifier to verify the program. + * + * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the + * corresponding entry for an given argument. + */ +struct bpf_struct_ops_arg_info { + struct bpf_ctx_arg_aux *info; + u32 cnt; +}; + struct bpf_struct_ops_desc { struct bpf_struct_ops *st_ops; @@ -1716,6 +1729,9 @@ struct bpf_struct_ops_desc { const struct btf_type *value_type; u32 type_id; u32 value_id; + + /* Collection of argument information for each member */ + struct bpf_struct_ops_arg_info *arg_info; }; enum bpf_struct_ops_state { @@ -1790,6 +1806,7 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, struct btf *btf, struct bpf_verifier_log *log); void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map); +void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc); #else #define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; }) static inline bool bpf_try_module_get(const void *data, struct module *owner) @@ -1814,6 +1831,10 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc { } +static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) +{ +} + #endif #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) diff --git a/include/linux/btf.h b/include/linux/btf.h index df76a14c64f6..cb96f6263638 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -498,6 +498,8 @@ static inline void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id) bool btf_param_match_suffix(const struct btf *btf, const struct btf_param *arg, const char *suffix); +int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto, + u32 arg_no); struct bpf_verifier_log; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index f98f580de77a..0d7be97a2411 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -116,17 +116,183 @@ static bool is_valid_value_type(struct btf *btf, s32 value_id, return true; } +#define MAYBE_NULL_SUFFIX "__nullable" +#define MAX_STUB_NAME 128 + +/* Return the type info of a stub function, if it exists. + * + * The name of a stub function is made up of the name of the struct_ops and + * the name of the function pointer member, separated by "__". For example, + * if the struct_ops type is named "foo_ops" and the function pointer + * member is named "bar", the stub function name would be "foo_ops__bar". + */ +static const struct btf_type * +find_stub_func_proto(const struct btf *btf, const char *st_op_name, + const char *member_name) +{ + char stub_func_name[MAX_STUB_NAME]; + const struct btf_type *func_type; + s32 btf_id; + int cp; + + cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s", + st_op_name, member_name); + if (cp >= MAX_STUB_NAME) { + pr_warn("Stub function name too long\n"); + return NULL; + } + btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC); + if (btf_id < 0) + return NULL; + func_type = btf_type_by_id(btf, btf_id); + if (!func_type) + return NULL; + + return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */ +} + +/* Prepare argument info for every nullable argument of a member of a + * struct_ops type. + * + * Initialize a struct bpf_struct_ops_arg_info according to type info of + * the arguments of a stub function. (Check kCFI for more information about + * stub functions.) + * + * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info + * to provide an array of struct bpf_ctx_arg_aux, which in turn provides + * the information that used by the verifier to check the arguments of the + * BPF struct_ops program assigned to the member. Here, we only care about + * the arguments that are marked as __nullable. + * + * The array of struct bpf_ctx_arg_aux is eventually assigned to + * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the + * verifier. (See check_struct_ops_btf_id()) + * + * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If + * fails, it will be kept untouched. + */ +static int prepare_arg_info(struct btf *btf, + const char *st_ops_name, + const char *member_name, + const struct btf_type *func_proto, + struct bpf_struct_ops_arg_info *arg_info) +{ + const struct btf_type *stub_func_proto, *pointed_type; + const struct btf_param *stub_args, *args; + struct bpf_ctx_arg_aux *info, *info_buf; + u32 nargs, arg_no, info_cnt = 0; + u32 arg_btf_id; + int offset; + + stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name); + if (!stub_func_proto) + return 0; + + /* Check if the number of arguments of the stub function is the same + * as the number of arguments of the function pointer. + */ + nargs = btf_type_vlen(func_proto); + if (nargs != btf_type_vlen(stub_func_proto)) { + pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n", + st_ops_name, member_name, member_name, st_ops_name); + return -EINVAL; + } + + if (!nargs) + return 0; + + args = btf_params(func_proto); + stub_args = btf_params(stub_func_proto); + + info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL); + if (!info_buf) + return -ENOMEM; + + /* Prepare info for every nullable argument */ + info = info_buf; + for (arg_no = 0; arg_no < nargs; arg_no++) { + /* Skip arguments that is not suffixed with + * "__nullable". + */ + if (!btf_param_match_suffix(btf, &stub_args[arg_no], + MAYBE_NULL_SUFFIX)) + continue; + + /* Should be a pointer to struct */ + pointed_type = btf_type_resolve_ptr(btf, + args[arg_no].type, + &arg_btf_id); + if (!pointed_type || + !btf_type_is_struct(pointed_type)) { + pr_warn("stub function %s__%s has %s tagging to an unsupported type\n", + st_ops_name, member_name, MAYBE_NULL_SUFFIX); + goto err_out; + } + + offset = btf_ctx_arg_offset(btf, func_proto, arg_no); + if (offset < 0) { + pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n", + st_ops_name, member_name, arg_no); + goto err_out; + } + + if (args[arg_no].type != stub_args[arg_no].type) { + pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n", + arg_no, st_ops_name, member_name); + goto err_out; + } + + /* Fill the information of the new argument */ + info->reg_type = + PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; + info->btf_id = arg_btf_id; + info->btf = btf; + info->offset = offset; + + info++; + info_cnt++; + } + + if (info_cnt) { + arg_info->info = info_buf; + arg_info->cnt = info_cnt; + } else { + kfree(info_buf); + } + + return 0; + +err_out: + kfree(info_buf); + + return -EINVAL; +} + +/* Clean up the arg_info in a struct bpf_struct_ops_desc. */ +void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) +{ + struct bpf_struct_ops_arg_info *arg_info; + int i; + + arg_info = st_ops_desc->arg_info; + for (i = 0; i < btf_type_vlen(st_ops_desc->type); i++) + kfree(arg_info[i].info); + + kfree(arg_info); +} + int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, struct btf *btf, struct bpf_verifier_log *log) { struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; + struct bpf_struct_ops_arg_info *arg_info; const struct btf_member *member; const struct btf_type *t; s32 type_id, value_id; char value_name[128]; const char *mname; - int i; + int i, err; if (strlen(st_ops->name) + VALUE_PREFIX_LEN >= sizeof(value_name)) { @@ -160,6 +326,17 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, if (!is_valid_value_type(btf, value_id, t, value_name)) return -EINVAL; + arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info), + GFP_KERNEL); + if (!arg_info) + return -ENOMEM; + + st_ops_desc->arg_info = arg_info; + st_ops_desc->type = t; + st_ops_desc->type_id = type_id; + st_ops_desc->value_id = value_id; + st_ops_desc->value_type = btf_type_by_id(btf, value_id); + for_each_member(i, t, member) { const struct btf_type *func_proto; @@ -167,40 +344,52 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, if (!*mname) { pr_warn("anon member in struct %s is not supported\n", st_ops->name); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto errout; } if (__btf_member_bitfield_size(t, member)) { pr_warn("bit field member %s in struct %s is not supported\n", mname, st_ops->name); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto errout; } func_proto = btf_type_resolve_func_ptr(btf, member->type, NULL); - if (func_proto && - btf_distill_func_proto(log, btf, + if (!func_proto) + continue; + + if (btf_distill_func_proto(log, btf, func_proto, mname, &st_ops->func_models[i])) { pr_warn("Error in parsing func ptr %s in struct %s\n", mname, st_ops->name); - return -EINVAL; + err = -EINVAL; + goto errout; } + + err = prepare_arg_info(btf, st_ops->name, mname, + func_proto, + arg_info + i); + if (err) + goto errout; } if (st_ops->init(btf)) { pr_warn("Error in init bpf_struct_ops %s\n", st_ops->name); - return -EINVAL; + err = -EINVAL; + goto errout; } - st_ops_desc->type_id = type_id; - st_ops_desc->type = t; - st_ops_desc->value_id = value_id; - st_ops_desc->value_type = btf_type_by_id(btf, value_id); - return 0; + +errout: + bpf_struct_ops_desc_release(st_ops_desc); + + return err; } static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 447da964f217..efd9bc274be0 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1699,6 +1699,13 @@ static void btf_free_struct_meta_tab(struct btf *btf) static void btf_free_struct_ops_tab(struct btf *btf) { struct btf_struct_ops_tab *tab = btf->struct_ops_tab; + u32 i; + + if (!tab) + return; + + for (i = 0; i < tab->cnt; i++) + bpf_struct_ops_desc_release(&tab->ops[i]); kfree(tab); btf->struct_ops_tab = NULL; @@ -6130,6 +6137,26 @@ static bool prog_args_trusted(const struct bpf_prog *prog) } } +int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto, + u32 arg_no) +{ + const struct btf_param *args; + const struct btf_type *t; + int off = 0, i; + u32 sz; + + args = btf_params(func_proto); + for (i = 0; i < arg_no; i++) { + t = btf_type_by_id(btf, args[i].type); + t = btf_resolve_size(btf, t, &sz); + if (IS_ERR(t)) + return PTR_ERR(t); + off += roundup(sz, 8); + } + + return off; +} + bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c92d6af7d975..72ca27f49616 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20419,6 +20419,12 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) } } + /* btf_ctx_access() used this to provide argument type info */ + prog->aux->ctx_arg_info = + st_ops_desc->arg_info[member_idx].info; + prog->aux->ctx_arg_info_size = + st_ops_desc->arg_info[member_idx].cnt; + prog->aux->attach_func_proto = func_proto; prog->aux->attach_func_name = mname; env->ops = st_ops->verifier_ops; -- cgit v1.2.3 From 32e18e7688c6847b0c9db073aafb00639ecf576c Mon Sep 17 00:00:00 2001 From: Oliver Crumrine Date: Fri, 9 Feb 2024 14:41:22 -0500 Subject: bpf: remove check in __cgroup_bpf_run_filter_skb Originally, this patch removed a redundant check in BPF_CGROUP_RUN_PROG_INET_EGRESS, as the check was already being done in the function it called, __cgroup_bpf_run_filter_skb. For v2, it was reccomended that I remove the check from __cgroup_bpf_run_filter_skb, and add the checks to the other macro that calls that function, BPF_CGROUP_RUN_PROG_INET_INGRESS. To sum it up, checking that the socket exists and that it is a full socket is now part of both macros BPF_CGROUP_RUN_PROG_INET_EGRESS and BPF_CGROUP_RUN_PROG_INET_INGRESS, and it is no longer part of the function they call, __cgroup_bpf_run_filter_skb. v3->v4: Fixed weird merge conflict. v2->v3: Sent to bpf-next instead of generic patch v1->v2: Addressed feedback about where check should be removed. Signed-off-by: Oliver Crumrine Acked-by: Stanislav Fomichev Link: https://lore.kernel.org/r/7lv62yiyvmj5a7eozv2iznglpkydkdfancgmbhiptrgvgan5sy@3fl3onchgdz3 Signed-off-by: Martin KaFai Lau --- include/linux/bpf-cgroup.h | 3 ++- kernel/bpf/cgroup.c | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index a789266feac3..fb3c3e7181e6 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -196,7 +196,8 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, ({ \ int __ret = 0; \ if (cgroup_bpf_enabled(CGROUP_INET_INGRESS) && \ - cgroup_bpf_sock_enabled(sk, CGROUP_INET_INGRESS)) \ + cgroup_bpf_sock_enabled(sk, CGROUP_INET_INGRESS) && sk && \ + sk_fullsock(sk)) \ __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ CGROUP_INET_INGRESS); \ \ diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 98e0e3835b28..5a568bbbeaeb 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1364,9 +1364,6 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk, struct cgroup *cgrp; int ret; - if (!sk || !sk_fullsock(sk)) - return 0; - if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6) return 0; -- cgit v1.2.3 From fb5b86cfd4ef21ea18966718f6bf6c8f1b9df12e Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 12 Feb 2024 15:32:18 -0800 Subject: bpf: simplify btf_get_prog_ctx_type() into btf_is_prog_ctx_type() Return result of btf_get_prog_ctx_type() is never used and callers only check NULL vs non-NULL case to determine if given type matches expected PTR_TO_CTX type. So rename function to `btf_is_prog_ctx_type()` and return a simple true/false. We'll use this simpler interface to handle kprobe program type's special typedef case in the next patch. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20240212233221.2575350-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 17 ++++++++--------- kernel/bpf/btf.c | 27 +++++++++++++-------------- kernel/bpf/verifier.c | 2 +- 3 files changed, 22 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index cb96f6263638..f9e56fd12a9f 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -531,10 +531,9 @@ s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id); int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt, struct module *owner); struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id); -const struct btf_type * -btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, - const struct btf_type *t, enum bpf_prog_type prog_type, - int arg); +bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, + const struct btf_type *t, enum bpf_prog_type prog_type, + int arg); int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type); bool btf_types_are_same(const struct btf *btf1, u32 id1, const struct btf *btf2, u32 id2); @@ -574,12 +573,12 @@ static inline struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf { return NULL; } -static inline const struct btf_member * -btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, - const struct btf_type *t, enum bpf_prog_type prog_type, - int arg) +static inline bool +btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, + const struct btf_type *t, enum bpf_prog_type prog_type, + int arg) { - return NULL; + return false; } static inline int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type) { diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index efd9bc274be0..405f95722905 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5694,10 +5694,9 @@ static int find_kern_ctx_type_id(enum bpf_prog_type prog_type) return ctx_type->type; } -const struct btf_type * -btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, - const struct btf_type *t, enum bpf_prog_type prog_type, - int arg) +bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, + const struct btf_type *t, enum bpf_prog_type prog_type, + int arg) { const struct btf_type *ctx_type; const char *tname, *ctx_tname; @@ -5711,26 +5710,26 @@ btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, * is not supported yet. * BPF_PROG_TYPE_RAW_TRACEPOINT is fine. */ - return NULL; + return false; } tname = btf_name_by_offset(btf, t->name_off); if (!tname) { bpf_log(log, "arg#%d struct doesn't have a name\n", arg); - return NULL; + return false; } ctx_type = find_canonical_prog_ctx_type(prog_type); if (!ctx_type) { bpf_log(log, "btf_vmlinux is malformed\n"); /* should not happen */ - return NULL; + return false; } again: ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off); if (!ctx_tname) { /* should not happen */ bpf_log(log, "Please fix kernel include/linux/bpf_types.h\n"); - return NULL; + return false; } /* only compare that prog's ctx type name is the same as * kernel expects. No need to compare field by field. @@ -5740,20 +5739,20 @@ again: * { // no fields of skb are ever used } */ if (strcmp(ctx_tname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0) - return ctx_type; + return true; if (strcmp(ctx_tname, "xdp_md") == 0 && strcmp(tname, "xdp_buff") == 0) - return ctx_type; + return true; if (strcmp(ctx_tname, tname)) { /* bpf_user_pt_regs_t is a typedef, so resolve it to * underlying struct and check name again */ if (!btf_type_is_modifier(ctx_type)) - return NULL; + return false; while (btf_type_is_modifier(ctx_type)) ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type); goto again; } - return ctx_type; + return true; } /* forward declarations for arch-specific underlying types of @@ -5905,7 +5904,7 @@ static int btf_translate_to_vmlinux(struct bpf_verifier_log *log, enum bpf_prog_type prog_type, int arg) { - if (!btf_get_prog_ctx_type(log, btf, t, prog_type, arg)) + if (!btf_is_prog_ctx_type(log, btf, t, prog_type, arg)) return -ENOENT; return find_kern_ctx_type_id(prog_type); } @@ -7211,7 +7210,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (!btf_type_is_ptr(t)) goto skip_pointer; - if ((tags & ARG_TAG_CTX) || btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { + if ((tags & ARG_TAG_CTX) || btf_is_prog_ctx_type(log, btf, t, prog_type, i)) { if (tags & ~ARG_TAG_CTX) { bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 72ca27f49616..aa192dc735a9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11015,7 +11015,7 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) + if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) return KF_ARG_PTR_TO_CTX; if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) -- cgit v1.2.3 From 4d2bb0bfe8741a8778e0053f31a4e0f0cba80e8b Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Mon, 12 Feb 2024 10:50:55 +0100 Subject: xdp: rely on skb pointer reference in do_xdp_generic and netif_receive_generic_xdp Rely on skb pointer reference instead of the skb pointer in do_xdp_generic and netif_receive_generic_xdp routine signatures. This is a preliminary patch to add multi-buff support for xdp running in generic mode where we will need to reallocate the skb to avoid linearization and we will need to make it visible to do_xdp_generic() caller. Acked-by: Jesper Dangaard Brouer Reviewed-by: Toke Hoiland-Jorgensen Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/c09415b1f48c8620ef4d76deed35050a7bddf7c2.1707729884.git.lorenzo@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/tun.c | 4 ++-- include/linux/netdevice.h | 2 +- net/core/dev.c | 16 +++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/tun.c b/drivers/net/tun.c index b472f2c972d8..bc80fc1d576e 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1926,7 +1926,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, rcu_read_lock(); xdp_prog = rcu_dereference(tun->xdp_prog); if (xdp_prog) { - ret = do_xdp_generic(xdp_prog, skb); + ret = do_xdp_generic(xdp_prog, &skb); if (ret != XDP_PASS) { rcu_read_unlock(); local_bh_enable(); @@ -2516,7 +2516,7 @@ build: skb_record_rx_queue(skb, tfile->queue_index); if (skb_xdp) { - ret = do_xdp_generic(xdp_prog, skb); + ret = do_xdp_generic(xdp_prog, &skb); if (ret != XDP_PASS) { ret = 0; goto out; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 07cefa32eafa..a3f9c95da51e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3958,7 +3958,7 @@ static inline void dev_consume_skb_any(struct sk_buff *skb) u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, struct bpf_prog *xdp_prog); void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog); -int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb); +int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb); int netif_rx(struct sk_buff *skb); int __netif_rx(struct sk_buff *skb); diff --git a/net/core/dev.c b/net/core/dev.c index e19bdf1421e0..ffeb0e0279fe 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4936,10 +4936,11 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, return act; } -static u32 netif_receive_generic_xdp(struct sk_buff *skb, +static u32 netif_receive_generic_xdp(struct sk_buff **pskb, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { + struct sk_buff *skb = *pskb; u32 act = XDP_DROP; /* Reinjected packets coming from act_mirred or similar should @@ -5020,24 +5021,24 @@ void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); -int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb) +int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb) { if (xdp_prog) { struct xdp_buff xdp; u32 act; int err; - act = netif_receive_generic_xdp(skb, &xdp, xdp_prog); + act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog); if (act != XDP_PASS) { switch (act) { case XDP_REDIRECT: - err = xdp_do_generic_redirect(skb->dev, skb, + err = xdp_do_generic_redirect((*pskb)->dev, *pskb, &xdp, xdp_prog); if (err) goto out_redir; break; case XDP_TX: - generic_xdp_tx(skb, xdp_prog); + generic_xdp_tx(*pskb, xdp_prog); break; } return XDP_DROP; @@ -5045,7 +5046,7 @@ int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb) } return XDP_PASS; out_redir: - kfree_skb_reason(skb, SKB_DROP_REASON_XDP); + kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP); return XDP_DROP; } EXPORT_SYMBOL_GPL(do_xdp_generic); @@ -5368,7 +5369,8 @@ another_round: int ret2; migrate_disable(); - ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb); + ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), + &skb); migrate_enable(); if (ret2 != XDP_PASS) { -- cgit v1.2.3 From e6d5dbdd20aa6a86974af51deb9414cd2e7794cb Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Mon, 12 Feb 2024 10:50:56 +0100 Subject: xdp: add multi-buff support for xdp running in generic mode Similar to native xdp, do not always linearize the skb in netif_receive_generic_xdp routine but create a non-linear xdp_buff to be processed by the eBPF program. This allow to add multi-buffer support for xdp running in generic mode. Acked-by: Jesper Dangaard Brouer Reviewed-by: Toke Hoiland-Jorgensen Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/1044d6412b1c3e95b40d34993fd5f37cd2f319fd.1707729884.git.lorenzo@kernel.org Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 2 ++ net/core/dev.c | 70 +++++++++++++++++++++++++++----------- net/core/skbuff.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 2dde34c29203..def3d8689c3d 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3446,6 +3446,8 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f) __skb_frag_ref(&skb_shinfo(skb)->frags[f]); } +int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, + struct bpf_prog *prog); bool napi_pp_put_page(struct page *page, bool napi_safe); static inline void diff --git a/net/core/dev.c b/net/core/dev.c index ffeb0e0279fe..2d02ca8a3da5 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4874,6 +4874,12 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, skb_headlen(skb) + mac_len, true); + if (skb_is_nonlinear(skb)) { + skb_shinfo(skb)->xdp_frags_size = skb->data_len; + xdp_buff_set_frags_flag(xdp); + } else { + xdp_buff_clear_frags_flag(xdp); + } orig_data_end = xdp->data_end; orig_data = xdp->data; @@ -4903,6 +4909,14 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, skb->len += off; /* positive on grow, negative on shrink */ } + /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers + * (e.g. bpf_xdp_adjust_tail), we need to update data_len here. + */ + if (xdp_buff_has_frags(xdp)) + skb->data_len = skb_shinfo(skb)->xdp_frags_size; + else + skb->data_len = 0; + /* check if XDP changed eth hdr such SKB needs update */ eth = (struct ethhdr *)xdp->data; if ((orig_eth_type != eth->h_proto) || @@ -4936,12 +4950,35 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, return act; } +static int +netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog) +{ + struct sk_buff *skb = *pskb; + int err, hroom, troom; + + if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog)) + return 0; + + /* In case we have to go down the path and also linearize, + * then lets do the pskb_expand_head() work just once here. + */ + hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); + troom = skb->tail + skb->data_len - skb->end; + err = pskb_expand_head(skb, + hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, + troom > 0 ? troom + 128 : 0, GFP_ATOMIC); + if (err) + return err; + + return skb_linearize(skb); +} + static u32 netif_receive_generic_xdp(struct sk_buff **pskb, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { struct sk_buff *skb = *pskb; - u32 act = XDP_DROP; + u32 mac_len, act = XDP_DROP; /* Reinjected packets coming from act_mirred or similar should * not get XDP generic processing. @@ -4949,41 +4986,36 @@ static u32 netif_receive_generic_xdp(struct sk_buff **pskb, if (skb_is_redirected(skb)) return XDP_PASS; - /* XDP packets must be linear and must have sufficient headroom - * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also - * native XDP provides, thus we need to do it here as well. + /* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM + * bytes. This is the guarantee that also native XDP provides, + * thus we need to do it here as well. */ + mac_len = skb->data - skb_mac_header(skb); + __skb_push(skb, mac_len); + if (skb_cloned(skb) || skb_is_nonlinear(skb) || skb_headroom(skb) < XDP_PACKET_HEADROOM) { - int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); - int troom = skb->tail + skb->data_len - skb->end; - - /* In case we have to go down the path and also linearize, - * then lets do the pskb_expand_head() work just once here. - */ - if (pskb_expand_head(skb, - hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, - troom > 0 ? troom + 128 : 0, GFP_ATOMIC)) - goto do_drop; - if (skb_linearize(skb)) + if (netif_skb_check_for_xdp(pskb, xdp_prog)) goto do_drop; } - act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog); + __skb_pull(*pskb, mac_len); + + act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog); switch (act) { case XDP_REDIRECT: case XDP_TX: case XDP_PASS: break; default: - bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act); + bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: - trace_xdp_exception(skb->dev, xdp_prog, act); + trace_xdp_exception((*pskb)->dev, xdp_prog, act); fallthrough; case XDP_DROP: do_drop: - kfree_skb(skb); + kfree_skb(*pskb); break; } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 9e5eb47b4025..bdb94749f05d 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -895,6 +895,97 @@ static bool is_pp_page(struct page *page) return (page->pp_magic & ~0x3UL) == PP_SIGNATURE; } +static int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, + unsigned int headroom) +{ +#if IS_ENABLED(CONFIG_PAGE_POOL) + u32 size, truesize, len, max_head_size, off; + struct sk_buff *skb = *pskb, *nskb; + int err, i, head_off; + void *data; + + /* XDP does not support fraglist so we need to linearize + * the skb. + */ + if (skb_has_frag_list(skb)) + return -EOPNOTSUPP; + + max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom); + if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE) + return -ENOMEM; + + size = min_t(u32, skb->len, max_head_size); + truesize = SKB_HEAD_ALIGN(size) + headroom; + data = page_pool_dev_alloc_va(pool, &truesize); + if (!data) + return -ENOMEM; + + nskb = napi_build_skb(data, truesize); + if (!nskb) { + page_pool_free_va(pool, data, true); + return -ENOMEM; + } + + skb_reserve(nskb, headroom); + skb_copy_header(nskb, skb); + skb_mark_for_recycle(nskb); + + err = skb_copy_bits(skb, 0, nskb->data, size); + if (err) { + consume_skb(nskb); + return err; + } + skb_put(nskb, size); + + head_off = skb_headroom(nskb) - skb_headroom(skb); + skb_headers_offset_update(nskb, head_off); + + off = size; + len = skb->len - off; + for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) { + struct page *page; + u32 page_off; + + size = min_t(u32, len, PAGE_SIZE); + truesize = size; + + page = page_pool_dev_alloc(pool, &page_off, &truesize); + if (!data) { + consume_skb(nskb); + return -ENOMEM; + } + + skb_add_rx_frag(nskb, i, page, page_off, size, truesize); + err = skb_copy_bits(skb, off, page_address(page) + page_off, + size); + if (err) { + consume_skb(nskb); + return err; + } + + len -= size; + off += size; + } + + consume_skb(skb); + *pskb = nskb; + + return 0; +#else + return -EOPNOTSUPP; +#endif +} + +int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, + struct bpf_prog *prog) +{ + if (!prog->aux->xdp_has_frags) + return -EINVAL; + + return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM); +} +EXPORT_SYMBOL(skb_cow_data_for_xdp); + #if IS_ENABLED(CONFIG_PAGE_POOL) bool napi_pp_put_page(struct page *page, bool napi_safe) { -- cgit v1.2.3 From 27accb3cc08a0ec4e348356774042d5fa5f30cce Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Mon, 12 Feb 2024 10:50:57 +0100 Subject: veth: rely on skb_pp_cow_data utility routine Rely on skb_pp_cow_data utility routine and remove duplicated code. Acked-by: Jesper Dangaard Brouer Reviewed-by: Toke Hoiland-Jorgensen Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/029cc14cce41cb242ee7efdcf32acc81f1ce4e9f.1707729884.git.lorenzo@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/veth.c | 74 ++------------------------------------------------ include/linux/skbuff.h | 2 ++ net/core/skbuff.c | 5 ++-- 3 files changed, 7 insertions(+), 74 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/veth.c b/drivers/net/veth.c index de1f13837782..500b9dfccd08 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -729,80 +729,10 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, if (skb_shared(skb) || skb_head_is_locked(skb) || skb_shinfo(skb)->nr_frags || skb_headroom(skb) < XDP_PACKET_HEADROOM) { - u32 size, len, max_head_size, off, truesize, page_offset; - struct sk_buff *nskb; - struct page *page; - int i, head_off; - void *va; - - /* We need a private copy of the skb and data buffers since - * the ebpf program can modify it. We segment the original skb - * into order-0 pages without linearize it. - * - * Make sure we have enough space for linear and paged area - */ - max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - - VETH_XDP_HEADROOM); - if (skb->len > PAGE_SIZE * MAX_SKB_FRAGS + max_head_size) - goto drop; - - size = min_t(u32, skb->len, max_head_size); - truesize = SKB_HEAD_ALIGN(size) + VETH_XDP_HEADROOM; - - /* Allocate skb head */ - va = page_pool_dev_alloc_va(rq->page_pool, &truesize); - if (!va) - goto drop; - - nskb = napi_build_skb(va, truesize); - if (!nskb) { - page_pool_free_va(rq->page_pool, va, true); + if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM)) goto drop; - } - - skb_reserve(nskb, VETH_XDP_HEADROOM); - skb_copy_header(nskb, skb); - skb_mark_for_recycle(nskb); - - if (skb_copy_bits(skb, 0, nskb->data, size)) { - consume_skb(nskb); - goto drop; - } - skb_put(nskb, size); - head_off = skb_headroom(nskb) - skb_headroom(skb); - skb_headers_offset_update(nskb, head_off); - - /* Allocate paged area of new skb */ - off = size; - len = skb->len - off; - - for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) { - size = min_t(u32, len, PAGE_SIZE); - truesize = size; - - page = page_pool_dev_alloc(rq->page_pool, &page_offset, - &truesize); - if (!page) { - consume_skb(nskb); - goto drop; - } - - skb_add_rx_frag(nskb, i, page, page_offset, size, - truesize); - if (skb_copy_bits(skb, off, - page_address(page) + page_offset, - size)) { - consume_skb(nskb); - goto drop; - } - - len -= size; - off += size; - } - - consume_skb(skb); - skb = nskb; + skb = *pskb; } /* SKB "head" area always have tailroom for skb_shared_info */ diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index def3d8689c3d..696e7680656f 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3446,6 +3446,8 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f) __skb_frag_ref(&skb_shinfo(skb)->frags[f]); } +int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, + unsigned int headroom); int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, struct bpf_prog *prog); bool napi_pp_put_page(struct page *page, bool napi_safe); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index bdb94749f05d..0d9a489e6ae1 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -895,8 +895,8 @@ static bool is_pp_page(struct page *page) return (page->pp_magic & ~0x3UL) == PP_SIGNATURE; } -static int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, - unsigned int headroom) +int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, + unsigned int headroom) { #if IS_ENABLED(CONFIG_PAGE_POOL) u32 size, truesize, len, max_head_size, off; @@ -975,6 +975,7 @@ static int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, return -EOPNOTSUPP; #endif } +EXPORT_SYMBOL(skb_pp_cow_data); int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, struct bpf_prog *prog) -- cgit v1.2.3 From d2e8899de71cd0a3c22a0eadfb9d54604d34eb96 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 13 Feb 2024 11:44:01 -0300 Subject: soc: qcom: apr: make aprbus const Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the aprbus variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240213-bus_cleanup-apr-v1-1-50c824eec06d@marliere.net Signed-off-by: Bjorn Andersson --- drivers/soc/qcom/apr.c | 2 +- include/linux/soc/qcom/apr.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/soc/qcom/apr.c b/drivers/soc/qcom/apr.c index 1f8b315576a4..50749e870efa 100644 --- a/drivers/soc/qcom/apr.c +++ b/drivers/soc/qcom/apr.c @@ -399,7 +399,7 @@ static int apr_uevent(const struct device *dev, struct kobj_uevent_env *env) return add_uevent_var(env, "MODALIAS=apr:%s", adev->name); } -struct bus_type aprbus = { +const struct bus_type aprbus = { .name = "aprbus", .match = apr_device_match, .probe = apr_device_probe, diff --git a/include/linux/soc/qcom/apr.h b/include/linux/soc/qcom/apr.h index be98aebcb3e1..7161a3183eda 100644 --- a/include/linux/soc/qcom/apr.h +++ b/include/linux/soc/qcom/apr.h @@ -9,7 +9,7 @@ #include #include -extern struct bus_type aprbus; +extern const struct bus_type aprbus; #define APR_HDR_LEN(hdr_len) ((hdr_len)/4) -- cgit v1.2.3 From 75fa9b7e375e35739663cde0252d31e586c6314a Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 12 Feb 2024 10:06:09 +0100 Subject: video: Add helpers for decoding screen_info The plain values as stored in struct screen_info need to be decoded before being used. Add helpers that decode the type of video output and the framebuffer I/O aperture. Old or non-x86 systems may not set the type of video directly, but only indicate the presence by storing 0x01 in orig_video_isVGA. The decoding logic in screen_info_video_type() takes this into account. It then follows similar code in vgacon's vgacon_startup() to detect the video type from the given values. A call to screen_info_resources() returns all known resources of the given screen_info. The resources' values have been taken from existing code in vgacon and vga16fb. These drivers can later be converted to use the new interfaces. v2: * return ssize_t from screen_info_resources() * don't call __screen_info_has_lfb() unnecessarily Signed-off-by: Thomas Zimmermann Reviewed-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20240212090736.11464-2-tzimmermann@suse.de --- drivers/firmware/Kconfig | 1 + drivers/video/Kconfig | 4 + drivers/video/Makefile | 3 + drivers/video/screen_info_generic.c | 146 ++++++++++++++++++++++++++++++++++++ include/linux/screen_info.h | 100 ++++++++++++++++++++++++ 5 files changed, 254 insertions(+) create mode 100644 drivers/video/screen_info_generic.c (limited to 'include/linux') diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index afd38539b92e..71d8b26c4103 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -182,6 +182,7 @@ config MTK_ADSP_IPC config SYSFB bool select BOOT_VESA_SUPPORT + select SCREEN_INFO config SYSFB_SIMPLEFB bool "Mark VGA/VBE/EFI FB as generic system framebuffer" diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 130ebccb8338..44c9ef1435a2 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -11,6 +11,10 @@ config APERTURE_HELPERS Support tracking and hand-over of aperture ownership. Required by graphics drivers for firmware-provided framebuffers. +config SCREEN_INFO + bool + default n + config STI_CORE bool depends on PARISC diff --git a/drivers/video/Makefile b/drivers/video/Makefile index 9eb5557911de..117cbdbb58c2 100644 --- a/drivers/video/Makefile +++ b/drivers/video/Makefile @@ -1,11 +1,14 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_APERTURE_HELPERS) += aperture.o +obj-$(CONFIG_SCREEN_INFO) += screen_info.o obj-$(CONFIG_STI_CORE) += sticore.o obj-$(CONFIG_VGASTATE) += vgastate.o obj-$(CONFIG_VIDEO) += cmdline.o nomodeset.o obj-$(CONFIG_HDMI) += hdmi.o +screen_info-y := screen_info_generic.o + obj-$(CONFIG_VT) += console/ obj-$(CONFIG_FB_STI) += console/ obj-$(CONFIG_LOGO) += logo/ diff --git a/drivers/video/screen_info_generic.c b/drivers/video/screen_info_generic.c new file mode 100644 index 000000000000..64117c6367ab --- /dev/null +++ b/drivers/video/screen_info_generic.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +static void resource_init_named(struct resource *r, + resource_size_t start, resource_size_t size, + const char *name, unsigned int flags) +{ + memset(r, 0, sizeof(*r)); + + r->start = start; + r->end = start + size - 1; + r->name = name; + r->flags = flags; +} + +static void resource_init_io_named(struct resource *r, + resource_size_t start, resource_size_t size, + const char *name) +{ + resource_init_named(r, start, size, name, IORESOURCE_IO); +} + +static void resource_init_mem_named(struct resource *r, + resource_size_t start, resource_size_t size, + const char *name) +{ + resource_init_named(r, start, size, name, IORESOURCE_MEM); +} + +static inline bool __screen_info_has_ega_gfx(unsigned int mode) +{ + switch (mode) { + case 0x0d: /* 320x200-4 */ + case 0x0e: /* 640x200-4 */ + case 0x0f: /* 640x350-1 */ + case 0x10: /* 640x350-4 */ + return true; + default: + return false; + } +} + +static inline bool __screen_info_has_vga_gfx(unsigned int mode) +{ + switch (mode) { + case 0x10: /* 640x480-1 */ + case 0x12: /* 640x480-4 */ + case 0x13: /* 320-200-8 */ + case 0x6a: /* 800x600-4 (VESA) */ + return true; + default: + return __screen_info_has_ega_gfx(mode); + } +} + +/** + * screen_info_resources() - Get resources from screen_info structure + * @si: the screen_info + * @r: pointer to an array of resource structures + * @num: number of elements in @r: + * + * Returns: + * The number of resources stored in @r on success, or a negative errno code otherwise. + * + * A call to screen_info_resources() returns the resources consumed by the + * screen_info's device or framebuffer. The result is stored in the caller-supplied + * array @r with up to @num elements. The function returns the number of + * initialized elements. + */ +ssize_t screen_info_resources(const struct screen_info *si, struct resource *r, size_t num) +{ + struct resource *pos = r; + unsigned int type = screen_info_video_type(si); + u64 base, size; + + switch (type) { + case VIDEO_TYPE_MDA: + if (num > 0) + resource_init_io_named(pos++, 0x3b0, 12, "mda"); + if (num > 1) + resource_init_io_named(pos++, 0x3bf, 0x01, "mda"); + if (num > 2) + resource_init_mem_named(pos++, 0xb0000, 0x2000, "mda"); + break; + case VIDEO_TYPE_CGA: + if (num > 0) + resource_init_io_named(pos++, 0x3d4, 0x02, "cga"); + if (num > 1) + resource_init_mem_named(pos++, 0xb8000, 0x2000, "cga"); + break; + case VIDEO_TYPE_EGAM: + if (num > 0) + resource_init_io_named(pos++, 0x3bf, 0x10, "ega"); + if (num > 1) + resource_init_mem_named(pos++, 0xb0000, 0x8000, "ega"); + break; + case VIDEO_TYPE_EGAC: + if (num > 0) + resource_init_io_named(pos++, 0x3c0, 0x20, "ega"); + if (num > 1) { + if (__screen_info_has_ega_gfx(si->orig_video_mode)) + resource_init_mem_named(pos++, 0xa0000, 0x10000, "ega"); + else + resource_init_mem_named(pos++, 0xb8000, 0x8000, "ega"); + } + break; + case VIDEO_TYPE_VGAC: + if (num > 0) + resource_init_io_named(pos++, 0x3c0, 0x20, "vga+"); + if (num > 1) { + if (__screen_info_has_vga_gfx(si->orig_video_mode)) + resource_init_mem_named(pos++, 0xa0000, 0x10000, "vga+"); + else + resource_init_mem_named(pos++, 0xb8000, 0x8000, "vga+"); + } + break; + case VIDEO_TYPE_VLFB: + case VIDEO_TYPE_EFI: + base = __screen_info_lfb_base(si); + if (!base) + break; + size = __screen_info_lfb_size(si, type); + if (!size) + break; + if (num > 0) + resource_init_mem_named(pos++, base, size, "lfb"); + break; + case VIDEO_TYPE_PICA_S3: + case VIDEO_TYPE_MIPS_G364: + case VIDEO_TYPE_SGI: + case VIDEO_TYPE_TGAC: + case VIDEO_TYPE_SUN: + case VIDEO_TYPE_SUNPCI: + case VIDEO_TYPE_PMAC: + default: + /* not supported */ + return -EINVAL; + } + + return pos - r; +} +EXPORT_SYMBOL(screen_info_resources); diff --git a/include/linux/screen_info.h b/include/linux/screen_info.h index eab7081392d5..e7a02c5609d1 100644 --- a/include/linux/screen_info.h +++ b/include/linux/screen_info.h @@ -4,6 +4,106 @@ #include +/** + * SCREEN_INFO_MAX_RESOURCES - maximum number of resources per screen_info + */ +#define SCREEN_INFO_MAX_RESOURCES 3 + +struct resource; + +static inline bool __screen_info_has_lfb(unsigned int type) +{ + return (type == VIDEO_TYPE_VLFB) || (type == VIDEO_TYPE_EFI); +} + +static inline u64 __screen_info_lfb_base(const struct screen_info *si) +{ + u64 lfb_base = si->lfb_base; + + if (si->capabilities & VIDEO_CAPABILITY_64BIT_BASE) + lfb_base |= (u64)si->ext_lfb_base << 32; + + return lfb_base; +} + +static inline u64 __screen_info_lfb_size(const struct screen_info *si, unsigned int type) +{ + u64 lfb_size = si->lfb_size; + + if (type == VIDEO_TYPE_VLFB) + lfb_size <<= 16; + return lfb_size; +} + +static inline unsigned int __screen_info_video_type(unsigned int type) +{ + switch (type) { + case VIDEO_TYPE_MDA: + case VIDEO_TYPE_CGA: + case VIDEO_TYPE_EGAM: + case VIDEO_TYPE_EGAC: + case VIDEO_TYPE_VGAC: + case VIDEO_TYPE_VLFB: + case VIDEO_TYPE_PICA_S3: + case VIDEO_TYPE_MIPS_G364: + case VIDEO_TYPE_SGI: + case VIDEO_TYPE_TGAC: + case VIDEO_TYPE_SUN: + case VIDEO_TYPE_SUNPCI: + case VIDEO_TYPE_PMAC: + case VIDEO_TYPE_EFI: + return type; + default: + return 0; + } +} + +/** + * screen_info_video_type() - Decodes the video type from struct screen_info + * @si: an instance of struct screen_info + * + * Returns: + * A VIDEO_TYPE_ constant representing si's type of video display, or 0 otherwise. + */ +static inline unsigned int screen_info_video_type(const struct screen_info *si) +{ + unsigned int type; + + // check if display output is on + if (!si->orig_video_isVGA) + return 0; + + // check for a known VIDEO_TYPE_ constant + type = __screen_info_video_type(si->orig_video_isVGA); + if (type) + return si->orig_video_isVGA; + + // check if text mode has been initialized + if (!si->orig_video_lines || !si->orig_video_cols) + return 0; + + // 80x25 text, mono + if (si->orig_video_mode == 0x07) { + if ((si->orig_video_ega_bx & 0xff) != 0x10) + return VIDEO_TYPE_EGAM; + else + return VIDEO_TYPE_MDA; + } + + // EGA/VGA, 16 colors + if ((si->orig_video_ega_bx & 0xff) != 0x10) { + if (si->orig_video_isVGA) + return VIDEO_TYPE_VGAC; + else + return VIDEO_TYPE_EGAC; + } + + // the rest... + return VIDEO_TYPE_CGA; +} + +ssize_t screen_info_resources(const struct screen_info *si, struct resource *r, size_t num); + extern struct screen_info screen_info; #endif /* _SCREEN_INFO_H */ -- cgit v1.2.3 From 036105e3a776b6fc2fe0d262896a23ff2cc2e6b1 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 12 Feb 2024 10:06:10 +0100 Subject: video: Provide screen_info_get_pci_dev() to find screen_info's PCI device Add screen_info_get_pci_dev() to find the PCI device of an instance of screen_info. Does nothing on systems without PCI bus. v3: * search PCI device with pci_get_base_class() (Sui) v2: * remove ret from screen_info_pci_dev() (Javier) Signed-off-by: Thomas Zimmermann Reviewed-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20240212090736.11464-3-tzimmermann@suse.de --- drivers/video/Makefile | 1 + drivers/video/screen_info_pci.c | 48 +++++++++++++++++++++++++++++++++++++++++ include/linux/screen_info.h | 10 +++++++++ 3 files changed, 59 insertions(+) create mode 100644 drivers/video/screen_info_pci.c (limited to 'include/linux') diff --git a/drivers/video/Makefile b/drivers/video/Makefile index 117cbdbb58c2..ffbac4387c67 100644 --- a/drivers/video/Makefile +++ b/drivers/video/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_VIDEO) += cmdline.o nomodeset.o obj-$(CONFIG_HDMI) += hdmi.o screen_info-y := screen_info_generic.o +screen_info-$(CONFIG_PCI) += screen_info_pci.o obj-$(CONFIG_VT) += console/ obj-$(CONFIG_FB_STI) += console/ diff --git a/drivers/video/screen_info_pci.c b/drivers/video/screen_info_pci.c new file mode 100644 index 000000000000..d8985a54ce71 --- /dev/null +++ b/drivers/video/screen_info_pci.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +static struct pci_dev *__screen_info_pci_dev(struct resource *res) +{ + struct pci_dev *pdev = NULL; + const struct resource *r = NULL; + + if (!(res->flags & IORESOURCE_MEM)) + return NULL; + + while (!r && (pdev = pci_get_base_class(PCI_BASE_CLASS_DISPLAY, pdev))) { + r = pci_find_resource(pdev, res); + } + + return pdev; +} + +/** + * screen_info_pci_dev() - Return PCI parent device that contains screen_info's framebuffer + * @si: the screen_info + * + * Returns: + * The screen_info's parent device or NULL on success, or a pointer-encoded + * errno value otherwise. The value NULL is not an error. It signals that no + * PCI device has been found. + */ +struct pci_dev *screen_info_pci_dev(const struct screen_info *si) +{ + struct resource res[SCREEN_INFO_MAX_RESOURCES]; + ssize_t i, numres; + + numres = screen_info_resources(si, res, ARRAY_SIZE(res)); + if (numres < 0) + return ERR_PTR(numres); + + for (i = 0; i < numres; ++i) { + struct pci_dev *pdev = __screen_info_pci_dev(&res[i]); + + if (pdev) + return pdev; + } + + return NULL; +} +EXPORT_SYMBOL(screen_info_pci_dev); diff --git a/include/linux/screen_info.h b/include/linux/screen_info.h index e7a02c5609d1..0eae08e3c6f9 100644 --- a/include/linux/screen_info.h +++ b/include/linux/screen_info.h @@ -9,6 +9,7 @@ */ #define SCREEN_INFO_MAX_RESOURCES 3 +struct pci_dev; struct resource; static inline bool __screen_info_has_lfb(unsigned int type) @@ -104,6 +105,15 @@ static inline unsigned int screen_info_video_type(const struct screen_info *si) ssize_t screen_info_resources(const struct screen_info *si, struct resource *r, size_t num); +#if defined(CONFIG_PCI) +struct pci_dev *screen_info_pci_dev(const struct screen_info *si); +#else +static inline struct pci_dev *screen_info_pci_dev(const struct screen_info *si) +{ + return NULL; +} +#endif + extern struct screen_info screen_info; #endif /* _SCREEN_INFO_H */ -- cgit v1.2.3 From 9eac534db0013aff9b9124985dab114600df9081 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 12 Feb 2024 10:06:11 +0100 Subject: firmware/sysfb: Set firmware-framebuffer parent device Set the firmware framebuffer's parent device, which usually is the graphics hardware's physical device. Integrates the framebuffer in the Linux device hierarchy and lets Linux handle dependencies among devices. For example, the graphics hardware won't be suspended while the firmware device is still active. v4: * fix build for CONFIG_SYSFB_SIMPLEFB=n, again v3: * fix build for CONFIG_SYSFB_SIMPLEFB=n (Sui) * test result of screen_info_pci_dev() for errors (Sui) v2: * detect parent device in sysfb_parent_dev() Signed-off-by: Thomas Zimmermann Reviewed-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20240212090736.11464-4-tzimmermann@suse.de --- drivers/firmware/sysfb.c | 21 ++++++++++++++++++++- drivers/firmware/sysfb_simplefb.c | 5 ++++- include/linux/sysfb.h | 6 ++++-- 3 files changed, 28 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/sysfb.c b/drivers/firmware/sysfb.c index 3c197db42c9d..4e104f3de4b9 100644 --- a/drivers/firmware/sysfb.c +++ b/drivers/firmware/sysfb.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -69,9 +70,23 @@ void sysfb_disable(void) } EXPORT_SYMBOL_GPL(sysfb_disable); +static __init struct device *sysfb_parent_dev(const struct screen_info *si) +{ + struct pci_dev *pdev; + + pdev = screen_info_pci_dev(si); + if (IS_ERR(pdev)) + return ERR_CAST(pdev); + else if (pdev) + return &pdev->dev; + + return NULL; +} + static __init int sysfb_init(void) { struct screen_info *si = &screen_info; + struct device *parent; struct simplefb_platform_data mode; const char *name; bool compatible; @@ -83,10 +98,12 @@ static __init int sysfb_init(void) sysfb_apply_efi_quirks(); + parent = sysfb_parent_dev(si); + /* try to create a simple-framebuffer device */ compatible = sysfb_parse_mode(si, &mode); if (compatible) { - pd = sysfb_create_simplefb(si, &mode); + pd = sysfb_create_simplefb(si, &mode, parent); if (!IS_ERR(pd)) goto unlock_mutex; } @@ -109,6 +126,8 @@ static __init int sysfb_init(void) goto unlock_mutex; } + pd->dev.parent = parent; + sysfb_set_efifb_fwnode(pd); ret = platform_device_add_data(pd, si, sizeof(*si)); diff --git a/drivers/firmware/sysfb_simplefb.c b/drivers/firmware/sysfb_simplefb.c index 74363ed7501f..75a186bf8f8e 100644 --- a/drivers/firmware/sysfb_simplefb.c +++ b/drivers/firmware/sysfb_simplefb.c @@ -91,7 +91,8 @@ __init bool sysfb_parse_mode(const struct screen_info *si, } __init struct platform_device *sysfb_create_simplefb(const struct screen_info *si, - const struct simplefb_platform_data *mode) + const struct simplefb_platform_data *mode, + struct device *parent) { struct platform_device *pd; struct resource res; @@ -143,6 +144,8 @@ __init struct platform_device *sysfb_create_simplefb(const struct screen_info *s if (!pd) return ERR_PTR(-ENOMEM); + pd->dev.parent = parent; + sysfb_set_efifb_fwnode(pd); ret = platform_device_add_resources(pd, &res, 1); diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h index 19cb803dd5ec..c9cb657dad08 100644 --- a/include/linux/sysfb.h +++ b/include/linux/sysfb.h @@ -91,7 +91,8 @@ static inline void sysfb_set_efifb_fwnode(struct platform_device *pd) bool sysfb_parse_mode(const struct screen_info *si, struct simplefb_platform_data *mode); struct platform_device *sysfb_create_simplefb(const struct screen_info *si, - const struct simplefb_platform_data *mode); + const struct simplefb_platform_data *mode, + struct device *parent); #else /* CONFIG_SYSFB_SIMPLE */ @@ -102,7 +103,8 @@ static inline bool sysfb_parse_mode(const struct screen_info *si, } static inline struct platform_device *sysfb_create_simplefb(const struct screen_info *si, - const struct simplefb_platform_data *mode) + const struct simplefb_platform_data *mode, + struct device *parent) { return ERR_PTR(-EINVAL); } -- cgit v1.2.3 From 78aa89d1dfba1e3cf4a2e053afa3b4c4ec622371 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 12 Feb 2024 10:06:15 +0100 Subject: firmware/sysfb: Update screen_info for relocated EFI framebuffers On ARM PCI systems, the PCI hierarchy might be reconfigured during boot and the firmware framebuffer might move as a result of that. The values in screen_info will then be invalid. Work around this problem by tracking the framebuffer's initial location before it get relocated; then fix the screen_info state between reloaction and creating the firmware framebuffer's device. This functionality has been lifted from efifb. See the commit message of commit 55d728a40d36 ("efi/fb: Avoid reconfiguration of BAR that covers the framebuffer") for more information. Signed-off-by: Thomas Zimmermann Reviewed-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20240212090736.11464-8-tzimmermann@suse.de --- drivers/firmware/sysfb.c | 2 + drivers/video/screen_info_pci.c | 88 +++++++++++++++++++++++++++++++++++++++++ include/linux/screen_info.h | 16 ++++++++ 3 files changed, 106 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/sysfb.c b/drivers/firmware/sysfb.c index 170b7cd0cfcb..a6b48703dc9e 100644 --- a/drivers/firmware/sysfb.c +++ b/drivers/firmware/sysfb.c @@ -118,6 +118,8 @@ static __init int sysfb_init(void) bool compatible; int ret = 0; + screen_info_apply_fixups(); + mutex_lock(&disable_lock); if (disabled) goto unlock_mutex; diff --git a/drivers/video/screen_info_pci.c b/drivers/video/screen_info_pci.c index d8985a54ce71..6c5833517141 100644 --- a/drivers/video/screen_info_pci.c +++ b/drivers/video/screen_info_pci.c @@ -1,7 +1,95 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include +#include + +static struct pci_dev *screen_info_lfb_pdev; +static size_t screen_info_lfb_bar; +static resource_size_t screen_info_lfb_offset; +static struct resource screen_info_lfb_res = DEFINE_RES_MEM(0, 0); + +static bool __screen_info_relocation_is_valid(const struct screen_info *si, struct resource *pr) +{ + u64 size = __screen_info_lfb_size(si, screen_info_video_type(si)); + + if (screen_info_lfb_offset > resource_size(pr)) + return false; + if (size > resource_size(pr)) + return false; + if (resource_size(pr) - size < screen_info_lfb_offset) + return false; + + return true; +} + +void screen_info_apply_fixups(void) +{ + struct screen_info *si = &screen_info; + + if (screen_info_lfb_pdev) { + struct resource *pr = &screen_info_lfb_pdev->resource[screen_info_lfb_bar]; + + if (pr->start != screen_info_lfb_res.start) { + if (__screen_info_relocation_is_valid(si, pr)) { + /* + * Only update base if we have an actual + * relocation to a valid I/O range. + */ + __screen_info_set_lfb_base(si, pr->start + screen_info_lfb_offset); + pr_info("Relocating firmware framebuffer to offset %pa[d] within %pr\n", + &screen_info_lfb_offset, pr); + } else { + pr_warn("Invalid relocating, disabling firmware framebuffer\n"); + } + } + } +} + +static void screen_info_fixup_lfb(struct pci_dev *pdev) +{ + unsigned int type; + struct resource res[SCREEN_INFO_MAX_RESOURCES]; + size_t i, numres; + int ret; + const struct screen_info *si = &screen_info; + + if (screen_info_lfb_pdev) + return; // already found + + type = screen_info_video_type(si); + if (type != VIDEO_TYPE_EFI) + return; // only applies to EFI + + ret = screen_info_resources(si, res, ARRAY_SIZE(res)); + if (ret < 0) + return; + numres = ret; + + for (i = 0; i < numres; ++i) { + struct resource *r = &res[i]; + const struct resource *pr; + + if (!(r->flags & IORESOURCE_MEM)) + continue; + pr = pci_find_resource(pdev, r); + if (!pr) + continue; + + /* + * We've found a PCI device with the framebuffer + * resource. Store away the parameters to track + * relocation of the framebuffer aperture. + */ + screen_info_lfb_pdev = pdev; + screen_info_lfb_bar = pr - pdev->resource; + screen_info_lfb_offset = r->start - pr->start; + memcpy(&screen_info_lfb_res, r, sizeof(screen_info_lfb_res)); + } +} +DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY, 16, + screen_info_fixup_lfb); static struct pci_dev *__screen_info_pci_dev(struct resource *res) { diff --git a/include/linux/screen_info.h b/include/linux/screen_info.h index 0eae08e3c6f9..75303c126285 100644 --- a/include/linux/screen_info.h +++ b/include/linux/screen_info.h @@ -4,6 +4,8 @@ #include +#include + /** * SCREEN_INFO_MAX_RESOURCES - maximum number of resources per screen_info */ @@ -27,6 +29,17 @@ static inline u64 __screen_info_lfb_base(const struct screen_info *si) return lfb_base; } +static inline void __screen_info_set_lfb_base(struct screen_info *si, u64 lfb_base) +{ + si->lfb_base = lfb_base & GENMASK_ULL(31, 0); + si->ext_lfb_base = (lfb_base & GENMASK_ULL(63, 32)) >> 32; + + if (si->ext_lfb_base) + si->capabilities |= VIDEO_CAPABILITY_64BIT_BASE; + else + si->capabilities &= ~VIDEO_CAPABILITY_64BIT_BASE; +} + static inline u64 __screen_info_lfb_size(const struct screen_info *si, unsigned int type) { u64 lfb_size = si->lfb_size; @@ -106,8 +119,11 @@ static inline unsigned int screen_info_video_type(const struct screen_info *si) ssize_t screen_info_resources(const struct screen_info *si, struct resource *r, size_t num); #if defined(CONFIG_PCI) +void screen_info_apply_fixups(void); struct pci_dev *screen_info_pci_dev(const struct screen_info *si); #else +static inline void screen_info_apply_fixups(void) +{ } static inline struct pci_dev *screen_info_pci_dev(const struct screen_info *si) { return NULL; -- cgit v1.2.3 From 161e16a5e50a82d219b3df3ce32286b0a2ae08bd Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 Jan 2024 13:39:47 +0100 Subject: PM: domains: Add helper functions to attach/detach multiple PM domains Attaching/detaching of a device to multiple PM domains has started to become a common operation for many drivers, typically during ->probe() and ->remove(). In most cases, this has lead to lots of boilerplate code in the drivers. To fixup up the situation, let's introduce a pair of helper functions, dev_pm_domain_attach|detach_list(), that driver can use instead of the open-coding. Note that, it seems reasonable to limit the support for these helpers to DT based platforms, at it's the only valid use case for now. Suggested-by: Daniel Baluta Tested-by: Bryan O'Donoghue Tested-by: Iuliana Prodan Signed-off-by: Ulf Hansson Link: https://lore.kernel.org/r/20240130123951.236243-2-ulf.hansson@linaro.org --- drivers/base/power/common.c | 134 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/pm_domain.h | 38 +++++++++++++ 2 files changed, 172 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index 44ec20918a4d..327d168dd37a 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -167,6 +167,115 @@ struct device *dev_pm_domain_attach_by_name(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_name); +/** + * dev_pm_domain_attach_list - Associate a device with its PM domains. + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * This function helps to attach a device to its multiple PM domains. The + * caller, which is typically a driver's probe function, may provide a list of + * names for the PM domains that we should try to attach the device to, but it + * may also provide an empty list, in case the attach should be done for all of + * the available PM domains. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. Note that, to detach the list of PM domains, the driver shall call + * dev_pm_domain_detach_list(), typically during the remove phase. + */ +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + struct device_node *np = dev->of_node; + struct dev_pm_domain_list *pds; + struct device *pd_dev = NULL; + int ret, i, num_pds = 0; + bool by_id = true; + u32 pd_flags = data ? data->pd_flags : 0; + u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 : + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME; + + if (dev->pm_domain) + return -EEXIST; + + /* For now this is limited to OF based platforms. */ + if (!np) + return 0; + + if (data && data->pd_names) { + num_pds = data->num_pd_names; + by_id = false; + } else { + num_pds = of_count_phandle_with_args(np, "power-domains", + "#power-domain-cells"); + } + + if (num_pds <= 0) + return 0; + + pds = devm_kzalloc(dev, sizeof(*pds), GFP_KERNEL); + if (!pds) + return -ENOMEM; + + pds->pd_devs = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_devs), + GFP_KERNEL); + if (!pds->pd_devs) + return -ENOMEM; + + pds->pd_links = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_links), + GFP_KERNEL); + if (!pds->pd_links) + return -ENOMEM; + + if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON) + link_flags |= DL_FLAG_RPM_ACTIVE; + + for (i = 0; i < num_pds; i++) { + if (by_id) + pd_dev = dev_pm_domain_attach_by_id(dev, i); + else + pd_dev = dev_pm_domain_attach_by_name(dev, + data->pd_names[i]); + if (IS_ERR_OR_NULL(pd_dev)) { + ret = pd_dev ? PTR_ERR(pd_dev) : -ENODEV; + goto err_attach; + } + + if (link_flags) { + struct device_link *link; + + link = device_link_add(dev, pd_dev, link_flags); + if (!link) { + ret = -ENODEV; + goto err_link; + } + + pds->pd_links[i] = link; + } + + pds->pd_devs[i] = pd_dev; + } + + pds->num_pds = num_pds; + *list = pds; + return num_pds; + +err_link: + dev_pm_domain_detach(pd_dev, true); +err_attach: + while (--i >= 0) { + if (pds->pd_links[i]) + device_link_del(pds->pd_links[i]); + dev_pm_domain_detach(pds->pd_devs[i], true); + } + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list); + /** * dev_pm_domain_detach - Detach a device from its PM domain. * @dev: Device to detach. @@ -187,6 +296,31 @@ void dev_pm_domain_detach(struct device *dev, bool power_off) } EXPORT_SYMBOL_GPL(dev_pm_domain_detach); +/** + * dev_pm_domain_detach_list - Detach a list of PM domains. + * @list: The list of PM domains to detach. + * + * This function reverse the actions from dev_pm_domain_attach_list(). + * Typically it should be invoked during the remove phase from drivers. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + */ +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) +{ + int i; + + if (!list) + return; + + for (i = 0; i < list->num_pds; i++) { + if (list->pd_links[i]) + device_link_del(list->pd_links[i]); + dev_pm_domain_detach(list->pd_devs[i], true); + } +} +EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list); + /** * dev_pm_domain_start - Start the device through its PM domain. * @dev: Device to start. diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index b97c5e9820f9..fb91770ba4ba 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -19,6 +19,33 @@ #include #include +/* + * Flags to control the behaviour when attaching a device to its PM domains. + * + * PD_FLAG_NO_DEV_LINK: As the default behaviour creates a device-link + * for every PM domain that gets attached, this + * flag can be used to skip that. + * + * PD_FLAG_DEV_LINK_ON: Add the DL_FLAG_RPM_ACTIVE to power-on the + * supplier and its PM domain when creating the + * device-links. + * + */ +#define PD_FLAG_NO_DEV_LINK BIT(0) +#define PD_FLAG_DEV_LINK_ON BIT(1) + +struct dev_pm_domain_attach_data { + const char * const *pd_names; + const u32 num_pd_names; + const u32 pd_flags; +}; + +struct dev_pm_domain_list { + struct device **pd_devs; + struct device_link **pd_links; + u32 num_pds; +}; + /* * Flags to control the behaviour of a genpd. * @@ -420,7 +447,11 @@ struct device *dev_pm_domain_attach_by_id(struct device *dev, unsigned int index); struct device *dev_pm_domain_attach_by_name(struct device *dev, const char *name); +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list); void dev_pm_domain_detach(struct device *dev, bool power_off); +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list); int dev_pm_domain_start(struct device *dev); void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd); int dev_pm_domain_set_performance_state(struct device *dev, unsigned int state); @@ -439,7 +470,14 @@ static inline struct device *dev_pm_domain_attach_by_name(struct device *dev, { return NULL; } +static inline int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + return 0; +} static inline void dev_pm_domain_detach(struct device *dev, bool power_off) {} +static inline void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) {} static inline int dev_pm_domain_start(struct device *dev) { return 0; -- cgit v1.2.3 From 4d0824608a636b64373e52d3ef1516a86048e0e7 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:28:21 +0100 Subject: pmdomain: core: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202822.631449-1-krzysztof.kozlowski@linaro.org Signed-off-by: Ulf Hansson --- drivers/pmdomain/core.c | 4 ++-- drivers/pmdomain/imx/scu-pd.c | 2 +- drivers/pmdomain/tegra/powergate-bpmp.c | 2 +- drivers/pmdomain/ti/ti_sci_pm_domains.c | 2 +- drivers/pmdomain/xilinx/zynqmp-pm-domains.c | 2 +- include/linux/pm_domain.h | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 46331e71108a..ea4b29475206 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -2266,7 +2266,7 @@ static DEFINE_MUTEX(of_genpd_mutex); * to be a valid pointer to struct generic_pm_domain. */ static struct generic_pm_domain *genpd_xlate_simple( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { return data; @@ -2283,7 +2283,7 @@ static struct generic_pm_domain *genpd_xlate_simple( * the genpd_onecell_data struct when registering the provider. */ static struct generic_pm_domain *genpd_xlate_onecell( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; diff --git a/drivers/pmdomain/imx/scu-pd.c b/drivers/pmdomain/imx/scu-pd.c index 891c1d925a9d..05841b0bf7f3 100644 --- a/drivers/pmdomain/imx/scu-pd.c +++ b/drivers/pmdomain/imx/scu-pd.c @@ -393,7 +393,7 @@ static int imx_sc_pd_power_off(struct generic_pm_domain *domain) return imx_sc_pd_power(domain, false); } -static struct generic_pm_domain *imx_scu_pd_xlate(struct of_phandle_args *spec, +static struct generic_pm_domain *imx_scu_pd_xlate(const struct of_phandle_args *spec, void *data) { struct generic_pm_domain *domain = ERR_PTR(-ENOENT); diff --git a/drivers/pmdomain/tegra/powergate-bpmp.c b/drivers/pmdomain/tegra/powergate-bpmp.c index 179ed895c279..b0138ca9f851 100644 --- a/drivers/pmdomain/tegra/powergate-bpmp.c +++ b/drivers/pmdomain/tegra/powergate-bpmp.c @@ -305,7 +305,7 @@ static void tegra_bpmp_remove_powergates(struct tegra_bpmp *bpmp) } static struct generic_pm_domain * -tegra_powergate_xlate(struct of_phandle_args *spec, void *data) +tegra_powergate_xlate(const struct of_phandle_args *spec, void *data) { struct generic_pm_domain *domain = ERR_PTR(-ENOENT); struct genpd_onecell_data *genpd = data; diff --git a/drivers/pmdomain/ti/ti_sci_pm_domains.c b/drivers/pmdomain/ti/ti_sci_pm_domains.c index c091d569ecd5..9dddf227a3a6 100644 --- a/drivers/pmdomain/ti/ti_sci_pm_domains.c +++ b/drivers/pmdomain/ti/ti_sci_pm_domains.c @@ -85,7 +85,7 @@ static int ti_sci_pd_power_on(struct generic_pm_domain *domain) * @data: genpd core data for all the powerdomains on the device */ static struct generic_pm_domain *ti_sci_pd_xlate( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; diff --git a/drivers/pmdomain/xilinx/zynqmp-pm-domains.c b/drivers/pmdomain/xilinx/zynqmp-pm-domains.c index 6fd514286d82..0b5831e5ba1b 100644 --- a/drivers/pmdomain/xilinx/zynqmp-pm-domains.c +++ b/drivers/pmdomain/xilinx/zynqmp-pm-domains.c @@ -210,7 +210,7 @@ static void zynqmp_gpd_detach_dev(struct generic_pm_domain *domain, } static struct generic_pm_domain *zynqmp_gpd_xlate - (struct of_phandle_args *genpdspec, void *data) + (const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; unsigned int i, idx = genpdspec->args[0]; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index fb91770ba4ba..1a391ef1b6f8 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -349,7 +349,7 @@ static inline void dev_pm_genpd_resume(struct device *dev) {} /* OF PM domain providers */ struct of_device_id; -typedef struct generic_pm_domain *(*genpd_xlate_t)(struct of_phandle_args *args, +typedef struct generic_pm_domain *(*genpd_xlate_t)(const struct of_phandle_args *args, void *data); struct genpd_onecell_data { -- cgit v1.2.3 From 4af6bc163c4d841cf60bc9bf6e98603a232889b6 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:28:22 +0100 Subject: pmdomain: core: constify of_phandle_args in add device and subdomain Pointer to of_phandle_args is not modified by of_genpd_add_device() and of_genpd_add_subdomain(), so it can be made pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202822.631449-2-krzysztof.kozlowski@linaro.org Signed-off-by: Ulf Hansson --- drivers/pmdomain/core.c | 12 ++++++------ include/linux/pm_domain.h | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index ea4b29475206..4215ffd9b11c 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -2526,7 +2526,7 @@ EXPORT_SYMBOL_GPL(of_genpd_del_provider); * on failure. */ static struct generic_pm_domain *genpd_get_from_provider( - struct of_phandle_args *genpdspec) + const struct of_phandle_args *genpdspec) { struct generic_pm_domain *genpd = ERR_PTR(-ENOENT); struct of_genpd_provider *provider; @@ -2557,7 +2557,7 @@ static struct generic_pm_domain *genpd_get_from_provider( * Looks-up an I/O PM domain based upon phandle args provided and adds * the device to the PM domain. Returns a negative error code on failure. */ -int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev) +int of_genpd_add_device(const struct of_phandle_args *genpdspec, struct device *dev) { struct generic_pm_domain *genpd; int ret; @@ -2591,8 +2591,8 @@ EXPORT_SYMBOL_GPL(of_genpd_add_device); * provided and adds the subdomain to the parent PM domain. Returns a * negative error code on failure. */ -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { struct generic_pm_domain *parent, *subdomain; int ret; @@ -2629,8 +2629,8 @@ EXPORT_SYMBOL_GPL(of_genpd_add_subdomain); * provided and removes the subdomain from the parent PM domain. Returns a * negative error code on failure. */ -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { struct generic_pm_domain *parent, *subdomain; int ret; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 1a391ef1b6f8..772d3280d35f 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -364,11 +364,11 @@ int of_genpd_add_provider_simple(struct device_node *np, int of_genpd_add_provider_onecell(struct device_node *np, struct genpd_onecell_data *data); void of_genpd_del_provider(struct device_node *np); -int of_genpd_add_device(struct of_phandle_args *args, struct device *dev); -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec); -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec); +int of_genpd_add_device(const struct of_phandle_args *args, struct device *dev); +int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec); +int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec); struct generic_pm_domain *of_genpd_remove_last(struct device_node *np); int of_genpd_parse_idle_states(struct device_node *dn, struct genpd_power_state **states, int *n); @@ -393,20 +393,20 @@ static inline int of_genpd_add_provider_onecell(struct device_node *np, static inline void of_genpd_del_provider(struct device_node *np) {} -static inline int of_genpd_add_device(struct of_phandle_args *args, +static inline int of_genpd_add_device(const struct of_phandle_args *args, struct device *dev) { return -ENODEV; } -static inline int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +static inline int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { return -ENODEV; } -static inline int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +static inline int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { return -ENODEV; } -- cgit v1.2.3 From 4276d28e1da689e0cad3e88521c2cddbea42eb6e Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 13 Feb 2024 14:42:19 -0800 Subject: iosys-map: fix typo Correct a spello/typo in comments. Signed-off-by: Randy Dunlap Cc: Thomas Zimmermann Cc: dri-devel@lists.freedesktop.org Reviewed-by: Thomas Zimmermann Signed-off-by: Thomas Zimmermann Link: https://patchwork.freedesktop.org/patch/msgid/20240213224219.10644-1-rdunlap@infradead.org --- include/linux/iosys-map.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h index e3649a6563dd..4696abfd311c 100644 --- a/include/linux/iosys-map.h +++ b/include/linux/iosys-map.h @@ -34,7 +34,7 @@ * the same driver for allocation, read and write operations. * * Open-coding access to :c:type:`struct iosys_map ` is considered - * bad style. Rather then accessing its fields directly, use one of the provided + * bad style. Rather than accessing its fields directly, use one of the provided * helper functions, or implement your own. For example, instances of * :c:type:`struct iosys_map ` can be initialized statically with * IOSYS_MAP_INIT_VADDR(), or at runtime with iosys_map_set_vaddr(). These -- cgit v1.2.3 From 4d42b37def70327b2bb19f823d42289aed2cd7c7 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 13 Feb 2024 06:32:36 +0000 Subject: net: convert dev->reg_state to u8 Prepares things so that dev->reg_state reads can be lockless, by adding WRITE_ONCE() on write side. READ_ONCE()/WRITE_ONCE() do not support bitfields. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 23 ++++++++++++++--------- net/core/dev.c | 8 ++++---- 2 files changed, 18 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index a3f9c95da51e..631124655107 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1815,6 +1815,15 @@ enum netdev_stat_type { NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */ }; +enum netdev_reg_state { + NETREG_UNINITIALIZED = 0, + NETREG_REGISTERED, /* completed register_netdevice */ + NETREG_UNREGISTERING, /* called unregister_netdevice */ + NETREG_UNREGISTERED, /* completed unregister todo */ + NETREG_RELEASED, /* called free_netdev */ + NETREG_DUMMY, /* dummy device for NAPI poll */ +}; + /** * struct net_device - The DEVICE structure. * @@ -2372,13 +2381,7 @@ struct net_device { struct list_head link_watch_list; - enum { NETREG_UNINITIALIZED=0, - NETREG_REGISTERED, /* completed register_netdevice */ - NETREG_UNREGISTERING, /* called unregister_netdevice */ - NETREG_UNREGISTERED, /* completed unregister todo */ - NETREG_RELEASED, /* called free_netdev */ - NETREG_DUMMY, /* dummy device for NAPI poll */ - } reg_state:8; + u8 reg_state; bool dismantle; @@ -5254,7 +5257,9 @@ static inline const char *netdev_name(const struct net_device *dev) static inline const char *netdev_reg_state(const struct net_device *dev) { - switch (dev->reg_state) { + u8 reg_state = READ_ONCE(dev->reg_state); + + switch (reg_state) { case NETREG_UNINITIALIZED: return " (uninitialized)"; case NETREG_REGISTERED: return ""; case NETREG_UNREGISTERING: return " (unregistering)"; @@ -5263,7 +5268,7 @@ static inline const char *netdev_reg_state(const struct net_device *dev) case NETREG_DUMMY: return " (dummy)"; } - WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state); + WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, reg_state); return " (unknown)"; } diff --git a/net/core/dev.c b/net/core/dev.c index 720bd6838212..9c95cae9d6ab 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10339,7 +10339,7 @@ int register_netdevice(struct net_device *dev) ret = netdev_register_kobject(dev); write_lock(&dev_base_lock); - dev->reg_state = ret ? NETREG_UNREGISTERED : NETREG_REGISTERED; + WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); write_unlock(&dev_base_lock); if (ret) goto err_uninit_notify; @@ -10630,7 +10630,7 @@ void netdev_run_todo(void) } write_lock(&dev_base_lock); - dev->reg_state = NETREG_UNREGISTERED; + WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); write_unlock(&dev_base_lock); linkwatch_sync_dev(dev); } @@ -11050,7 +11050,7 @@ void free_netdev(struct net_device *dev) } BUG_ON(dev->reg_state != NETREG_UNREGISTERED); - dev->reg_state = NETREG_RELEASED; + WRITE_ONCE(dev->reg_state, NETREG_RELEASED); /* will free via device release */ put_device(&dev->dev); @@ -11140,7 +11140,7 @@ void unregister_netdevice_many_notify(struct list_head *head, /* And unlink it from device chain. */ write_lock(&dev_base_lock); unlist_netdevice(dev, false); - dev->reg_state = NETREG_UNREGISTERING; + WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); write_unlock(&dev_base_lock); } flush_all_backlogs(); -- cgit v1.2.3 From 6a2968ee1ee2cc6fce30f6f5724442b34b1483b3 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 13 Feb 2024 06:32:42 +0000 Subject: net: add netdev_set_operstate() helper dev_base_lock is going away, add netdev_set_operstate() helper so that hsr does not have to know core internals. Remove dev_base_lock acquisition from rfc2863_policy() v3: use an "unsigned int" for dev->operstate, so that try_cmpxchg() can work on all arches. ( https://lore.kernel.org/oe-kbuild-all/202402081918.OLyGaea3-lkp@intel.com/ ) Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 2 +- include/linux/rtnetlink.h | 2 ++ net/core/link_watch.c | 9 ++------- net/core/rtnetlink.c | 22 +++++++++++++++------- net/hsr/hsr_device.c | 22 ++++++---------------- 5 files changed, 26 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 631124655107..697370706a82 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2258,7 +2258,7 @@ struct net_device { const struct tlsdev_ops *tlsdev_ops; #endif - unsigned char operstate; + unsigned int operstate; unsigned char link_mode; unsigned char if_port; diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 21780608cf47..cdfc897f1e3c 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -172,4 +172,6 @@ rtnl_notify_needed(const struct net *net, u16 nlflags, u32 group) return (nlflags & NLM_F_ECHO) || rtnl_has_listeners(net, group); } +void netdev_set_operstate(struct net_device *dev, int newstate); + #endif /* __LINUX_RTNETLINK_H */ diff --git a/net/core/link_watch.c b/net/core/link_watch.c index 1b93e054c9a3..8ec35194bfcb 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -33,7 +33,7 @@ static DECLARE_DELAYED_WORK(linkwatch_work, linkwatch_event); static LIST_HEAD(lweventlist); static DEFINE_SPINLOCK(lweventlist_lock); -static unsigned char default_operstate(const struct net_device *dev) +static unsigned int default_operstate(const struct net_device *dev) { if (netif_testing(dev)) return IF_OPER_TESTING; @@ -62,16 +62,13 @@ static unsigned char default_operstate(const struct net_device *dev) return IF_OPER_UP; } - static void rfc2863_policy(struct net_device *dev) { - unsigned char operstate = default_operstate(dev); + unsigned int operstate = default_operstate(dev); if (operstate == READ_ONCE(dev->operstate)) return; - write_lock(&dev_base_lock); - switch(dev->link_mode) { case IF_LINK_MODE_TESTING: if (operstate == IF_OPER_UP) @@ -88,8 +85,6 @@ static void rfc2863_policy(struct net_device *dev) } WRITE_ONCE(dev->operstate, operstate); - - write_unlock(&dev_base_lock); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 43d92de8601c..e484ba44f23b 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -842,9 +842,22 @@ int rtnl_put_cacheinfo(struct sk_buff *skb, struct dst_entry *dst, u32 id, } EXPORT_SYMBOL_GPL(rtnl_put_cacheinfo); +void netdev_set_operstate(struct net_device *dev, int newstate) +{ + unsigned int old = READ_ONCE(dev->operstate); + + do { + if (old == newstate) + return; + } while (!try_cmpxchg(&dev->operstate, &old, newstate)); + + netdev_state_change(dev); +} +EXPORT_SYMBOL(netdev_set_operstate); + static void set_operstate(struct net_device *dev, unsigned char transition) { - unsigned char operstate = dev->operstate; + unsigned char operstate = READ_ONCE(dev->operstate); switch (transition) { case IF_OPER_UP: @@ -866,12 +879,7 @@ static void set_operstate(struct net_device *dev, unsigned char transition) break; } - if (READ_ONCE(dev->operstate) != operstate) { - write_lock(&dev_base_lock); - WRITE_ONCE(dev->operstate, operstate); - write_unlock(&dev_base_lock); - netdev_state_change(dev); - } + netdev_set_operstate(dev, operstate); } static unsigned int rtnl_dev_get_flags(const struct net_device *dev) diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index be0e43f46556..5ef6d437db72 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -28,29 +28,19 @@ static bool is_slave_up(struct net_device *dev) return dev && is_admin_up(dev) && netif_oper_up(dev); } -static void __hsr_set_operstate(struct net_device *dev, int transition) -{ - write_lock(&dev_base_lock); - if (READ_ONCE(dev->operstate) != transition) { - WRITE_ONCE(dev->operstate, transition); - write_unlock(&dev_base_lock); - netdev_state_change(dev); - } else { - write_unlock(&dev_base_lock); - } -} - static void hsr_set_operstate(struct hsr_port *master, bool has_carrier) { - if (!is_admin_up(master->dev)) { - __hsr_set_operstate(master->dev, IF_OPER_DOWN); + struct net_device *dev = master->dev; + + if (!is_admin_up(dev)) { + netdev_set_operstate(dev, IF_OPER_DOWN); return; } if (has_carrier) - __hsr_set_operstate(master->dev, IF_OPER_UP); + netdev_set_operstate(dev, IF_OPER_UP); else - __hsr_set_operstate(master->dev, IF_OPER_LOWERLAYERDOWN); + netdev_set_operstate(dev, IF_OPER_LOWERLAYERDOWN); } static bool hsr_check_carrier(struct hsr_port *master) -- cgit v1.2.3 From 1b3ef46cb7f2618cc0b507393220a69810f6da12 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 13 Feb 2024 06:32:45 +0000 Subject: net: remove dev_base_lock dev_base_lock is not needed anymore, all remaining users also hold RTNL. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 2 -- net/core/dev.c | 39 ++++----------------------------------- 2 files changed, 4 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 697370706a82..c541550b0e6e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3077,8 +3077,6 @@ int call_netdevice_notifiers(unsigned long val, struct net_device *dev); int call_netdevice_notifiers_info(unsigned long val, struct netdev_notifier_info *info); -extern rwlock_t dev_base_lock; /* Device list lock */ - #define for_each_netdev(net, d) \ list_for_each_entry(d, &(net)->dev_base_head, dev_list) #define for_each_netdev_reverse(net, d) \ diff --git a/net/core/dev.c b/net/core/dev.c index 02cf9fd68da6..d8dd293a7a27 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -168,28 +168,6 @@ static int call_netdevice_notifiers_extack(unsigned long val, struct net_device *dev, struct netlink_ext_ack *extack); -/* - * The @dev_base_head list is protected by @dev_base_lock and the rtnl - * semaphore. - * - * Pure readers hold dev_base_lock for reading, or rcu_read_lock() - * - * Writers must hold the rtnl semaphore while they loop through the - * dev_base_head list, and hold dev_base_lock for writing when they do the - * actual updates. This allows pure readers to access the list even - * while a writer is preparing to update it. - * - * To put it another way, dev_base_lock is held for writing only to - * protect against pure readers; the rtnl semaphore provides the - * protection against other writers. - * - * See, for example usages, register_netdevice() and - * unregister_netdevice(), which must be called with the rtnl - * semaphore held. - */ -DEFINE_RWLOCK(dev_base_lock); -EXPORT_SYMBOL(dev_base_lock); - static DEFINE_MUTEX(ifalias_mutex); /* protects napi_hash addition/deletion and napi_gen_id */ @@ -395,12 +373,10 @@ static void list_netdevice(struct net_device *dev) ASSERT_RTNL(); - write_lock(&dev_base_lock); list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); netdev_name_node_add(net, dev->name_node); hlist_add_head_rcu(&dev->index_hlist, dev_index_hash(net, dev->ifindex)); - write_unlock(&dev_base_lock); netdev_for_each_altname(dev, name_node) netdev_name_node_add(net, name_node); @@ -427,11 +403,9 @@ static void unlist_netdevice(struct net_device *dev) netdev_name_node_del(name_node); /* Unlink dev from the device chain */ - write_lock(&dev_base_lock); list_del_rcu(&dev->dev_list); netdev_name_node_del(dev->name_node); hlist_del_rcu(&dev->index_hlist); - write_unlock(&dev_base_lock); dev_base_seq_inc(dev_net(dev)); } @@ -752,9 +726,9 @@ EXPORT_SYMBOL_GPL(dev_fill_forward_path); * @net: the applicable net namespace * @name: name to find * - * Find an interface by name. Must be called under RTNL semaphore - * or @dev_base_lock. If the name is found a pointer to the device - * is returned. If the name is not found then %NULL is returned. The + * Find an interface by name. Must be called under RTNL semaphore. + * If the name is found a pointer to the device is returned. + * If the name is not found then %NULL is returned. The * reference counters are not incremented so the caller must be * careful with locks. */ @@ -835,8 +809,7 @@ EXPORT_SYMBOL(netdev_get_by_name); * Search for an interface by index. Returns %NULL if the device * is not found or a pointer to the device. The device has not * had its reference counter increased so the caller must be careful - * about locking. The caller must hold either the RTNL semaphore - * or @dev_base_lock. + * about locking. The caller must hold the RTNL semaphore. */ struct net_device *__dev_get_by_index(struct net *net, int ifindex) @@ -1241,15 +1214,11 @@ rollback: netdev_adjacent_rename_links(dev, oldname); - write_lock(&dev_base_lock); netdev_name_node_del(dev->name_node); - write_unlock(&dev_base_lock); synchronize_net(); - write_lock(&dev_base_lock); netdev_name_node_add(net, dev->name_node); - write_unlock(&dev_base_lock); ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); ret = notifier_to_errno(ret); -- cgit v1.2.3 From 4e77431cda4973f03d063c47f6ea313dfceebf16 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Tue, 9 Jan 2024 22:34:32 +0100 Subject: pwm: Drop useless member .of_pwm_n_cells of struct pwm_chip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apart from the two of_xlate implementations this member is write-only. In the of_xlate functions of_pwm_xlate_with_flags() and of_pwm_single_xlate() it's more sensible to check for args->args_count because this is what is actually used in the device tree. Acked-by: Douglas Anderson Link: https://lore.kernel.org/r/53d8c545aa8f79a920358be9e72e382b3981bdc4.1704835845.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/gpu/drm/bridge/ti-sn65dsi86.c | 1 - drivers/pwm/core.c | 22 +++------------------- drivers/pwm/pwm-clps711x.c | 1 - drivers/pwm/pwm-cros-ec.c | 1 - drivers/pwm/pwm-pxa.c | 4 +--- include/linux/pwm.h | 2 -- 6 files changed, 4 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index 62cc3893dca5..1f6e929c2f6a 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -1591,7 +1591,6 @@ static int ti_sn_pwm_probe(struct auxiliary_device *adev, pdata->pchip.ops = &ti_sn_pwm_ops; pdata->pchip.npwm = 1; pdata->pchip.of_xlate = of_pwm_single_xlate; - pdata->pchip.of_pwm_n_cells = 1; devm_pm_runtime_enable(&adev->dev); diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index f2728ee787d7..31f210872a07 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -107,9 +107,6 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg { struct pwm_device *pwm; - if (chip->of_pwm_n_cells < 2) - return ERR_PTR(-EINVAL); - /* flags in the third cell are optional */ if (args->args_count < 2) return ERR_PTR(-EINVAL); @@ -124,10 +121,8 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg pwm->args.period = args->args[1]; pwm->args.polarity = PWM_POLARITY_NORMAL; - if (chip->of_pwm_n_cells >= 3) { - if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) - pwm->args.polarity = PWM_POLARITY_INVERSED; - } + if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; return pwm; } @@ -138,9 +133,6 @@ of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) { struct pwm_device *pwm; - if (chip->of_pwm_n_cells < 1) - return ERR_PTR(-EINVAL); - /* validate that one cell is specified, optionally with flags */ if (args->args_count != 1 && args->args_count != 2) return ERR_PTR(-EINVAL); @@ -164,16 +156,8 @@ static void of_pwmchip_add(struct pwm_chip *chip) if (!chip->dev || !chip->dev->of_node) return; - if (!chip->of_xlate) { - u32 pwm_cells; - - if (of_property_read_u32(chip->dev->of_node, "#pwm-cells", - &pwm_cells)) - pwm_cells = 2; - + if (!chip->of_xlate) chip->of_xlate = of_pwm_xlate_with_flags; - chip->of_pwm_n_cells = pwm_cells; - } of_node_get(chip->dev->of_node); } diff --git a/drivers/pwm/pwm-clps711x.c b/drivers/pwm/pwm-clps711x.c index 42179b3f7ec3..06562d4bb963 100644 --- a/drivers/pwm/pwm-clps711x.c +++ b/drivers/pwm/pwm-clps711x.c @@ -103,7 +103,6 @@ static int clps711x_pwm_probe(struct platform_device *pdev) priv->chip.dev = &pdev->dev; priv->chip.npwm = 2; priv->chip.of_xlate = clps711x_pwm_xlate; - priv->chip.of_pwm_n_cells = 1; spin_lock_init(&priv->lock); diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 5fe303b8656d..339cedf3a7b1 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -279,7 +279,6 @@ static int cros_ec_pwm_probe(struct platform_device *pdev) chip->dev = dev; chip->ops = &cros_ec_pwm_ops; chip->of_xlate = cros_ec_pwm_xlate; - chip->of_pwm_n_cells = 1; if (ec_pwm->use_pwm_type) { chip->npwm = CROS_EC_PWM_DT_COUNT; diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c index 76685f926c75..61b74fa1d348 100644 --- a/drivers/pwm/pwm-pxa.c +++ b/drivers/pwm/pwm-pxa.c @@ -180,10 +180,8 @@ static int pwm_probe(struct platform_device *pdev) pc->chip.ops = &pxa_pwm_ops; pc->chip.npwm = (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1; - if (IS_ENABLED(CONFIG_OF)) { + if (IS_ENABLED(CONFIG_OF)) pc->chip.of_xlate = of_pwm_single_xlate; - pc->chip.of_pwm_n_cells = 1; - } pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) diff --git a/include/linux/pwm.h b/include/linux/pwm.h index fcc2c4496f73..8ffe9ae7a23a 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -271,7 +271,6 @@ struct pwm_ops { * @id: unique number of this PWM chip * @npwm: number of PWMs controlled by this chip * @of_xlate: request a PWM device given a device tree PWM specifier - * @of_pwm_n_cells: number of cells expected in the device tree PWM specifier * @atomic: can the driver's ->apply() be called in atomic context * @pwms: array of PWM devices allocated by the framework */ @@ -284,7 +283,6 @@ struct pwm_chip { struct pwm_device * (*of_xlate)(struct pwm_chip *chip, const struct of_phandle_args *args); - unsigned int of_pwm_n_cells; bool atomic; /* only used internally by the PWM framework */ -- cgit v1.2.3 From 4e59267c7a20eb1c1ad9106e682cb6a0d8eb3111 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 14 Feb 2024 10:30:48 +0100 Subject: pwm: Provide an inline function to get the parent device of a given chip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently a pwm_chip stores in its struct device *dev member a pointer to the parent device. Preparing a change that embeds a full struct device in struct pwm_chip, this accessor function should be used in all drivers directly accessing chip->dev now. This way struct pwm_chip and this new function can be changed without having to touch all drivers in the same change set. Make use of this function in the framework's core sources. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/cc30090d2f9762bed9854a55612144bccc910781.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/core.c | 42 +++++++++++++++++++++--------------------- drivers/pwm/sysfs.c | 4 ++-- include/linux/pwm.h | 5 +++++ 3 files changed, 28 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 1b4c3d0caa82..830a697826af 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -71,18 +71,18 @@ static void pwm_apply_debug(struct pwm_device *pwm, if (s2.polarity != state->polarity && state->duty_cycle < state->period) - dev_warn(chip->dev, ".apply ignored .polarity\n"); + dev_warn(pwmchip_parent(chip), ".apply ignored .polarity\n"); if (state->enabled && last->polarity == state->polarity && last->period > s2.period && last->period <= state->period) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply didn't pick the best available period (requested: %llu, applied: %llu, possible: %llu)\n", state->period, s2.period, last->period); if (state->enabled && state->period < s2.period) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply is supposed to round down period (requested: %llu, applied: %llu)\n", state->period, s2.period); @@ -91,20 +91,20 @@ static void pwm_apply_debug(struct pwm_device *pwm, last->period == s2.period && last->duty_cycle > s2.duty_cycle && last->duty_cycle <= state->duty_cycle) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply didn't pick the best available duty cycle (requested: %llu/%llu, applied: %llu/%llu, possible: %llu/%llu)\n", state->duty_cycle, state->period, s2.duty_cycle, s2.period, last->duty_cycle, last->period); if (state->enabled && state->duty_cycle < s2.duty_cycle) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply is supposed to round down duty_cycle (requested: %llu/%llu, applied: %llu/%llu)\n", state->duty_cycle, state->period, s2.duty_cycle, s2.period); if (!state->enabled && s2.enabled && s2.duty_cycle > 0) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "requested disabled, but yielded enabled with duty > 0\n"); /* reapply the state that the driver reported being configured. */ @@ -112,7 +112,7 @@ static void pwm_apply_debug(struct pwm_device *pwm, trace_pwm_apply(pwm, &s1, err); if (err) { *last = s1; - dev_err(chip->dev, "failed to reapply current setting\n"); + dev_err(pwmchip_parent(chip), "failed to reapply current setting\n"); return; } @@ -127,7 +127,7 @@ static void pwm_apply_debug(struct pwm_device *pwm, s1.polarity != last->polarity || (s1.enabled && s1.period != last->period) || (s1.enabled && s1.duty_cycle != last->duty_cycle)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), ".apply is not idempotent (ena=%d pol=%d %llu/%llu) -> (ena=%d pol=%d %llu/%llu)\n", s1.enabled, s1.polarity, s1.duty_cycle, s1.period, last->enabled, last->polarity, last->duty_cycle, @@ -318,7 +318,7 @@ static struct pwm_chip *pwmchip_find_by_name(const char *name) mutex_lock(&pwm_lock); idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) { - const char *chip_name = dev_name(chip->dev); + const char *chip_name = dev_name(pwmchip_parent(chip)); if (chip_name && strcmp(chip_name, name) == 0) { mutex_unlock(&pwm_lock); @@ -456,19 +456,19 @@ EXPORT_SYMBOL_GPL(of_pwm_single_xlate); static void of_pwmchip_add(struct pwm_chip *chip) { - if (!chip->dev || !chip->dev->of_node) + if (!pwmchip_parent(chip) || !pwmchip_parent(chip)->of_node) return; if (!chip->of_xlate) chip->of_xlate = of_pwm_xlate_with_flags; - of_node_get(chip->dev->of_node); + of_node_get(pwmchip_parent(chip)->of_node); } static void of_pwmchip_remove(struct pwm_chip *chip) { - if (chip->dev) - of_node_put(chip->dev->of_node); + if (pwmchip_parent(chip)) + of_node_put(pwmchip_parent(chip)->of_node); } static bool pwm_ops_check(const struct pwm_chip *chip) @@ -479,7 +479,7 @@ static bool pwm_ops_check(const struct pwm_chip *chip) return false; if (IS_ENABLED(CONFIG_PWM_DEBUG) && !ops->get_state) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "Please implement the .get_state() callback\n"); return true; @@ -500,7 +500,7 @@ int __pwmchip_add(struct pwm_chip *chip, struct module *owner) unsigned int i; int ret; - if (!chip || !chip->dev || !chip->ops || !chip->npwm) + if (!chip || !pwmchip_parent(chip) || !chip->ops || !chip->npwm) return -EINVAL; if (!pwm_ops_check(chip)) @@ -594,15 +594,15 @@ static struct device_link *pwm_device_link_add(struct device *dev, * impact the PM sequence ordering: the PWM supplier may get * suspended before the consumer. */ - dev_warn(pwm->chip->dev, + dev_warn(pwmchip_parent(pwm->chip), "No consumer device specified to create a link to\n"); return NULL; } - dl = device_link_add(dev, pwm->chip->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + dl = device_link_add(dev, pwmchip_parent(pwm->chip), DL_FLAG_AUTOREMOVE_CONSUMER); if (!dl) { dev_err(dev, "failed to create device link to %s\n", - dev_name(pwm->chip->dev)); + dev_name(pwmchip_parent(pwm->chip))); return ERR_PTR(-EINVAL); } @@ -617,7 +617,7 @@ static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) mutex_lock(&pwm_lock); idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) - if (chip->dev && device_match_fwnode(chip->dev, fwnode)) { + if (pwmchip_parent(chip) && device_match_fwnode(pwmchip_parent(chip), fwnode)) { mutex_unlock(&pwm_lock); return chip; } @@ -1085,8 +1085,8 @@ static int pwm_seq_show(struct seq_file *s, void *v) seq_printf(s, "%s%d: %s/%s, %d PWM device%s\n", (char *)s->private, chip->id, - chip->dev->bus ? chip->dev->bus->name : "no-bus", - dev_name(chip->dev), chip->npwm, + pwmchip_parent(chip)->bus ? pwmchip_parent(chip)->bus->name : "no-bus", + dev_name(pwmchip_parent(chip)), chip->npwm, (chip->npwm != 1) ? "s" : ""); pwm_dbg_show(chip, s); diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index 1698609d91c8..3f434a771fb5 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -509,10 +509,10 @@ void pwmchip_sysfs_export(struct pwm_chip *chip) * If device_create() fails the pwm_chip is still usable by * the kernel it's just not exported. */ - parent = device_create(&pwm_class, chip->dev, MKDEV(0, 0), chip, + parent = device_create(&pwm_class, pwmchip_parent(chip), MKDEV(0, 0), chip, "pwmchip%d", chip->id); if (IS_ERR(parent)) { - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "device_create failed for pwm_chip sysfs export\n"); } } diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 8ffe9ae7a23a..07af6910bdce 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -289,6 +289,11 @@ struct pwm_chip { struct pwm_device *pwms; }; +static inline struct device *pwmchip_parent(const struct pwm_chip *chip) +{ + return chip->dev; +} + #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ int pwm_apply_might_sleep(struct pwm_device *pwm, const struct pwm_state *state); -- cgit v1.2.3 From 24003d501f112a9827aa8c59c00e12b2d5cbddf6 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 14 Feb 2024 10:30:49 +0100 Subject: pwm: Provide wrappers for storing and getting driver private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These functions are useful to store and query driver private data depending on the pwm_chip. After struct pwm_chip got its own struct device, this can make use of dev_get_drvdata() and dev_set_drvdata() on that device. These functions are required already now to convert drivers to pwmchip_alloc() which must happen before changing pwm_chip::dev. Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/67514cdf29d29bd8b4ad8d44fac87f6ae6dca1e5.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- include/linux/pwm.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 07af6910bdce..29a7d9140f77 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -272,6 +272,7 @@ struct pwm_ops { * @npwm: number of PWMs controlled by this chip * @of_xlate: request a PWM device given a device tree PWM specifier * @atomic: can the driver's ->apply() be called in atomic context + * @driver_data: Private pointer for driver specific info * @pwms: array of PWM devices allocated by the framework */ struct pwm_chip { @@ -286,6 +287,7 @@ struct pwm_chip { bool atomic; /* only used internally by the PWM framework */ + void *driver_data; struct pwm_device *pwms; }; @@ -294,6 +296,24 @@ static inline struct device *pwmchip_parent(const struct pwm_chip *chip) return chip->dev; } +static inline void *pwmchip_get_drvdata(struct pwm_chip *chip) +{ + /* + * After pwm_chip got a dedicated struct device, this can be replaced by + * dev_get_drvdata(&chip->dev); + */ + return chip->driver_data; +} + +static inline void pwmchip_set_drvdata(struct pwm_chip *chip, void *data) +{ + /* + * After pwm_chip got a dedicated struct device, this can be replaced by + * dev_set_drvdata(&chip->dev, data); + */ + chip->driver_data = data; +} + #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ int pwm_apply_might_sleep(struct pwm_device *pwm, const struct pwm_state *state); -- cgit v1.2.3 From 024913dbf99f06f09d5c06c940ca39821697d41c Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 14 Feb 2024 10:30:50 +0100 Subject: pwm: Provide pwmchip_alloc() function and a devm variant of it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function allocates a struct pwm_chip and driver data. Compared to the status quo the split into pwm_chip and driver data is new, otherwise it doesn't change anything relevant (yet). The intention is that after all drivers are switched to use this allocation function, its possible to add a struct device to struct pwm_chip to properly track the latter's lifetime without touching all drivers again. Proper lifetime tracking is a necessary precondition to introduce character device support for PWMs (that implements atomic setting and doesn't suffer from the sysfs overhead of the /sys/class/pwm userspace support). The new function pwmchip_priv() (obviously?) only works for chips allocated with pwmchip_alloc(). Link: https://lore.kernel.org/r/9577d6053a5a52536057dc8654ff567181c2da82.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- Documentation/driver-api/driver-model/devres.rst | 1 + Documentation/driver-api/pwm.rst | 11 +++-- drivers/pwm/core.c | 58 ++++++++++++++++++++++++ include/linux/pwm.h | 22 +++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index c5f99d834ec5..e4df72c408d2 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -420,6 +420,7 @@ POWER devm_reboot_mode_unregister() PWM + devm_pwmchip_alloc() devm_pwmchip_add() devm_pwm_get() devm_fwnode_pwm_get() diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst index 3c28ccc4b611..b41b1c56477f 100644 --- a/Documentation/driver-api/pwm.rst +++ b/Documentation/driver-api/pwm.rst @@ -143,11 +143,12 @@ to implement the pwm_*() functions itself. This means that it's impossible to have multiple PWM drivers in the system. For this reason it's mandatory for new drivers to use the generic PWM framework. -A new PWM controller/chip can be added using pwmchip_add() and removed -again with pwmchip_remove(). pwmchip_add() takes a filled in struct -pwm_chip as argument which provides a description of the PWM chip, the -number of PWM devices provided by the chip and the chip-specific -implementation of the supported PWM operations to the framework. +A new PWM controller/chip can be allocated using pwmchip_alloc(), then +registered using pwmchip_add() and removed again with pwmchip_remove(). To undo +pwmchip_alloc() use pwmchip_put(). pwmchip_add() takes a filled in struct +pwm_chip as argument which provides a description of the PWM chip, the number +of PWM devices provided by the chip and the chip-specific implementation of the +supported PWM operations to the framework. When implementing polarity support in a PWM driver, make sure to respect the signal conventions in the PWM framework. By definition, normal polarity diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 830a697826af..d70f793ce4b3 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -454,6 +454,64 @@ of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) } EXPORT_SYMBOL_GPL(of_pwm_single_xlate); +#define PWMCHIP_ALIGN ARCH_DMA_MINALIGN + +static void *pwmchip_priv(struct pwm_chip *chip) +{ + return (void *)chip + ALIGN(sizeof(*chip), PWMCHIP_ALIGN); +} + +/* This is the counterpart to pwmchip_alloc() */ +void pwmchip_put(struct pwm_chip *chip) +{ + kfree(chip); +} +EXPORT_SYMBOL_GPL(pwmchip_put); + +struct pwm_chip *pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv) +{ + struct pwm_chip *chip; + size_t alloc_size; + + alloc_size = size_add(ALIGN(sizeof(*chip), PWMCHIP_ALIGN), sizeof_priv); + + chip = kzalloc(alloc_size, GFP_KERNEL); + if (!chip) + return ERR_PTR(-ENOMEM); + + chip->dev = parent; + chip->npwm = npwm; + + pwmchip_set_drvdata(chip, pwmchip_priv(chip)); + + return chip; +} +EXPORT_SYMBOL_GPL(pwmchip_alloc); + +static void devm_pwmchip_put(void *data) +{ + struct pwm_chip *chip = data; + + pwmchip_put(chip); +} + +struct pwm_chip *devm_pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv) +{ + struct pwm_chip *chip; + int ret; + + chip = pwmchip_alloc(parent, npwm, sizeof_priv); + if (IS_ERR(chip)) + return chip; + + ret = devm_add_action_or_reset(parent, devm_pwmchip_put, chip); + if (ret) + return ERR_PTR(ret); + + return chip; +} +EXPORT_SYMBOL_GPL(devm_pwmchip_alloc); + static void of_pwmchip_add(struct pwm_chip *chip) { if (!pwmchip_parent(chip) || !pwmchip_parent(chip)->of_node) diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 29a7d9140f77..4a6568dfdf3f 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -403,6 +403,10 @@ static inline bool pwm_might_sleep(struct pwm_device *pwm) int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, unsigned long timeout); +void pwmchip_put(struct pwm_chip *chip); +struct pwm_chip *pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv); +struct pwm_chip *devm_pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv); + int __pwmchip_add(struct pwm_chip *chip, struct module *owner); #define pwmchip_add(chip) __pwmchip_add(chip, THIS_MODULE) void pwmchip_remove(struct pwm_chip *chip); @@ -475,6 +479,24 @@ static inline int pwm_capture(struct pwm_device *pwm, return -EINVAL; } +static inline void pwmchip_put(struct pwm_chip *chip) +{ +} + +static inline struct pwm_chip *pwmchip_alloc(struct device *parent, + unsigned int npwm, + size_t sizeof_priv) +{ + return ERR_PTR(-EINVAL); +} + +static inline struct pwm_chip *devm_pwmchip_alloc(struct device *parent, + unsigned int npwm, + size_t sizeof_priv) +{ + return pwmchip_alloc(parent, npwm, sizeof_priv); +} + static inline int pwmchip_add(struct pwm_chip *chip) { return -EINVAL; -- cgit v1.2.3 From 81800aef0eba33df2b30f2e29a0137078b9ba256 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 13 Feb 2024 11:48:00 -0300 Subject: net: mdio_bus: make mdio_bus_type const Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the mdio_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Andrew Lunn Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240213-bus_cleanup-mdio-v1-1-f9e799da7fda@marliere.net Signed-off-by: Paolo Abeni --- drivers/net/phy/mdio_bus.c | 2 +- drivers/net/phy/phy_device.c | 3 +-- include/linux/phy.h | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 08624f073014..b765466e767f 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -1426,7 +1426,7 @@ static const struct attribute_group *mdio_bus_dev_groups[] = { NULL, }; -struct bus_type mdio_bus_type = { +const struct bus_type mdio_bus_type = { .name = "mdio_bus", .dev_groups = mdio_bus_dev_groups, .match = mdio_bus_match, diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 9f37c0bfbf8d..d63dca535746 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1604,7 +1604,6 @@ EXPORT_SYMBOL(phy_attach_direct); struct phy_device *phy_attach(struct net_device *dev, const char *bus_id, phy_interface_t interface) { - struct bus_type *bus = &mdio_bus_type; struct phy_device *phydev; struct device *d; int rc; @@ -1615,7 +1614,7 @@ struct phy_device *phy_attach(struct net_device *dev, const char *bus_id, /* Search the list of PHY devices on the mdio bus for the * PHY with the requested name */ - d = bus_find_device_by_name(bus, NULL, bus_id); + d = bus_find_device_by_name(&mdio_bus_type, NULL, bus_id); if (!d) { pr_err("PHY %s not found\n", bus_id); return ERR_PTR(-ENODEV); diff --git a/include/linux/phy.h b/include/linux/phy.h index 2249cdb5957a..c2dda21b39e1 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -2129,7 +2129,7 @@ static inline bool phy_package_probe_once(struct phy_device *phydev) return __phy_package_set_once(phydev, PHY_SHARED_F_PROBE_DONE); } -extern struct bus_type mdio_bus_type; +extern const struct bus_type mdio_bus_type; struct mdio_board_info { const char *bus_id; -- cgit v1.2.3 From a5fcea2d2f790aa90b6e996d411ae2cf8db55186 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Tue, 13 Feb 2024 23:26:31 -0700 Subject: net: ipv6/addrconf: introduce a regen_min_advance sysctl In RFC 8981, REGEN_ADVANCE cannot be less than 2 seconds, and the RFC does not permit the creation of temporary addresses with lifetimes shorter than that: > When processing a Router Advertisement with a > Prefix Information option carrying a prefix for the purposes of > address autoconfiguration (i.e., the A bit is set), the host MUST > perform the following steps: > 5. A temporary address is created only if this calculated preferred > lifetime is greater than REGEN_ADVANCE time units. However, some users want to change their IPv6 address as frequently as possible regardless of the RFC's arbitrary minimum lifetime. For the benefit of those users, add a regen_min_advance sysctl parameter that can be set to below or above 2 seconds. Link: https://datatracker.ietf.org/doc/html/rfc8981 Signed-off-by: Alex Henrie Reviewed-by: David Ahern Signed-off-by: Paolo Abeni --- Documentation/networking/ip-sysctl.rst | 10 ++++++++++ include/linux/ipv6.h | 1 + include/net/addrconf.h | 5 +++-- net/ipv6/addrconf.c | 11 ++++++++++- 4 files changed, 24 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 458305931345..407d917d1a36 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -2535,6 +2535,16 @@ max_desync_factor - INTEGER Default: 600 +regen_min_advance - INTEGER + How far in advance (in seconds), at minimum, to create a new temporary + address before the current one is deprecated. This value is added to + the amount of time that may be required for duplicate address detection + to determine when to create a new address. Linux permits setting this + value to less than the default of 2 seconds, but a value less than 2 + does not conform to RFC 8981. + + Default: 2 + regen_max_retry - INTEGER Number of attempts before give up attempting to generate valid temporary addresses. diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 5e605e384aac..ef3aa060a289 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -27,6 +27,7 @@ struct ipv6_devconf { __s32 use_tempaddr; __s32 temp_valid_lft; __s32 temp_prefered_lft; + __s32 regen_min_advance; __s32 regen_max_retry; __s32 max_desync_factor; __s32 max_addresses; diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 61ebe723ee4d..30d6f1e84e46 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -8,8 +8,9 @@ #define MIN_VALID_LIFETIME (2*3600) /* 2 hours */ -#define TEMP_VALID_LIFETIME (7*86400) -#define TEMP_PREFERRED_LIFETIME (86400) +#define TEMP_VALID_LIFETIME (7*86400) /* 1 week */ +#define TEMP_PREFERRED_LIFETIME (86400) /* 24 hours */ +#define REGEN_MIN_ADVANCE (2) /* 2 seconds */ #define REGEN_MAX_RETRY (3) #define MAX_DESYNC_FACTOR (600) diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 65e886d7d80c..283823fba96a 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -195,6 +195,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .use_tempaddr = 0, .temp_valid_lft = TEMP_VALID_LIFETIME, .temp_prefered_lft = TEMP_PREFERRED_LIFETIME, + .regen_min_advance = REGEN_MIN_ADVANCE, .regen_max_retry = REGEN_MAX_RETRY, .max_desync_factor = MAX_DESYNC_FACTOR, .max_addresses = IPV6_MAX_ADDRESSES, @@ -257,6 +258,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .use_tempaddr = 0, .temp_valid_lft = TEMP_VALID_LIFETIME, .temp_prefered_lft = TEMP_PREFERRED_LIFETIME, + .regen_min_advance = REGEN_MIN_ADVANCE, .regen_max_retry = REGEN_MAX_RETRY, .max_desync_factor = MAX_DESYNC_FACTOR, .max_addresses = IPV6_MAX_ADDRESSES, @@ -1341,7 +1343,7 @@ out: static unsigned long ipv6_get_regen_advance(struct inet6_dev *idev) { - return 2 + idev->cnf.regen_max_retry * + return idev->cnf.regen_min_advance + idev->cnf.regen_max_retry * idev->cnf.dad_transmits * max(NEIGH_VAR(idev->nd_parms, RETRANS_TIME), HZ/100) / HZ; } @@ -6819,6 +6821,13 @@ static const struct ctl_table addrconf_sysctl[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "regen_min_advance", + .data = &ipv6_devconf.regen_min_advance, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { .procname = "regen_max_retry", .data = &ipv6_devconf.regen_max_retry, -- cgit v1.2.3 From 4ff4c745a16c4c151a71863420811e7f406c3ec2 Mon Sep 17 00:00:00 2001 From: Andrea Parri Date: Wed, 31 Jan 2024 15:49:35 +0100 Subject: locking: Introduce prepare_sync_core_cmd() Introduce an architecture function that architectures can use to set up ("prepare") SYNC_CORE commands. The function will be used by RISC-V to update its "deferred icache- flush" data structures (icache_stale_mask). Architectures defining prepare_sync_core_cmd() static inline need to select ARCH_HAS_PREPARE_SYNC_CORE_CMD. Suggested-by: Mathieu Desnoyers Signed-off-by: Andrea Parri Reviewed-by: Mathieu Desnoyers Link: https://lore.kernel.org/r/20240131144936.29190-4-parri.andrea@gmail.com Signed-off-by: Palmer Dabbelt --- include/linux/sync_core.h | 16 +++++++++++++++- init/Kconfig | 3 +++ kernel/sched/membarrier.c | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sync_core.h b/include/linux/sync_core.h index 013da4b8b327..67bb9794b875 100644 --- a/include/linux/sync_core.h +++ b/include/linux/sync_core.h @@ -17,5 +17,19 @@ static inline void sync_core_before_usermode(void) } #endif -#endif /* _LINUX_SYNC_CORE_H */ +#ifdef CONFIG_ARCH_HAS_PREPARE_SYNC_CORE_CMD +#include +#else +/* + * This is a dummy prepare_sync_core_cmd() implementation that can be used on + * all architectures which provide unconditional core serializing instructions + * in switch_mm(). + * If your architecture doesn't provide such core serializing instructions in + * switch_mm(), you may need to write your own functions. + */ +static inline void prepare_sync_core_cmd(struct mm_struct *mm) +{ +} +#endif +#endif /* _LINUX_SYNC_CORE_H */ diff --git a/init/Kconfig b/init/Kconfig index 8df18f3a9748..c3994b92333d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1970,6 +1970,9 @@ source "kernel/Kconfig.locks" config ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE bool +config ARCH_HAS_PREPARE_SYNC_CORE_CMD + bool + config ARCH_HAS_SYNC_CORE_BEFORE_USERMODE bool diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index f3d91628d6b8..6d1f31b3a967 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -320,6 +320,7 @@ static int membarrier_private_expedited(int flags, int cpu_id) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY)) return -EPERM; ipi_func = ipi_sync_core; + prepare_sync_core_cmd(mm); } else if (flags == MEMBARRIER_FLAG_RSEQ) { if (!IS_ENABLED(CONFIG_RSEQ)) return -EINVAL; -- cgit v1.2.3 From 1269b6d7222f761b6f5fb85b19f7ab76a5bbf803 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 13 Feb 2024 15:46:05 +0200 Subject: ACPI: utils: Make acpi_handle_path() not static acpi_handle_path() will soon be required for node name comparison elsewhere in ACPI framework. Remove the static keyword and add the prototype to include/linux/acpi.h. Signed-off-by: Sakari Ailus Signed-off-by: Rafael J. Wysocki --- drivers/acpi/utils.c | 2 +- include/linux/acpi.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index abac5cc25477..202234ba54bd 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -559,7 +559,7 @@ EXPORT_SYMBOL(acpi_evaluate_ost); * * Caller must free the returned buffer */ -static char *acpi_handle_path(acpi_handle handle) +char *acpi_handle_path(acpi_handle handle) { struct acpi_buffer buffer = { .length = ACPI_ALLOCATE_BUFFER, diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b7165e52b3c6..a170c389dd74 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1170,6 +1170,7 @@ static inline void acpi_ec_set_gpe_wake_mask(u8 action) {} #endif #ifdef CONFIG_ACPI +char *acpi_handle_path(acpi_handle handle); __printf(3, 4) void acpi_handle_printk(const char *level, acpi_handle handle, const char *fmt, ...); -- cgit v1.2.3 From bad5247a2c4f7eab6fb922af3362740a562dc665 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:49 +0100 Subject: ima: Align ima_inode_post_setattr() definition with LSM infrastructure Change ima_inode_post_setattr() definition, so that it can be registered as implementation of the inode_post_setattr hook (to be introduced). Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 +- include/linux/ima.h | 4 ++-- security/integrity/ima/ima_appraise.c | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/attr.c b/fs/attr.c index 5a13f0c8495f..b53ae408ad4f 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -502,7 +502,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); - ima_inode_post_setattr(idmap, dentry); + ima_inode_post_setattr(idmap, dentry, ia_valid); evm_inode_post_setattr(dentry, ia_valid); } diff --git a/include/linux/ima.h b/include/linux/ima.h index 86b57757c7b1..910a2f11a906 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -186,7 +186,7 @@ static inline void ima_post_key_create_or_update(struct key *keyring, #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); extern void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry); + struct dentry *dentry, int ia_valid); extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len); extern int ima_inode_set_acl(struct mnt_idmap *idmap, @@ -206,7 +206,7 @@ static inline bool is_ima_appraise_enabled(void) } static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry) + struct dentry *dentry, int ia_valid) { return; } diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 870dde67707b..36c2938a5c69 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -629,6 +629,7 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * ima_inode_post_setattr - reflect file metadata changes * @idmap: idmap of the mount the inode was found from * @dentry: pointer to the affected dentry + * @ia_valid: for the UID and GID status * * Changes to a dentry's metadata might result in needing to appraise. * @@ -636,7 +637,7 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * to lock the inode's i_mutex. */ void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry) + struct dentry *dentry, int ia_valid) { struct inode *inode = d_backing_inode(dentry); struct integrity_iint_cache *iint; -- cgit v1.2.3 From 0298c5a9b168f0d74ea3bf881301c4bd9252d367 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:50 +0100 Subject: ima: Align ima_file_mprotect() definition with LSM infrastructure Change ima_file_mprotect() definition, so that it can be registered as implementation of the file_mprotect hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 5 +++-- security/integrity/ima/ima_main.c | 6 ++++-- security/security.c | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ima.h b/include/linux/ima.h index 910a2f11a906..b66353f679e8 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -23,7 +23,8 @@ extern void ima_post_create_tmpfile(struct mnt_idmap *idmap, extern void ima_file_free(struct file *file); extern int ima_file_mmap(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags); -extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot); +extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot); extern int ima_load_data(enum kernel_load_data_id id, bool contents); extern int ima_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description); @@ -84,7 +85,7 @@ static inline int ima_file_mmap(struct file *file, unsigned long reqprot, } static inline int ima_file_mprotect(struct vm_area_struct *vma, - unsigned long prot) + unsigned long reqprot, unsigned long prot) { return 0; } diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index cc1217ac2c6f..b3f5e8401056 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -455,7 +455,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, /** * ima_file_mprotect - based on policy, limit mprotect change * @vma: vm_area_struct protection is set to - * @prot: contains the protection that will be applied by the kernel. + * @reqprot: protection requested by the application + * @prot: protection that will be applied by the kernel * * Files can be mmap'ed read/write and later changed to execute to circumvent * IMA's mmap appraisal policy rules. Due to locking issues (mmap semaphore @@ -465,7 +466,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, * * On mprotect change success, return 0. On failure, return -EACESS. */ -int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) +int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot) { struct ima_template_desc *template = NULL; struct file *file; diff --git a/security/security.c b/security/security.c index 3aaad75c9ce8..36933ac45d3b 100644 --- a/security/security.c +++ b/security/security.c @@ -2831,7 +2831,7 @@ int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, ret = call_int_hook(file_mprotect, 0, vma, reqprot, prot); if (ret) return ret; - return ima_file_mprotect(vma, prot); + return ima_file_mprotect(vma, reqprot, prot); } /** -- cgit v1.2.3 From fbd0506e5c5874fd52403fd38e3e77d895689870 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:51 +0100 Subject: ima: Align ima_inode_setxattr() definition with LSM infrastructure Change ima_inode_setxattr() definition, so that it can be registered as implementation of the inode_setxattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 11 +++++++---- security/integrity/ima/ima_appraise.c | 5 +++-- security/security.c | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ima.h b/include/linux/ima.h index b66353f679e8..077324309c11 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -188,8 +188,9 @@ static inline void ima_post_key_create_or_update(struct key *keyring, extern bool is_ima_appraise_enabled(void); extern void ima_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid); -extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len); +extern int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags); extern int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); @@ -212,10 +213,12 @@ static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, return; } -static inline int ima_inode_setxattr(struct dentry *dentry, +static inline int ima_inode_setxattr(struct mnt_idmap *idmap, + struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) + size_t xattr_value_len, + int flags) { return 0; } diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 36c2938a5c69..cb2d0d11aa77 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -750,8 +750,9 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; } -int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) +int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xvalue = xattr_value; int digsig = 0; diff --git a/security/security.c b/security/security.c index 36933ac45d3b..b4f6b7467e08 100644 --- a/security/security.c +++ b/security/security.c @@ -2269,7 +2269,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap, ret = cap_inode_setxattr(dentry, name, value, size, flags); if (ret) return ret; - ret = ima_inode_setxattr(dentry, name, value, size); + ret = ima_inode_setxattr(idmap, dentry, name, value, size, flags); if (ret) return ret; return evm_inode_setxattr(idmap, dentry, name, value, size); -- cgit v1.2.3 From 526864dd2f60c16bcdc84e9e7bc6d69d08cfee21 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:52 +0100 Subject: ima: Align ima_inode_removexattr() definition with LSM infrastructure Change ima_inode_removexattr() definition, so that it can be registered as implementation of the inode_removexattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 7 +++++-- security/integrity/ima/ima_appraise.c | 3 ++- security/security.c | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ima.h b/include/linux/ima.h index 077324309c11..678a03fddd7e 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -200,7 +200,9 @@ static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, { return ima_inode_set_acl(idmap, dentry, acl_name, NULL); } -extern int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name); + +extern int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name); #else static inline bool is_ima_appraise_enabled(void) { @@ -231,7 +233,8 @@ static inline int ima_inode_set_acl(struct mnt_idmap *idmap, return 0; } -static inline int ima_inode_removexattr(struct dentry *dentry, +static inline int ima_inode_removexattr(struct mnt_idmap *idmap, + struct dentry *dentry, const char *xattr_name) { return 0; diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index cb2d0d11aa77..36abc84ba299 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -790,7 +790,8 @@ int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return 0; } -int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name) +int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { int result; diff --git a/security/security.c b/security/security.c index b4f6b7467e08..58e20362b679 100644 --- a/security/security.c +++ b/security/security.c @@ -2430,7 +2430,7 @@ int security_inode_removexattr(struct mnt_idmap *idmap, ret = cap_inode_removexattr(idmap, dentry, name); if (ret) return ret; - ret = ima_inode_removexattr(dentry, name); + ret = ima_inode_removexattr(idmap, dentry, name); if (ret) return ret; return evm_inode_removexattr(idmap, dentry, name); -- cgit v1.2.3 From fec5f85e468d6f47851d531ec81f9da821768e00 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:53 +0100 Subject: ima: Align ima_post_read_file() definition with LSM infrastructure Change ima_post_read_file() definition, by making "void *buf" a "char *buf", so that it can be registered as implementation of the post_read_file hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 4 ++-- security/integrity/ima/ima_main.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ima.h b/include/linux/ima.h index 678a03fddd7e..31ef6c3c3207 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -30,7 +30,7 @@ extern int ima_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description); extern int ima_read_file(struct file *file, enum kernel_read_file_id id, bool contents); -extern int ima_post_read_file(struct file *file, void *buf, loff_t size, +extern int ima_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id); extern void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry); @@ -108,7 +108,7 @@ static inline int ima_read_file(struct file *file, enum kernel_read_file_id id, return 0; } -static inline int ima_post_read_file(struct file *file, void *buf, loff_t size, +static inline int ima_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id) { return 0; diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index b3f5e8401056..02021ee467d3 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -803,7 +803,7 @@ const int read_idmap[READING_MAX_ID] = { * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_read_file(struct file *file, void *buf, loff_t size, +int ima_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id read_id) { enum ima_hooks func; -- cgit v1.2.3 From 784111d0093e007950cc20033daf3d74ac388821 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:54 +0100 Subject: evm: Align evm_inode_post_setattr() definition with LSM infrastructure Change evm_inode_post_setattr() definition, so that it can be registered as implementation of the inode_post_setattr hook (to be introduced). Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 +- include/linux/evm.h | 6 ++++-- security/integrity/evm/evm_main.c | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/attr.c b/fs/attr.c index b53ae408ad4f..adeba0ec40f1 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -503,7 +503,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); ima_inode_post_setattr(idmap, dentry, ia_valid); - evm_inode_post_setattr(dentry, ia_valid); + evm_inode_post_setattr(idmap, dentry, ia_valid); } return error; diff --git a/include/linux/evm.h b/include/linux/evm.h index 36ec884320d9..5cc386312b5a 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -23,7 +23,8 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry, struct integrity_iint_cache *iint); extern int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); -extern void evm_inode_post_setattr(struct dentry *dentry, int ia_valid); +extern void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid); extern int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, size_t size); @@ -98,7 +99,8 @@ static inline int evm_inode_setattr(struct mnt_idmap *idmap, return 0; } -static inline void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) +static inline void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { return; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index cc7956d7878b..ac34f21122cd 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -870,6 +870,7 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /** * evm_inode_post_setattr - update 'security.evm' after modifying metadata + * @idmap: idmap of the idmapped mount * @dentry: pointer to the affected dentry * @ia_valid: for the UID and GID status * @@ -879,7 +880,8 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) +void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) { if (!evm_revalidate_status(NULL)) return; -- cgit v1.2.3 From 2b6a4054f8c2758cf5c1d78f6ba7006a940b31ce Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:55 +0100 Subject: evm: Align evm_inode_setxattr() definition with LSM infrastructure Change evm_inode_setxattr() definition, so that it can be registered as implementation of the inode_setxattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/evm.h | 4 ++-- security/integrity/evm/evm_main.c | 3 ++- security/security.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/evm.h b/include/linux/evm.h index 5cc386312b5a..7de24c1ada90 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -27,7 +27,7 @@ extern void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid); extern int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, - const void *value, size_t size); + const void *value, size_t size, int flags); extern void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, @@ -107,7 +107,7 @@ static inline void evm_inode_post_setattr(struct mnt_idmap *idmap, static inline int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, - const void *value, size_t size) + const void *value, size_t size, int flags) { return 0; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index ac34f21122cd..12ba3207fd31 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -581,6 +581,7 @@ out: * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value * @xattr_value_len: pointer to the new extended attribute value length + * @flags: flags to pass into filesystem operations * * Before allowing the 'security.evm' protected xattr to be updated, * verify the existing value is valid. As only the kernel should have @@ -590,7 +591,7 @@ out: */ int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xattr_data = xattr_value; diff --git a/security/security.c b/security/security.c index 58e20362b679..b99c9a69d4c2 100644 --- a/security/security.c +++ b/security/security.c @@ -2272,7 +2272,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap, ret = ima_inode_setxattr(idmap, dentry, name, value, size, flags); if (ret) return ret; - return evm_inode_setxattr(idmap, dentry, name, value, size); + return evm_inode_setxattr(idmap, dentry, name, value, size, flags); } /** -- cgit v1.2.3 From 779cb1947e270504c66a96fc0b7e6e31c748b1e1 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:56 +0100 Subject: evm: Align evm_inode_post_setxattr() definition with LSM infrastructure Change evm_inode_post_setxattr() definition, so that it can be registered as implementation of the inode_post_setxattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/evm.h | 8 +++++--- security/integrity/evm/evm_main.c | 4 +++- security/security.c | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/evm.h b/include/linux/evm.h index 7de24c1ada90..3faabdd47852 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -31,7 +31,8 @@ extern int evm_inode_setxattr(struct mnt_idmap *idmap, extern void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len); + size_t xattr_value_len, + int flags); extern int evm_inode_copy_up_xattr(const char *name); extern int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *xattr_name); @@ -56,7 +57,7 @@ static inline void evm_inode_post_set_acl(struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) { - return evm_inode_post_setxattr(dentry, acl_name, NULL, 0); + return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); } int evm_inode_init_security(struct inode *inode, struct inode *dir, @@ -115,7 +116,8 @@ static inline int evm_inode_setxattr(struct mnt_idmap *idmap, static inline void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) + size_t xattr_value_len, + int flags) { return; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 12ba3207fd31..d35143179699 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -753,6 +753,7 @@ bool evm_revalidate_status(const char *xattr_name) * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value * @xattr_value_len: pointer to the new extended attribute value length + * @flags: flags to pass into filesystem operations * * Update the HMAC stored in 'security.evm' to reflect the change. * @@ -761,7 +762,8 @@ bool evm_revalidate_status(const char *xattr_name) * i_mutex lock. */ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) + const void *xattr_value, size_t xattr_value_len, + int flags) { if (!evm_revalidate_status(xattr_name)) return; diff --git a/security/security.c b/security/security.c index b99c9a69d4c2..dc188201a4f5 100644 --- a/security/security.c +++ b/security/security.c @@ -2367,7 +2367,7 @@ void security_inode_post_setxattr(struct dentry *dentry, const char *name, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return; call_void_hook(inode_post_setxattr, dentry, name, value, size, flags); - evm_inode_post_setxattr(dentry, name, value, size); + evm_inode_post_setxattr(dentry, name, value, size, flags); } /** -- cgit v1.2.3 From 314a8dc728d038378795236f6b5199265f921f45 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:57 +0100 Subject: security: Align inode_setattr hook definition with EVM Add the idmap parameter to the definition, so that evm_inode_setattr() can be registered as this hook implementation. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 3 ++- security/security.c | 2 +- security/selinux/hooks.c | 3 ++- security/smack/smack_lsm.c | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 76458b6d53da..b00b16d58413 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -135,7 +135,8 @@ LSM_HOOK(int, 0, inode_readlink, struct dentry *dentry) LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, bool rcu) LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) -LSM_HOOK(int, 0, inode_setattr, struct dentry *dentry, struct iattr *attr) +LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) LSM_HOOK(int, 0, inode_getattr, const struct path *path) LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, diff --git a/security/security.c b/security/security.c index dc188201a4f5..9dc601d45960 100644 --- a/security/security.c +++ b/security/security.c @@ -2215,7 +2215,7 @@ int security_inode_setattr(struct mnt_idmap *idmap, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_setattr, 0, dentry, attr); + ret = call_int_hook(inode_setattr, 0, idmap, dentry, attr); if (ret) return ret; return evm_inode_setattr(idmap, dentry, attr); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index a6bf90ace84c..cedb4cbf072e 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3136,7 +3136,8 @@ static int selinux_inode_permission(struct inode *inode, int mask) return rc; } -static int selinux_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int selinux_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) { const struct cred *cred = current_cred(); struct inode *inode = d_backing_inode(dentry); diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 0fdbf04cc258..1b6abfdf7173 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -1233,12 +1233,14 @@ static int smack_inode_permission(struct inode *inode, int mask) /** * smack_inode_setattr - Smack check for setting attributes + * @idmap: idmap of the mount * @dentry: the object * @iattr: for the force flag * * Returns 0 if access is permitted, an error code otherwise */ -static int smack_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int smack_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) { struct smk_audit_info ad; int rc; -- cgit v1.2.3 From 77fa6f314f0376176ef6bf3d84403e0d8b54ce28 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:58 +0100 Subject: security: Introduce inode_post_setattr hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_setattr hook. At inode_setattr hook, EVM verifies the file's existing HMAC value. At inode_post_setattr, EVM re-calculates the file's HMAC based on the modified file attributes and other file metadata. Other LSMs could similarly take some action after successful file attribute change. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Casey Schaufler Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/attr.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 7 +++++++ security/security.c | 16 ++++++++++++++++ 4 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/fs/attr.c b/fs/attr.c index adeba0ec40f1..990e1b3a3c91 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -502,6 +502,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); + security_inode_post_setattr(idmap, dentry, ia_valid); ima_inode_post_setattr(idmap, dentry, ia_valid); evm_inode_post_setattr(idmap, dentry, ia_valid); } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index b00b16d58413..a0e9e48015a4 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -137,6 +137,8 @@ LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) +LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) LSM_HOOK(int, 0, inode_getattr, const struct path *path) LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, diff --git a/include/linux/security.h b/include/linux/security.h index d0eb20f90b26..56c841aa3994 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -361,6 +361,8 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode, int security_inode_permission(struct inode *inode, int mask); int security_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); +void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid); int security_inode_getattr(const struct path *path); int security_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, @@ -879,6 +881,11 @@ static inline int security_inode_setattr(struct mnt_idmap *idmap, return 0; } +static inline void +security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) +{ } + static inline int security_inode_getattr(const struct path *path) { return 0; diff --git a/security/security.c b/security/security.c index 9dc601d45960..56527d5415e2 100644 --- a/security/security.c +++ b/security/security.c @@ -2222,6 +2222,22 @@ int security_inode_setattr(struct mnt_idmap *idmap, } EXPORT_SYMBOL_GPL(security_inode_setattr); +/** + * security_inode_post_setattr() - Update the inode after a setattr operation + * @idmap: idmap of the mount + * @dentry: file + * @ia_valid: file attributes set + * + * Update inode security field after successful setting file attributes. + */ +void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_setattr, idmap, dentry, ia_valid); +} + /** * security_inode_getattr() - Check if getting file attributes is allowed * @path: file -- cgit v1.2.3 From dae52cbf5887ac51c3574648124cfe475a9b3246 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:59 +0100 Subject: security: Introduce inode_post_removexattr hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_removexattr hook. At inode_removexattr hook, EVM verifies the file's existing HMAC value. At inode_post_removexattr, EVM re-calculates the file's HMAC with the passed xattr removed and other file metadata. Other LSMs could similarly take some action after successful xattr removal. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/xattr.c | 9 +++++---- include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 5 +++++ security/security.c | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/xattr.c b/fs/xattr.c index 09d927603433..f891c260a971 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -552,11 +552,12 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap, goto out; error = __vfs_removexattr(idmap, dentry, name); + if (error) + return error; - if (!error) { - fsnotify_xattr(dentry); - evm_inode_post_removexattr(dentry, name); - } + fsnotify_xattr(dentry); + security_inode_post_removexattr(dentry, name); + evm_inode_post_removexattr(dentry, name); out: return error; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index a0e9e48015a4..f849f7d5bb53 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -149,6 +149,8 @@ LSM_HOOK(int, 0, inode_getxattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_listxattr, struct dentry *dentry) LSM_HOOK(int, 0, inode_removexattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name) +LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry, + const char *name) LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, diff --git a/include/linux/security.h b/include/linux/security.h index 56c841aa3994..84ae03690340 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -380,6 +380,7 @@ int security_inode_getxattr(struct dentry *dentry, const char *name); int security_inode_listxattr(struct dentry *dentry); int security_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name); +void security_inode_post_removexattr(struct dentry *dentry, const char *name); int security_inode_need_killpriv(struct dentry *dentry); int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry); int security_inode_getsecurity(struct mnt_idmap *idmap, @@ -942,6 +943,10 @@ static inline int security_inode_removexattr(struct mnt_idmap *idmap, return cap_inode_removexattr(idmap, dentry, name); } +static inline void security_inode_post_removexattr(struct dentry *dentry, + const char *name) +{ } + static inline int security_inode_need_killpriv(struct dentry *dentry) { return cap_inode_need_killpriv(dentry); diff --git a/security/security.c b/security/security.c index 56527d5415e2..d77fdd3ca1bc 100644 --- a/security/security.c +++ b/security/security.c @@ -2452,6 +2452,20 @@ int security_inode_removexattr(struct mnt_idmap *idmap, return evm_inode_removexattr(idmap, dentry, name); } +/** + * security_inode_post_removexattr() - Update the inode after a removexattr op + * @dentry: file + * @name: xattr name + * + * Update the inode after a successful removexattr operation. + */ +void security_inode_post_removexattr(struct dentry *dentry, const char *name) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_removexattr, dentry, name); +} + /** * security_inode_need_killpriv() - Check if security_inode_killpriv() required * @dentry: associated dentry -- cgit v1.2.3 From 8f46ff5767b0b18329140d80d6bcabd818f42c4c Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:00 +0100 Subject: security: Introduce file_post_open hook In preparation to move IMA and EVM to the LSM infrastructure, introduce the file_post_open hook. Also, export security_file_post_open() for NFS. Based on policy, IMA calculates the digest of the file content and extends the TPM with the digest, verifies the file's integrity based on the digest, and/or includes the file digest in the audit log. LSMs could similarly take action depending on the file content and the access mask requested with open(). The new hook returns a value and can cause the open to be aborted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/namei.c | 2 ++ fs/nfsd/vfs.c | 6 ++++++ include/linux/lsm_hook_defs.h | 1 + include/linux/security.h | 6 ++++++ security/security.c | 17 +++++++++++++++++ 5 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/fs/namei.c b/fs/namei.c index 4e0de939fea1..ef867f1d6704 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3639,6 +3639,8 @@ static int do_open(struct nameidata *nd, error = may_open(idmap, &nd->path, acc_mode, open_flag); if (!error && !(file->f_mode & FMODE_OPENED)) error = vfs_open(&nd->path, file); + if (!error) + error = security_file_post_open(file, op->acc_mode); if (!error) error = ima_file_check(file, op->acc_mode); if (!error && do_truncate) diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index b7c7a9273ea0..e44d8239545b 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -877,6 +877,12 @@ __nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, goto out; } + host_err = security_file_post_open(file, may_flags); + if (host_err) { + fput(file); + goto out; + } + host_err = ima_file_check(file, may_flags); if (host_err) { fput(file); diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index f849f7d5bb53..3c84942d2818 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -191,6 +191,7 @@ LSM_HOOK(int, 0, file_send_sigiotask, struct task_struct *tsk, struct fown_struct *fown, int sig) LSM_HOOK(int, 0, file_receive, struct file *file) LSM_HOOK(int, 0, file_open, struct file *file) +LSM_HOOK(int, 0, file_post_open, struct file *file, int mask) LSM_HOOK(int, 0, file_truncate, struct file *file) LSM_HOOK(int, 0, task_alloc, struct task_struct *task, unsigned long clone_flags) diff --git a/include/linux/security.h b/include/linux/security.h index 84ae03690340..97f2212c13b6 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -411,6 +411,7 @@ int security_file_send_sigiotask(struct task_struct *tsk, struct fown_struct *fown, int sig); int security_file_receive(struct file *file); int security_file_open(struct file *file); +int security_file_post_open(struct file *file, int mask); int security_file_truncate(struct file *file); int security_task_alloc(struct task_struct *task, unsigned long clone_flags); void security_task_free(struct task_struct *task); @@ -1074,6 +1075,11 @@ static inline int security_file_open(struct file *file) return 0; } +static inline int security_file_post_open(struct file *file, int mask) +{ + return 0; +} + static inline int security_file_truncate(struct file *file) { return 0; diff --git a/security/security.c b/security/security.c index d77fdd3ca1bc..207405a84902 100644 --- a/security/security.c +++ b/security/security.c @@ -2966,6 +2966,23 @@ int security_file_open(struct file *file) return fsnotify_open_perm(file); } +/** + * security_file_post_open() - Evaluate a file after it has been opened + * @file: the file + * @mask: access mask + * + * Evaluate an opened file and the access mask requested with open(). The hook + * is useful for LSMs that require the file content to be available in order to + * make decisions. + * + * Return: Returns 0 if permission is granted. + */ +int security_file_post_open(struct file *file, int mask) +{ + return call_int_hook(file_post_open, 0, file, mask); +} +EXPORT_SYMBOL_GPL(security_file_post_open); + /** * security_file_truncate() - Check if truncating a file is allowed * @file: file -- cgit v1.2.3 From f09068b5a114ed28d2df2e82a7d30dde0145dc69 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:01 +0100 Subject: security: Introduce file_release hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the file_release hook. IMA calculates at file close the new digest of the file content and writes it to security.ima, so that appraisal at next file access succeeds. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Acked-by: Christian Brauner Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/file_table.c | 1 + include/linux/lsm_hook_defs.h | 1 + include/linux/security.h | 4 ++++ security/security.c | 11 +++++++++++ 4 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/fs/file_table.c b/fs/file_table.c index b991f90571b4..725407f374fa 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -367,6 +367,7 @@ static void __fput(struct file *file) eventpoll_release(file); locks_remove_file(file); + security_file_release(file); ima_file_free(file); if (unlikely(file->f_flags & FASYNC)) { if (file->f_op->fasync) diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 3c84942d2818..7f9e9240606e 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -173,6 +173,7 @@ LSM_HOOK(int, 0, kernfs_init_security, struct kernfs_node *kn_dir, struct kernfs_node *kn) LSM_HOOK(int, 0, file_permission, struct file *file, int mask) LSM_HOOK(int, 0, file_alloc_security, struct file *file) +LSM_HOOK(void, LSM_RET_VOID, file_release, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file) LSM_HOOK(int, 0, file_ioctl, struct file *file, unsigned int cmd, unsigned long arg) diff --git a/include/linux/security.h b/include/linux/security.h index 97f2212c13b6..2997348afcb7 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -395,6 +395,7 @@ int security_kernfs_init_security(struct kernfs_node *kn_dir, struct kernfs_node *kn); int security_file_permission(struct file *file, int mask); int security_file_alloc(struct file *file); +void security_file_release(struct file *file); void security_file_free(struct file *file); int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); int security_file_ioctl_compat(struct file *file, unsigned int cmd, @@ -1008,6 +1009,9 @@ static inline int security_file_alloc(struct file *file) return 0; } +static inline void security_file_release(struct file *file) +{ } + static inline void security_file_free(struct file *file) { } diff --git a/security/security.c b/security/security.c index 207405a84902..99004c44ce55 100644 --- a/security/security.c +++ b/security/security.c @@ -2718,6 +2718,17 @@ int security_file_alloc(struct file *file) return rc; } +/** + * security_file_release() - Perform actions before releasing the file ref + * @file: the file + * + * Perform actions before releasing the last reference to a file. + */ +void security_file_release(struct file *file) +{ + call_void_hook(file_release, file); +} + /** * security_file_free() - Free a file's LSM blob * @file: the file -- cgit v1.2.3 From 08abce60d63fb55f440c393f4508e99064f2fd91 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:02 +0100 Subject: security: Introduce path_post_mknod hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the path_post_mknod hook. IMA-appraisal requires all existing files in policy to have a file hash/signature stored in security.ima. An exception is made for empty files created by mknod, by tagging them as new files. LSMs could also take some action after files are created. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Reviewed-by: Stefan Berger Signed-off-by: Paul Moore --- fs/namei.c | 5 +++++ include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 5 +++++ security/security.c | 14 ++++++++++++++ 4 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/fs/namei.c b/fs/namei.c index ef867f1d6704..9280aa5d60a7 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -4063,6 +4063,11 @@ retry: dentry, mode, 0); break; } + + if (error) + goto out2; + + security_path_post_mknod(idmap, dentry); out2: done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 7f9e9240606e..dba5d8204dc5 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -94,6 +94,8 @@ LSM_HOOK(int, 0, path_mkdir, const struct path *dir, struct dentry *dentry, LSM_HOOK(int, 0, path_rmdir, const struct path *dir, struct dentry *dentry) LSM_HOOK(int, 0, path_mknod, const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev) +LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, struct mnt_idmap *idmap, + struct dentry *dentry) LSM_HOOK(int, 0, path_truncate, const struct path *path) LSM_HOOK(int, 0, path_symlink, const struct path *dir, struct dentry *dentry, const char *old_name) diff --git a/include/linux/security.h b/include/linux/security.h index 2997348afcb7..977dd9f7f51a 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1893,6 +1893,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, umode_t m int security_path_rmdir(const struct path *dir, struct dentry *dentry); int security_path_mknod(const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev); +void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry); int security_path_truncate(const struct path *path); int security_path_symlink(const struct path *dir, struct dentry *dentry, const char *old_name); @@ -1927,6 +1928,10 @@ static inline int security_path_mknod(const struct path *dir, struct dentry *den return 0; } +static inline void security_path_post_mknod(struct mnt_idmap *idmap, + struct dentry *dentry) +{ } + static inline int security_path_truncate(const struct path *path) { return 0; diff --git a/security/security.c b/security/security.c index 99004c44ce55..ee4582e1d4d0 100644 --- a/security/security.c +++ b/security/security.c @@ -1800,6 +1800,20 @@ int security_path_mknod(const struct path *dir, struct dentry *dentry, } EXPORT_SYMBOL(security_path_mknod); +/** + * security_path_post_mknod() - Update inode security field after file creation + * @idmap: idmap of the mount + * @dentry: new file + * + * Update inode security field after a file has been created. + */ +void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(path_post_mknod, idmap, dentry); +} + /** * security_path_mkdir() - Check if creating a new directory is allowed * @dir: parent directory -- cgit v1.2.3 From a7811e34d100acf24870eb949c5ae3e49dde18b9 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:03 +0100 Subject: security: Introduce inode_post_create_tmpfile hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_create_tmpfile hook. As temp files can be made persistent, treat new temp files like other new files, so that the file hash is calculated and stored in the security xattr. LSMs could also take some action after temp files have been created. The new hook cannot return an error and cannot cause the operation to be canceled. Signed-off-by: Roberto Sassu Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Reviewed-by: Stefan Berger Signed-off-by: Paul Moore --- fs/namei.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 6 ++++++ security/security.c | 15 +++++++++++++++ 4 files changed, 24 insertions(+) (limited to 'include/linux') diff --git a/fs/namei.c b/fs/namei.c index 9280aa5d60a7..c7ac7fdf0f25 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3705,6 +3705,7 @@ static int vfs_tmpfile(struct mnt_idmap *idmap, inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } + security_inode_post_create_tmpfile(idmap, inode); ima_post_create_tmpfile(idmap, inode); return 0; } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index dba5d8204dc5..87f60b47dfca 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -121,6 +121,8 @@ LSM_HOOK(int, 0, inode_init_security_anon, struct inode *inode, const struct qstr *name, const struct inode *context_inode) LSM_HOOK(int, 0, inode_create, struct inode *dir, struct dentry *dentry, umode_t mode) +LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, struct mnt_idmap *idmap, + struct inode *inode) LSM_HOOK(int, 0, inode_link, struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) LSM_HOOK(int, 0, inode_unlink, struct inode *dir, struct dentry *dentry) diff --git a/include/linux/security.h b/include/linux/security.h index 977dd9f7f51a..1cb604282617 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -344,6 +344,8 @@ int security_inode_init_security_anon(struct inode *inode, const struct qstr *name, const struct inode *context_inode); int security_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode); +void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode); int security_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry); int security_inode_unlink(struct inode *dir, struct dentry *dentry); @@ -811,6 +813,10 @@ static inline int security_inode_create(struct inode *dir, return 0; } +static inline void +security_inode_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode) +{ } + static inline int security_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) diff --git a/security/security.c b/security/security.c index ee4582e1d4d0..710db090aa8b 100644 --- a/security/security.c +++ b/security/security.c @@ -2013,6 +2013,21 @@ int security_inode_create(struct inode *dir, struct dentry *dentry, } EXPORT_SYMBOL_GPL(security_inode_create); +/** + * security_inode_post_create_tmpfile() - Update inode security of new tmpfile + * @idmap: idmap of the mount + * @inode: inode of the new tmpfile + * + * Update inode security data after a tmpfile has been created. + */ +void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode) +{ + if (unlikely(IS_PRIVATE(inode))) + return; + call_void_hook(inode_post_create_tmpfile, idmap, inode); +} + /** * security_inode_link() - Check if creating a hard link is allowed * @old_dentry: existing file -- cgit v1.2.3 From 8b9d0b825c6573d654c8b8039ea79920926305c2 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:04 +0100 Subject: security: Introduce inode_post_set_acl hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_set_acl hook. At inode_set_acl hook, EVM verifies the file's existing HMAC value. At inode_post_set_acl, EVM re-calculates the file's HMAC based on the modified POSIX ACL and other file metadata. Other LSMs could similarly take some action after successful POSIX ACL change. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/posix_acl.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 7 +++++++ security/security.c | 17 +++++++++++++++++ 4 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/fs/posix_acl.c b/fs/posix_acl.c index e1af20893ebe..7556ee9f8a51 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -1137,6 +1137,7 @@ retry_deleg: error = -EIO; if (!error) { fsnotify_xattr(dentry); + security_inode_post_set_acl(dentry, acl_name, kacl); evm_inode_post_set_acl(dentry, acl_name, kacl); } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 87f60b47dfca..b0125c99f80a 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -157,6 +157,8 @@ LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) +LSM_HOOK(void, LSM_RET_VOID, inode_post_set_acl, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap, diff --git a/include/linux/security.h b/include/linux/security.h index 1cb604282617..c372797e1617 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -372,6 +372,8 @@ int security_inode_setxattr(struct mnt_idmap *idmap, int security_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); +void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl); int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); int security_inode_remove_acl(struct mnt_idmap *idmap, @@ -915,6 +917,11 @@ static inline int security_inode_set_acl(struct mnt_idmap *idmap, return 0; } +static inline void security_inode_post_set_acl(struct dentry *dentry, + const char *acl_name, + struct posix_acl *kacl) +{ } + static inline int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) diff --git a/security/security.c b/security/security.c index 710db090aa8b..52f62f785087 100644 --- a/security/security.c +++ b/security/security.c @@ -2350,6 +2350,23 @@ int security_inode_set_acl(struct mnt_idmap *idmap, return evm_inode_set_acl(idmap, dentry, acl_name, kacl); } +/** + * security_inode_post_set_acl() - Update inode security from posix acls set + * @dentry: file + * @acl_name: acl name + * @kacl: acl struct + * + * Update inode security data after successfully setting posix acls on @dentry. + * The posix acls in @kacl are identified by @acl_name. + */ +void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_set_acl, dentry, acl_name, kacl); +} + /** * security_inode_get_acl() - Check if reading posix acls is allowed * @idmap: idmap of the mount -- cgit v1.2.3 From 2d705d8024143c272a764320c880ccd3230bb699 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:05 +0100 Subject: security: Introduce inode_post_remove_acl hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_remove_acl hook. At inode_remove_acl hook, EVM verifies the file's existing HMAC value. At inode_post_remove_acl, EVM re-calculates the file's HMAC with the passed POSIX ACL removed and other file metadata. Other LSMs could similarly take some action after successful POSIX ACL removal. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/posix_acl.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 8 ++++++++ security/security.c | 17 +++++++++++++++++ 4 files changed, 28 insertions(+) (limited to 'include/linux') diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 7556ee9f8a51..0d2371240c1b 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -1246,6 +1246,7 @@ retry_deleg: error = -EIO; if (!error) { fsnotify_xattr(dentry); + security_inode_post_remove_acl(idmap, dentry, acl_name); evm_inode_post_remove_acl(idmap, dentry, acl_name); } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index b0125c99f80a..7e414ba26333 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -163,6 +163,8 @@ LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) +LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_need_killpriv, struct dentry *dentry) LSM_HOOK(int, 0, inode_killpriv, struct mnt_idmap *idmap, struct dentry *dentry) diff --git a/include/linux/security.h b/include/linux/security.h index c372797e1617..4b03c76b91f1 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -378,6 +378,9 @@ int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); int security_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); +void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name); void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); int security_inode_getxattr(struct dentry *dentry, const char *name); @@ -936,6 +939,11 @@ static inline int security_inode_remove_acl(struct mnt_idmap *idmap, return 0; } +static inline void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name) +{ } + static inline void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { } diff --git a/security/security.c b/security/security.c index 52f62f785087..3bed660fc950 100644 --- a/security/security.c +++ b/security/security.c @@ -2413,6 +2413,23 @@ int security_inode_remove_acl(struct mnt_idmap *idmap, return evm_inode_remove_acl(idmap, dentry, acl_name); } +/** + * security_inode_post_remove_acl() - Update inode security after rm posix acls + * @idmap: idmap of the mount + * @dentry: file + * @acl_name: acl name + * + * Update inode security data after successfully removing posix acls on + * @dentry in @idmap. The posix acls are identified by @acl_name. + */ +void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_remove_acl, idmap, dentry, acl_name); +} + /** * security_inode_post_setxattr() - Update the inode after a setxattr operation * @dentry: file -- cgit v1.2.3 From b8d997032a46fcf47d5bda011c0d1e87b20c08ba Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:06 +0100 Subject: security: Introduce key_post_create_or_update hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the key_post_create_or_update hook. Depending on policy, IMA measures the key content after creation or update, so that remote verifiers are aware of the operation. Other LSMs could similarly take some action after successful key creation or update. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 3 +++ include/linux/security.h | 11 +++++++++++ security/keys/key.c | 7 ++++++- security/security.c | 19 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 7e414ba26333..7e4683035d34 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -405,6 +405,9 @@ LSM_HOOK(void, LSM_RET_VOID, key_free, struct key *key) LSM_HOOK(int, 0, key_permission, key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm) LSM_HOOK(int, 0, key_getsecurity, struct key *key, char **buffer) +LSM_HOOK(void, LSM_RET_VOID, key_post_create_or_update, struct key *keyring, + struct key *key, const void *payload, size_t payload_len, + unsigned long flags, bool create) #endif /* CONFIG_KEYS */ #ifdef CONFIG_AUDIT diff --git a/include/linux/security.h b/include/linux/security.h index 4b03c76b91f1..8436f9abf43d 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2004,6 +2004,9 @@ void security_key_free(struct key *key); int security_key_permission(key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm); int security_key_getsecurity(struct key *key, char **_buffer); +void security_key_post_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t payload_len, + unsigned long flags, bool create); #else @@ -2031,6 +2034,14 @@ static inline int security_key_getsecurity(struct key *key, char **_buffer) return 0; } +static inline void security_key_post_create_or_update(struct key *keyring, + struct key *key, + const void *payload, + size_t payload_len, + unsigned long flags, + bool create) +{ } + #endif #endif /* CONFIG_KEYS */ diff --git a/security/keys/key.c b/security/keys/key.c index 5b10641debd5..31a8b9408b7c 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -930,6 +930,8 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, goto error_link_end; } + security_key_post_create_or_update(keyring, key, payload, plen, flags, + true); ima_post_key_create_or_update(keyring, key, payload, plen, flags, true); @@ -963,10 +965,13 @@ error: key_ref = __key_update(key_ref, &prep); - if (!IS_ERR(key_ref)) + if (!IS_ERR(key_ref)) { + security_key_post_create_or_update(keyring, key, payload, plen, + flags, false); ima_post_key_create_or_update(keyring, key, payload, plen, flags, false); + } goto error_free_prep; } diff --git a/security/security.c b/security/security.c index 3bed660fc950..6c23c620e3c1 100644 --- a/security/security.c +++ b/security/security.c @@ -5453,6 +5453,25 @@ int security_key_getsecurity(struct key *key, char **buffer) *buffer = NULL; return call_int_hook(key_getsecurity, 0, key, buffer); } + +/** + * security_key_post_create_or_update() - Notification of key create or update + * @keyring: keyring to which the key is linked to + * @key: created or updated key + * @payload: data used to instantiate or update the key + * @payload_len: length of payload + * @flags: key flags + * @create: flag indicating whether the key was created or updated + * + * Notify the caller of a key creation or update. + */ +void security_key_post_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t payload_len, + unsigned long flags, bool create) +{ + call_void_hook(key_post_create_or_update, keyring, key, payload, + payload_len, flags, create); +} #endif /* CONFIG_KEYS */ #ifdef CONFIG_AUDIT -- cgit v1.2.3 From 06cca5110774f7b59a1685431ac697865588f4ca Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:07 +0100 Subject: integrity: Move integrity_kernel_module_request() to IMA In preparation for removing the 'integrity' LSM, move integrity_kernel_module_request() to IMA, and rename it to ima_kernel_module_request(). Rewrite the function documentation, to explain better what the problem is. Compile it conditionally if CONFIG_INTEGRITY_ASYMMETRIC_KEYS is enabled, and call it from security.c (removed afterwards with the move of IMA to the LSM infrastructure). Adding this hook cannot be avoided, since IMA has no control on the flags passed to crypto_alloc_sig() in public_key_verify_signature(), and thus cannot pass CRYPTO_NOLOAD, which solved the problem for EVM hashing with commit e2861fa71641 ("evm: Don't deadlock if a crypto algorithm is unavailable"). EVM alone does not need to implement this hook, first because there is no mutex to deadlock, and second because even if it had it, there should be a recursive call. However, since verification from EVM can be initiated only by setting inode metadata, deadlock would occur if modprobe would do the same while loading a kernel module (which is unlikely). Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 10 ++++++++++ include/linux/integrity.h | 13 ------------- security/integrity/digsig_asymmetric.c | 23 ----------------------- security/integrity/ima/ima_main.c | 33 +++++++++++++++++++++++++++++++++ security/security.c | 2 +- 5 files changed, 44 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ima.h b/include/linux/ima.h index 31ef6c3c3207..0f9af283cbc8 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -256,4 +256,14 @@ static inline bool ima_appraise_signature(enum kernel_read_file_id func) return false; } #endif /* CONFIG_IMA_APPRAISE && CONFIG_INTEGRITY_TRUSTED_KEYRING */ + +#if defined(CONFIG_IMA) && defined(CONFIG_INTEGRITY_ASYMMETRIC_KEYS) +extern int ima_kernel_module_request(char *kmod_name); +#else +static inline int ima_kernel_module_request(char *kmod_name) +{ + return 0; +} + +#endif #endif /* _LINUX_IMA_H */ diff --git a/include/linux/integrity.h b/include/linux/integrity.h index 2ea0f2f65ab6..ef0f63ef5ebc 100644 --- a/include/linux/integrity.h +++ b/include/linux/integrity.h @@ -42,17 +42,4 @@ static inline void integrity_load_keys(void) } #endif /* CONFIG_INTEGRITY */ -#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS - -extern int integrity_kernel_module_request(char *kmod_name); - -#else - -static inline int integrity_kernel_module_request(char *kmod_name) -{ - return 0; -} - -#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */ - #endif /* _LINUX_INTEGRITY_H */ diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index 895f4b9ce8c6..de603cf42ac7 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -132,26 +132,3 @@ out: pr_debug("%s() = %d\n", __func__, ret); return ret; } - -/** - * integrity_kernel_module_request - prevent crypto-pkcs1pad(rsa,*) requests - * @kmod_name: kernel module name - * - * We have situation, when public_key_verify_signature() in case of RSA - * algorithm use alg_name to store internal information in order to - * construct an algorithm on the fly, but crypto_larval_lookup() will try - * to use alg_name in order to load kernel module with same name. - * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, - * we are safe to fail such module request from crypto_larval_lookup(). - * - * In this way we prevent modprobe execution during digsig verification - * and avoid possible deadlock if modprobe and/or it's dependencies - * also signed with digsig. - */ -int integrity_kernel_module_request(char *kmod_name) -{ - if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) - return -EINVAL; - - return 0; -} diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 02021ee467d3..3891b83efdb3 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -1091,6 +1091,39 @@ int ima_measure_critical_data(const char *event_label, } EXPORT_SYMBOL_GPL(ima_measure_critical_data); +#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS + +/** + * ima_kernel_module_request - Prevent crypto-pkcs1pad(rsa,*) requests + * @kmod_name: kernel module name + * + * Avoid a verification loop where verifying the signature of the modprobe + * binary requires executing modprobe itself. Since the modprobe iint->mutex + * is already held when the signature verification is performed, a deadlock + * occurs as soon as modprobe is executed within the critical region, since + * the same lock cannot be taken again. + * + * This happens when public_key_verify_signature(), in case of RSA algorithm, + * use alg_name to store internal information in order to construct an + * algorithm on the fly, but crypto_larval_lookup() will try to use alg_name + * in order to load a kernel module with same name. + * + * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, + * we are safe to fail such module request from crypto_larval_lookup(), and + * avoid the verification loop. + * + * Return: Zero if it is safe to load the kernel module, -EINVAL otherwise. + */ +int ima_kernel_module_request(char *kmod_name) +{ + if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) + return -EINVAL; + + return 0; +} + +#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */ + static int __init init_ima(void) { int error; diff --git a/security/security.c b/security/security.c index 6c23c620e3c1..bc3c6790aeaf 100644 --- a/security/security.c +++ b/security/security.c @@ -3249,7 +3249,7 @@ int security_kernel_module_request(char *kmod_name) ret = call_int_hook(kernel_module_request, 0, kmod_name); if (ret) return ret; - return integrity_kernel_module_request(kmod_name); + return ima_kernel_module_request(kmod_name); } /** -- cgit v1.2.3 From cd3cec0a02c7338ce2901c574f3935b8f6984aab Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:08 +0100 Subject: ima: Move to LSM infrastructure Move hardcoded IMA function calls (not appraisal-specific functions) from various places in the kernel to the LSM infrastructure, by introducing a new LSM named 'ima' (at the end of the LSM list and always enabled like 'integrity'). Having IMA before EVM in the Makefile is sufficient to preserve the relative order of the new 'ima' LSM in respect to the upcoming 'evm' LSM, and thus the order of IMA and EVM function calls as when they were hardcoded. Make moved functions as static (except ima_post_key_create_or_update(), which is not in ima_main.c), and register them as implementation of the respective hooks in the new function init_ima_lsm(). Select CONFIG_SECURITY_PATH, to ensure that the path-based LSM hook path_post_mknod is always available and ima_post_path_mknod() is always executed to mark files as new, as before the move. A slight difference is that IMA and EVM functions registered for the inode_post_setattr, inode_post_removexattr, path_post_mknod, inode_post_create_tmpfile, inode_post_set_acl and inode_post_remove_acl won't be executed for private inodes. Since those inodes are supposed to be fs-internal, they should not be of interest to IMA or EVM. The S_PRIVATE flag is used for anonymous inodes, hugetlbfs, reiserfs xattrs, XFS scrub and kernel-internal tmpfs files. Conditionally register ima_post_key_create_or_update() if CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS is enabled. Also, conditionally register ima_kernel_module_request() if CONFIG_INTEGRITY_ASYMMETRIC_KEYS is enabled. Finally, add the LSM_ID_IMA case in lsm_list_modules_test.c. Signed-off-by: Roberto Sassu Acked-by: Chuck Lever Acked-by: Casey Schaufler Acked-by: Christian Brauner Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/file_table.c | 2 - fs/namei.c | 6 -- fs/nfsd/vfs.c | 7 -- fs/open.c | 1 - include/linux/ima.h | 104 --------------------- include/uapi/linux/lsm.h | 1 + security/integrity/Makefile | 1 + security/integrity/ima/Kconfig | 1 + security/integrity/ima/ima.h | 6 ++ security/integrity/ima/ima_main.c | 78 +++++++++++----- security/integrity/integrity.h | 1 + security/keys/key.c | 9 +- security/security.c | 63 +++---------- .../testing/selftests/lsm/lsm_list_modules_test.c | 3 + 14 files changed, 83 insertions(+), 200 deletions(-) (limited to 'include/linux') diff --git a/fs/file_table.c b/fs/file_table.c index 725407f374fa..1a0962981b48 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -368,7 +367,6 @@ static void __fput(struct file *file) locks_remove_file(file); security_file_release(file); - ima_file_free(file); if (unlikely(file->f_flags & FASYNC)) { if (file->f_op->fasync) file->f_op->fasync(-1, file, 0); diff --git a/fs/namei.c b/fs/namei.c index c7ac7fdf0f25..1bf7435cdf76 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -3641,8 +3640,6 @@ static int do_open(struct nameidata *nd, error = vfs_open(&nd->path, file); if (!error) error = security_file_post_open(file, op->acc_mode); - if (!error) - error = ima_file_check(file, op->acc_mode); if (!error && do_truncate) error = handle_truncate(idmap, file); if (unlikely(error > 0)) { @@ -3706,7 +3703,6 @@ static int vfs_tmpfile(struct mnt_idmap *idmap, spin_unlock(&inode->i_lock); } security_inode_post_create_tmpfile(idmap, inode); - ima_post_create_tmpfile(idmap, inode); return 0; } @@ -4052,8 +4048,6 @@ retry: case 0: case S_IFREG: error = vfs_create(idmap, path.dentry->d_inode, dentry, mode, true); - if (!error) - ima_post_path_mknod(idmap, dentry); break; case S_IFCHR: case S_IFBLK: error = vfs_mknod(idmap, path.dentry->d_inode, diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index e44d8239545b..4349dde92154 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -883,12 +882,6 @@ __nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, goto out; } - host_err = ima_file_check(file, may_flags); - if (host_err) { - fput(file); - goto out; - } - if (may_flags & NFSD_MAY_64BIT_COOKIE) file->f_mode |= FMODE_64BITHASH; else diff --git a/fs/open.c b/fs/open.c index a84d21e55c39..1cf05a1413aa 100644 --- a/fs/open.c +++ b/fs/open.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include diff --git a/include/linux/ima.h b/include/linux/ima.h index 0f9af283cbc8..23ae24b60ecf 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -16,24 +16,6 @@ struct linux_binprm; #ifdef CONFIG_IMA extern enum hash_algo ima_get_current_hash_algo(void); -extern int ima_bprm_check(struct linux_binprm *bprm); -extern int ima_file_check(struct file *file, int mask); -extern void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode); -extern void ima_file_free(struct file *file); -extern int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags); -extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, - unsigned long prot); -extern int ima_load_data(enum kernel_load_data_id id, bool contents); -extern int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id id, char *description); -extern int ima_read_file(struct file *file, enum kernel_read_file_id id, - bool contents); -extern int ima_post_read_file(struct file *file, char *buf, loff_t size, - enum kernel_read_file_id id); -extern void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry); extern int ima_file_hash(struct file *file, char *buf, size_t buf_size); extern int ima_inode_hash(struct inode *inode, char *buf, size_t buf_size); extern void ima_kexec_cmdline(int kernel_fd, const void *buf, int size); @@ -58,68 +40,6 @@ static inline enum hash_algo ima_get_current_hash_algo(void) return HASH_ALGO__LAST; } -static inline int ima_bprm_check(struct linux_binprm *bprm) -{ - return 0; -} - -static inline int ima_file_check(struct file *file, int mask) -{ - return 0; -} - -static inline void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode) -{ -} - -static inline void ima_file_free(struct file *file) -{ - return; -} - -static inline int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags) -{ - return 0; -} - -static inline int ima_file_mprotect(struct vm_area_struct *vma, - unsigned long reqprot, unsigned long prot) -{ - return 0; -} - -static inline int ima_load_data(enum kernel_load_data_id id, bool contents) -{ - return 0; -} - -static inline int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id id, - char *description) -{ - return 0; -} - -static inline int ima_read_file(struct file *file, enum kernel_read_file_id id, - bool contents) -{ - return 0; -} - -static inline int ima_post_read_file(struct file *file, char *buf, loff_t size, - enum kernel_read_file_id id) -{ - return 0; -} - -static inline void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry) -{ - return; -} - static inline int ima_file_hash(struct file *file, char *buf, size_t buf_size) { return -EOPNOTSUPP; @@ -170,20 +90,6 @@ static inline void ima_add_kexec_buffer(struct kimage *image) {} #endif -#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS -extern void ima_post_key_create_or_update(struct key *keyring, - struct key *key, - const void *payload, size_t plen, - unsigned long flags, bool create); -#else -static inline void ima_post_key_create_or_update(struct key *keyring, - struct key *key, - const void *payload, - size_t plen, - unsigned long flags, - bool create) {} -#endif /* CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS */ - #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); extern void ima_inode_post_setattr(struct mnt_idmap *idmap, @@ -256,14 +162,4 @@ static inline bool ima_appraise_signature(enum kernel_read_file_id func) return false; } #endif /* CONFIG_IMA_APPRAISE && CONFIG_INTEGRITY_TRUSTED_KEYRING */ - -#if defined(CONFIG_IMA) && defined(CONFIG_INTEGRITY_ASYMMETRIC_KEYS) -extern int ima_kernel_module_request(char *kmod_name); -#else -static inline int ima_kernel_module_request(char *kmod_name) -{ - return 0; -} - -#endif #endif /* _LINUX_IMA_H */ diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h index f8aef9ade549..b3b7fd699b63 100644 --- a/include/uapi/linux/lsm.h +++ b/include/uapi/linux/lsm.h @@ -62,6 +62,7 @@ struct lsm_ctx { #define LSM_ID_LOCKDOWN 108 #define LSM_ID_BPF 109 #define LSM_ID_LANDLOCK 110 +#define LSM_ID_IMA 111 /* * LSM_ATTR_XXX definitions identify different LSM attributes diff --git a/security/integrity/Makefile b/security/integrity/Makefile index d0ffe37dc1d6..92b63039c654 100644 --- a/security/integrity/Makefile +++ b/security/integrity/Makefile @@ -18,5 +18,6 @@ integrity-$(CONFIG_LOAD_IPL_KEYS) += platform_certs/load_ipl_s390.o integrity-$(CONFIG_LOAD_PPC_KEYS) += platform_certs/efi_parser.o \ platform_certs/load_powerpc.o \ platform_certs/keyring_handler.o +# The relative order of the 'ima' and 'evm' LSMs depends on the order below. obj-$(CONFIG_IMA) += ima/ obj-$(CONFIG_EVM) += evm/ diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index b98bfe9efd0c..475c32615006 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -8,6 +8,7 @@ config IMA select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_HASH_INFO + select SECURITY_PATH select TCG_TPM if HAS_IOMEM select TCG_TIS if TCG_TPM && X86 select TCG_CRB if TCG_TPM && ACPI diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index c29db699c996..c0412100023e 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -127,6 +127,12 @@ void ima_load_kexec_buffer(void); static inline void ima_load_kexec_buffer(void) {} #endif /* CONFIG_HAVE_IMA_KEXEC */ +#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS +void ima_post_key_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t plen, + unsigned long flags, bool create); +#endif + /* * The default binary_runtime_measurements list format is defined as the * platform native format. The canonical format is defined as little-endian. diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 3891b83efdb3..d0826e864c29 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -189,7 +189,7 @@ static void ima_check_last_writer(struct integrity_iint_cache *iint, * * Flag files that changed, based on i_version */ -void ima_file_free(struct file *file) +static void ima_file_free(struct file *file) { struct inode *inode = file_inode(file); struct integrity_iint_cache *iint; @@ -427,8 +427,8 @@ out: * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags) +static int ima_file_mmap(struct file *file, unsigned long reqprot, + unsigned long prot, unsigned long flags) { u32 secid; int ret; @@ -466,8 +466,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, * * On mprotect change success, return 0. On failure, return -EACESS. */ -int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, - unsigned long prot) +static int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot) { struct ima_template_desc *template = NULL; struct file *file; @@ -525,7 +525,7 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_bprm_check(struct linux_binprm *bprm) +static int ima_bprm_check(struct linux_binprm *bprm) { int ret; u32 secid; @@ -551,7 +551,7 @@ int ima_bprm_check(struct linux_binprm *bprm) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_file_check(struct file *file, int mask) +static int ima_file_check(struct file *file, int mask) { u32 secid; @@ -560,7 +560,6 @@ int ima_file_check(struct file *file, int mask) mask & (MAY_READ | MAY_WRITE | MAY_EXEC | MAY_APPEND), FILE_CHECK); } -EXPORT_SYMBOL_GPL(ima_file_check); static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, size_t buf_size) @@ -685,8 +684,9 @@ EXPORT_SYMBOL_GPL(ima_inode_hash); * Skip calling process_measurement(), but indicate which newly, created * tmpfiles are in policy. */ -void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode) +static void ima_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode) + { struct integrity_iint_cache *iint; int must_appraise; @@ -717,8 +717,7 @@ void ima_post_create_tmpfile(struct mnt_idmap *idmap, * Mark files created via the mknodat syscall as new, so that the * file data can be written later. */ -void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry) +static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) { struct integrity_iint_cache *iint; struct inode *inode = dentry->d_inode; @@ -753,8 +752,8 @@ void ima_post_path_mknod(struct mnt_idmap *idmap, * * For permission return 0, otherwise return -EACCES. */ -int ima_read_file(struct file *file, enum kernel_read_file_id read_id, - bool contents) +static int ima_read_file(struct file *file, enum kernel_read_file_id read_id, + bool contents) { enum ima_hooks func; u32 secid; @@ -803,8 +802,8 @@ const int read_idmap[READING_MAX_ID] = { * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_read_file(struct file *file, char *buf, loff_t size, - enum kernel_read_file_id read_id) +static int ima_post_read_file(struct file *file, char *buf, loff_t size, + enum kernel_read_file_id read_id) { enum ima_hooks func; u32 secid; @@ -837,7 +836,7 @@ int ima_post_read_file(struct file *file, char *buf, loff_t size, * * For permission return 0, otherwise return -EACCES. */ -int ima_load_data(enum kernel_load_data_id id, bool contents) +static int ima_load_data(enum kernel_load_data_id id, bool contents) { bool ima_enforce, sig_enforce; @@ -891,9 +890,9 @@ int ima_load_data(enum kernel_load_data_id id, bool contents) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id load_id, - char *description) +static int ima_post_load_data(char *buf, loff_t size, + enum kernel_load_data_id load_id, + char *description) { if (load_id == LOADING_FIRMWARE) { if ((ima_appraise & IMA_APPRAISE_FIRMWARE) && @@ -1114,7 +1113,7 @@ EXPORT_SYMBOL_GPL(ima_measure_critical_data); * * Return: Zero if it is safe to load the kernel module, -EINVAL otherwise. */ -int ima_kernel_module_request(char *kmod_name) +static int ima_kernel_module_request(char *kmod_name) { if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) return -EINVAL; @@ -1155,4 +1154,41 @@ static int __init init_ima(void) return error; } +static struct security_hook_list ima_hooks[] __ro_after_init = { + LSM_HOOK_INIT(bprm_check_security, ima_bprm_check), + LSM_HOOK_INIT(file_post_open, ima_file_check), + LSM_HOOK_INIT(inode_post_create_tmpfile, ima_post_create_tmpfile), + LSM_HOOK_INIT(file_release, ima_file_free), + LSM_HOOK_INIT(mmap_file, ima_file_mmap), + LSM_HOOK_INIT(file_mprotect, ima_file_mprotect), + LSM_HOOK_INIT(kernel_load_data, ima_load_data), + LSM_HOOK_INIT(kernel_post_load_data, ima_post_load_data), + LSM_HOOK_INIT(kernel_read_file, ima_read_file), + LSM_HOOK_INIT(kernel_post_read_file, ima_post_read_file), + LSM_HOOK_INIT(path_post_mknod, ima_post_path_mknod), +#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS + LSM_HOOK_INIT(key_post_create_or_update, ima_post_key_create_or_update), +#endif +#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS + LSM_HOOK_INIT(kernel_module_request, ima_kernel_module_request), +#endif +}; + +static const struct lsm_id ima_lsmid = { + .name = "ima", + .id = LSM_ID_IMA, +}; + +static int __init init_ima_lsm(void) +{ + security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid); + return 0; +} + +DEFINE_LSM(ima) = { + .name = "ima", + .init = init_ima_lsm, + .order = LSM_ORDER_LAST, +}; + late_initcall(init_ima); /* Start IMA after the TPM is available */ diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 9561db7cf6b4..59eaddd84434 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -18,6 +18,7 @@ #include #include #include +#include /* iint action cache flags */ #define IMA_MEASURE 0x00000001 diff --git a/security/keys/key.c b/security/keys/key.c index 31a8b9408b7c..560790038329 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "internal.h" @@ -932,8 +931,6 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, security_key_post_create_or_update(keyring, key, payload, plen, flags, true); - ima_post_key_create_or_update(keyring, key, payload, plen, - flags, true); key_ref = make_key_ref(key, is_key_possessed(keyring_ref)); @@ -965,13 +962,9 @@ error: key_ref = __key_update(key_ref, &prep); - if (!IS_ERR(key_ref)) { + if (!IS_ERR(key_ref)) security_key_post_create_or_update(keyring, key, payload, plen, flags, false); - ima_post_key_create_or_update(keyring, key, - payload, plen, - flags, false); - } goto error_free_prep; } diff --git a/security/security.c b/security/security.c index bc3c6790aeaf..8495890a5a37 100644 --- a/security/security.c +++ b/security/security.c @@ -50,7 +50,8 @@ (IS_ENABLED(CONFIG_SECURITY_SAFESETID) ? 1 : 0) + \ (IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \ - (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0)) + (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0) + \ + (IS_ENABLED(CONFIG_IMA) ? 1 : 0)) /* * These are descriptions of the reasons that can be passed to the @@ -1182,12 +1183,7 @@ int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file * */ int security_bprm_check(struct linux_binprm *bprm) { - int ret; - - ret = call_int_hook(bprm_check_security, 0, bprm); - if (ret) - return ret; - return ima_bprm_check(bprm); + return call_int_hook(bprm_check_security, 0, bprm); } /** @@ -2895,13 +2891,8 @@ static inline unsigned long mmap_prot(struct file *file, unsigned long prot) int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags) { - unsigned long prot_adj = mmap_prot(file, prot); - int ret; - - ret = call_int_hook(mmap_file, 0, file, prot, prot_adj, flags); - if (ret) - return ret; - return ima_file_mmap(file, prot, prot_adj, flags); + return call_int_hook(mmap_file, 0, file, prot, mmap_prot(file, prot), + flags); } /** @@ -2930,12 +2921,7 @@ int security_mmap_addr(unsigned long addr) int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { - int ret; - - ret = call_int_hook(file_mprotect, 0, vma, reqprot, prot); - if (ret) - return ret; - return ima_file_mprotect(vma, reqprot, prot); + return call_int_hook(file_mprotect, 0, vma, reqprot, prot); } /** @@ -3244,12 +3230,7 @@ int security_kernel_create_files_as(struct cred *new, struct inode *inode) */ int security_kernel_module_request(char *kmod_name) { - int ret; - - ret = call_int_hook(kernel_module_request, 0, kmod_name); - if (ret) - return ret; - return ima_kernel_module_request(kmod_name); + return call_int_hook(kernel_module_request, 0, kmod_name); } /** @@ -3265,12 +3246,7 @@ int security_kernel_module_request(char *kmod_name) int security_kernel_read_file(struct file *file, enum kernel_read_file_id id, bool contents) { - int ret; - - ret = call_int_hook(kernel_read_file, 0, file, id, contents); - if (ret) - return ret; - return ima_read_file(file, id, contents); + return call_int_hook(kernel_read_file, 0, file, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_read_file); @@ -3290,12 +3266,7 @@ EXPORT_SYMBOL_GPL(security_kernel_read_file); int security_kernel_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id) { - int ret; - - ret = call_int_hook(kernel_post_read_file, 0, file, buf, size, id); - if (ret) - return ret; - return ima_post_read_file(file, buf, size, id); + return call_int_hook(kernel_post_read_file, 0, file, buf, size, id); } EXPORT_SYMBOL_GPL(security_kernel_post_read_file); @@ -3310,12 +3281,7 @@ EXPORT_SYMBOL_GPL(security_kernel_post_read_file); */ int security_kernel_load_data(enum kernel_load_data_id id, bool contents) { - int ret; - - ret = call_int_hook(kernel_load_data, 0, id, contents); - if (ret) - return ret; - return ima_load_data(id, contents); + return call_int_hook(kernel_load_data, 0, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_load_data); @@ -3337,13 +3303,8 @@ int security_kernel_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description) { - int ret; - - ret = call_int_hook(kernel_post_load_data, 0, buf, size, id, - description); - if (ret) - return ret; - return ima_post_load_data(buf, size, id, description); + return call_int_hook(kernel_post_load_data, 0, buf, size, id, + description); } EXPORT_SYMBOL_GPL(security_kernel_post_load_data); diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 9df29b1e3497..17333787cb2f 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -122,6 +122,9 @@ TEST(correct_lsm_list_modules) case LSM_ID_LANDLOCK: name = "landlock"; break; + case LSM_ID_IMA: + name = "ima"; + break; default: name = "INVALID"; break; -- cgit v1.2.3 From 84594c9ecdca7ca595bc50e315093cb76921fd8e Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:09 +0100 Subject: ima: Move IMA-Appraisal to LSM infrastructure A few additional IMA hooks are needed to reset the cached appraisal status, causing the file's integrity to be re-evaluated on next access. Register these IMA-appraisal only functions separately from the rest of IMA functions, as appraisal is a separate feature not necessarily enabled in the kernel configuration. Reuse the same approach as for other IMA functions, move hardcoded calls from various places in the kernel to the LSM infrastructure. Declare the functions as static and register them as hook implementations in init_ima_appraise_lsm(), called by init_ima_lsm(). Also move the inline function ima_inode_remove_acl() from the public ima.h header to ima_appraise.c. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Christian Brauner Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 -- include/linux/ima.h | 55 ----------------------------------- security/integrity/ima/ima.h | 5 ++++ security/integrity/ima/ima_appraise.c | 38 ++++++++++++++++++------ security/integrity/ima/ima_main.c | 1 + security/security.c | 13 --------- 6 files changed, 35 insertions(+), 79 deletions(-) (limited to 'include/linux') diff --git a/fs/attr.c b/fs/attr.c index 990e1b3a3c91..7e97313e7f70 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -17,7 +17,6 @@ #include #include #include -#include #include "internal.h" @@ -503,7 +502,6 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); security_inode_post_setattr(idmap, dentry, ia_valid); - ima_inode_post_setattr(idmap, dentry, ia_valid); evm_inode_post_setattr(idmap, dentry, ia_valid); } diff --git a/include/linux/ima.h b/include/linux/ima.h index 23ae24b60ecf..0bae61a15b60 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -92,66 +92,11 @@ static inline void ima_add_kexec_buffer(struct kimage *image) #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); -extern void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid); -extern int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len, int flags); -extern int ima_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl); -static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return ima_inode_set_acl(idmap, dentry, acl_name, NULL); -} - -extern int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name); #else static inline bool is_ima_appraise_enabled(void) { return 0; } - -static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid) -{ - return; -} - -static inline int ima_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len, - int flags) -{ - return 0; -} - -static inline int ima_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl) -{ - - return 0; -} - -static inline int ima_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name) -{ - return 0; -} - -static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return 0; -} #endif /* CONFIG_IMA_APPRAISE */ #if defined(CONFIG_IMA_APPRAISE) && defined(CONFIG_INTEGRITY_TRUSTED_KEYRING) diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index c0412100023e..a27fc10f84f7 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -334,6 +334,7 @@ enum hash_algo ima_get_hash_algo(const struct evm_ima_xattr_data *xattr_value, int xattr_len); int ima_read_xattr(struct dentry *dentry, struct evm_ima_xattr_data **xattr_value, int xattr_len); +void __init init_ima_appraise_lsm(const struct lsm_id *lsmid); #else static inline int ima_check_blacklist(struct integrity_iint_cache *iint, @@ -385,6 +386,10 @@ static inline int ima_read_xattr(struct dentry *dentry, return 0; } +static inline void __init init_ima_appraise_lsm(const struct lsm_id *lsmid) +{ +} + #endif /* CONFIG_IMA_APPRAISE */ #ifdef CONFIG_IMA_APPRAISE_MODSIG diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 36abc84ba299..076451109637 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -636,8 +636,8 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid) +static void ima_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { struct inode *inode = d_backing_inode(dentry); struct integrity_iint_cache *iint; @@ -750,9 +750,9 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; } -int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len, int flags) +static int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xvalue = xattr_value; int digsig = 0; @@ -781,8 +781,8 @@ int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, return result; } -int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, - const char *acl_name, struct posix_acl *kacl) +static int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) { if (evm_revalidate_status(acl_name)) ima_reset_appraise_flags(d_backing_inode(dentry), 0); @@ -790,8 +790,8 @@ int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return 0; } -int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name) +static int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { int result; @@ -803,3 +803,23 @@ int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, } return result; } + +static int ima_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + return ima_inode_set_acl(idmap, dentry, acl_name, NULL); +} + +static struct security_hook_list ima_appraise_hooks[] __ro_after_init = { + LSM_HOOK_INIT(inode_post_setattr, ima_inode_post_setattr), + LSM_HOOK_INIT(inode_setxattr, ima_inode_setxattr), + LSM_HOOK_INIT(inode_set_acl, ima_inode_set_acl), + LSM_HOOK_INIT(inode_removexattr, ima_inode_removexattr), + LSM_HOOK_INIT(inode_remove_acl, ima_inode_remove_acl), +}; + +void __init init_ima_appraise_lsm(const struct lsm_id *lsmid) +{ + security_add_hooks(ima_appraise_hooks, ARRAY_SIZE(ima_appraise_hooks), + lsmid); +} diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index d0826e864c29..b8b8891a9379 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -1182,6 +1182,7 @@ static const struct lsm_id ima_lsmid = { static int __init init_ima_lsm(void) { security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid); + init_ima_appraise_lsm(&ima_lsmid); return 0; } diff --git a/security/security.c b/security/security.c index 8495890a5a37..bed659b53e59 100644 --- a/security/security.c +++ b/security/security.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -2308,9 +2307,6 @@ int security_inode_setxattr(struct mnt_idmap *idmap, if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); - if (ret) - return ret; - ret = ima_inode_setxattr(idmap, dentry, name, value, size, flags); if (ret) return ret; return evm_inode_setxattr(idmap, dentry, name, value, size, flags); @@ -2338,9 +2334,6 @@ int security_inode_set_acl(struct mnt_idmap *idmap, return 0; ret = call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, kacl); - if (ret) - return ret; - ret = ima_inode_set_acl(idmap, dentry, acl_name, kacl); if (ret) return ret; return evm_inode_set_acl(idmap, dentry, acl_name, kacl); @@ -2401,9 +2394,6 @@ int security_inode_remove_acl(struct mnt_idmap *idmap, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; ret = call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); - if (ret) - return ret; - ret = ima_inode_remove_acl(idmap, dentry, acl_name); if (ret) return ret; return evm_inode_remove_acl(idmap, dentry, acl_name); @@ -2503,9 +2493,6 @@ int security_inode_removexattr(struct mnt_idmap *idmap, ret = call_int_hook(inode_removexattr, 1, idmap, dentry, name); if (ret == 1) ret = cap_inode_removexattr(idmap, dentry, name); - if (ret) - return ret; - ret = ima_inode_removexattr(idmap, dentry, name); if (ret) return ret; return evm_inode_removexattr(idmap, dentry, name); -- cgit v1.2.3 From 9238311176115aac1b1a86e8e968c04ebec747a1 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:10 +0100 Subject: evm: Move to LSM infrastructure As for IMA, move hardcoded EVM function calls from various places in the kernel to the LSM infrastructure, by introducing a new LSM named 'evm' (last and always enabled like 'ima'). The order in the Makefile ensures that 'evm' hooks are executed after 'ima' ones. Make EVM functions as static (except for evm_inode_init_security(), which is exported), and register them as hook implementations in init_evm_lsm(). Also move the inline functions evm_inode_remove_acl(), evm_inode_post_remove_acl(), and evm_inode_post_set_acl() from the public evm.h header to evm_main.c. Unlike before (see commit to move IMA to the LSM infrastructure), evm_inode_post_setattr(), evm_inode_post_set_acl(), evm_inode_post_remove_acl(), and evm_inode_post_removexattr() are not executed for private inodes. Finally, add the LSM_ID_EVM case in lsm_list_modules_test.c Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Acked-by: Christian Brauner Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 - fs/posix_acl.c | 3 - fs/xattr.c | 2 - include/linux/evm.h | 113 -------------------- include/uapi/linux/lsm.h | 1 + security/integrity/evm/evm_main.c | 118 ++++++++++++++++++--- security/security.c | 43 ++------ .../testing/selftests/lsm/lsm_list_modules_test.c | 3 + 8 files changed, 116 insertions(+), 169 deletions(-) (limited to 'include/linux') diff --git a/fs/attr.c b/fs/attr.c index 7e97313e7f70..4d0d75953107 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -16,7 +16,6 @@ #include #include #include -#include #include "internal.h" @@ -502,7 +501,6 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); security_inode_post_setattr(idmap, dentry, ia_valid); - evm_inode_post_setattr(idmap, dentry, ia_valid); } return error; diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 0d2371240c1b..5c90239e3f2b 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -1138,7 +1137,6 @@ retry_deleg: if (!error) { fsnotify_xattr(dentry); security_inode_post_set_acl(dentry, acl_name, kacl); - evm_inode_post_set_acl(dentry, acl_name, kacl); } out_inode_unlock: @@ -1247,7 +1245,6 @@ retry_deleg: if (!error) { fsnotify_xattr(dentry); security_inode_post_remove_acl(idmap, dentry, acl_name); - evm_inode_post_remove_acl(idmap, dentry, acl_name); } out_inode_unlock: diff --git a/fs/xattr.c b/fs/xattr.c index f891c260a971..f8b643f91a98 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -557,7 +556,6 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap, fsnotify_xattr(dentry); security_inode_post_removexattr(dentry, name); - evm_inode_post_removexattr(dentry, name); out: return error; diff --git a/include/linux/evm.h b/include/linux/evm.h index 3faabdd47852..cb481eccc967 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -21,45 +21,6 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry, void *xattr_value, size_t xattr_value_len, struct integrity_iint_cache *iint); -extern int evm_inode_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, struct iattr *attr); -extern void evm_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid); -extern int evm_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *name, - const void *value, size_t size, int flags); -extern void evm_inode_post_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len, - int flags); -extern int evm_inode_copy_up_xattr(const char *name); -extern int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *xattr_name); -extern void evm_inode_post_removexattr(struct dentry *dentry, - const char *xattr_name); -static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - evm_inode_post_removexattr(dentry, acl_name); -} -extern int evm_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl); -static inline int evm_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return evm_inode_set_acl(idmap, dentry, acl_name, NULL); -} -static inline void evm_inode_post_set_acl(struct dentry *dentry, - const char *acl_name, - struct posix_acl *kacl) -{ - return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); -} - int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, int *xattr_count); @@ -94,80 +55,6 @@ static inline enum integrity_status evm_verifyxattr(struct dentry *dentry, } #endif -static inline int evm_inode_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, struct iattr *attr) -{ - return 0; -} - -static inline void evm_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid) -{ - return; -} - -static inline int evm_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *name, - const void *value, size_t size, int flags) -{ - return 0; -} - -static inline void evm_inode_post_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len, - int flags) -{ - return; -} - -static inline int evm_inode_copy_up_xattr(const char *name) -{ - return 0; -} - -static inline int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name) -{ - return 0; -} - -static inline void evm_inode_post_removexattr(struct dentry *dentry, - const char *xattr_name) -{ - return; -} - -static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return; -} - -static inline int evm_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl) -{ - return 0; -} - -static inline int evm_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return 0; -} - -static inline void evm_inode_post_set_acl(struct dentry *dentry, - const char *acl_name, - struct posix_acl *kacl) -{ - return; -} - static inline int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h index b3b7fd699b63..33d8c9f4aa6b 100644 --- a/include/uapi/linux/lsm.h +++ b/include/uapi/linux/lsm.h @@ -63,6 +63,7 @@ struct lsm_ctx { #define LSM_ID_BPF 109 #define LSM_ID_LANDLOCK 110 #define LSM_ID_IMA 111 +#define LSM_ID_EVM 112 /* * LSM_ATTR_XXX definitions identify different LSM attributes diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index d35143179699..0a089af83a45 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -589,9 +589,9 @@ out: * userspace from writing HMAC value. Writing 'security.evm' requires * requires CAP_SYS_ADMIN privileges. */ -int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len, int flags) +static int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xattr_data = xattr_value; @@ -621,8 +621,8 @@ int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, * Removing 'security.evm' requires CAP_SYS_ADMIN privileges and that * the current value is valid. */ -int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *xattr_name) +static int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { /* Policy permits modification of the protected xattrs even though * there's no HMAC key loaded @@ -672,9 +672,11 @@ static inline int evm_inode_set_acl_change(struct mnt_idmap *idmap, * Prevent modifying posix acls causing the EVM HMAC to be re-calculated * and 'security.evm' xattr updated, unless the existing 'security.evm' is * valid. + * + * Return: zero on success, -EPERM on failure. */ -int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, - const char *acl_name, struct posix_acl *kacl) +static int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) { enum integrity_status evm_status; @@ -713,6 +715,24 @@ int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; } +/** + * evm_inode_remove_acl - Protect the EVM extended attribute from posix acls + * @idmap: idmap of the mount + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * + * Prevent removing posix acls causing the EVM HMAC to be re-calculated + * and 'security.evm' xattr updated, unless the existing 'security.evm' is + * valid. + * + * Return: zero on success, -EPERM on failure. + */ +static int evm_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + return evm_inode_set_acl(idmap, dentry, acl_name, NULL); +} + static void evm_reset_status(struct inode *inode) { struct integrity_iint_cache *iint; @@ -761,9 +781,11 @@ bool evm_revalidate_status(const char *xattr_name) * __vfs_setxattr_noperm(). The caller of which has taken the inode's * i_mutex lock. */ -void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len, - int flags) +static void evm_inode_post_setxattr(struct dentry *dentry, + const char *xattr_name, + const void *xattr_value, + size_t xattr_value_len, + int flags) { if (!evm_revalidate_status(xattr_name)) return; @@ -782,6 +804,21 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, evm_update_evmxattr(dentry, xattr_name, xattr_value, xattr_value_len); } +/** + * evm_inode_post_set_acl - Update the EVM extended attribute from posix acls + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * @kacl: pointer to the posix acls + * + * Update the 'security.evm' xattr with the EVM HMAC re-calculated after setting + * posix acls. + */ +static void evm_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl) +{ + return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); +} + /** * evm_inode_post_removexattr - update 'security.evm' after removing the xattr * @dentry: pointer to the affected dentry @@ -792,7 +829,8 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, * No need to take the i_mutex lock here, as this function is called from * vfs_removexattr() which takes the i_mutex. */ -void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) +static void evm_inode_post_removexattr(struct dentry *dentry, + const char *xattr_name) { if (!evm_revalidate_status(xattr_name)) return; @@ -808,6 +846,22 @@ void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) evm_update_evmxattr(dentry, xattr_name, NULL, 0); } +/** + * evm_inode_post_remove_acl - Update the EVM extended attribute from posix acls + * @idmap: idmap of the mount + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * + * Update the 'security.evm' xattr with the EVM HMAC re-calculated after + * removing posix acls. + */ +static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name) +{ + evm_inode_post_removexattr(dentry, acl_name); +} + static int evm_attr_change(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -831,8 +885,8 @@ static int evm_attr_change(struct mnt_idmap *idmap, * Permit update of file attributes when files have a valid EVM signature, * except in the case of them having an immutable portable signature. */ -int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, - struct iattr *attr) +static int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; enum integrity_status evm_status; @@ -883,8 +937,8 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, - int ia_valid) +static void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { if (!evm_revalidate_status(NULL)) return; @@ -901,7 +955,7 @@ void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, evm_update_evmxattr(dentry, NULL, NULL, 0); } -int evm_inode_copy_up_xattr(const char *name) +static int evm_inode_copy_up_xattr(const char *name) { if (strcmp(name, XATTR_NAME_EVM) == 0) return 1; /* Discard */ @@ -1004,4 +1058,36 @@ error: return error; } +static struct security_hook_list evm_hooks[] __ro_after_init = { + LSM_HOOK_INIT(inode_setattr, evm_inode_setattr), + LSM_HOOK_INIT(inode_post_setattr, evm_inode_post_setattr), + LSM_HOOK_INIT(inode_copy_up_xattr, evm_inode_copy_up_xattr), + LSM_HOOK_INIT(inode_setxattr, evm_inode_setxattr), + LSM_HOOK_INIT(inode_post_setxattr, evm_inode_post_setxattr), + LSM_HOOK_INIT(inode_set_acl, evm_inode_set_acl), + LSM_HOOK_INIT(inode_post_set_acl, evm_inode_post_set_acl), + LSM_HOOK_INIT(inode_remove_acl, evm_inode_remove_acl), + LSM_HOOK_INIT(inode_post_remove_acl, evm_inode_post_remove_acl), + LSM_HOOK_INIT(inode_removexattr, evm_inode_removexattr), + LSM_HOOK_INIT(inode_post_removexattr, evm_inode_post_removexattr), + LSM_HOOK_INIT(inode_init_security, evm_inode_init_security), +}; + +static const struct lsm_id evm_lsmid = { + .name = "evm", + .id = LSM_ID_EVM, +}; + +static int __init init_evm_lsm(void) +{ + security_add_hooks(evm_hooks, ARRAY_SIZE(evm_hooks), &evm_lsmid); + return 0; +} + +DEFINE_LSM(evm) = { + .name = "evm", + .init = init_evm_lsm, + .order = LSM_ORDER_LAST, +}; + late_initcall(init_evm); diff --git a/security/security.c b/security/security.c index bed659b53e59..59cb0e6cbae3 100644 --- a/security/security.c +++ b/security/security.c @@ -20,13 +20,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include @@ -50,7 +50,8 @@ (IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0) + \ - (IS_ENABLED(CONFIG_IMA) ? 1 : 0)) + (IS_ENABLED(CONFIG_IMA) ? 1 : 0) + \ + (IS_ENABLED(CONFIG_EVM) ? 1 : 0)) /* * These are descriptions of the reasons that can be passed to the @@ -1740,10 +1741,6 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, if (!xattr_count) goto out; - ret = evm_inode_init_security(inode, dir, qstr, new_xattrs, - &xattr_count); - if (ret) - goto out; ret = initxattrs(inode, new_xattrs, fs_data); out: for (; xattr_count > 0; xattr_count--) @@ -2235,14 +2232,9 @@ int security_inode_permission(struct inode *inode, int mask) int security_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_setattr, 0, idmap, dentry, attr); - if (ret) - return ret; - return evm_inode_setattr(idmap, dentry, attr); + return call_int_hook(inode_setattr, 0, idmap, dentry, attr); } EXPORT_SYMBOL_GPL(security_inode_setattr); @@ -2307,9 +2299,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap, if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); - if (ret) - return ret; - return evm_inode_setxattr(idmap, dentry, name, value, size, flags); + return ret; } /** @@ -2328,15 +2318,10 @@ int security_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, - kacl); - if (ret) - return ret; - return evm_inode_set_acl(idmap, dentry, acl_name, kacl); + return call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, + kacl); } /** @@ -2389,14 +2374,9 @@ int security_inode_get_acl(struct mnt_idmap *idmap, int security_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); - if (ret) - return ret; - return evm_inode_remove_acl(idmap, dentry, acl_name); + return call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); } /** @@ -2432,7 +2412,6 @@ void security_inode_post_setxattr(struct dentry *dentry, const char *name, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return; call_void_hook(inode_post_setxattr, dentry, name, value, size, flags); - evm_inode_post_setxattr(dentry, name, value, size, flags); } /** @@ -2493,9 +2472,7 @@ int security_inode_removexattr(struct mnt_idmap *idmap, ret = call_int_hook(inode_removexattr, 1, idmap, dentry, name); if (ret == 1) ret = cap_inode_removexattr(idmap, dentry, name); - if (ret) - return ret; - return evm_inode_removexattr(idmap, dentry, name); + return ret; } /** @@ -2699,7 +2676,7 @@ int security_inode_copy_up_xattr(const char *name) return rc; } - return evm_inode_copy_up_xattr(name); + return LSM_RET_DEFAULT(inode_copy_up_xattr); } EXPORT_SYMBOL(security_inode_copy_up_xattr); diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 17333787cb2f..4d5d4cee2586 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -125,6 +125,9 @@ TEST(correct_lsm_list_modules) case LSM_ID_IMA: name = "ima"; break; + case LSM_ID_EVM: + name = "evm"; + break; default: name = "INVALID"; break; -- cgit v1.2.3 From 75a323e604fc77c50c7ef2af6f0eeef221637642 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:11 +0100 Subject: evm: Make it independent from 'integrity' LSM Define a new structure for EVM-specific metadata, called evm_iint_cache, and embed it in the inode security blob. Introduce evm_iint_inode() to retrieve metadata, and register evm_inode_alloc_security() for the inode_alloc_security LSM hook, to initialize the structure (before splitting metadata, this task was done by iint_init_always()). Keep the non-NULL checks after calling evm_iint_inode() except in evm_inode_alloc_security(), to take into account inodes for which security_inode_alloc() was not called. When using shared metadata, obtaining a NULL pointer from integrity_iint_find() meant that the file wasn't in the IMA policy. Now, because IMA and EVM use disjoint metadata, the EVM status has to be stored for every inode regardless of the IMA policy. Given that from now on EVM relies on its own metadata, remove the iint parameter from evm_verifyxattr(). Also, directly retrieve the iint in evm_verify_hmac(), called by both evm_verifyxattr() and evm_verify_current_integrity(), since now there is no performance penalty in retrieving EVM metadata (constant time). Replicate the management of the IMA_NEW_FILE flag, by introducing evm_post_path_mknod() and evm_file_release() to respectively set and clear the newly introduced flag EVM_NEW_FILE, at the same time IMA does. Like for IMA, select CONFIG_SECURITY_PATH when EVM is enabled, to ensure that files are marked as new. Unlike ima_post_path_mknod(), evm_post_path_mknod() cannot check if a file must be appraised. Thus, it marks all affected files. Also, it does not clear EVM_NEW_FILE depending on i_version, but that is not a problem because IMA_NEW_FILE is always cleared when set in ima_check_last_writer(). Move the EVM-specific flag EVM_IMMUTABLE_DIGSIG to security/integrity/evm/evm.h, since that definition is now unnecessary in the common integrity layer. Finally, switch to the LSM reservation mechanism for the EVM xattr, and consequently decrement by one the number of xattrs to allocate in security_inode_init_security(). Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/evm.h | 8 +--- security/integrity/evm/Kconfig | 1 + security/integrity/evm/evm.h | 19 +++++++++ security/integrity/evm/evm_crypto.c | 4 +- security/integrity/evm/evm_main.c | 76 ++++++++++++++++++++++++++--------- security/integrity/ima/ima_appraise.c | 2 +- security/integrity/integrity.h | 1 - security/security.c | 4 +- 8 files changed, 83 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/include/linux/evm.h b/include/linux/evm.h index cb481eccc967..d48d6da32315 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -12,15 +12,12 @@ #include #include -struct integrity_iint_cache; - #ifdef CONFIG_EVM extern int evm_set_key(void *key, size_t keylen); extern enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint); + size_t xattr_value_len); int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, int *xattr_count); @@ -48,8 +45,7 @@ static inline int evm_set_key(void *key, size_t keylen) static inline enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint) + size_t xattr_value_len) { return INTEGRITY_UNKNOWN; } diff --git a/security/integrity/evm/Kconfig b/security/integrity/evm/Kconfig index fba9ee359bc9..861b3bacab82 100644 --- a/security/integrity/evm/Kconfig +++ b/security/integrity/evm/Kconfig @@ -6,6 +6,7 @@ config EVM select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_HASH_INFO + select SECURITY_PATH default n help EVM protects a file's security extended attributes against diff --git a/security/integrity/evm/evm.h b/security/integrity/evm/evm.h index 53bd7fec93fa..eb1a2c343bd7 100644 --- a/security/integrity/evm/evm.h +++ b/security/integrity/evm/evm.h @@ -32,6 +32,25 @@ struct xattr_list { bool enabled; }; +#define EVM_NEW_FILE 0x00000001 +#define EVM_IMMUTABLE_DIGSIG 0x00000002 + +/* EVM integrity metadata associated with an inode */ +struct evm_iint_cache { + unsigned long flags; + enum integrity_status evm_status:4; +}; + +extern struct lsm_blob_sizes evm_blob_sizes; + +static inline struct evm_iint_cache *evm_iint_inode(const struct inode *inode) +{ + if (unlikely(!inode->i_security)) + return NULL; + + return inode->i_security + evm_blob_sizes.lbs_inode; +} + extern int evm_initialized; #define EVM_ATTR_FSUUID 0x0001 diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index b1ffd4cc0b44..7552d49d0725 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -322,10 +322,10 @@ int evm_calc_hash(struct dentry *dentry, const char *req_xattr_name, static int evm_is_immutable(struct dentry *dentry, struct inode *inode) { const struct evm_ima_xattr_data *xattr_data = NULL; - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; int rc = 0; - iint = integrity_iint_find(inode); + iint = evm_iint_inode(inode); if (iint && (iint->flags & EVM_IMMUTABLE_DIGSIG)) return 1; diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 0a089af83a45..81dbade5b9b3 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -178,14 +178,14 @@ static int is_unsupported_fs(struct dentry *dentry) static enum integrity_status evm_verify_hmac(struct dentry *dentry, const char *xattr_name, char *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint) + size_t xattr_value_len) { struct evm_ima_xattr_data *xattr_data = NULL; struct signature_v2_hdr *hdr; enum integrity_status evm_status = INTEGRITY_PASS; struct evm_digest digest; - struct inode *inode; + struct inode *inode = d_backing_inode(dentry); + struct evm_iint_cache *iint = evm_iint_inode(inode); int rc, xattr_len, evm_immutable = 0; if (iint && (iint->evm_status == INTEGRITY_PASS || @@ -254,8 +254,6 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, (const char *)xattr_data, xattr_len, digest.digest, digest.hdr.length); if (!rc) { - inode = d_backing_inode(dentry); - if (xattr_data->type == EVM_XATTR_PORTABLE_DIGSIG) { if (iint) iint->flags |= EVM_IMMUTABLE_DIGSIG; @@ -403,7 +401,6 @@ int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, * @xattr_name: requested xattr * @xattr_value: requested xattr value * @xattr_value_len: requested xattr value length - * @iint: inode integrity metadata * * Calculate the HMAC for the given dentry and verify it against the stored * security.evm xattr. For performance, use the xattr value and length @@ -416,8 +413,7 @@ int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, */ enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, - void *xattr_value, size_t xattr_value_len, - struct integrity_iint_cache *iint) + void *xattr_value, size_t xattr_value_len) { if (!evm_key_loaded() || !evm_protected_xattr(xattr_name)) return INTEGRITY_UNKNOWN; @@ -425,13 +421,8 @@ enum integrity_status evm_verifyxattr(struct dentry *dentry, if (is_unsupported_fs(dentry)) return INTEGRITY_UNKNOWN; - if (!iint) { - iint = integrity_iint_find(d_backing_inode(dentry)); - if (!iint) - return INTEGRITY_UNKNOWN; - } return evm_verify_hmac(dentry, xattr_name, xattr_value, - xattr_value_len, iint); + xattr_value_len); } EXPORT_SYMBOL_GPL(evm_verifyxattr); @@ -448,7 +439,7 @@ static enum integrity_status evm_verify_current_integrity(struct dentry *dentry) if (!evm_key_loaded() || !S_ISREG(inode->i_mode) || evm_fixmode) return INTEGRITY_PASS; - return evm_verify_hmac(dentry, NULL, NULL, 0, NULL); + return evm_verify_hmac(dentry, NULL, NULL, 0); } /* @@ -526,14 +517,14 @@ static int evm_protect_xattr(struct mnt_idmap *idmap, evm_status = evm_verify_current_integrity(dentry); if (evm_status == INTEGRITY_NOXATTRS) { - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; /* Exception if the HMAC is not going to be calculated. */ if (evm_hmac_disabled()) return 0; - iint = integrity_iint_find(d_backing_inode(dentry)); - if (iint && (iint->flags & IMA_NEW_FILE)) + iint = evm_iint_inode(d_backing_inode(dentry)); + if (iint && (iint->flags & EVM_NEW_FILE)) return 0; /* exception for pseudo filesystems */ @@ -735,9 +726,9 @@ static int evm_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, static void evm_reset_status(struct inode *inode) { - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; - iint = integrity_iint_find(inode); + iint = evm_iint_inode(inode); if (iint) iint->evm_status = INTEGRITY_UNKNOWN; } @@ -1019,6 +1010,42 @@ out: } EXPORT_SYMBOL_GPL(evm_inode_init_security); +static int evm_inode_alloc_security(struct inode *inode) +{ + struct evm_iint_cache *iint = evm_iint_inode(inode); + + /* Called by security_inode_alloc(), it cannot be NULL. */ + iint->flags = 0UL; + iint->evm_status = INTEGRITY_UNKNOWN; + + return 0; +} + +static void evm_file_release(struct file *file) +{ + struct inode *inode = file_inode(file); + struct evm_iint_cache *iint = evm_iint_inode(inode); + fmode_t mode = file->f_mode; + + if (!S_ISREG(inode->i_mode) || !(mode & FMODE_WRITE)) + return; + + if (iint && atomic_read(&inode->i_writecount) == 1) + iint->flags &= ~EVM_NEW_FILE; +} + +static void evm_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + struct inode *inode = d_backing_inode(dentry); + struct evm_iint_cache *iint = evm_iint_inode(inode); + + if (!S_ISREG(inode->i_mode)) + return; + + if (iint) + iint->flags |= EVM_NEW_FILE; +} + #ifdef CONFIG_EVM_LOAD_X509 void __init evm_load_x509(void) { @@ -1071,6 +1098,9 @@ static struct security_hook_list evm_hooks[] __ro_after_init = { LSM_HOOK_INIT(inode_removexattr, evm_inode_removexattr), LSM_HOOK_INIT(inode_post_removexattr, evm_inode_post_removexattr), LSM_HOOK_INIT(inode_init_security, evm_inode_init_security), + LSM_HOOK_INIT(inode_alloc_security, evm_inode_alloc_security), + LSM_HOOK_INIT(file_release, evm_file_release), + LSM_HOOK_INIT(path_post_mknod, evm_post_path_mknod), }; static const struct lsm_id evm_lsmid = { @@ -1084,10 +1114,16 @@ static int __init init_evm_lsm(void) return 0; } +struct lsm_blob_sizes evm_blob_sizes __ro_after_init = { + .lbs_inode = sizeof(struct evm_iint_cache), + .lbs_xattr_count = 1, +}; + DEFINE_LSM(evm) = { .name = "evm", .init = init_evm_lsm, .order = LSM_ORDER_LAST, + .blobs = &evm_blob_sizes, }; late_initcall(init_evm); diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 076451109637..1dd6ee72a20a 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -520,7 +520,7 @@ int ima_appraise_measurement(enum ima_hooks func, } status = evm_verifyxattr(dentry, XATTR_NAME_IMA, xattr_value, - rc < 0 ? 0 : rc, iint); + rc < 0 ? 0 : rc); switch (status) { case INTEGRITY_PASS: case INTEGRITY_PASS_IMMUTABLE: diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 59eaddd84434..7a97c269a072 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -37,7 +37,6 @@ #define IMA_DIGSIG_REQUIRED 0x01000000 #define IMA_PERMIT_DIRECTIO 0x02000000 #define IMA_NEW_FILE 0x04000000 -#define EVM_IMMUTABLE_DIGSIG 0x08000000 #define IMA_FAIL_UNVERIFIABLE_SIGS 0x10000000 #define IMA_MODSIG_ALLOWED 0x20000000 #define IMA_CHECK_BLACKLIST 0x40000000 diff --git a/security/security.c b/security/security.c index 59cb0e6cbae3..e08c1aa82edf 100644 --- a/security/security.c +++ b/security/security.c @@ -1716,8 +1716,8 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, return 0; if (initxattrs) { - /* Allocate +1 for EVM and +1 as terminator. */ - new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 2, + /* Allocate +1 as terminator. */ + new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 1, sizeof(*new_xattrs), GFP_NOFS); if (!new_xattrs) return -ENOMEM; -- cgit v1.2.3 From b6c0dec9f78bc691166d60347b52b3adf03d5875 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:13 +0100 Subject: integrity: Remove LSM Since now IMA and EVM use their own integrity metadata, it is safe to remove the 'integrity' LSM, with its management of integrity metadata. Keep the iint.c file only for loading IMA and EVM keys at boot, and for creating the integrity directory in securityfs (we need to keep it for retrocompatibility reasons). Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/integrity.h | 14 --- security/integrity/iint.c | 197 +---------------------------------------- security/integrity/integrity.h | 25 ------ security/security.c | 2 - 4 files changed, 2 insertions(+), 236 deletions(-) (limited to 'include/linux') diff --git a/include/linux/integrity.h b/include/linux/integrity.h index ef0f63ef5ebc..459b79683783 100644 --- a/include/linux/integrity.h +++ b/include/linux/integrity.h @@ -19,24 +19,10 @@ enum integrity_status { INTEGRITY_UNKNOWN, }; -/* List of EVM protected security xattrs */ #ifdef CONFIG_INTEGRITY -extern struct integrity_iint_cache *integrity_inode_get(struct inode *inode); -extern void integrity_inode_free(struct inode *inode); extern void __init integrity_load_keys(void); #else -static inline struct integrity_iint_cache * - integrity_inode_get(struct inode *inode) -{ - return NULL; -} - -static inline void integrity_inode_free(struct inode *inode) -{ - return; -} - static inline void integrity_load_keys(void) { } diff --git a/security/integrity/iint.c b/security/integrity/iint.c index d4419a2a1e24..068ac6c2ae1e 100644 --- a/security/integrity/iint.c +++ b/security/integrity/iint.c @@ -6,207 +6,14 @@ * Mimi Zohar * * File: integrity_iint.c - * - implements the integrity hooks: integrity_inode_alloc, - * integrity_inode_free - * - cache integrity information associated with an inode - * using a rbtree tree. + * - initialize the integrity directory in securityfs + * - load IMA and EVM keys */ -#include -#include -#include -#include -#include -#include #include -#include #include "integrity.h" -static struct rb_root integrity_iint_tree = RB_ROOT; -static DEFINE_RWLOCK(integrity_iint_lock); -static struct kmem_cache *iint_cache __ro_after_init; - struct dentry *integrity_dir; -/* - * __integrity_iint_find - return the iint associated with an inode - */ -static struct integrity_iint_cache *__integrity_iint_find(struct inode *inode) -{ - struct integrity_iint_cache *iint; - struct rb_node *n = integrity_iint_tree.rb_node; - - while (n) { - iint = rb_entry(n, struct integrity_iint_cache, rb_node); - - if (inode < iint->inode) - n = n->rb_left; - else if (inode > iint->inode) - n = n->rb_right; - else - return iint; - } - - return NULL; -} - -/* - * integrity_iint_find - return the iint associated with an inode - */ -struct integrity_iint_cache *integrity_iint_find(struct inode *inode) -{ - struct integrity_iint_cache *iint; - - if (!IS_IMA(inode)) - return NULL; - - read_lock(&integrity_iint_lock); - iint = __integrity_iint_find(inode); - read_unlock(&integrity_iint_lock); - - return iint; -} - -#define IMA_MAX_NESTING (FILESYSTEM_MAX_STACK_DEPTH+1) - -/* - * It is not clear that IMA should be nested at all, but as long is it measures - * files both on overlayfs and on underlying fs, we need to annotate the iint - * mutex to avoid lockdep false positives related to IMA + overlayfs. - * See ovl_lockdep_annotate_inode_mutex_key() for more details. - */ -static inline void iint_lockdep_annotate(struct integrity_iint_cache *iint, - struct inode *inode) -{ -#ifdef CONFIG_LOCKDEP - static struct lock_class_key iint_mutex_key[IMA_MAX_NESTING]; - - int depth = inode->i_sb->s_stack_depth; - - if (WARN_ON_ONCE(depth < 0 || depth >= IMA_MAX_NESTING)) - depth = 0; - - lockdep_set_class(&iint->mutex, &iint_mutex_key[depth]); -#endif -} - -static void iint_init_always(struct integrity_iint_cache *iint, - struct inode *inode) -{ - iint->ima_hash = NULL; - iint->version = 0; - iint->flags = 0UL; - iint->atomic_flags = 0UL; - iint->ima_file_status = INTEGRITY_UNKNOWN; - iint->ima_mmap_status = INTEGRITY_UNKNOWN; - iint->ima_bprm_status = INTEGRITY_UNKNOWN; - iint->ima_read_status = INTEGRITY_UNKNOWN; - iint->ima_creds_status = INTEGRITY_UNKNOWN; - iint->evm_status = INTEGRITY_UNKNOWN; - iint->measured_pcrs = 0; - mutex_init(&iint->mutex); - iint_lockdep_annotate(iint, inode); -} - -static void iint_free(struct integrity_iint_cache *iint) -{ - kfree(iint->ima_hash); - mutex_destroy(&iint->mutex); - kmem_cache_free(iint_cache, iint); -} - -/** - * integrity_inode_get - find or allocate an iint associated with an inode - * @inode: pointer to the inode - * @return: allocated iint - * - * Caller must lock i_mutex - */ -struct integrity_iint_cache *integrity_inode_get(struct inode *inode) -{ - struct rb_node **p; - struct rb_node *node, *parent = NULL; - struct integrity_iint_cache *iint, *test_iint; - - iint = integrity_iint_find(inode); - if (iint) - return iint; - - iint = kmem_cache_alloc(iint_cache, GFP_NOFS); - if (!iint) - return NULL; - - iint_init_always(iint, inode); - - write_lock(&integrity_iint_lock); - - p = &integrity_iint_tree.rb_node; - while (*p) { - parent = *p; - test_iint = rb_entry(parent, struct integrity_iint_cache, - rb_node); - if (inode < test_iint->inode) { - p = &(*p)->rb_left; - } else if (inode > test_iint->inode) { - p = &(*p)->rb_right; - } else { - write_unlock(&integrity_iint_lock); - kmem_cache_free(iint_cache, iint); - return test_iint; - } - } - - iint->inode = inode; - node = &iint->rb_node; - inode->i_flags |= S_IMA; - rb_link_node(node, parent, p); - rb_insert_color(node, &integrity_iint_tree); - - write_unlock(&integrity_iint_lock); - return iint; -} - -/** - * integrity_inode_free - called on security_inode_free - * @inode: pointer to the inode - * - * Free the integrity information(iint) associated with an inode. - */ -void integrity_inode_free(struct inode *inode) -{ - struct integrity_iint_cache *iint; - - if (!IS_IMA(inode)) - return; - - write_lock(&integrity_iint_lock); - iint = __integrity_iint_find(inode); - rb_erase(&iint->rb_node, &integrity_iint_tree); - write_unlock(&integrity_iint_lock); - - iint_free(iint); -} - -static void iint_init_once(void *foo) -{ - struct integrity_iint_cache *iint = (struct integrity_iint_cache *) foo; - - memset(iint, 0, sizeof(*iint)); -} - -static int __init integrity_iintcache_init(void) -{ - iint_cache = - kmem_cache_create("iint_cache", sizeof(struct integrity_iint_cache), - 0, SLAB_PANIC, iint_init_once); - return 0; -} -DEFINE_LSM(integrity) = { - .name = "integrity", - .init = integrity_iintcache_init, - .order = LSM_ORDER_LAST, -}; - - /* * integrity_kernel_read - read data from the file * diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 671fc50255f9..50d6f798e613 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -102,31 +102,6 @@ struct ima_file_id { __u8 hash[HASH_MAX_DIGESTSIZE]; } __packed; -/* integrity data associated with an inode */ -struct integrity_iint_cache { - struct rb_node rb_node; /* rooted in integrity_iint_tree */ - struct mutex mutex; /* protects: version, flags, digest */ - struct inode *inode; /* back pointer to inode in question */ - u64 version; /* track inode changes */ - unsigned long flags; - unsigned long measured_pcrs; - unsigned long atomic_flags; - unsigned long real_ino; - dev_t real_dev; - enum integrity_status ima_file_status:4; - enum integrity_status ima_mmap_status:4; - enum integrity_status ima_bprm_status:4; - enum integrity_status ima_read_status:4; - enum integrity_status ima_creds_status:4; - enum integrity_status evm_status:4; - struct ima_digest_data *ima_hash; -}; - -/* rbtree tree calls to lookup, insert, delete - * integrity data associated with an inode. - */ -struct integrity_iint_cache *integrity_iint_find(struct inode *inode); - int integrity_kernel_read(struct file *file, loff_t offset, void *addr, unsigned long count); diff --git a/security/security.c b/security/security.c index e08c1aa82edf..cbdc9bebe802 100644 --- a/security/security.c +++ b/security/security.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -1597,7 +1596,6 @@ static void inode_free_by_rcu(struct rcu_head *head) */ void security_inode_free(struct inode *inode) { - integrity_inode_free(inode); call_void_hook(inode_free_security, inode); /* * The inode may still be referenced in a path walk and -- cgit v1.2.3 From 81a7d0c4d059cb5c122110acbeec7bedfb91a741 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 13 Feb 2024 11:36:56 -0300 Subject: soundwire: bus_type: make sdw_bus_type const Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the sdw_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240213-bus_cleanup-soundwire-v1-1-3878b00f6f57@marliere.net Signed-off-by: Vinod Koul --- drivers/soundwire/bus_type.c | 2 +- include/linux/soundwire/sdw_type.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c index 9fa93bb923d7..fd65b2360fc1 100644 --- a/drivers/soundwire/bus_type.c +++ b/drivers/soundwire/bus_type.c @@ -72,7 +72,7 @@ int sdw_slave_uevent(const struct device *dev, struct kobj_uevent_env *env) return 0; } -struct bus_type sdw_bus_type = { +const struct bus_type sdw_bus_type = { .name = "soundwire", .match = sdw_bus_match, }; diff --git a/include/linux/soundwire/sdw_type.h b/include/linux/soundwire/sdw_type.h index d8c27f1e5559..b445f7200f06 100644 --- a/include/linux/soundwire/sdw_type.h +++ b/include/linux/soundwire/sdw_type.h @@ -4,7 +4,7 @@ #ifndef __SOUNDWIRE_TYPES_H #define __SOUNDWIRE_TYPES_H -extern struct bus_type sdw_bus_type; +extern const struct bus_type sdw_bus_type; extern struct device_type sdw_slave_type; extern struct device_type sdw_master_type; -- cgit v1.2.3 From 00a9bc6070434814d39118a0de70c1645f64bf60 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:12 +0800 Subject: iommu: Move iommu fault data to linux/iommu.h The iommu fault data is currently defined in uapi/linux/iommu.h, but is only used inside the iommu subsystem. Move it to linux/iommu.h, where it will be more accessible to kernel drivers. With this done, uapi/linux/iommu.h becomes empty and can be removed from the tree. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- MAINTAINERS | 1 - include/linux/iommu.h | 152 +++++++++++++++++++++++++++++++++++++++++- include/uapi/linux/iommu.h | 161 --------------------------------------------- 3 files changed, 151 insertions(+), 163 deletions(-) delete mode 100644 include/uapi/linux/iommu.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 960512bec428..1bcdf3144415 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11240,7 +11240,6 @@ F: drivers/iommu/ F: include/linux/iommu.h F: include/linux/iova.h F: include/linux/of_iommu.h -F: include/uapi/linux/iommu.h IOMMUFD M: Jason Gunthorpe diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1ea2a820e1eb..472a8ce029b1 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -14,7 +14,6 @@ #include #include #include -#include #define IOMMU_READ (1 << 0) #define IOMMU_WRITE (1 << 1) @@ -44,6 +43,157 @@ struct iommu_sva; struct iommu_fault_event; struct iommu_dma_cookie; +#define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ +#define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ +#define IOMMU_FAULT_PERM_EXEC (1 << 2) /* exec */ +#define IOMMU_FAULT_PERM_PRIV (1 << 3) /* privileged */ + +/* Generic fault types, can be expanded IRQ remapping fault */ +enum iommu_fault_type { + IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ + IOMMU_FAULT_PAGE_REQ, /* page request fault */ +}; + +enum iommu_fault_reason { + IOMMU_FAULT_REASON_UNKNOWN = 0, + + /* Could not access the PASID table (fetch caused external abort) */ + IOMMU_FAULT_REASON_PASID_FETCH, + + /* PASID entry is invalid or has configuration errors */ + IOMMU_FAULT_REASON_BAD_PASID_ENTRY, + + /* + * PASID is out of range (e.g. exceeds the maximum PASID + * supported by the IOMMU) or disabled. + */ + IOMMU_FAULT_REASON_PASID_INVALID, + + /* + * An external abort occurred fetching (or updating) a translation + * table descriptor + */ + IOMMU_FAULT_REASON_WALK_EABT, + + /* + * Could not access the page table entry (Bad address), + * actual translation fault + */ + IOMMU_FAULT_REASON_PTE_FETCH, + + /* Protection flag check failed */ + IOMMU_FAULT_REASON_PERMISSION, + + /* access flag check failed */ + IOMMU_FAULT_REASON_ACCESS, + + /* Output address of a translation stage caused Address Size fault */ + IOMMU_FAULT_REASON_OOR_ADDRESS, +}; + +/** + * struct iommu_fault_unrecoverable - Unrecoverable fault data + * @reason: reason of the fault, from &enum iommu_fault_reason + * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) + * @pasid: Process Address Space ID + * @perm: requested permission access using by the incoming transaction + * (IOMMU_FAULT_PERM_* values) + * @addr: offending page address + * @fetch_addr: address that caused a fetch abort, if any + */ +struct iommu_fault_unrecoverable { + __u32 reason; +#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) +#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) +#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) + __u32 flags; + __u32 pasid; + __u32 perm; + __u64 addr; + __u64 fetch_addr; +}; + +/** + * struct iommu_fault_page_request - Page Request data + * @flags: encodes whether the corresponding fields are valid and whether this + * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). + * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response + * must have the same PASID value as the page request. When it is clear, + * the page response should not have a PASID. + * @pasid: Process Address Space ID + * @grpid: Page Request Group Index + * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) + * @addr: page address + * @private_data: device-specific private information + */ +struct iommu_fault_page_request { +#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) +#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) +#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) +#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) + __u32 flags; + __u32 pasid; + __u32 grpid; + __u32 perm; + __u64 addr; + __u64 private_data[2]; +}; + +/** + * struct iommu_fault - Generic fault data + * @type: fault type from &enum iommu_fault_type + * @padding: reserved for future use (should be zero) + * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV + * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ + * @padding2: sets the fault size to allow for future extensions + */ +struct iommu_fault { + __u32 type; + __u32 padding; + union { + struct iommu_fault_unrecoverable event; + struct iommu_fault_page_request prm; + __u8 padding2[56]; + }; +}; + +/** + * enum iommu_page_response_code - Return status of fault handlers + * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables + * populated, retry the access. This is "Success" in PCI PRI. + * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from + * this device if possible. This is "Response Failure" in PCI PRI. + * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the + * access. This is "Invalid Request" in PCI PRI. + */ +enum iommu_page_response_code { + IOMMU_PAGE_RESP_SUCCESS = 0, + IOMMU_PAGE_RESP_INVALID, + IOMMU_PAGE_RESP_FAILURE, +}; + +/** + * struct iommu_page_response - Generic page response information + * @argsz: User filled size of this data + * @version: API version of this structure + * @flags: encodes whether the corresponding fields are valid + * (IOMMU_FAULT_PAGE_RESPONSE_* values) + * @pasid: Process Address Space ID + * @grpid: Page Request Group Index + * @code: response code from &enum iommu_page_response_code + */ +struct iommu_page_response { + __u32 argsz; +#define IOMMU_PAGE_RESP_VERSION_1 1 + __u32 version; +#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) + __u32 flags; + __u32 pasid; + __u32 grpid; + __u32 code; +}; + + /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 #define IOMMU_FAULT_WRITE 0x1 diff --git a/include/uapi/linux/iommu.h b/include/uapi/linux/iommu.h deleted file mode 100644 index 65d8b0234f69..000000000000 --- a/include/uapi/linux/iommu.h +++ /dev/null @@ -1,161 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * IOMMU user API definitions - */ - -#ifndef _UAPI_IOMMU_H -#define _UAPI_IOMMU_H - -#include - -#define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ -#define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ -#define IOMMU_FAULT_PERM_EXEC (1 << 2) /* exec */ -#define IOMMU_FAULT_PERM_PRIV (1 << 3) /* privileged */ - -/* Generic fault types, can be expanded IRQ remapping fault */ -enum iommu_fault_type { - IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ - IOMMU_FAULT_PAGE_REQ, /* page request fault */ -}; - -enum iommu_fault_reason { - IOMMU_FAULT_REASON_UNKNOWN = 0, - - /* Could not access the PASID table (fetch caused external abort) */ - IOMMU_FAULT_REASON_PASID_FETCH, - - /* PASID entry is invalid or has configuration errors */ - IOMMU_FAULT_REASON_BAD_PASID_ENTRY, - - /* - * PASID is out of range (e.g. exceeds the maximum PASID - * supported by the IOMMU) or disabled. - */ - IOMMU_FAULT_REASON_PASID_INVALID, - - /* - * An external abort occurred fetching (or updating) a translation - * table descriptor - */ - IOMMU_FAULT_REASON_WALK_EABT, - - /* - * Could not access the page table entry (Bad address), - * actual translation fault - */ - IOMMU_FAULT_REASON_PTE_FETCH, - - /* Protection flag check failed */ - IOMMU_FAULT_REASON_PERMISSION, - - /* access flag check failed */ - IOMMU_FAULT_REASON_ACCESS, - - /* Output address of a translation stage caused Address Size fault */ - IOMMU_FAULT_REASON_OOR_ADDRESS, -}; - -/** - * struct iommu_fault_unrecoverable - Unrecoverable fault data - * @reason: reason of the fault, from &enum iommu_fault_reason - * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) - * @pasid: Process Address Space ID - * @perm: requested permission access using by the incoming transaction - * (IOMMU_FAULT_PERM_* values) - * @addr: offending page address - * @fetch_addr: address that caused a fetch abort, if any - */ -struct iommu_fault_unrecoverable { - __u32 reason; -#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) -#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) -#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) - __u32 flags; - __u32 pasid; - __u32 perm; - __u64 addr; - __u64 fetch_addr; -}; - -/** - * struct iommu_fault_page_request - Page Request data - * @flags: encodes whether the corresponding fields are valid and whether this - * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). - * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response - * must have the same PASID value as the page request. When it is clear, - * the page response should not have a PASID. - * @pasid: Process Address Space ID - * @grpid: Page Request Group Index - * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) - * @addr: page address - * @private_data: device-specific private information - */ -struct iommu_fault_page_request { -#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) -#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) -#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) -#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 perm; - __u64 addr; - __u64 private_data[2]; -}; - -/** - * struct iommu_fault - Generic fault data - * @type: fault type from &enum iommu_fault_type - * @padding: reserved for future use (should be zero) - * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV - * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ - * @padding2: sets the fault size to allow for future extensions - */ -struct iommu_fault { - __u32 type; - __u32 padding; - union { - struct iommu_fault_unrecoverable event; - struct iommu_fault_page_request prm; - __u8 padding2[56]; - }; -}; - -/** - * enum iommu_page_response_code - Return status of fault handlers - * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables - * populated, retry the access. This is "Success" in PCI PRI. - * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from - * this device if possible. This is "Response Failure" in PCI PRI. - * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the - * access. This is "Invalid Request" in PCI PRI. - */ -enum iommu_page_response_code { - IOMMU_PAGE_RESP_SUCCESS = 0, - IOMMU_PAGE_RESP_INVALID, - IOMMU_PAGE_RESP_FAILURE, -}; - -/** - * struct iommu_page_response - Generic page response information - * @argsz: User filled size of this data - * @version: API version of this structure - * @flags: encodes whether the corresponding fields are valid - * (IOMMU_FAULT_PAGE_RESPONSE_* values) - * @pasid: Process Address Space ID - * @grpid: Page Request Group Index - * @code: response code from &enum iommu_page_response_code - */ -struct iommu_page_response { - __u32 argsz; -#define IOMMU_PAGE_RESP_VERSION_1 1 - __u32 version; -#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 code; -}; - -#endif /* _UAPI_IOMMU_H */ -- cgit v1.2.3 From 0edeab66eba88947dabe8634a3efd136cc771750 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:14 +0800 Subject: iommu: Remove unrecoverable fault data The unrecoverable fault data is not used anywhere. Remove it to avoid dead code. Suggested-by: Kevin Tian Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 72 ++------------------------------------------------- 1 file changed, 2 insertions(+), 70 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 472a8ce029b1..c960c4fae3bc 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -50,67 +50,7 @@ struct iommu_dma_cookie; /* Generic fault types, can be expanded IRQ remapping fault */ enum iommu_fault_type { - IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ - IOMMU_FAULT_PAGE_REQ, /* page request fault */ -}; - -enum iommu_fault_reason { - IOMMU_FAULT_REASON_UNKNOWN = 0, - - /* Could not access the PASID table (fetch caused external abort) */ - IOMMU_FAULT_REASON_PASID_FETCH, - - /* PASID entry is invalid or has configuration errors */ - IOMMU_FAULT_REASON_BAD_PASID_ENTRY, - - /* - * PASID is out of range (e.g. exceeds the maximum PASID - * supported by the IOMMU) or disabled. - */ - IOMMU_FAULT_REASON_PASID_INVALID, - - /* - * An external abort occurred fetching (or updating) a translation - * table descriptor - */ - IOMMU_FAULT_REASON_WALK_EABT, - - /* - * Could not access the page table entry (Bad address), - * actual translation fault - */ - IOMMU_FAULT_REASON_PTE_FETCH, - - /* Protection flag check failed */ - IOMMU_FAULT_REASON_PERMISSION, - - /* access flag check failed */ - IOMMU_FAULT_REASON_ACCESS, - - /* Output address of a translation stage caused Address Size fault */ - IOMMU_FAULT_REASON_OOR_ADDRESS, -}; - -/** - * struct iommu_fault_unrecoverable - Unrecoverable fault data - * @reason: reason of the fault, from &enum iommu_fault_reason - * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) - * @pasid: Process Address Space ID - * @perm: requested permission access using by the incoming transaction - * (IOMMU_FAULT_PERM_* values) - * @addr: offending page address - * @fetch_addr: address that caused a fetch abort, if any - */ -struct iommu_fault_unrecoverable { - __u32 reason; -#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) -#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) -#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) - __u32 flags; - __u32 pasid; - __u32 perm; - __u64 addr; - __u64 fetch_addr; + IOMMU_FAULT_PAGE_REQ = 1, /* page request fault */ }; /** @@ -142,19 +82,11 @@ struct iommu_fault_page_request { /** * struct iommu_fault - Generic fault data * @type: fault type from &enum iommu_fault_type - * @padding: reserved for future use (should be zero) - * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ - * @padding2: sets the fault size to allow for future extensions */ struct iommu_fault { __u32 type; - __u32 padding; - union { - struct iommu_fault_unrecoverable event; - struct iommu_fault_page_request prm; - __u8 padding2[56]; - }; + struct iommu_fault_page_request prm; }; /** -- cgit v1.2.3 From 8b32a3bea2629049c484f595af7aad797e24453e Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:15 +0800 Subject: iommu: Cleanup iopf data structure definitions struct iommu_fault_page_request and struct iommu_page_response are not part of uAPI anymore. Convert them to data structures for kAPI. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 1 - drivers/iommu/iommu.c | 4 ---- include/linux/iommu.h | 27 +++++++++++---------------- 3 files changed, 11 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index e5b8b9110c13..24b5545352ae 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -56,7 +56,6 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, enum iommu_page_response_code status) { struct iommu_page_response resp = { - .version = IOMMU_PAGE_RESP_VERSION_1, .pasid = iopf->fault.prm.pasid, .grpid = iopf->fault.prm.grpid, .code = status, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..6a4d9bc0641d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1494,10 +1494,6 @@ int iommu_page_response(struct device *dev, if (!param || !param->fault_param) return -EINVAL; - if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || - msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) - return -EINVAL; - /* Only send response if there is a fault report pending */ mutex_lock(¶m->fault_param->lock); if (list_empty(¶m->fault_param->faults)) { diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c960c4fae3bc..829bcb5a8e23 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -71,12 +71,12 @@ struct iommu_fault_page_request { #define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) #define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) #define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 perm; - __u64 addr; - __u64 private_data[2]; + u32 flags; + u32 pasid; + u32 grpid; + u32 perm; + u64 addr; + u64 private_data[2]; }; /** @@ -85,7 +85,7 @@ struct iommu_fault_page_request { * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ */ struct iommu_fault { - __u32 type; + u32 type; struct iommu_fault_page_request prm; }; @@ -106,8 +106,6 @@ enum iommu_page_response_code { /** * struct iommu_page_response - Generic page response information - * @argsz: User filled size of this data - * @version: API version of this structure * @flags: encodes whether the corresponding fields are valid * (IOMMU_FAULT_PAGE_RESPONSE_* values) * @pasid: Process Address Space ID @@ -115,14 +113,11 @@ enum iommu_page_response_code { * @code: response code from &enum iommu_page_response_code */ struct iommu_page_response { - __u32 argsz; -#define IOMMU_PAGE_RESP_VERSION_1 1 - __u32 version; #define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 code; + u32 flags; + u32 pasid; + u32 grpid; + u32 code; }; -- cgit v1.2.3 From 15fc60cdd2d236a73b32c99d21fc0f7b7ce6cbbb Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:16 +0800 Subject: iommu: Merge iopf_device_param into iommu_fault_param The struct dev_iommu contains two pointers, fault_param and iopf_param. The fault_param pointer points to a data structure that is used to store pending faults that are awaiting responses. The iopf_param pointer points to a data structure that is used to store partial faults that are part of a Page Request Group. The fault_param and iopf_param pointers are essentially duplicate. This causes memory waste. Merge the iopf_device_param pointer into the iommu_fault_param pointer to consolidate the code and save memory. The consolidated pointer would be allocated on demand when the device driver enables the iopf on device, and would be freed after iopf is disabled. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 110 +++++++++++++++++++++------------------------ drivers/iommu/iommu.c | 34 +++----------- include/linux/iommu.h | 18 ++++++-- 3 files changed, 72 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 24b5545352ae..f948303b2a91 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -25,21 +25,6 @@ struct iopf_queue { struct mutex lock; }; -/** - * struct iopf_device_param - IO Page Fault data attached to a device - * @dev: the device that owns this param - * @queue: IOPF queue - * @queue_list: index into queue->devices - * @partial: faults that are part of a Page Request Group for which the last - * request hasn't been submitted yet. - */ -struct iopf_device_param { - struct device *dev; - struct iopf_queue *queue; - struct list_head queue_list; - struct list_head partial; -}; - struct iopf_fault { struct iommu_fault fault; struct list_head list; @@ -144,7 +129,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) int ret; struct iopf_group *group; struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; struct device *dev = cookie; struct dev_iommu *param = dev->iommu; @@ -159,7 +144,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) * As long as we're holding param->lock, the queue can't be unlinked * from the device and therefore cannot disappear. */ - iopf_param = param->iopf_param; + iopf_param = param->fault_param; if (!iopf_param) return -ENODEV; @@ -229,14 +214,14 @@ EXPORT_SYMBOL_GPL(iommu_queue_iopf); int iopf_queue_flush_dev(struct device *dev) { int ret = 0; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; struct dev_iommu *param = dev->iommu; if (!param) return -ENODEV; mutex_lock(¶m->lock); - iopf_param = param->iopf_param; + iopf_param = param->fault_param; if (iopf_param) flush_workqueue(iopf_param->queue->wq); else @@ -260,7 +245,7 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); int iopf_queue_discard_partial(struct iopf_queue *queue) { struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; if (!queue) return -EINVAL; @@ -287,34 +272,36 @@ EXPORT_SYMBOL_GPL(iopf_queue_discard_partial); */ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) { - int ret = -EBUSY; - struct iopf_device_param *iopf_param; + int ret = 0; struct dev_iommu *param = dev->iommu; - - if (!param) - return -ENODEV; - - iopf_param = kzalloc(sizeof(*iopf_param), GFP_KERNEL); - if (!iopf_param) - return -ENOMEM; - - INIT_LIST_HEAD(&iopf_param->partial); - iopf_param->queue = queue; - iopf_param->dev = dev; + struct iommu_fault_param *fault_param; mutex_lock(&queue->lock); mutex_lock(¶m->lock); - if (!param->iopf_param) { - list_add(&iopf_param->queue_list, &queue->devices); - param->iopf_param = iopf_param; - ret = 0; + if (param->fault_param) { + ret = -EBUSY; + goto done_unlock; } + + fault_param = kzalloc(sizeof(*fault_param), GFP_KERNEL); + if (!fault_param) { + ret = -ENOMEM; + goto done_unlock; + } + + mutex_init(&fault_param->lock); + INIT_LIST_HEAD(&fault_param->faults); + INIT_LIST_HEAD(&fault_param->partial); + fault_param->dev = dev; + list_add(&fault_param->queue_list, &queue->devices); + fault_param->queue = queue; + + param->fault_param = fault_param; + +done_unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); - if (ret) - kfree(iopf_param); - return ret; } EXPORT_SYMBOL_GPL(iopf_queue_add_device); @@ -330,34 +317,41 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); */ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - int ret = -EINVAL; + int ret = 0; struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; struct dev_iommu *param = dev->iommu; - - if (!param || !queue) - return -EINVAL; + struct iommu_fault_param *fault_param = param->fault_param; mutex_lock(&queue->lock); mutex_lock(¶m->lock); - iopf_param = param->iopf_param; - if (iopf_param && iopf_param->queue == queue) { - list_del(&iopf_param->queue_list); - param->iopf_param = NULL; - ret = 0; + if (!fault_param) { + ret = -ENODEV; + goto unlock; } - mutex_unlock(¶m->lock); - mutex_unlock(&queue->lock); - if (ret) - return ret; + + if (fault_param->queue != queue) { + ret = -EINVAL; + goto unlock; + } + + if (!list_empty(&fault_param->faults)) { + ret = -EBUSY; + goto unlock; + } + + list_del(&fault_param->queue_list); /* Just in case some faults are still stuck */ - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) + list_for_each_entry_safe(iopf, next, &fault_param->partial, list) kfree(iopf); - kfree(iopf_param); + param->fault_param = NULL; + kfree(fault_param); +unlock: + mutex_unlock(¶m->lock); + mutex_unlock(&queue->lock); - return 0; + return ret; } EXPORT_SYMBOL_GPL(iopf_queue_remove_device); @@ -403,7 +397,7 @@ EXPORT_SYMBOL_GPL(iopf_queue_alloc); */ void iopf_queue_free(struct iopf_queue *queue) { - struct iopf_device_param *iopf_param, *next; + struct iommu_fault_param *iopf_param, *next; if (!queue) return; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 6a4d9bc0641d..9575cb75732a 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1355,27 +1355,18 @@ int iommu_register_device_fault_handler(struct device *dev, struct dev_iommu *param = dev->iommu; int ret = 0; - if (!param) + if (!param || !param->fault_param) return -EINVAL; mutex_lock(¶m->lock); /* Only allow one fault handler registered for each device */ - if (param->fault_param) { + if (param->fault_param->handler) { ret = -EBUSY; goto done_unlock; } - get_device(dev); - param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); - if (!param->fault_param) { - put_device(dev); - ret = -ENOMEM; - goto done_unlock; - } param->fault_param->handler = handler; param->fault_param->data = data; - mutex_init(¶m->fault_param->lock); - INIT_LIST_HEAD(¶m->fault_param->faults); done_unlock: mutex_unlock(¶m->lock); @@ -1396,29 +1387,16 @@ EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); int iommu_unregister_device_fault_handler(struct device *dev) { struct dev_iommu *param = dev->iommu; - int ret = 0; - if (!param) + if (!param || !param->fault_param) return -EINVAL; mutex_lock(¶m->lock); - - if (!param->fault_param) - goto unlock; - - /* we cannot unregister handler if there are pending faults */ - if (!list_empty(¶m->fault_param->faults)) { - ret = -EBUSY; - goto unlock; - } - - kfree(param->fault_param); - param->fault_param = NULL; - put_device(dev); -unlock: + param->fault_param->handler = NULL; + param->fault_param->data = NULL; mutex_unlock(¶m->lock); - return ret; + return 0; } EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 829bcb5a8e23..bbb7c2ad5184 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -42,6 +42,7 @@ struct notifier_block; struct iommu_sva; struct iommu_fault_event; struct iommu_dma_cookie; +struct iopf_queue; #define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ #define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ @@ -672,21 +673,31 @@ struct iommu_fault_event { * struct iommu_fault_param - per-device IOMMU fault data * @handler: Callback function to handle IOMMU faults at device level * @data: handler private data - * @faults: holds the pending faults which needs response * @lock: protect pending faults list + * @dev: the device that owns this param + * @queue: IOPF queue + * @queue_list: index into queue->devices + * @partial: faults that are part of a Page Request Group for which the last + * request hasn't been submitted yet. + * @faults: holds the pending faults which need response */ struct iommu_fault_param { iommu_dev_fault_handler_t handler; void *data; - struct list_head faults; struct mutex lock; + + struct device *dev; + struct iopf_queue *queue; + struct list_head queue_list; + + struct list_head partial; + struct list_head faults; }; /** * struct dev_iommu - Collection of per-device IOMMU data * * @fault_param: IOMMU detected device fault reporting data - * @iopf_param: I/O Page Fault queue and data * @fwspec: IOMMU fwspec data * @iommu_dev: IOMMU device this device is linked to * @priv: IOMMU Driver private data @@ -702,7 +713,6 @@ struct iommu_fault_param { struct dev_iommu { struct mutex lock; struct iommu_fault_param *fault_param; - struct iopf_device_param *iopf_param; struct iommu_fwspec *fwspec; struct iommu_device *iommu_dev; void *priv; -- cgit v1.2.3 From 1ff25d798e52943d037accf15c675a6845d9776f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:17 +0800 Subject: iommu: Remove iommu_[un]register_device_fault_handler() The individual iommu driver reports the iommu page faults by calling iommu_report_device_fault(), where a pre-registered device fault handler is called to route the fault to another fault handler installed on the corresponding iommu domain. The pre-registered device fault handler is static and won't be dynamic as the fault handler is eventually per iommu domain. Replace calling device fault handler with iommu_queue_iopf(). After this replacement, the registering and unregistering fault handler interfaces are not needed anywhere. Remove the interfaces and the related data structures to avoid dead code. Convert cookie parameter of iommu_queue_iopf() into a device pointer that is really passed. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 13 +---- drivers/iommu/intel/iommu.c | 24 +++----- drivers/iommu/io-pgfault.c | 6 +- drivers/iommu/iommu-sva.h | 4 +- drivers/iommu/iommu.c | 76 +------------------------ include/linux/iommu.h | 23 -------- 6 files changed, 13 insertions(+), 133 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 05722121f00e..ab2b0a5e4369 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -487,7 +487,6 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) { - int ret; struct device *dev = master->dev; /* @@ -500,16 +499,7 @@ static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) if (!master->iopf_enabled) return -EINVAL; - ret = iopf_queue_add_device(master->smmu->evtq.iopf, dev); - if (ret) - return ret; - - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) { - iopf_queue_remove_device(master->smmu->evtq.iopf, dev); - return ret; - } - return 0; + return iopf_queue_add_device(master->smmu->evtq.iopf, dev); } static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) @@ -519,7 +509,6 @@ static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) if (!master->iopf_enabled) return; - iommu_unregister_device_fault_handler(dev); iopf_queue_remove_device(master->smmu->evtq.iopf, dev); } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 6fb5f6fceea1..df6ceefc09ee 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4427,23 +4427,15 @@ static int intel_iommu_enable_iopf(struct device *dev) if (ret) return ret; - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) - goto iopf_remove_device; - ret = pci_enable_pri(pdev, PRQ_DEPTH); - if (ret) - goto iopf_unregister_handler; + if (ret) { + iopf_queue_remove_device(iommu->iopf_queue, dev); + return ret; + } + info->pri_enabled = 1; return 0; - -iopf_unregister_handler: - iommu_unregister_device_fault_handler(dev); -iopf_remove_device: - iopf_queue_remove_device(iommu->iopf_queue, dev); - - return ret; } static int intel_iommu_disable_iopf(struct device *dev) @@ -4466,11 +4458,9 @@ static int intel_iommu_disable_iopf(struct device *dev) info->pri_enabled = 0; /* - * With PRI disabled and outstanding PRQs drained, unregistering - * fault handler and removing device from iopf queue should never - * fail. + * With PRI disabled and outstanding PRQs drained, removing device + * from iopf queue should never fail. */ - WARN_ON(iommu_unregister_device_fault_handler(dev)); WARN_ON(iopf_queue_remove_device(iommu->iopf_queue, dev)); return 0; diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index f948303b2a91..4fda01de5589 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -87,7 +87,7 @@ static void iopf_handler(struct work_struct *work) /** * iommu_queue_iopf - IO Page Fault handler * @fault: fault event - * @cookie: struct device, passed to iommu_register_device_fault_handler. + * @dev: struct device. * * Add a fault to the device workqueue, to be handled by mm. * @@ -124,14 +124,12 @@ static void iopf_handler(struct work_struct *work) * * Return: 0 on success and <0 on error. */ -int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) { int ret; struct iopf_group *group; struct iopf_fault *iopf, *next; struct iommu_fault_param *iopf_param; - - struct device *dev = cookie; struct dev_iommu *param = dev->iommu; lockdep_assert_held(¶m->lock); diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h index 54946b5a7caf..de7819c796ce 100644 --- a/drivers/iommu/iommu-sva.h +++ b/drivers/iommu/iommu-sva.h @@ -13,7 +13,7 @@ struct iommu_fault; struct iopf_queue; #ifdef CONFIG_IOMMU_SVA -int iommu_queue_iopf(struct iommu_fault *fault, void *cookie); +int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev); int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); int iopf_queue_remove_device(struct iopf_queue *queue, @@ -26,7 +26,7 @@ enum iommu_page_response_code iommu_sva_handle_iopf(struct iommu_fault *fault, void *data); #else /* CONFIG_IOMMU_SVA */ -static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +static inline int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) { return -ENODEV; } diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 9575cb75732a..5cb81dbe8d9d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1330,76 +1330,6 @@ void iommu_group_put(struct iommu_group *group) } EXPORT_SYMBOL_GPL(iommu_group_put); -/** - * iommu_register_device_fault_handler() - Register a device fault handler - * @dev: the device - * @handler: the fault handler - * @data: private data passed as argument to the handler - * - * When an IOMMU fault event is received, this handler gets called with the - * fault event and data as argument. The handler should return 0 on success. If - * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also - * complete the fault by calling iommu_page_response() with one of the following - * response code: - * - IOMMU_PAGE_RESP_SUCCESS: retry the translation - * - IOMMU_PAGE_RESP_INVALID: terminate the fault - * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting - * page faults if possible. - * - * Return 0 if the fault handler was installed successfully, or an error. - */ -int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data) -{ - struct dev_iommu *param = dev->iommu; - int ret = 0; - - if (!param || !param->fault_param) - return -EINVAL; - - mutex_lock(¶m->lock); - /* Only allow one fault handler registered for each device */ - if (param->fault_param->handler) { - ret = -EBUSY; - goto done_unlock; - } - - param->fault_param->handler = handler; - param->fault_param->data = data; - -done_unlock: - mutex_unlock(¶m->lock); - - return ret; -} -EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); - -/** - * iommu_unregister_device_fault_handler() - Unregister the device fault handler - * @dev: the device - * - * Remove the device fault handler installed with - * iommu_register_device_fault_handler(). - * - * Return 0 on success, or an error. - */ -int iommu_unregister_device_fault_handler(struct device *dev) -{ - struct dev_iommu *param = dev->iommu; - - if (!param || !param->fault_param) - return -EINVAL; - - mutex_lock(¶m->lock); - param->fault_param->handler = NULL; - param->fault_param->data = NULL; - mutex_unlock(¶m->lock); - - return 0; -} -EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); - /** * iommu_report_device_fault() - Report fault event to device driver * @dev: the device @@ -1424,10 +1354,6 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) /* we only report device fault if there is a handler registered */ mutex_lock(¶m->lock); fparam = param->fault_param; - if (!fparam || !fparam->handler) { - ret = -EINVAL; - goto done_unlock; - } if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { @@ -1442,7 +1368,7 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) mutex_unlock(&fparam->lock); } - ret = fparam->handler(&evt->fault, fparam->data); + ret = iommu_queue_iopf(&evt->fault, dev); if (ret && evt_pending) { mutex_lock(&fparam->lock); list_del(&evt_pending->list); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index bbb7c2ad5184..70176c1c5573 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -128,7 +128,6 @@ struct iommu_page_response { typedef int (*iommu_fault_handler_t)(struct iommu_domain *, struct device *, unsigned long, int, void *); -typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); struct iommu_domain_geometry { dma_addr_t aperture_start; /* First address that can be mapped */ @@ -671,8 +670,6 @@ struct iommu_fault_event { /** * struct iommu_fault_param - per-device IOMMU fault data - * @handler: Callback function to handle IOMMU faults at device level - * @data: handler private data * @lock: protect pending faults list * @dev: the device that owns this param * @queue: IOPF queue @@ -682,8 +679,6 @@ struct iommu_fault_event { * @faults: holds the pending faults which need response */ struct iommu_fault_param { - iommu_dev_fault_handler_t handler; - void *data; struct mutex lock; struct device *dev; @@ -806,11 +801,6 @@ extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data); - -extern int iommu_unregister_device_fault_handler(struct device *dev); extern int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt); @@ -1222,19 +1212,6 @@ static inline void iommu_group_put(struct iommu_group *group) { } -static inline -int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data) -{ - return -ENODEV; -} - -static inline int iommu_unregister_device_fault_handler(struct device *dev) -{ - return 0; -} - static inline int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) { -- cgit v1.2.3 From 3f02a9dc70007c0e6299fda9c4f7a1e2277ec3d2 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:18 +0800 Subject: iommu: Merge iommu_fault_event and iopf_fault The iommu_fault_event and iopf_fault data structures store the same information about an iopf fault. They are also used in the same way. Merge these two data structures into a single one to make the code more concise and easier to maintain. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 ++-- drivers/iommu/intel/iommu.h | 2 +- drivers/iommu/intel/svm.c | 5 ++--- drivers/iommu/io-pgfault.c | 5 ----- drivers/iommu/iommu.c | 8 ++++---- include/linux/iommu.h | 27 ++++++++------------------- 6 files changed, 17 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4cf1054ed321..ab4f04c7f932 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -922,7 +922,7 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, } static int arm_smmu_page_response(struct device *dev, - struct iommu_fault_event *unused, + struct iopf_fault *unused, struct iommu_page_response *resp) { struct arm_smmu_cmdq_ent cmd = {0}; @@ -1465,7 +1465,7 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) struct arm_smmu_master *master; bool ssid_valid = evt[0] & EVTQ_0_SSV; u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]); - struct iommu_fault_event fault_evt = { }; + struct iopf_fault fault_evt = { }; struct iommu_fault *flt = &fault_evt.fault; switch (FIELD_GET(EVTQ_0_ID, evt[0])) { diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index d02f916d8e59..696d95293a69 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1079,7 +1079,7 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); -int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, +int intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 40edd282903f..9751f037e188 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -565,13 +565,12 @@ static int prq_to_iommu_prot(struct page_req_dsc *req) static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, struct page_req_dsc *desc) { - struct iommu_fault_event event; + struct iopf_fault event = { }; if (!dev || !dev_is_pci(dev)) return -ENODEV; /* Fill in event data for device specific processing */ - memset(&event, 0, sizeof(struct iommu_fault_event)); event.fault.type = IOMMU_FAULT_PAGE_REQ; event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; event.fault.prm.pasid = desc->pasid; @@ -743,7 +742,7 @@ prq_advance: } int intel_svm_page_response(struct device *dev, - struct iommu_fault_event *evt, + struct iopf_fault *evt, struct iommu_page_response *msg) { struct device_domain_info *info = dev_iommu_priv_get(dev); diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 4fda01de5589..10d48eb72608 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -25,11 +25,6 @@ struct iopf_queue { struct mutex lock; }; -struct iopf_fault { - struct iommu_fault fault; - struct list_head list; -}; - struct iopf_group { struct iopf_fault last_fault; struct list_head faults; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 5cb81dbe8d9d..15d7b2914780 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1341,10 +1341,10 @@ EXPORT_SYMBOL_GPL(iommu_group_put); * * Return 0 on success, or an error. */ -int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { struct dev_iommu *param = dev->iommu; - struct iommu_fault_event *evt_pending = NULL; + struct iopf_fault *evt_pending = NULL; struct iommu_fault_param *fparam; int ret = 0; @@ -1357,7 +1357,7 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), + evt_pending = kmemdup(evt, sizeof(struct iopf_fault), GFP_KERNEL); if (!evt_pending) { ret = -ENOMEM; @@ -1386,7 +1386,7 @@ int iommu_page_response(struct device *dev, { bool needs_pasid; int ret = -EINVAL; - struct iommu_fault_event *evt; + struct iopf_fault *evt; struct iommu_fault_page_request *prm; struct dev_iommu *param = dev->iommu; const struct iommu_ops *ops = dev_iommu_ops(dev); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 70176c1c5573..2320548a90f8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -40,7 +40,6 @@ struct iommu_domain_ops; struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; -struct iommu_fault_event; struct iommu_dma_cookie; struct iopf_queue; @@ -121,6 +120,11 @@ struct iommu_page_response { u32 code; }; +struct iopf_fault { + struct iommu_fault fault; + /* node for pending lists */ + struct list_head list; +}; /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 @@ -553,7 +557,7 @@ struct iommu_ops { int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); int (*page_response)(struct device *dev, - struct iommu_fault_event *evt, + struct iopf_fault *evt, struct iommu_page_response *msg); int (*def_domain_type)(struct device *dev); @@ -654,20 +658,6 @@ struct iommu_device { u32 max_pasids; }; -/** - * struct iommu_fault_event - Generic fault event - * - * Can represent recoverable faults such as a page requests or - * unrecoverable faults such as DMA or IRQ remapping faults. - * - * @fault: fault descriptor - * @list: pending fault event list, used for tracking responses - */ -struct iommu_fault_event { - struct iommu_fault fault; - struct list_head list; -}; - /** * struct iommu_fault_param - per-device IOMMU fault data * @lock: protect pending faults list @@ -802,8 +792,7 @@ extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_report_device_fault(struct device *dev, - struct iommu_fault_event *evt); +extern int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); extern int iommu_page_response(struct device *dev, struct iommu_page_response *msg); @@ -1213,7 +1202,7 @@ static inline void iommu_group_put(struct iommu_group *group) } static inline -int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { return -ENODEV; } -- cgit v1.2.3 From 24b5d268b5ab95c12b5ae58a054d04bfa442f58f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:19 +0800 Subject: iommu: Prepare for separating SVA and IOPF Move iopf_group data structure to iommu.h to make it a minimal set of faults that a domain's page fault handler should handle. Add a new function, iopf_free_group(), to free a fault group after all faults in the group are handled. This function will be made global so that it can be called from other files, such as iommu-sva.c. Move iopf_queue data structure to iommu.h to allow the workqueue to be scheduled out of this file. This will simplify the sequential patches. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 39 ++++++++++++++------------------------- include/linux/iommu.h | 20 +++++++++++++++++++- 2 files changed, 33 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 10d48eb72608..c7e6bbed5c05 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -13,24 +13,17 @@ #include "iommu-sva.h" -/** - * struct iopf_queue - IO Page Fault queue - * @wq: the fault workqueue - * @devices: devices attached to this queue - * @lock: protects the device list - */ -struct iopf_queue { - struct workqueue_struct *wq; - struct list_head devices; - struct mutex lock; -}; - -struct iopf_group { - struct iopf_fault last_fault; - struct list_head faults; - struct work_struct work; - struct device *dev; -}; +static void iopf_free_group(struct iopf_group *group) +{ + struct iopf_fault *iopf, *next; + + list_for_each_entry_safe(iopf, next, &group->faults, list) { + if (!(iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) + kfree(iopf); + } + + kfree(group); +} static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, enum iommu_page_response_code status) @@ -50,9 +43,9 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, static void iopf_handler(struct work_struct *work) { + struct iopf_fault *iopf; struct iopf_group *group; struct iommu_domain *domain; - struct iopf_fault *iopf, *next; enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; group = container_of(work, struct iopf_group, work); @@ -61,7 +54,7 @@ static void iopf_handler(struct work_struct *work) if (!domain || !domain->iopf_handler) status = IOMMU_PAGE_RESP_INVALID; - list_for_each_entry_safe(iopf, next, &group->faults, list) { + list_for_each_entry(iopf, &group->faults, list) { /* * For the moment, errors are sticky: don't handle subsequent * faults in the group if there is an error. @@ -69,14 +62,10 @@ static void iopf_handler(struct work_struct *work) if (status == IOMMU_PAGE_RESP_SUCCESS) status = domain->iopf_handler(&iopf->fault, domain->fault_data); - - if (!(iopf->fault.prm.flags & - IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) - kfree(iopf); } iopf_complete_group(group->dev, &group->last_fault, status); - kfree(group); + iopf_free_group(group); } /** diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 2320548a90f8..c9d4f175f121 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -41,7 +41,6 @@ struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; struct iommu_dma_cookie; -struct iopf_queue; #define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ #define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ @@ -126,6 +125,25 @@ struct iopf_fault { struct list_head list; }; +struct iopf_group { + struct iopf_fault last_fault; + struct list_head faults; + struct work_struct work; + struct device *dev; +}; + +/** + * struct iopf_queue - IO Page Fault queue + * @wq: the fault workqueue + * @devices: devices attached to this queue + * @lock: protects the device list + */ +struct iopf_queue { + struct workqueue_struct *wq; + struct list_head devices; + struct mutex lock; +}; + /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 #define IOMMU_FAULT_WRITE 0x1 -- cgit v1.2.3 From 351ffcb11ca0ff64e399982e279cfa131e7cb1aa Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:20 +0800 Subject: iommu: Make iommu_queue_iopf() more generic Make iommu_queue_iopf() more generic by making the iopf_group a minimal set of iopf's that an iopf handler of domain should handle and respond to. Add domain parameter to struct iopf_group so that the handler can retrieve and use it directly. Change iommu_queue_iopf() to forward groups of iopf's to the domain's iopf handler. This is also a necessary step to decouple the sva iopf handling code from this interface. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 68 ++++++++++++++++++++++++++++++++++++++-------- drivers/iommu/iommu-sva.c | 3 +- drivers/iommu/iommu-sva.h | 6 ++-- include/linux/iommu.h | 4 +-- 4 files changed, 61 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index c7e6bbed5c05..13cd0929e766 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -13,6 +13,9 @@ #include "iommu-sva.h" +enum iommu_page_response_code +iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm); + static void iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -45,29 +48,48 @@ static void iopf_handler(struct work_struct *work) { struct iopf_fault *iopf; struct iopf_group *group; - struct iommu_domain *domain; enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; group = container_of(work, struct iopf_group, work); - domain = iommu_get_domain_for_dev_pasid(group->dev, - group->last_fault.fault.prm.pasid, 0); - if (!domain || !domain->iopf_handler) - status = IOMMU_PAGE_RESP_INVALID; - list_for_each_entry(iopf, &group->faults, list) { /* * For the moment, errors are sticky: don't handle subsequent * faults in the group if there is an error. */ - if (status == IOMMU_PAGE_RESP_SUCCESS) - status = domain->iopf_handler(&iopf->fault, - domain->fault_data); + if (status != IOMMU_PAGE_RESP_SUCCESS) + break; + + status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); } iopf_complete_group(group->dev, &group->last_fault, status); iopf_free_group(group); } +static struct iommu_domain *get_domain_for_iopf(struct device *dev, + struct iommu_fault *fault) +{ + struct iommu_domain *domain; + + if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) { + domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0); + if (IS_ERR(domain)) + domain = NULL; + } else { + domain = iommu_get_domain_for_dev(dev); + } + + if (!domain || !domain->iopf_handler) { + dev_warn_ratelimited(dev, + "iopf (pasid %d) without domain attached or handler installed\n", + fault->prm.pasid); + + return NULL; + } + + return domain; +} + /** * iommu_queue_iopf - IO Page Fault handler * @fault: fault event @@ -112,6 +134,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) { int ret; struct iopf_group *group; + struct iommu_domain *domain; struct iopf_fault *iopf, *next; struct iommu_fault_param *iopf_param; struct dev_iommu *param = dev->iommu; @@ -143,6 +166,12 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) return 0; } + domain = get_domain_for_iopf(dev, fault); + if (!domain) { + ret = -EINVAL; + goto cleanup_partial; + } + group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) { /* @@ -157,8 +186,8 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) group->dev = dev; group->last_fault.fault = *fault; INIT_LIST_HEAD(&group->faults); + group->domain = domain; list_add(&group->last_fault.list, &group->faults); - INIT_WORK(&group->work, iopf_handler); /* See if we have partial faults for this group */ list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { @@ -167,9 +196,13 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) list_move(&iopf->list, &group->faults); } - queue_work(iopf_param->queue->wq, &group->work); - return 0; + mutex_unlock(&iopf_param->lock); + ret = domain->iopf_handler(group); + mutex_lock(&iopf_param->lock); + if (ret) + iopf_free_group(group); + return ret; cleanup_partial: list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { if (iopf->fault.prm.grpid == fault->prm.grpid) { @@ -181,6 +214,17 @@ cleanup_partial: } EXPORT_SYMBOL_GPL(iommu_queue_iopf); +int iommu_sva_handle_iopf(struct iopf_group *group) +{ + struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + + INIT_WORK(&group->work, iopf_handler); + if (!queue_work(fault_param->queue->wq, &group->work)) + return -EBUSY; + + return 0; +} + /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed * @dev: the endpoint whose faults need to be flushed. diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index c3fc9201d0be..fcae7308fcb7 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -163,11 +163,10 @@ EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); * I/O page fault handler for SVA */ enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data) +iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) { vm_fault_t ret; struct vm_area_struct *vma; - struct mm_struct *mm = data; unsigned int access_flags = 0; unsigned int fault_flags = FAULT_FLAG_REMOTE; struct iommu_fault_page_request *prm = &fault->prm; diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h index de7819c796ce..27c8da115b41 100644 --- a/drivers/iommu/iommu-sva.h +++ b/drivers/iommu/iommu-sva.h @@ -22,8 +22,7 @@ int iopf_queue_flush_dev(struct device *dev); struct iopf_queue *iopf_queue_alloc(const char *name); void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); -enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data); +int iommu_sva_handle_iopf(struct iopf_group *group); #else /* CONFIG_IOMMU_SVA */ static inline int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) @@ -62,8 +61,7 @@ static inline int iopf_queue_discard_partial(struct iopf_queue *queue) return -ENODEV; } -static inline enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data) +static inline int iommu_sva_handle_iopf(struct iopf_group *group) { return IOMMU_PAGE_RESP_INVALID; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c9d4f175f121..791f183a988e 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -130,6 +130,7 @@ struct iopf_group { struct list_head faults; struct work_struct work; struct device *dev; + struct iommu_domain *domain; }; /** @@ -209,8 +210,7 @@ struct iommu_domain { unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ struct iommu_domain_geometry geometry; struct iommu_dma_cookie *iova_cookie; - enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault, - void *data); + int (*iopf_handler)(struct iopf_group *group); void *fault_data; union { struct { -- cgit v1.2.3 From 17c51a0ea36b800e7a5998a92d83016c82935dff Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:21 +0800 Subject: iommu: Separate SVA and IOPF Add CONFIG_IOMMU_IOPF for page fault handling framework and select it from its real consumer. Move iopf function declaration from iommu-sva.h to iommu.h and remove iommu-sva.h as it's empty now. Consolidate all SVA related code into iommu-sva.c: - Move iommu_sva_domain_alloc() from iommu.c to iommu-sva.c. - Move sva iopf handling code from io-pgfault.c to iommu-sva.c. Consolidate iommu_report_device_fault() and iommu_page_response() into io-pgfault.c. Export iopf_free_group() and iopf_group_response() for iopf handlers implemented in modules. Some functions are renamed with more meaningful names. No other intentional functionality changes. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-11-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 4 + drivers/iommu/Makefile | 3 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 1 - drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 1 - drivers/iommu/intel/Kconfig | 1 + drivers/iommu/intel/iommu.c | 1 - drivers/iommu/intel/svm.c | 1 - drivers/iommu/io-pgfault.c | 188 +++++++++++++++++------- drivers/iommu/iommu-sva.c | 68 ++++++++- drivers/iommu/iommu-sva.h | 69 --------- drivers/iommu/iommu.c | 133 ----------------- include/linux/iommu.h | 98 +++++++++--- 12 files changed, 277 insertions(+), 291 deletions(-) delete mode 100644 drivers/iommu/iommu-sva.h (limited to 'include/linux') diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 9a29d742617e..d9ed5ad129f2 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -163,6 +163,9 @@ config IOMMU_SVA select IOMMU_MM_DATA bool +config IOMMU_IOPF + bool + config FSL_PAMU bool "Freescale IOMMU support" depends on PCI @@ -398,6 +401,7 @@ config ARM_SMMU_V3_SVA bool "Shared Virtual Addressing support for the ARM SMMUv3" depends on ARM_SMMU_V3 select IOMMU_SVA + select IOMMU_IOPF select MMU_NOTIFIER help Support for sharing process address spaces with devices using the diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 95ad9dbfbda0..542760d963ec 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o -obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o +obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o +obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index ab2b0a5e4369..6513a98fcb72 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -10,7 +10,6 @@ #include #include "arm-smmu-v3.h" -#include "../../iommu-sva.h" #include "../../io-pgtable-arm.h" struct arm_smmu_mmu_notifier { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index ab4f04c7f932..4e93e845458c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -29,7 +29,6 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -#include "../../iommu-sva.h" static bool disable_bypass = true; module_param(disable_bypass, bool, 0444); diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 012cd2541a68..a4a125666293 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -51,6 +51,7 @@ config INTEL_IOMMU_SVM depends on X86_64 select MMU_NOTIFIER select IOMMU_SVA + select IOMMU_IOPF help Shared Virtual Memory (SVM) provides a facility for devices to access DMA resources through process address space by diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index df6ceefc09ee..29a12f289e2e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -27,7 +27,6 @@ #include "iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" -#include "../iommu-sva.h" #include "pasid.h" #include "cap_audit.h" #include "perfmon.h" diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 9751f037e188..e1cbcb9515f0 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -22,7 +22,6 @@ #include "iommu.h" #include "pasid.h" #include "perf.h" -#include "../iommu-sva.h" #include "trace.h" static irqreturn_t prq_event_thread(int irq, void *d); diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 13cd0929e766..c1e88da973ce 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -11,12 +11,9 @@ #include #include -#include "iommu-sva.h" +#include "iommu-priv.h" -enum iommu_page_response_code -iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm); - -static void iopf_free_group(struct iopf_group *group) +void iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -27,44 +24,7 @@ static void iopf_free_group(struct iopf_group *group) kfree(group); } - -static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, - enum iommu_page_response_code status) -{ - struct iommu_page_response resp = { - .pasid = iopf->fault.prm.pasid, - .grpid = iopf->fault.prm.grpid, - .code = status, - }; - - if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && - (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; - - return iommu_page_response(dev, &resp); -} - -static void iopf_handler(struct work_struct *work) -{ - struct iopf_fault *iopf; - struct iopf_group *group; - enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; - - group = container_of(work, struct iopf_group, work); - list_for_each_entry(iopf, &group->faults, list) { - /* - * For the moment, errors are sticky: don't handle subsequent - * faults in the group if there is an error. - */ - if (status != IOMMU_PAGE_RESP_SUCCESS) - break; - - status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); - } - - iopf_complete_group(group->dev, &group->last_fault, status); - iopf_free_group(group); -} +EXPORT_SYMBOL_GPL(iopf_free_group); static struct iommu_domain *get_domain_for_iopf(struct device *dev, struct iommu_fault *fault) @@ -91,7 +51,7 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, } /** - * iommu_queue_iopf - IO Page Fault handler + * iommu_handle_iopf - IO Page Fault handler * @fault: fault event * @dev: struct device. * @@ -130,7 +90,7 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, * * Return: 0 on success and <0 on error. */ -int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) +static int iommu_handle_iopf(struct iommu_fault *fault, struct device *dev) { int ret; struct iopf_group *group; @@ -212,18 +172,117 @@ cleanup_partial: } return ret; } -EXPORT_SYMBOL_GPL(iommu_queue_iopf); -int iommu_sva_handle_iopf(struct iopf_group *group) +/** + * iommu_report_device_fault() - Report fault event to device driver + * @dev: the device + * @evt: fault event data + * + * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ + * handler. When this function fails and the fault is recoverable, it is the + * caller's responsibility to complete the fault. + * + * Return 0 on success, or an error. + */ +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + struct dev_iommu *param = dev->iommu; + struct iopf_fault *evt_pending = NULL; + struct iommu_fault_param *fparam; + int ret = 0; - INIT_WORK(&group->work, iopf_handler); - if (!queue_work(fault_param->queue->wq, &group->work)) - return -EBUSY; + if (!param || !evt) + return -EINVAL; - return 0; + /* we only report device fault if there is a handler registered */ + mutex_lock(¶m->lock); + fparam = param->fault_param; + + if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && + (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + evt_pending = kmemdup(evt, sizeof(struct iopf_fault), + GFP_KERNEL); + if (!evt_pending) { + ret = -ENOMEM; + goto done_unlock; + } + mutex_lock(&fparam->lock); + list_add_tail(&evt_pending->list, &fparam->faults); + mutex_unlock(&fparam->lock); + } + + ret = iommu_handle_iopf(&evt->fault, dev); + if (ret && evt_pending) { + mutex_lock(&fparam->lock); + list_del(&evt_pending->list); + mutex_unlock(&fparam->lock); + kfree(evt_pending); + } +done_unlock: + mutex_unlock(¶m->lock); + return ret; +} +EXPORT_SYMBOL_GPL(iommu_report_device_fault); + +int iommu_page_response(struct device *dev, + struct iommu_page_response *msg) +{ + bool needs_pasid; + int ret = -EINVAL; + struct iopf_fault *evt; + struct iommu_fault_page_request *prm; + struct dev_iommu *param = dev->iommu; + const struct iommu_ops *ops = dev_iommu_ops(dev); + bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; + + if (!ops->page_response) + return -ENODEV; + + if (!param || !param->fault_param) + return -EINVAL; + + /* Only send response if there is a fault report pending */ + mutex_lock(¶m->fault_param->lock); + if (list_empty(¶m->fault_param->faults)) { + dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); + goto done_unlock; + } + /* + * Check if we have a matching page request pending to respond, + * otherwise return -EINVAL + */ + list_for_each_entry(evt, ¶m->fault_param->faults, list) { + prm = &evt->fault.prm; + if (prm->grpid != msg->grpid) + continue; + + /* + * If the PASID is required, the corresponding request is + * matched using the group ID, the PASID valid bit and the PASID + * value. Otherwise only the group ID matches request and + * response. + */ + needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; + if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) + continue; + + if (!needs_pasid && has_pasid) { + /* No big deal, just clear it. */ + msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; + msg->pasid = 0; + } + + ret = ops->page_response(dev, evt, msg); + list_del(&evt->list); + kfree(evt); + break; + } + +done_unlock: + mutex_unlock(¶m->fault_param->lock); + return ret; } +EXPORT_SYMBOL_GPL(iommu_page_response); /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed @@ -258,6 +317,31 @@ int iopf_queue_flush_dev(struct device *dev) } EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); +/** + * iopf_group_response - Respond a group of page faults + * @group: the group of faults with the same group id + * @status: the response code + * + * Return 0 on success and <0 on error. + */ +int iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) +{ + struct iopf_fault *iopf = &group->last_fault; + struct iommu_page_response resp = { + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = status, + }; + + if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && + (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) + resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + + return iommu_page_response(group->dev, &resp); +} +EXPORT_SYMBOL_GPL(iopf_group_response); + /** * iopf_queue_discard_partial - Remove all pending partial fault * @queue: the queue whose partial faults need to be discarded diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index fcae7308fcb7..9de878e40413 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -7,7 +7,7 @@ #include #include -#include "iommu-sva.h" +#include "iommu-priv.h" static DEFINE_MUTEX(iommu_sva_lock); @@ -159,10 +159,21 @@ u32 iommu_sva_get_pasid(struct iommu_sva *handle) } EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); +void mm_pasid_drop(struct mm_struct *mm) +{ + struct iommu_mm_data *iommu_mm = mm->iommu_mm; + + if (!iommu_mm) + return; + + iommu_free_global_pasid(iommu_mm->pasid); + kfree(iommu_mm); +} + /* * I/O page fault handler for SVA */ -enum iommu_page_response_code +static enum iommu_page_response_code iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) { vm_fault_t ret; @@ -216,13 +227,54 @@ out_put_mm: return status; } -void mm_pasid_drop(struct mm_struct *mm) +static void iommu_sva_handle_iopf(struct work_struct *work) { - struct iommu_mm_data *iommu_mm = mm->iommu_mm; + struct iopf_fault *iopf; + struct iopf_group *group; + enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; + + group = container_of(work, struct iopf_group, work); + list_for_each_entry(iopf, &group->faults, list) { + /* + * For the moment, errors are sticky: don't handle subsequent + * faults in the group if there is an error. + */ + if (status != IOMMU_PAGE_RESP_SUCCESS) + break; + + status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); + } - if (!iommu_mm) - return; + iopf_group_response(group, status); + iopf_free_group(group); +} - iommu_free_global_pasid(iommu_mm->pasid); - kfree(iommu_mm); +static int iommu_sva_iopf_handler(struct iopf_group *group) +{ + struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + + INIT_WORK(&group->work, iommu_sva_handle_iopf); + if (!queue_work(fault_param->queue->wq, &group->work)) + return -EBUSY; + + return 0; +} + +struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, + struct mm_struct *mm) +{ + const struct iommu_ops *ops = dev_iommu_ops(dev); + struct iommu_domain *domain; + + domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); + if (!domain) + return NULL; + + domain->type = IOMMU_DOMAIN_SVA; + mmgrab(mm); + domain->mm = mm; + domain->owner = ops; + domain->iopf_handler = iommu_sva_iopf_handler; + + return domain; } diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h deleted file mode 100644 index 27c8da115b41..000000000000 --- a/drivers/iommu/iommu-sva.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * SVA library for IOMMU drivers - */ -#ifndef _IOMMU_SVA_H -#define _IOMMU_SVA_H - -#include - -/* I/O Page fault */ -struct device; -struct iommu_fault; -struct iopf_queue; - -#ifdef CONFIG_IOMMU_SVA -int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev); - -int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); -int iopf_queue_remove_device(struct iopf_queue *queue, - struct device *dev); -int iopf_queue_flush_dev(struct device *dev); -struct iopf_queue *iopf_queue_alloc(const char *name); -void iopf_queue_free(struct iopf_queue *queue); -int iopf_queue_discard_partial(struct iopf_queue *queue); -int iommu_sva_handle_iopf(struct iopf_group *group); - -#else /* CONFIG_IOMMU_SVA */ -static inline int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_add_device(struct iopf_queue *queue, - struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_remove_device(struct iopf_queue *queue, - struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_flush_dev(struct device *dev) -{ - return -ENODEV; -} - -static inline struct iopf_queue *iopf_queue_alloc(const char *name) -{ - return NULL; -} - -static inline void iopf_queue_free(struct iopf_queue *queue) -{ -} - -static inline int iopf_queue_discard_partial(struct iopf_queue *queue) -{ - return -ENODEV; -} - -static inline int iommu_sva_handle_iopf(struct iopf_group *group) -{ - return IOMMU_PAGE_RESP_INVALID; -} -#endif /* CONFIG_IOMMU_SVA */ -#endif /* _IOMMU_SVA_H */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 15d7b2914780..35dac3e4629a 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -36,8 +36,6 @@ #include "dma-iommu.h" #include "iommu-priv.h" -#include "iommu-sva.h" - static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); static DEFINE_IDA(iommu_global_pasid_ida); @@ -1330,117 +1328,6 @@ void iommu_group_put(struct iommu_group *group) } EXPORT_SYMBOL_GPL(iommu_group_put); -/** - * iommu_report_device_fault() - Report fault event to device driver - * @dev: the device - * @evt: fault event data - * - * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. - * - * Return 0 on success, or an error. - */ -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) -{ - struct dev_iommu *param = dev->iommu; - struct iopf_fault *evt_pending = NULL; - struct iommu_fault_param *fparam; - int ret = 0; - - if (!param || !evt) - return -EINVAL; - - /* we only report device fault if there is a handler registered */ - mutex_lock(¶m->lock); - fparam = param->fault_param; - - if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - evt_pending = kmemdup(evt, sizeof(struct iopf_fault), - GFP_KERNEL); - if (!evt_pending) { - ret = -ENOMEM; - goto done_unlock; - } - mutex_lock(&fparam->lock); - list_add_tail(&evt_pending->list, &fparam->faults); - mutex_unlock(&fparam->lock); - } - - ret = iommu_queue_iopf(&evt->fault, dev); - if (ret && evt_pending) { - mutex_lock(&fparam->lock); - list_del(&evt_pending->list); - mutex_unlock(&fparam->lock); - kfree(evt_pending); - } -done_unlock: - mutex_unlock(¶m->lock); - return ret; -} -EXPORT_SYMBOL_GPL(iommu_report_device_fault); - -int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) -{ - bool needs_pasid; - int ret = -EINVAL; - struct iopf_fault *evt; - struct iommu_fault_page_request *prm; - struct dev_iommu *param = dev->iommu; - const struct iommu_ops *ops = dev_iommu_ops(dev); - bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - - if (!ops->page_response) - return -ENODEV; - - if (!param || !param->fault_param) - return -EINVAL; - - /* Only send response if there is a fault report pending */ - mutex_lock(¶m->fault_param->lock); - if (list_empty(¶m->fault_param->faults)) { - dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); - goto done_unlock; - } - /* - * Check if we have a matching page request pending to respond, - * otherwise return -EINVAL - */ - list_for_each_entry(evt, ¶m->fault_param->faults, list) { - prm = &evt->fault.prm; - if (prm->grpid != msg->grpid) - continue; - - /* - * If the PASID is required, the corresponding request is - * matched using the group ID, the PASID valid bit and the PASID - * value. Otherwise only the group ID matches request and - * response. - */ - needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; - if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) - continue; - - if (!needs_pasid && has_pasid) { - /* No big deal, just clear it. */ - msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; - msg->pasid = 0; - } - - ret = ops->page_response(dev, evt, msg); - list_del(&evt->list); - kfree(evt); - break; - } - -done_unlock: - mutex_unlock(¶m->fault_param->lock); - return ret; -} -EXPORT_SYMBOL_GPL(iommu_page_response); - /** * iommu_group_id - Return ID for a group * @group: the group to ID @@ -3523,26 +3410,6 @@ struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev, } EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid); -struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, - struct mm_struct *mm) -{ - const struct iommu_ops *ops = dev_iommu_ops(dev); - struct iommu_domain *domain; - - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); - if (!domain) - return NULL; - - domain->type = IOMMU_DOMAIN_SVA; - mmgrab(mm); - domain->mm = mm; - domain->owner = ops; - domain->iopf_handler = iommu_sva_handle_iopf; - domain->fault_data = mm; - - return domain; -} - ioasid_t iommu_alloc_global_pasid(struct device *dev) { int ret; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 791f183a988e..fc912aed7886 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -810,10 +810,6 @@ extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); -extern int iommu_page_response(struct device *dev, - struct iommu_page_response *msg); - extern int iommu_group_id(struct iommu_group *group); extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); @@ -1029,8 +1025,6 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group); int iommu_device_claim_dma_owner(struct device *dev, void *owner); void iommu_device_release_dma_owner(struct device *dev); -struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, - struct mm_struct *mm); int iommu_attach_device_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid); void iommu_detach_device_pasid(struct iommu_domain *domain, @@ -1219,18 +1213,6 @@ static inline void iommu_group_put(struct iommu_group *group) { } -static inline -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) -{ - return -ENODEV; -} - -static inline int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) -{ - return -ENODEV; -} - static inline int iommu_group_id(struct iommu_group *group) { return -ENODEV; @@ -1379,12 +1361,6 @@ static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner) return -ENODEV; } -static inline struct iommu_domain * -iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) -{ - return NULL; -} - static inline int iommu_attach_device_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid) { @@ -1524,6 +1500,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm); void iommu_sva_unbind_device(struct iommu_sva *handle); u32 iommu_sva_get_pasid(struct iommu_sva *handle); +struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, + struct mm_struct *mm); #else static inline struct iommu_sva * iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) @@ -1548,6 +1526,78 @@ static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm) } static inline void mm_pasid_drop(struct mm_struct *mm) {} + +static inline struct iommu_domain * +iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) +{ + return NULL; +} #endif /* CONFIG_IOMMU_SVA */ +#ifdef CONFIG_IOMMU_IOPF +int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); +int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); +int iopf_queue_flush_dev(struct device *dev); +struct iopf_queue *iopf_queue_alloc(const char *name); +void iopf_queue_free(struct iopf_queue *queue); +int iopf_queue_discard_partial(struct iopf_queue *queue); +void iopf_free_group(struct iopf_group *group); +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); +int iommu_page_response(struct device *dev, struct iommu_page_response *msg); +int iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status); +#else +static inline int +iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) +{ + return -ENODEV; +} + +static inline int +iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +{ + return -ENODEV; +} + +static inline int iopf_queue_flush_dev(struct device *dev) +{ + return -ENODEV; +} + +static inline struct iopf_queue *iopf_queue_alloc(const char *name) +{ + return NULL; +} + +static inline void iopf_queue_free(struct iopf_queue *queue) +{ +} + +static inline int iopf_queue_discard_partial(struct iopf_queue *queue) +{ + return -ENODEV; +} + +static inline void iopf_free_group(struct iopf_group *group) +{ +} + +static inline int +iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) +{ + return -ENODEV; +} + +static inline int +iommu_page_response(struct device *dev, struct iommu_page_response *msg) +{ + return -ENODEV; +} + +static inline int iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) +{ + return -ENODEV; +} +#endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ -- cgit v1.2.3 From a74c077b9021b36c785095c571336e5b204d3c2d Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:23 +0800 Subject: iommu: Use refcount for fault data access The per-device fault data structure stores information about faults occurring on a device. Its lifetime spans from IOPF enablement to disablement. Multiple paths, including IOPF reporting, handling, and responding, may access it concurrently. Previously, a mutex protected the fault data from use after free. But this is not performance friendly due to the critical nature of IOPF handling paths. Refine this with a refcount-based approach. The fault data pointer is obtained within an RCU read region with a refcount. The fault data pointer is returned for usage only when the pointer is valid and a refcount is successfully obtained. The fault data is freed with kfree_rcu(), ensuring data is only freed after all RCU critical regions complete. An iopf handling work starts once an iopf group is created. The handling work continues until iommu_page_response() is called to respond to the iopf and the iopf group is freed. During this time, the device fault parameter should always be available. Add a pointer to the device fault parameter in the iopf_group structure and hold the reference until the iopf_group is freed. Make iommu_page_response() static as it is only used in io-pgfault.c. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Link: https://lore.kernel.org/r/20240212012227.119381-13-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 127 ++++++++++++++++++++++++++++----------------- drivers/iommu/iommu-sva.c | 2 +- include/linux/iommu.h | 17 +++--- 3 files changed, 88 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 5aea8402be47..ce7058892b59 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -13,6 +13,32 @@ #include "iommu-priv.h" +/* + * Return the fault parameter of a device if it exists. Otherwise, return NULL. + * On a successful return, the caller takes a reference of this parameter and + * should put it after use by calling iopf_put_dev_fault_param(). + */ +static struct iommu_fault_param *iopf_get_dev_fault_param(struct device *dev) +{ + struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *fault_param; + + rcu_read_lock(); + fault_param = rcu_dereference(param->fault_param); + if (fault_param && !refcount_inc_not_zero(&fault_param->users)) + fault_param = NULL; + rcu_read_unlock(); + + return fault_param; +} + +/* Caller must hold a reference of the fault parameter. */ +static void iopf_put_dev_fault_param(struct iommu_fault_param *fault_param) +{ + if (refcount_dec_and_test(&fault_param->users)) + kfree_rcu(fault_param, rcu); +} + void iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -22,6 +48,8 @@ void iopf_free_group(struct iopf_group *group) kfree(iopf); } + /* Pair with iommu_report_device_fault(). */ + iopf_put_dev_fault_param(group->fault_param); kfree(group); } EXPORT_SYMBOL_GPL(iopf_free_group); @@ -135,7 +163,7 @@ static int iommu_handle_iopf(struct iommu_fault *fault, goto cleanup_partial; } - group->dev = dev; + group->fault_param = iopf_param; group->last_fault.fault = *fault; INIT_LIST_HEAD(&group->faults); group->domain = domain; @@ -178,64 +206,61 @@ cleanup_partial: */ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { + bool last_prq = evt->fault.type == IOMMU_FAULT_PAGE_REQ && + (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE); struct iommu_fault_param *fault_param; - struct iopf_fault *evt_pending = NULL; - struct dev_iommu *param = dev->iommu; - int ret = 0; + struct iopf_fault *evt_pending; + int ret; - mutex_lock(¶m->lock); - fault_param = param->fault_param; - if (!fault_param) { - mutex_unlock(¶m->lock); + fault_param = iopf_get_dev_fault_param(dev); + if (!fault_param) return -EINVAL; - } mutex_lock(&fault_param->lock); - if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + if (last_prq) { evt_pending = kmemdup(evt, sizeof(struct iopf_fault), GFP_KERNEL); if (!evt_pending) { ret = -ENOMEM; - goto done_unlock; + goto err_unlock; } list_add_tail(&evt_pending->list, &fault_param->faults); } ret = iommu_handle_iopf(&evt->fault, fault_param); - if (ret && evt_pending) { + if (ret) + goto err_free; + + mutex_unlock(&fault_param->lock); + /* The reference count of fault_param is now held by iopf_group. */ + if (!last_prq) + iopf_put_dev_fault_param(fault_param); + + return 0; +err_free: + if (last_prq) { list_del(&evt_pending->list); kfree(evt_pending); } -done_unlock: +err_unlock: mutex_unlock(&fault_param->lock); - mutex_unlock(¶m->lock); + iopf_put_dev_fault_param(fault_param); return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); -int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) +static int iommu_page_response(struct iopf_group *group, + struct iommu_page_response *msg) { bool needs_pasid; int ret = -EINVAL; struct iopf_fault *evt; struct iommu_fault_page_request *prm; - struct dev_iommu *param = dev->iommu; - struct iommu_fault_param *fault_param; + struct device *dev = group->fault_param->dev; const struct iommu_ops *ops = dev_iommu_ops(dev); bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - - if (!ops->page_response) - return -ENODEV; - - mutex_lock(¶m->lock); - fault_param = param->fault_param; - if (!fault_param) { - mutex_unlock(¶m->lock); - return -EINVAL; - } + struct iommu_fault_param *fault_param = group->fault_param; /* Only send response if there is a fault report pending */ mutex_lock(&fault_param->lock); @@ -276,10 +301,9 @@ int iommu_page_response(struct device *dev, done_unlock: mutex_unlock(&fault_param->lock); - mutex_unlock(¶m->lock); + return ret; } -EXPORT_SYMBOL_GPL(iommu_page_response); /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed @@ -295,22 +319,20 @@ EXPORT_SYMBOL_GPL(iommu_page_response); */ int iopf_queue_flush_dev(struct device *dev) { - int ret = 0; struct iommu_fault_param *iopf_param; - struct dev_iommu *param = dev->iommu; - if (!param) + /* + * It's a driver bug to be here after iopf_queue_remove_device(). + * Therefore, it's safe to dereference the fault parameter without + * holding the lock. + */ + iopf_param = rcu_dereference_check(dev->iommu->fault_param, true); + if (WARN_ON(!iopf_param)) return -ENODEV; - mutex_lock(¶m->lock); - iopf_param = param->fault_param; - if (iopf_param) - flush_workqueue(iopf_param->queue->wq); - else - ret = -ENODEV; - mutex_unlock(¶m->lock); + flush_workqueue(iopf_param->queue->wq); - return ret; + return 0; } EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); @@ -335,7 +357,7 @@ int iopf_group_response(struct iopf_group *group, (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) resp.flags = IOMMU_PAGE_RESP_PASID_VALID; - return iommu_page_response(group->dev, &resp); + return iommu_page_response(group, &resp); } EXPORT_SYMBOL_GPL(iopf_group_response); @@ -384,10 +406,15 @@ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) int ret = 0; struct dev_iommu *param = dev->iommu; struct iommu_fault_param *fault_param; + const struct iommu_ops *ops = dev_iommu_ops(dev); + + if (!ops->page_response) + return -ENODEV; mutex_lock(&queue->lock); mutex_lock(¶m->lock); - if (param->fault_param) { + if (rcu_dereference_check(param->fault_param, + lockdep_is_held(¶m->lock))) { ret = -EBUSY; goto done_unlock; } @@ -402,10 +429,11 @@ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) INIT_LIST_HEAD(&fault_param->faults); INIT_LIST_HEAD(&fault_param->partial); fault_param->dev = dev; + refcount_set(&fault_param->users, 1); list_add(&fault_param->queue_list, &queue->devices); fault_param->queue = queue; - param->fault_param = fault_param; + rcu_assign_pointer(param->fault_param, fault_param); done_unlock: mutex_unlock(¶m->lock); @@ -429,10 +457,12 @@ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) int ret = 0; struct iopf_fault *iopf, *next; struct dev_iommu *param = dev->iommu; - struct iommu_fault_param *fault_param = param->fault_param; + struct iommu_fault_param *fault_param; mutex_lock(&queue->lock); mutex_lock(¶m->lock); + fault_param = rcu_dereference_check(param->fault_param, + lockdep_is_held(¶m->lock)); if (!fault_param) { ret = -ENODEV; goto unlock; @@ -454,8 +484,9 @@ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) list_for_each_entry_safe(iopf, next, &fault_param->partial, list) kfree(iopf); - param->fault_param = NULL; - kfree(fault_param); + /* dec the ref owned by iopf_queue_add_device() */ + rcu_assign_pointer(param->fault_param, NULL); + iopf_put_dev_fault_param(fault_param); unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 9de878e40413..b51995b4fe90 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -251,7 +251,7 @@ static void iommu_sva_handle_iopf(struct work_struct *work) static int iommu_sva_iopf_handler(struct iopf_group *group) { - struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + struct iommu_fault_param *fault_param = group->fault_param; INIT_WORK(&group->work, iommu_sva_handle_iopf); if (!queue_work(fault_param->queue->wq, &group->work)) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index fc912aed7886..1e9161ae95da 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -41,6 +41,7 @@ struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; struct iommu_dma_cookie; +struct iommu_fault_param; #define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ #define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ @@ -129,8 +130,9 @@ struct iopf_group { struct iopf_fault last_fault; struct list_head faults; struct work_struct work; - struct device *dev; struct iommu_domain *domain; + /* The device's fault data parameter. */ + struct iommu_fault_param *fault_param; }; /** @@ -679,6 +681,8 @@ struct iommu_device { /** * struct iommu_fault_param - per-device IOMMU fault data * @lock: protect pending faults list + * @users: user counter to manage the lifetime of the data + * @rcu: rcu head for kfree_rcu() * @dev: the device that owns this param * @queue: IOPF queue * @queue_list: index into queue->devices @@ -688,6 +692,8 @@ struct iommu_device { */ struct iommu_fault_param { struct mutex lock; + refcount_t users; + struct rcu_head rcu; struct device *dev; struct iopf_queue *queue; @@ -715,7 +721,7 @@ struct iommu_fault_param { */ struct dev_iommu { struct mutex lock; - struct iommu_fault_param *fault_param; + struct iommu_fault_param __rcu *fault_param; struct iommu_fwspec *fwspec; struct iommu_device *iommu_dev; void *priv; @@ -1543,7 +1549,6 @@ void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); void iopf_free_group(struct iopf_group *group); int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); -int iommu_page_response(struct device *dev, struct iommu_page_response *msg); int iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status); #else @@ -1588,12 +1593,6 @@ iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) return -ENODEV; } -static inline int -iommu_page_response(struct device *dev, struct iommu_page_response *msg) -{ - return -ENODEV; -} - static inline int iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status) { -- cgit v1.2.3 From 0095bf83554f8e7a681961656608101bdf40e9ef Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:24 +0800 Subject: iommu: Improve iopf_queue_remove_device() Convert iopf_queue_remove_device() to return void instead of an error code, as the return value is never used. This removal helper is designed to be never-failed, so there's no need for error handling. Ack all outstanding page requests from the device with the response code of IOMMU_PAGE_RESP_INVALID, indicating device should not attempt any retry. Add comments to this helper explaining the steps involved in removing a device from the iopf queue and disabling its PRI. The individual drivers are expected to be adjusted accordingly. Here we just define the expected behaviors of the individual iommu driver from the core's perspective. Suggested-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Tested-by: Yan Zhao Link: https://lore.kernel.org/r/20240212012227.119381-14-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 7 +----- drivers/iommu/io-pgfault.c | 57 +++++++++++++++++++++++++++++---------------- include/linux/iommu.h | 5 ++-- 3 files changed, 40 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 29a12f289e2e..a81a2be9b870 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4455,12 +4455,7 @@ static int intel_iommu_disable_iopf(struct device *dev) */ pci_disable_pri(to_pci_dev(dev)); info->pri_enabled = 0; - - /* - * With PRI disabled and outstanding PRQs drained, removing device - * from iopf queue should never fail. - */ - WARN_ON(iopf_queue_remove_device(iommu->iopf_queue, dev)); + iopf_queue_remove_device(iommu->iopf_queue, dev); return 0; } diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index ce7058892b59..ece09552e5cf 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -448,41 +448,60 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); * @queue: IOPF queue * @dev: device to remove * - * Caller makes sure that no more faults are reported for this device. + * Removing a device from an iopf_queue. It's recommended to follow these + * steps when removing a device: * - * Return: 0 on success and <0 on error. + * - Disable new PRI reception: Turn off PRI generation in the IOMMU hardware + * and flush any hardware page request queues. This should be done before + * calling into this helper. + * - Acknowledge all outstanding PRQs to the device: Respond to all outstanding + * page requests with IOMMU_PAGE_RESP_INVALID, indicating the device should + * not retry. This helper function handles this. + * - Disable PRI on the device: After calling this helper, the caller could + * then disable PRI on the device. + * + * Calling iopf_queue_remove_device() essentially disassociates the device. + * The fault_param might still exist, but iommu_page_response() will do + * nothing. The device fault parameter reference count has been properly + * passed from iommu_report_device_fault() to the fault handling work, and + * will eventually be released after iommu_page_response(). */ -int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - int ret = 0; struct iopf_fault *iopf, *next; + struct iommu_page_response resp; struct dev_iommu *param = dev->iommu; struct iommu_fault_param *fault_param; + const struct iommu_ops *ops = dev_iommu_ops(dev); mutex_lock(&queue->lock); mutex_lock(¶m->lock); fault_param = rcu_dereference_check(param->fault_param, lockdep_is_held(¶m->lock)); - if (!fault_param) { - ret = -ENODEV; - goto unlock; - } - if (fault_param->queue != queue) { - ret = -EINVAL; + if (WARN_ON(!fault_param || fault_param->queue != queue)) goto unlock; - } - if (!list_empty(&fault_param->faults)) { - ret = -EBUSY; - goto unlock; - } + mutex_lock(&fault_param->lock); + list_for_each_entry_safe(iopf, next, &fault_param->partial, list) + kfree(iopf); - list_del(&fault_param->queue_list); + list_for_each_entry_safe(iopf, next, &fault_param->faults, list) { + memset(&resp, 0, sizeof(struct iommu_page_response)); + resp.pasid = iopf->fault.prm.pasid; + resp.grpid = iopf->fault.prm.grpid; + resp.code = IOMMU_PAGE_RESP_INVALID; - /* Just in case some faults are still stuck */ - list_for_each_entry_safe(iopf, next, &fault_param->partial, list) + if (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID) + resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + + ops->page_response(dev, iopf, &resp); + list_del(&iopf->list); kfree(iopf); + } + mutex_unlock(&fault_param->lock); + + list_del(&fault_param->queue_list); /* dec the ref owned by iopf_queue_add_device() */ rcu_assign_pointer(param->fault_param, NULL); @@ -490,8 +509,6 @@ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); - - return ret; } EXPORT_SYMBOL_GPL(iopf_queue_remove_device); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1e9161ae95da..92dfd9b94577 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1542,7 +1542,7 @@ iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) #ifdef CONFIG_IOMMU_IOPF int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); -int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); +void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); int iopf_queue_flush_dev(struct device *dev); struct iopf_queue *iopf_queue_alloc(const char *name); void iopf_queue_free(struct iopf_queue *queue); @@ -1558,10 +1558,9 @@ iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) return -ENODEV; } -static inline int +static inline void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - return -ENODEV; } static inline int iopf_queue_flush_dev(struct device *dev) -- cgit v1.2.3 From 19911232713573a2ebea84a25bd4d71d024ed86b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:25 +0800 Subject: iommu: Track iopf group instead of last fault Previously, before a group of page faults was passed to the domain's iopf handler, the last page fault of the group was kept in the list of iommu_fault_param::faults. In the page fault response path, the group's last page fault was used to look up the list, and the page faults were responded to device only if there was a matched fault. The previous approach seems unnecessarily complex and not performance friendly. Put the page fault group itself to the outstanding fault list. It can be removed in the page fault response path or in the iopf_queue_remove_device() path. The pending list is protected by iommu_fault_param::lock. To allow checking for the group's presence in the list using list_empty(), the iopf group should be removed from the list with list_del_init(). IOMMU_PAGE_RESP_PASID_VALID is set in the code but not used anywhere. Remove it to make the code clean. IOMMU_PAGE_RESP_PASID_VALID is set in the response message indicating that the response message includes a valid PASID value. Actually, we should keep this hardware detail in the individual driver. When the page fault handling framework in IOMMU and IOMMUFD subsystems includes a valid PASID in the fault message, the response message should always contain the same PASID value. Individual drivers should be responsible for deciding whether to include the PASID in the messages they provide for the hardware. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Link: https://lore.kernel.org/r/20240212012227.119381-15-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 242 ++++++++++++++++----------------------------- include/linux/iommu.h | 6 +- 2 files changed, 86 insertions(+), 162 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index ece09552e5cf..05e49e2e6a52 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -78,12 +78,33 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, return domain; } +/* Non-last request of a group. Postpone until the last one. */ +static int report_partial_fault(struct iommu_fault_param *fault_param, + struct iommu_fault *fault) +{ + struct iopf_fault *iopf; + + iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); + if (!iopf) + return -ENOMEM; + + iopf->fault = *fault; + + mutex_lock(&fault_param->lock); + list_add(&iopf->list, &fault_param->partial); + mutex_unlock(&fault_param->lock); + + return 0; +} + /** - * iommu_handle_iopf - IO Page Fault handler - * @fault: fault event - * @iopf_param: the fault parameter of the device. + * iommu_report_device_fault() - Report fault event to device driver + * @dev: the device + * @evt: fault event data * - * Add a fault to the device workqueue, to be handled by mm. + * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ + * handler. When this function fails and the fault is recoverable, it is the + * caller's responsibility to complete the fault. * * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't @@ -118,34 +139,37 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, * * Return: 0 on success and <0 on error. */ -static int iommu_handle_iopf(struct iommu_fault *fault, - struct iommu_fault_param *iopf_param) +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - int ret; - struct iopf_group *group; - struct iommu_domain *domain; + struct iommu_fault *fault = &evt->fault; + struct iommu_fault_param *iopf_param; struct iopf_fault *iopf, *next; - struct device *dev = iopf_param->dev; - - lockdep_assert_held(&iopf_param->lock); + struct iommu_domain *domain; + struct iopf_group *group; + int ret; if (fault->type != IOMMU_FAULT_PAGE_REQ) - /* Not a recoverable page fault */ return -EOPNOTSUPP; - if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); - if (!iopf) - return -ENOMEM; - - iopf->fault = *fault; + iopf_param = iopf_get_dev_fault_param(dev); + if (!iopf_param) + return -ENODEV; - /* Non-last request of a group. Postpone until the last one */ - list_add(&iopf->list, &iopf_param->partial); + if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + ret = report_partial_fault(iopf_param, fault); + iopf_put_dev_fault_param(iopf_param); - return 0; + return ret; } + /* + * This is the last page fault of a group. Allocate an iopf group and + * pass it to domain's page fault handler. The group holds a reference + * count of the fault parameter. It will be released after response or + * error path of this function. If an error is returned, the caller + * will send a response to the hardware. We need to clean up before + * leaving, otherwise partial faults will be stuck. + */ domain = get_domain_for_iopf(dev, fault); if (!domain) { ret = -EINVAL; @@ -154,11 +178,6 @@ static int iommu_handle_iopf(struct iommu_fault *fault, group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) { - /* - * The caller will send a response to the hardware. But we do - * need to clean up before leaving, otherwise partial faults - * will be stuck. - */ ret = -ENOMEM; goto cleanup_partial; } @@ -166,145 +185,45 @@ static int iommu_handle_iopf(struct iommu_fault *fault, group->fault_param = iopf_param; group->last_fault.fault = *fault; INIT_LIST_HEAD(&group->faults); + INIT_LIST_HEAD(&group->pending_node); group->domain = domain; list_add(&group->last_fault.list, &group->faults); /* See if we have partial faults for this group */ + mutex_lock(&iopf_param->lock); list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { if (iopf->fault.prm.grpid == fault->prm.grpid) /* Insert *before* the last fault */ list_move(&iopf->list, &group->faults); } - + list_add(&group->pending_node, &iopf_param->faults); mutex_unlock(&iopf_param->lock); + ret = domain->iopf_handler(group); - mutex_lock(&iopf_param->lock); - if (ret) + if (ret) { + mutex_lock(&iopf_param->lock); + list_del_init(&group->pending_node); + mutex_unlock(&iopf_param->lock); iopf_free_group(group); + } return ret; + cleanup_partial: + mutex_lock(&iopf_param->lock); list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { if (iopf->fault.prm.grpid == fault->prm.grpid) { list_del(&iopf->list); kfree(iopf); } } - return ret; -} - -/** - * iommu_report_device_fault() - Report fault event to device driver - * @dev: the device - * @evt: fault event data - * - * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. - * - * Return 0 on success, or an error. - */ -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) -{ - bool last_prq = evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE); - struct iommu_fault_param *fault_param; - struct iopf_fault *evt_pending; - int ret; - - fault_param = iopf_get_dev_fault_param(dev); - if (!fault_param) - return -EINVAL; - - mutex_lock(&fault_param->lock); - if (last_prq) { - evt_pending = kmemdup(evt, sizeof(struct iopf_fault), - GFP_KERNEL); - if (!evt_pending) { - ret = -ENOMEM; - goto err_unlock; - } - list_add_tail(&evt_pending->list, &fault_param->faults); - } - - ret = iommu_handle_iopf(&evt->fault, fault_param); - if (ret) - goto err_free; - - mutex_unlock(&fault_param->lock); - /* The reference count of fault_param is now held by iopf_group. */ - if (!last_prq) - iopf_put_dev_fault_param(fault_param); - - return 0; -err_free: - if (last_prq) { - list_del(&evt_pending->list); - kfree(evt_pending); - } -err_unlock: - mutex_unlock(&fault_param->lock); - iopf_put_dev_fault_param(fault_param); + mutex_unlock(&iopf_param->lock); + iopf_put_dev_fault_param(iopf_param); return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); -static int iommu_page_response(struct iopf_group *group, - struct iommu_page_response *msg) -{ - bool needs_pasid; - int ret = -EINVAL; - struct iopf_fault *evt; - struct iommu_fault_page_request *prm; - struct device *dev = group->fault_param->dev; - const struct iommu_ops *ops = dev_iommu_ops(dev); - bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - struct iommu_fault_param *fault_param = group->fault_param; - - /* Only send response if there is a fault report pending */ - mutex_lock(&fault_param->lock); - if (list_empty(&fault_param->faults)) { - dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); - goto done_unlock; - } - /* - * Check if we have a matching page request pending to respond, - * otherwise return -EINVAL - */ - list_for_each_entry(evt, &fault_param->faults, list) { - prm = &evt->fault.prm; - if (prm->grpid != msg->grpid) - continue; - - /* - * If the PASID is required, the corresponding request is - * matched using the group ID, the PASID valid bit and the PASID - * value. Otherwise only the group ID matches request and - * response. - */ - needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; - if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) - continue; - - if (!needs_pasid && has_pasid) { - /* No big deal, just clear it. */ - msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; - msg->pasid = 0; - } - - ret = ops->page_response(dev, evt, msg); - list_del(&evt->list); - kfree(evt); - break; - } - -done_unlock: - mutex_unlock(&fault_param->lock); - - return ret; -} - /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed * @dev: the endpoint whose faults need to be flushed. @@ -346,18 +265,26 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); int iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status) { + struct iommu_fault_param *fault_param = group->fault_param; struct iopf_fault *iopf = &group->last_fault; + struct device *dev = group->fault_param->dev; + const struct iommu_ops *ops = dev_iommu_ops(dev); struct iommu_page_response resp = { .pasid = iopf->fault.prm.pasid, .grpid = iopf->fault.prm.grpid, .code = status, }; + int ret = -EINVAL; - if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && - (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + /* Only send response if there is a fault report pending */ + mutex_lock(&fault_param->lock); + if (!list_empty(&group->pending_node)) { + ret = ops->page_response(dev, &group->last_fault, &resp); + list_del_init(&group->pending_node); + } + mutex_unlock(&fault_param->lock); - return iommu_page_response(group, &resp); + return ret; } EXPORT_SYMBOL_GPL(iopf_group_response); @@ -468,8 +395,9 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); */ void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - struct iopf_fault *iopf, *next; - struct iommu_page_response resp; + struct iopf_fault *partial_iopf; + struct iopf_fault *next; + struct iopf_group *group, *temp; struct dev_iommu *param = dev->iommu; struct iommu_fault_param *fault_param; const struct iommu_ops *ops = dev_iommu_ops(dev); @@ -483,21 +411,19 @@ void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) goto unlock; mutex_lock(&fault_param->lock); - list_for_each_entry_safe(iopf, next, &fault_param->partial, list) - kfree(iopf); - - list_for_each_entry_safe(iopf, next, &fault_param->faults, list) { - memset(&resp, 0, sizeof(struct iommu_page_response)); - resp.pasid = iopf->fault.prm.pasid; - resp.grpid = iopf->fault.prm.grpid; - resp.code = IOMMU_PAGE_RESP_INVALID; + list_for_each_entry_safe(partial_iopf, next, &fault_param->partial, list) + kfree(partial_iopf); - if (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + list_for_each_entry_safe(group, temp, &fault_param->faults, pending_node) { + struct iopf_fault *iopf = &group->last_fault; + struct iommu_page_response resp = { + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = IOMMU_PAGE_RESP_INVALID + }; ops->page_response(dev, iopf, &resp); - list_del(&iopf->list); - kfree(iopf); + list_del_init(&group->pending_node); } mutex_unlock(&fault_param->lock); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 92dfd9b94577..f8ed1cc7212e 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -106,15 +106,11 @@ enum iommu_page_response_code { /** * struct iommu_page_response - Generic page response information - * @flags: encodes whether the corresponding fields are valid - * (IOMMU_FAULT_PAGE_RESPONSE_* values) * @pasid: Process Address Space ID * @grpid: Page Request Group Index * @code: response code from &enum iommu_page_response_code */ struct iommu_page_response { -#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - u32 flags; u32 pasid; u32 grpid; u32 code; @@ -129,6 +125,8 @@ struct iopf_fault { struct iopf_group { struct iopf_fault last_fault; struct list_head faults; + /* list node for iommu_fault_param::faults */ + struct list_head pending_node; struct work_struct work; struct iommu_domain *domain; /* The device's fault data parameter. */ -- cgit v1.2.3 From b554e396e51ce3d378a560666f85c6836a8323fd Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:26 +0800 Subject: iommu: Make iopf_group_response() return void The iopf_group_response() should return void, as nothing can do anything with the failure. This implies that ops->page_response() must also return void; this is consistent with what the drivers do. The failure paths, which are all integrity validations of the fault, should be WARN_ON'd, not return codes. If the iommu core fails to enqueue the fault, it should respond the fault directly by calling ops->page_response() instead of returning an error number and relying on the iommu drivers to do so. Consolidate the error fault handling code in the core. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240212012227.119381-16-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 50 ++++------- drivers/iommu/intel/iommu.h | 4 +- drivers/iommu/intel/svm.c | 18 +--- drivers/iommu/io-pgfault.c | 132 +++++++++++++++------------- include/linux/iommu.h | 14 ++- 5 files changed, 98 insertions(+), 120 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4e93e845458c..42eb59cb99f4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -920,31 +920,29 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); } -static int arm_smmu_page_response(struct device *dev, - struct iopf_fault *unused, - struct iommu_page_response *resp) +static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused, + struct iommu_page_response *resp) { struct arm_smmu_cmdq_ent cmd = {0}; struct arm_smmu_master *master = dev_iommu_priv_get(dev); int sid = master->streams[0].id; - if (master->stall_enabled) { - cmd.opcode = CMDQ_OP_RESUME; - cmd.resume.sid = sid; - cmd.resume.stag = resp->grpid; - switch (resp->code) { - case IOMMU_PAGE_RESP_INVALID: - case IOMMU_PAGE_RESP_FAILURE: - cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; - break; - case IOMMU_PAGE_RESP_SUCCESS: - cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; - break; - default: - return -EINVAL; - } - } else { - return -ENODEV; + if (WARN_ON(!master->stall_enabled)) + return; + + cmd.opcode = CMDQ_OP_RESUME; + cmd.resume.sid = sid; + cmd.resume.stag = resp->grpid; + switch (resp->code) { + case IOMMU_PAGE_RESP_INVALID: + case IOMMU_PAGE_RESP_FAILURE: + cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; + break; + case IOMMU_PAGE_RESP_SUCCESS: + cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; + break; + default: + break; } arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); @@ -954,8 +952,6 @@ static int arm_smmu_page_response(struct device *dev, * terminated... at some point in the future. PRI_RESP is fire and * forget. */ - - return 0; } /* Context descriptor manipulation functions */ @@ -1516,16 +1512,6 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) } ret = iommu_report_device_fault(master->dev, &fault_evt); - if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) { - /* Nobody cared, abort the access */ - struct iommu_page_response resp = { - .pasid = flt->prm.pasid, - .grpid = flt->prm.grpid, - .code = IOMMU_PAGE_RESP_FAILURE, - }; - arm_smmu_page_response(master->dev, &fault_evt, &resp); - } - out_unlock: mutex_unlock(&smmu->streams_mutex); return ret; diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 696d95293a69..cf9a28c7fab8 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1079,8 +1079,8 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); -int intel_svm_page_response(struct device *dev, struct iopf_fault *evt, - struct iommu_page_response *msg); +void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); void intel_drain_pasid_prq(struct device *dev, u32 pasid); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e1cbcb9515f0..2f8716636dbb 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -740,9 +740,8 @@ prq_advance: return IRQ_RETVAL(handled); } -int intel_svm_page_response(struct device *dev, - struct iopf_fault *evt, - struct iommu_page_response *msg) +void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg) { struct device_domain_info *info = dev_iommu_priv_get(dev); struct intel_iommu *iommu = info->iommu; @@ -751,7 +750,6 @@ int intel_svm_page_response(struct device *dev, bool private_present; bool pasid_present; bool last_page; - int ret = 0; u16 sid; prm = &evt->fault.prm; @@ -760,16 +758,6 @@ int intel_svm_page_response(struct device *dev, private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - if (!pasid_present) { - ret = -EINVAL; - goto out; - } - - if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { - ret = -EINVAL; - goto out; - } - /* * Per VT-d spec. v3.0 ch7.7, system software must respond * with page group response if private data is present (PDP) @@ -798,8 +786,6 @@ int intel_svm_page_response(struct device *dev, qi_submit_sync(iommu, &desc, 1, 0); } -out: - return ret; } static int intel_svm_set_dev_pasid(struct iommu_domain *domain, diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 05e49e2e6a52..6a325bff8164 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -39,7 +39,7 @@ static void iopf_put_dev_fault_param(struct iommu_fault_param *fault_param) kfree_rcu(fault_param, rcu); } -void iopf_free_group(struct iopf_group *group) +static void __iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -50,6 +50,11 @@ void iopf_free_group(struct iopf_group *group) /* Pair with iommu_report_device_fault(). */ iopf_put_dev_fault_param(group->fault_param); +} + +void iopf_free_group(struct iopf_group *group) +{ + __iopf_free_group(group); kfree(group); } EXPORT_SYMBOL_GPL(iopf_free_group); @@ -97,14 +102,49 @@ static int report_partial_fault(struct iommu_fault_param *fault_param, return 0; } +static struct iopf_group *iopf_group_alloc(struct iommu_fault_param *iopf_param, + struct iopf_fault *evt, + struct iopf_group *abort_group) +{ + struct iopf_fault *iopf, *next; + struct iopf_group *group; + + group = kzalloc(sizeof(*group), GFP_KERNEL); + if (!group) { + /* + * We always need to construct the group as we need it to abort + * the request at the driver if it can't be handled. + */ + group = abort_group; + } + + group->fault_param = iopf_param; + group->last_fault.fault = evt->fault; + INIT_LIST_HEAD(&group->faults); + INIT_LIST_HEAD(&group->pending_node); + list_add(&group->last_fault.list, &group->faults); + + /* See if we have partial faults for this group */ + mutex_lock(&iopf_param->lock); + list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { + if (iopf->fault.prm.grpid == evt->fault.prm.grpid) + /* Insert *before* the last fault */ + list_move(&iopf->list, &group->faults); + } + list_add(&group->pending_node, &iopf_param->faults); + mutex_unlock(&iopf_param->lock); + + return group; +} + /** * iommu_report_device_fault() - Report fault event to device driver * @dev: the device * @evt: fault event data * * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. + * handler. If this function fails then ops->page_response() was called to + * complete evt if required. * * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't @@ -143,22 +183,18 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { struct iommu_fault *fault = &evt->fault; struct iommu_fault_param *iopf_param; - struct iopf_fault *iopf, *next; - struct iommu_domain *domain; + struct iopf_group abort_group = {}; struct iopf_group *group; int ret; - if (fault->type != IOMMU_FAULT_PAGE_REQ) - return -EOPNOTSUPP; - iopf_param = iopf_get_dev_fault_param(dev); - if (!iopf_param) + if (WARN_ON(!iopf_param)) return -ENODEV; if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { ret = report_partial_fault(iopf_param, fault); iopf_put_dev_fault_param(iopf_param); - + /* A request that is not the last does not need to be ack'd */ return ret; } @@ -170,56 +206,33 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) * will send a response to the hardware. We need to clean up before * leaving, otherwise partial faults will be stuck. */ - domain = get_domain_for_iopf(dev, fault); - if (!domain) { - ret = -EINVAL; - goto cleanup_partial; - } - - group = kzalloc(sizeof(*group), GFP_KERNEL); - if (!group) { + group = iopf_group_alloc(iopf_param, evt, &abort_group); + if (group == &abort_group) { ret = -ENOMEM; - goto cleanup_partial; + goto err_abort; } - group->fault_param = iopf_param; - group->last_fault.fault = *fault; - INIT_LIST_HEAD(&group->faults); - INIT_LIST_HEAD(&group->pending_node); - group->domain = domain; - list_add(&group->last_fault.list, &group->faults); - - /* See if we have partial faults for this group */ - mutex_lock(&iopf_param->lock); - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { - if (iopf->fault.prm.grpid == fault->prm.grpid) - /* Insert *before* the last fault */ - list_move(&iopf->list, &group->faults); - } - list_add(&group->pending_node, &iopf_param->faults); - mutex_unlock(&iopf_param->lock); - - ret = domain->iopf_handler(group); - if (ret) { - mutex_lock(&iopf_param->lock); - list_del_init(&group->pending_node); - mutex_unlock(&iopf_param->lock); - iopf_free_group(group); + group->domain = get_domain_for_iopf(dev, fault); + if (!group->domain) { + ret = -EINVAL; + goto err_abort; } - return ret; - -cleanup_partial: - mutex_lock(&iopf_param->lock); - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { - if (iopf->fault.prm.grpid == fault->prm.grpid) { - list_del(&iopf->list); - kfree(iopf); - } - } - mutex_unlock(&iopf_param->lock); - iopf_put_dev_fault_param(iopf_param); + /* + * On success iopf_handler must call iopf_group_response() and + * iopf_free_group() + */ + ret = group->domain->iopf_handler(group); + if (ret) + goto err_abort; + return 0; +err_abort: + iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE); + if (group == &abort_group) + __iopf_free_group(group); + else + iopf_free_group(group); return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); @@ -259,11 +272,9 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); * iopf_group_response - Respond a group of page faults * @group: the group of faults with the same group id * @status: the response code - * - * Return 0 on success and <0 on error. */ -int iopf_group_response(struct iopf_group *group, - enum iommu_page_response_code status) +void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) { struct iommu_fault_param *fault_param = group->fault_param; struct iopf_fault *iopf = &group->last_fault; @@ -274,17 +285,14 @@ int iopf_group_response(struct iopf_group *group, .grpid = iopf->fault.prm.grpid, .code = status, }; - int ret = -EINVAL; /* Only send response if there is a fault report pending */ mutex_lock(&fault_param->lock); if (!list_empty(&group->pending_node)) { - ret = ops->page_response(dev, &group->last_fault, &resp); + ops->page_response(dev, &group->last_fault, &resp); list_del_init(&group->pending_node); } mutex_unlock(&fault_param->lock); - - return ret; } EXPORT_SYMBOL_GPL(iopf_group_response); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index f8ed1cc7212e..f632775414a5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -574,9 +574,8 @@ struct iommu_ops { int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); - int (*page_response)(struct device *dev, - struct iopf_fault *evt, - struct iommu_page_response *msg); + void (*page_response)(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg); int (*def_domain_type)(struct device *dev); void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid); @@ -1547,8 +1546,8 @@ void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); void iopf_free_group(struct iopf_group *group); int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); -int iopf_group_response(struct iopf_group *group, - enum iommu_page_response_code status); +void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status); #else static inline int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) @@ -1590,10 +1589,9 @@ iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) return -ENODEV; } -static inline int iopf_group_response(struct iopf_group *group, - enum iommu_page_response_code status) +static inline void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) { - return -ENODEV; } #endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ -- cgit v1.2.3 From 3dfa64aecbafc288216b2790438d395add192c30 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:27 +0800 Subject: iommu: Make iommu_report_device_fault() return void As the iommu_report_device_fault() has been converted to auto-respond a page fault if it fails to enqueue it, there's no need to return a code in any case. Make it return void. Suggested-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240212012227.119381-17-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 ++-- drivers/iommu/intel/svm.c | 19 +++++++------------ drivers/iommu/io-pgfault.c | 25 ++++++++----------------- include/linux/iommu.h | 5 ++--- 4 files changed, 19 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 42eb59cb99f4..02580364acda 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1455,7 +1455,7 @@ arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) /* IRQ and event handlers */ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) { - int ret; + int ret = 0; u32 perm = 0; struct arm_smmu_master *master; bool ssid_valid = evt[0] & EVTQ_0_SSV; @@ -1511,7 +1511,7 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) goto out_unlock; } - ret = iommu_report_device_fault(master->dev, &fault_evt); + iommu_report_device_fault(master->dev, &fault_evt); out_unlock: mutex_unlock(&smmu->streams_mutex); return ret; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 2f8716636dbb..b644d57da841 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -561,14 +561,11 @@ static int prq_to_iommu_prot(struct page_req_dsc *req) return prot; } -static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, - struct page_req_dsc *desc) +static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, + struct page_req_dsc *desc) { struct iopf_fault event = { }; - if (!dev || !dev_is_pci(dev)) - return -ENODEV; - /* Fill in event data for device specific processing */ event.fault.type = IOMMU_FAULT_PAGE_REQ; event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; @@ -601,7 +598,7 @@ static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); } - return iommu_report_device_fault(dev, &event); + iommu_report_device_fault(dev, &event); } static void handle_bad_prq_event(struct intel_iommu *iommu, @@ -704,12 +701,10 @@ bad_req: if (!pdev) goto bad_req; - if (intel_svm_prq_report(iommu, &pdev->dev, req)) - handle_bad_prq_event(iommu, req, QI_RESP_INVALID); - else - trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, - req->priv_data[0], req->priv_data[1], - iommu->prq_seq_number++); + intel_svm_prq_report(iommu, &pdev->dev, req); + trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, + req->priv_data[0], req->priv_data[1], + iommu->prq_seq_number++); pci_dev_put(pdev); prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 6a325bff8164..06d78fcc79fd 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -176,26 +176,22 @@ static struct iopf_group *iopf_group_alloc(struct iommu_fault_param *iopf_param, * freed after the device has stopped generating page faults (or the iommu * hardware has been set to block the page faults) and the pending page faults * have been flushed. - * - * Return: 0 on success and <0 on error. */ -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) +void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { struct iommu_fault *fault = &evt->fault; struct iommu_fault_param *iopf_param; struct iopf_group abort_group = {}; struct iopf_group *group; - int ret; iopf_param = iopf_get_dev_fault_param(dev); if (WARN_ON(!iopf_param)) - return -ENODEV; + return; if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - ret = report_partial_fault(iopf_param, fault); + report_partial_fault(iopf_param, fault); iopf_put_dev_fault_param(iopf_param); /* A request that is not the last does not need to be ack'd */ - return ret; } /* @@ -207,25 +203,21 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) * leaving, otherwise partial faults will be stuck. */ group = iopf_group_alloc(iopf_param, evt, &abort_group); - if (group == &abort_group) { - ret = -ENOMEM; + if (group == &abort_group) goto err_abort; - } group->domain = get_domain_for_iopf(dev, fault); - if (!group->domain) { - ret = -EINVAL; + if (!group->domain) goto err_abort; - } /* * On success iopf_handler must call iopf_group_response() and * iopf_free_group() */ - ret = group->domain->iopf_handler(group); - if (ret) + if (group->domain->iopf_handler(group)) goto err_abort; - return 0; + + return; err_abort: iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE); @@ -233,7 +225,6 @@ err_abort: __iopf_free_group(group); else iopf_free_group(group); - return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index f632775414a5..7cc56cfe98dd 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1545,7 +1545,7 @@ struct iopf_queue *iopf_queue_alloc(const char *name); void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); void iopf_free_group(struct iopf_group *group); -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); +void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); void iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status); #else @@ -1583,10 +1583,9 @@ static inline void iopf_free_group(struct iopf_group *group) { } -static inline int +static inline void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - return -ENODEV; } static inline void iopf_group_response(struct iopf_group *group, -- cgit v1.2.3 From e01c9797c0ebb307c9bb196c677f6e571335773e Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Fri, 16 Feb 2024 14:45:14 +0100 Subject: PCI: endpoint: Clean up hardware description for BARs The hardware description for BARs is scattered in many different variables in pci_epc_features. Some of these things are mutually exclusive, so it can create confusion over which variable that has precedence over another. Improve the situation by creating a struct pci_epc_bar_desc, and a new enum pci_epc_bar_type, and convert the endpoint controller drivers to use this more well defined format. Additionally, some endpoint controller drivers mark the BAR succeeding a "64-bit only BAR" as reserved, while some do not. By definition, a 64-bit BAR uses the succeeding BAR for the upper 32-bits, so an EPF driver cannot use a BAR succeeding a 64-bit BAR. Ensure that all endpoint controller drivers are uniform, and actually describe a reserved BAR as reserved. Signed-off-by: Niklas Cassel Reviewed-by: Kishon Vijay Abraham I Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20240216134524.1142149-2-cassel@kernel.org Signed-off-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-imx6.c | 3 ++- drivers/pci/controller/dwc/pci-keystone.c | 12 ++++----- drivers/pci/controller/dwc/pci-layerscape-ep.c | 5 +++- drivers/pci/controller/dwc/pcie-keembay.c | 8 ++++-- drivers/pci/controller/dwc/pcie-rcar-gen4.c | 4 ++- drivers/pci/controller/dwc/pcie-tegra194.c | 10 +++++--- drivers/pci/controller/dwc/pcie-uniphier-ep.c | 15 +++++++++--- drivers/pci/controller/pcie-rcar-ep.c | 14 +++++++---- drivers/pci/endpoint/functions/pci-epf-ntb.c | 4 +-- drivers/pci/endpoint/functions/pci-epf-test.c | 8 +++--- drivers/pci/endpoint/functions/pci-epf-vntb.c | 2 +- drivers/pci/endpoint/pci-epc-core.c | 32 ++++++++++++------------ drivers/pci/endpoint/pci-epf-core.c | 15 ++++++------ include/linux/pci-epc.h | 34 +++++++++++++++++++++----- 14 files changed, 108 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index dc2c036ab28c..47a9a96484ed 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1081,7 +1081,8 @@ static const struct pci_epc_features imx8m_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = false, - .reserved_bar = 1 << BAR_1 | 1 << BAR_3, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, .align = SZ_64K, }; diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index c0c62533a3f1..b2b93b4fa82d 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -924,12 +924,12 @@ static const struct pci_epc_features ks_pcie_am654_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = true, - .reserved_bar = 1 << BAR_0 | 1 << BAR_1, - .bar_fixed_64bit = 1 << BAR_0, - .bar_fixed_size[2] = SZ_1M, - .bar_fixed_size[3] = SZ_64K, - .bar_fixed_size[4] = 256, - .bar_fixed_size[5] = SZ_1M, + .bar[BAR_0] = { .type = BAR_RESERVED, .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .type = BAR_FIXED, .fixed_size = SZ_1M, }, + .bar[BAR_3] = { .type = BAR_FIXED, .fixed_size = SZ_64K, }, + .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256, }, + .bar[BAR_5] = { .type = BAR_FIXED, .fixed_size = SZ_1M, }, .align = SZ_1M, }; diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 2e398494e7c0..1f6ee1460ec2 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -250,7 +250,10 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) pci->dev = dev; pci->ops = pcie->drvdata->dw_pcie_ops; - ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4); + ls_epc->bar[BAR_2].only_64bit = true; + ls_epc->bar[BAR_3].type = BAR_RESERVED; + ls_epc->bar[BAR_4].only_64bit = true; + ls_epc->bar[BAR_5].type = BAR_RESERVED; ls_epc->linkup_notifier = true; pcie->pci = pci; diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c index 208d3b0ba196..5e8e54f597dd 100644 --- a/drivers/pci/controller/dwc/pcie-keembay.c +++ b/drivers/pci/controller/dwc/pcie-keembay.c @@ -312,8 +312,12 @@ static const struct pci_epc_features keembay_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = true, - .reserved_bar = BIT(BAR_1) | BIT(BAR_3) | BIT(BAR_5), - .bar_fixed_64bit = BIT(BAR_0) | BIT(BAR_2) | BIT(BAR_4), + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .only_64bit = true, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_4] = { .only_64bit = true, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, .align = SZ_16K, }; diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index e9166619b1f9..0be760ed420b 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -383,7 +383,9 @@ static const struct pci_epc_features rcar_gen4_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = false, - .reserved_bar = 1 << BAR_1 | 1 << BAR_3 | 1 << BAR_5, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, .align = SZ_1M, }; diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 7afa9e9aabe2..1f7b662cb8e1 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -2007,9 +2007,13 @@ static const struct pci_epc_features tegra_pcie_epc_features = { .core_init_notifier = true, .msi_capable = false, .msix_capable = false, - .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5, - .bar_fixed_64bit = 1 << BAR_0, - .bar_fixed_size[0] = SZ_1M, + .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M, + .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .type = BAR_RESERVED, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_4] = { .type = BAR_RESERVED, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, }; static const struct pci_epc_features* diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c index 3fced0d3e851..265f65fc673f 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier-ep.c +++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c @@ -411,8 +411,12 @@ static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = { .msi_capable = true, .msix_capable = false, .align = 1 << 16, - .bar_fixed_64bit = BIT(BAR_0) | BIT(BAR_2) | BIT(BAR_4), - .reserved_bar = BIT(BAR_4), + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .only_64bit = true, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_4] = { .type = BAR_RESERVED, .only_64bit = true, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, }, }; @@ -425,7 +429,12 @@ static const struct uniphier_pcie_ep_soc_data uniphier_nx1_data = { .msi_capable = true, .msix_capable = false, .align = 1 << 12, - .bar_fixed_64bit = BIT(BAR_0) | BIT(BAR_2) | BIT(BAR_4), + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .only_64bit = true, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_4] = { .only_64bit = true, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, }, }; diff --git a/drivers/pci/controller/pcie-rcar-ep.c b/drivers/pci/controller/pcie-rcar-ep.c index e6909271def7..05967c6c0b42 100644 --- a/drivers/pci/controller/pcie-rcar-ep.c +++ b/drivers/pci/controller/pcie-rcar-ep.c @@ -440,11 +440,15 @@ static const struct pci_epc_features rcar_pcie_epc_features = { .msi_capable = true, .msix_capable = false, /* use 64-bit BARs so mark BAR[1,3,5] as reserved */ - .reserved_bar = 1 << BAR_1 | 1 << BAR_3 | 1 << BAR_5, - .bar_fixed_64bit = 1 << BAR_0 | 1 << BAR_2 | 1 << BAR_4, - .bar_fixed_size[0] = 128, - .bar_fixed_size[2] = 256, - .bar_fixed_size[4] = 256, + .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = 128, + .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .type = BAR_FIXED, .fixed_size = 256, + .only_64bit = true, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256, + .only_64bit = true, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, }; static const struct pci_epc_features* diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index 43cd309ce22f..e01a98e74d21 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -1012,13 +1012,13 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, epc_features = ntb_epc->epc_features; barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; - size = epc_features->bar_fixed_size[barno]; + size = epc_features->bar[barno].fixed_size; align = epc_features->align; peer_ntb_epc = ntb->epc[!type]; peer_epc_features = peer_ntb_epc->epc_features; peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; - peer_size = peer_epc_features->bar_fixed_size[peer_barno]; + peer_size = peer_epc_features->bar[peer_barno].fixed_size; /* Check if epc_features is populated incorrectly */ if ((!IS_ALIGNED(size, align))) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 981894e40681..cd4ffb39dcdc 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -729,7 +729,7 @@ static int pci_epf_test_set_bar(struct pci_epf *epf) */ add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1; - if (!!(epc_features->reserved_bar & (1 << bar))) + if (epc_features->bar[bar].type == BAR_RESERVED) continue; ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, @@ -856,7 +856,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) if (bar == test_reg_bar) continue; - if (!!(epc_features->reserved_bar & (1 << bar))) + if (epc_features->bar[bar].type == BAR_RESERVED) continue; base = pci_epf_alloc_space(epf, bar_size[bar], bar, @@ -874,13 +874,11 @@ static void pci_epf_configure_bar(struct pci_epf *epf, const struct pci_epc_features *epc_features) { struct pci_epf_bar *epf_bar; - bool bar_fixed_64bit; int i; for (i = 0; i < PCI_STD_NUM_BARS; i++) { epf_bar = &epf->bar[i]; - bar_fixed_64bit = !!(epc_features->bar_fixed_64bit & (1 << i)); - if (bar_fixed_64bit) + if (epc_features->bar[i].only_64bit) epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64; } } diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 0675929fc529..8e779eecd62d 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -422,7 +422,7 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb) epf->func_no, epf->vfunc_no); barno = ntb->epf_ntb_bar[BAR_CONFIG]; - size = epc_features->bar_fixed_size[barno]; + size = epc_features->bar[barno].fixed_size; align = epc_features->align; if ((!IS_ALIGNED(size, align))) diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index dcd4e66430c1..7fe8f4336765 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -87,7 +87,7 @@ EXPORT_SYMBOL_GPL(pci_epc_get); * @epc_features: pci_epc_features structure that holds the reserved bar bitmap * * Invoke to get the first unreserved BAR that can be used by the endpoint - * function. For any incorrect value in reserved_bar return '0'. + * function. */ enum pci_barno pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features) @@ -102,32 +102,34 @@ EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar); * @bar: the starting BAR number from where unreserved BAR should be searched * * Invoke to get the next unreserved BAR starting from @bar that can be used - * for endpoint function. For any incorrect value in reserved_bar return '0'. + * for endpoint function. */ enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features *epc_features, enum pci_barno bar) { - unsigned long free_bar; + int i; if (!epc_features) return BAR_0; /* If 'bar - 1' is a 64-bit BAR, move to the next BAR */ - if ((epc_features->bar_fixed_64bit << 1) & 1 << bar) + if (bar > 0 && epc_features->bar[bar - 1].only_64bit) bar++; - /* Find if the reserved BAR is also a 64-bit BAR */ - free_bar = epc_features->reserved_bar & epc_features->bar_fixed_64bit; - - /* Set the adjacent bit if the reserved BAR is also a 64-bit BAR */ - free_bar <<= 1; - free_bar |= epc_features->reserved_bar; - - free_bar = find_next_zero_bit(&free_bar, 6, bar); - if (free_bar > 5) - return NO_BAR; + for (i = bar; i < PCI_STD_NUM_BARS; i++) { + /* If the BAR is not reserved, return it. */ + if (epc_features->bar[i].type != BAR_RESERVED) + return i; + + /* + * If the BAR is reserved, and marked as 64-bit only, then the + * succeeding BAR is also reserved. + */ + if (epc_features->bar[i].only_64bit) + i++; + } - return free_bar; + return NO_BAR; } EXPORT_SYMBOL_GPL(pci_epc_get_next_free_bar); diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index f2b4d34454c4..0a28a0b0911b 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -260,7 +260,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, const struct pci_epc_features *epc_features, enum pci_epc_interface_type type) { - u64 bar_fixed_size = epc_features->bar_fixed_size[bar]; + u64 bar_fixed_size = epc_features->bar[bar].fixed_size; size_t align = epc_features->align; struct pci_epf_bar *epf_bar; dma_addr_t phys_addr; @@ -271,13 +271,14 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, if (size < 128) size = 128; - if (bar_fixed_size && size > bar_fixed_size) { - dev_err(&epf->dev, "requested BAR size is larger than fixed size\n"); - return NULL; - } - - if (bar_fixed_size) + if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) { + if (size > bar_fixed_size) { + dev_err(&epf->dev, + "requested BAR size is larger than fixed size\n"); + return NULL; + } size = bar_fixed_size; + } if (align) size = ALIGN(size, align); diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 40ea18f5aa02..4ccb4f4f3883 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -145,6 +145,32 @@ struct pci_epc { unsigned long function_num_map; }; +/** + * @BAR_PROGRAMMABLE: The BAR mask can be configured by the EPC. + * @BAR_FIXED: The BAR mask is fixed by the hardware. + * @BAR_RESERVED: The BAR should not be touched by an EPF driver. + */ +enum pci_epc_bar_type { + BAR_PROGRAMMABLE = 0, + BAR_FIXED, + BAR_RESERVED, +}; + +/** + * struct pci_epc_bar_desc - hardware description for a BAR + * @type: the type of the BAR + * @fixed_size: the fixed size, only applicable if type is BAR_FIXED_MASK. + * @only_64bit: if true, an EPF driver is not allowed to choose if this BAR + * should be configured as 32-bit or 64-bit, the EPF driver must + * configure this BAR as 64-bit. Additionally, the BAR succeeding + * this BAR must be set to type BAR_RESERVED. + */ +struct pci_epc_bar_desc { + enum pci_epc_bar_type type; + u64 fixed_size; + bool only_64bit; +}; + /** * struct pci_epc_features - features supported by a EPC device per function * @linkup_notifier: indicate if the EPC device can notify EPF driver on link up @@ -152,9 +178,7 @@ struct pci_epc { * for initialization * @msi_capable: indicate if the endpoint function has MSI capability * @msix_capable: indicate if the endpoint function has MSI-X capability - * @reserved_bar: bitmap to indicate reserved BAR unavailable to function driver - * @bar_fixed_64bit: bitmap to indicate fixed 64bit BARs - * @bar_fixed_size: Array specifying the size supported by each BAR + * @bar: array specifying the hardware description for each BAR * @align: alignment size required for BAR buffer allocation */ struct pci_epc_features { @@ -162,9 +186,7 @@ struct pci_epc_features { unsigned int core_init_notifier : 1; unsigned int msi_capable : 1; unsigned int msix_capable : 1; - u8 reserved_bar; - u8 bar_fixed_64bit; - u64 bar_fixed_size[PCI_STD_NUM_BARS]; + struct pci_epc_bar_desc bar[PCI_STD_NUM_BARS]; size_t align; }; -- cgit v1.2.3 From 9266514689fe6476423209ee40168db53134101d Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Fri, 16 Feb 2024 14:45:15 +0100 Subject: PCI: endpoint: Drop only_64bit on reserved BARs The definition of a reserved BAR is that EPF drivers should not touch them. The definition of only_64bit is that the EPF driver must configure this BAR as 64-bit. (An EPF driver is not allowed to choose if this BAR should be configured as 32-bit or 64-bit.) Thus, it does not make sense to put only_64bit of a BAR that EPF drivers are not allow to touch. Drop the only_64bit property from hardware descriptions that are of type reserved BAR. Signed-off-by: Niklas Cassel Reviewed-by: Kishon Vijay Abraham I Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20240216134524.1142149-3-cassel@kernel.org Signed-off-by: Manivannan Sadhasivam --- drivers/pci/controller/dwc/pci-keystone.c | 2 +- drivers/pci/controller/dwc/pcie-uniphier-ep.c | 2 +- drivers/pci/endpoint/pci-epc-core.c | 7 ------- include/linux/pci-epc.h | 5 +++++ 4 files changed, 7 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index b2b93b4fa82d..844de4418724 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -924,7 +924,7 @@ static const struct pci_epc_features ks_pcie_am654_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = true, - .bar[BAR_0] = { .type = BAR_RESERVED, .only_64bit = true, }, + .bar[BAR_0] = { .type = BAR_RESERVED, }, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_2] = { .type = BAR_FIXED, .fixed_size = SZ_1M, }, .bar[BAR_3] = { .type = BAR_FIXED, .fixed_size = SZ_64K, }, diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c index 265f65fc673f..639bc2e12476 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier-ep.c +++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c @@ -415,7 +415,7 @@ static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = { .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_2] = { .only_64bit = true, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, - .bar[BAR_4] = { .type = BAR_RESERVED, .only_64bit = true, }, + .bar[BAR_4] = { .type = BAR_RESERVED, }, .bar[BAR_5] = { .type = BAR_RESERVED, }, }, }; diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 7fe8f4336765..da3fc0795b0b 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -120,13 +120,6 @@ enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features /* If the BAR is not reserved, return it. */ if (epc_features->bar[i].type != BAR_RESERVED) return i; - - /* - * If the BAR is reserved, and marked as 64-bit only, then the - * succeeding BAR is also reserved. - */ - if (epc_features->bar[i].only_64bit) - i++; } return NO_BAR; diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 4ccb4f4f3883..cc2f70d061c8 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -164,6 +164,11 @@ enum pci_epc_bar_type { * should be configured as 32-bit or 64-bit, the EPF driver must * configure this BAR as 64-bit. Additionally, the BAR succeeding * this BAR must be set to type BAR_RESERVED. + * + * only_64bit should not be set on a BAR of type BAR_RESERVED. + * (If BARx is a 64-bit BAR that an EPF driver is not allowed to + * touch, then both BARx and BARx+1 must be set to type + * BAR_RESERVED.) */ struct pci_epc_bar_desc { enum pci_epc_bar_type type; -- cgit v1.2.3 From f1cebae1dbf85f9de65c13a2d9f5cc3be7e51dc4 Mon Sep 17 00:00:00 2001 From: "Nícolas F. R. A. Prado" Date: Mon, 12 Feb 2024 09:50:06 -0500 Subject: firmware: coreboot: Generate aliases for coreboot modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate aliases for coreboot modules to allow automatic module probing. Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Brian Norris Signed-off-by: Nícolas F. R. A. Prado Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240212-coreboot-mod-defconfig-v4-2-d14172676f6d@collabora.com Signed-off-by: Tzung-Bi Shih --- include/linux/mod_devicetable.h | 10 ++++++++++ scripts/mod/devicetable-offsets.c | 3 +++ scripts/mod/file2alias.c | 10 ++++++++++ 3 files changed, 23 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index f458469c5ce5..7a9a07ea451b 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -960,4 +960,14 @@ struct vchiq_device_id { char name[32]; }; +/** + * struct coreboot_device_id - Identifies a coreboot table entry + * @tag: tag ID + * @driver_data: driver specific data + */ +struct coreboot_device_id { + __u32 tag; + kernel_ulong_t driver_data; +}; + #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index e91a3c38143b..518200813d4e 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -274,5 +274,8 @@ int main(void) DEVID(vchiq_device_id); DEVID_FIELD(vchiq_device_id, name); + DEVID(coreboot_device_id); + DEVID_FIELD(coreboot_device_id, tag); + return 0; } diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 4829680a0a6d..5d1c61fa5a55 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1494,6 +1494,15 @@ static int do_vchiq_entry(const char *filename, void *symval, char *alias) return 1; } +/* Looks like: coreboot:tN */ +static int do_coreboot_entry(const char *filename, void *symval, char *alias) +{ + DEF_FIELD(symval, coreboot_device_id, tag); + sprintf(alias, "coreboot:t%08X", tag); + + return 1; +} + /* Does namelen bytes of name exactly match the symbol? */ static bool sym_is(const char *name, unsigned namelen, const char *symbol) { @@ -1575,6 +1584,7 @@ static const struct devtable devtable[] = { {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry}, {"cdx", SIZE_cdx_device_id, do_cdx_entry}, {"vchiq", SIZE_vchiq_device_id, do_vchiq_entry}, + {"coreboot", SIZE_coreboot_device_id, do_coreboot_entry}, }; /* Create MODULE_ALIAS() statements. -- cgit v1.2.3 From 12b8ae68f50de200c038246c2496822f38b18fe2 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Wed, 7 Feb 2024 17:50:59 +0800 Subject: crypto: hisilicon/qm - add stop function by hardware Hardware V3 could be able to drain function by sending mailbox to hardware which will trigger tasks in device to be flushed out. When the function is reset, the function can be stopped by this way. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 40 ++++++++++++++++++++++++++++++++-------- include/linux/hisi_acc_qm.h | 2 ++ 2 files changed, 34 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 4b20b94e6371..3b015482b4e6 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -312,6 +312,7 @@ static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, + {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, }; @@ -1674,6 +1675,11 @@ unlock: return ret; } +static int qm_drain_qm(struct hisi_qm *qm) +{ + return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0); +} + static int qm_stop_qp(struct hisi_qp *qp) { return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); @@ -2088,7 +2094,8 @@ static int qm_drain_qp(struct hisi_qp *qp) static int qm_stop_qp_nolock(struct hisi_qp *qp) { - struct device *dev = &qp->qm->pdev->dev; + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; int ret; /* @@ -2104,11 +2111,14 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) atomic_set(&qp->qp_status.flags, QP_STOP); - ret = qm_drain_qp(qp); - if (ret) - dev_err(dev, "Failed to drain out data for stopping!\n"); + /* V3 supports direct stop function when FLR prepare */ + if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) { + ret = qm_drain_qp(qp); + if (ret) + dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id); + } - flush_workqueue(qp->qm->wq); + flush_workqueue(qm->wq); if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) qp_stop_fail_cb(qp); @@ -3112,16 +3122,29 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) down_write(&qm->qps_lock); - qm->status.stop_reason = r; if (atomic_read(&qm->status.flags) == QM_STOP) goto err_unlock; /* Stop all the request sending at first. */ atomic_set(&qm->status.flags, QM_STOP); + qm->status.stop_reason = r; - if (qm->status.stop_reason == QM_SOFT_RESET || - qm->status.stop_reason == QM_DOWN) { + if (qm->status.stop_reason != QM_NORMAL) { hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + /* + * When performing soft reset, the hardware will no longer + * do tasks, and the tasks in the device will be flushed + * out directly since the master ooo is closed. + */ + if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) && + r != QM_SOFT_RESET) { + ret = qm_drain_qm(qm); + if (ret) { + dev_err(dev, "failed to drain qm!\n"); + goto err_unlock; + } + } + ret = qm_stop_started_qp(qm); if (ret < 0) { dev_err(dev, "Failed to stop started qp!\n"); @@ -3141,6 +3164,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) } qm_clear_queues(qm); + qm->status.stop_reason = QM_NORMAL; err_unlock: up_write(&qm->qps_lock); diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 5f4c74facf6a..720f10874a66 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -43,6 +43,7 @@ #define QM_MB_CMD_CQC_BT 0x5 #define QM_MB_CMD_SQC_VFT_V2 0x6 #define QM_MB_CMD_STOP_QP 0x8 +#define QM_MB_CMD_FLUSH_QM 0x9 #define QM_MB_CMD_SRC 0xc #define QM_MB_CMD_DST 0xd @@ -151,6 +152,7 @@ enum qm_cap_bits { QM_SUPPORT_DB_ISOLATION = 0x0, QM_SUPPORT_FUNC_QOS, QM_SUPPORT_STOP_QP, + QM_SUPPORT_STOP_FUNC, QM_SUPPORT_MB_COMMAND, QM_SUPPORT_SVA_PREFETCH, QM_SUPPORT_RPM, -- cgit v1.2.3 From ce133a22123055f5f988499cd9ac7953d2bf0677 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Wed, 7 Feb 2024 17:51:00 +0800 Subject: crypto: hisilicon/qm - obtain stop queue status The debugfs files 'dev_state' and 'dev_timeout' are added. Users can query the current queue stop status through these two files. And set the waiting timeout when the queue is released. dev_state: if dev_timeout is set, dev_state indicates the status of stopping the queue. 0 indicates that the queue is stopped successfully. Other values indicate that the queue stops fail. If dev_timeout is not set, the value of dev_state is 0; dev_timeout: if the queue fails to stop, the queue is released after waiting dev_timeout * 20ms. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- Documentation/ABI/testing/debugfs-hisi-hpre | 15 ++++ Documentation/ABI/testing/debugfs-hisi-sec | 15 ++++ Documentation/ABI/testing/debugfs-hisi-zip | 15 ++++ drivers/crypto/hisilicon/debugfs.c | 5 ++ drivers/crypto/hisilicon/qm.c | 108 +++++++++++++++++++++------- include/linux/hisi_acc_qm.h | 6 ++ 6 files changed, 138 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/debugfs-hisi-hpre b/Documentation/ABI/testing/debugfs-hisi-hpre index 6ed9258605c7..d4e16ef9ac9a 100644 --- a/Documentation/ABI/testing/debugfs-hisi-hpre +++ b/Documentation/ABI/testing/debugfs-hisi-hpre @@ -118,6 +118,21 @@ Description: Dump the state of the device. 0: busy, 1: idle. Only available for PF, and take no other effect on HPRE. +What: /sys/kernel/debug/hisi_hpre//qm/dev_timeout +Date: Feb 2024 +Contact: linux-crypto@vger.kernel.org +Description: Set the wait time when stop queue fails. Available for both PF + and VF, and take no other effect on HPRE. + 0: not wait(default), others value: wait dev_timeout * 20 microsecond. + +What: /sys/kernel/debug/hisi_hpre//qm/dev_state +Date: Feb 2024 +Contact: linux-crypto@vger.kernel.org +Description: Dump the stop queue status of the QM. The default value is 0, + if dev_timeout is set, when stop queue fails, the dev_state + will return non-zero value. Available for both PF and VF, + and take no other effect on HPRE. + What: /sys/kernel/debug/hisi_hpre//hpre_dfx/diff_regs Date: Mar 2022 Contact: linux-crypto@vger.kernel.org diff --git a/Documentation/ABI/testing/debugfs-hisi-sec b/Documentation/ABI/testing/debugfs-hisi-sec index 403f5de96318..6c6c9a6e150a 100644 --- a/Documentation/ABI/testing/debugfs-hisi-sec +++ b/Documentation/ABI/testing/debugfs-hisi-sec @@ -98,6 +98,21 @@ Description: Dump the state of the device. 0: busy, 1: idle. Only available for PF, and take no other effect on SEC. +What: /sys/kernel/debug/hisi_sec2//qm/dev_timeout +Date: Feb 2024 +Contact: linux-crypto@vger.kernel.org +Description: Set the wait time when stop queue fails. Available for both PF + and VF, and take no other effect on SEC. + 0: not wait(default), others value: wait dev_timeout * 20 microsecond. + +What: /sys/kernel/debug/hisi_sec2//qm/dev_state +Date: Feb 2024 +Contact: linux-crypto@vger.kernel.org +Description: Dump the stop queue status of the QM. The default value is 0, + if dev_timeout is set, when stop queue fails, the dev_state + will return non-zero value. Available for both PF and VF, + and take no other effect on SEC. + What: /sys/kernel/debug/hisi_sec2//sec_dfx/diff_regs Date: Mar 2022 Contact: linux-crypto@vger.kernel.org diff --git a/Documentation/ABI/testing/debugfs-hisi-zip b/Documentation/ABI/testing/debugfs-hisi-zip index 2394e6a3cfe2..a22dd6942219 100644 --- a/Documentation/ABI/testing/debugfs-hisi-zip +++ b/Documentation/ABI/testing/debugfs-hisi-zip @@ -111,6 +111,21 @@ Description: Dump the state of the device. 0: busy, 1: idle. Only available for PF, and take no other effect on ZIP. +What: /sys/kernel/debug/hisi_zip//qm/dev_timeout +Date: Feb 2024 +Contact: linux-crypto@vger.kernel.org +Description: Set the wait time when stop queue fails. Available for both PF + and VF, and take no other effect on ZIP. + 0: not wait(default), others value: wait dev_timeout * 20 microsecond. + +What: /sys/kernel/debug/hisi_zip//qm/dev_state +Date: Feb 2024 +Contact: linux-crypto@vger.kernel.org +Description: Dump the stop queue status of the QM. The default value is 0, + if dev_timeout is set, when stop queue fails, the dev_state + will return non-zero value. Available for both PF and VF, + and take no other effect on ZIP. + What: /sys/kernel/debug/hisi_zip//zip_dfx/diff_regs Date: Mar 2022 Contact: linux-crypto@vger.kernel.org diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index 06e67eda409f..cd67fa348ca7 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -1112,6 +1112,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, void hisi_qm_debug_init(struct hisi_qm *qm) { struct dfx_diff_registers *qm_regs = qm->debug.qm_diff_regs; + struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx; struct qm_dfx *dfx = &qm->debug.dfx; struct dentry *qm_d; void *data; @@ -1140,6 +1141,10 @@ void hisi_qm_debug_init(struct hisi_qm *qm) debugfs_create_file("status", 0444, qm->debug.qm_d, qm, &qm_status_fops); + + debugfs_create_u32("dev_state", 0444, qm->debug.qm_d, &dev_dfx->dev_state); + debugfs_create_u32("dev_timeout", 0644, qm->debug.qm_d, &dev_dfx->dev_timeout); + for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) { data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset); debugfs_create_file(qm_dfx_files[i].name, diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 3b015482b4e6..41dff28326f1 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -236,6 +236,12 @@ #define QM_DEV_ALG_MAX_LEN 256 + /* abnormal status value for stopping queue */ +#define QM_STOP_QUEUE_FAIL 1 +#define QM_DUMP_SQC_FAIL 3 +#define QM_DUMP_CQC_FAIL 4 +#define QM_FINISH_WAIT 5 + #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ @@ -2037,43 +2043,25 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) } } -/** - * qm_drain_qp() - Drain a qp. - * @qp: The qp we want to drain. - * - * Determine whether the queue is cleared by judging the tail pointers of - * sq and cq. - */ -static int qm_drain_qp(struct hisi_qp *qp) +static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) { - struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; struct qm_sqc sqc; struct qm_cqc cqc; int ret, i = 0; - /* No need to judge if master OOO is blocked. */ - if (qm_check_dev_error(qm)) - return 0; - - /* Kunpeng930 supports drain qp by device */ - if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { - ret = qm_stop_qp(qp); - if (ret) - dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id); - return ret; - } - while (++i) { - ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp->qp_id, 1); + ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1); if (ret) { dev_err_ratelimited(dev, "Failed to dump sqc!\n"); + *state = QM_DUMP_SQC_FAIL; return ret; } - ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp->qp_id, 1); + ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1); if (ret) { dev_err_ratelimited(dev, "Failed to dump cqc!\n"); + *state = QM_DUMP_CQC_FAIL; return ret; } @@ -2082,8 +2070,9 @@ static int qm_drain_qp(struct hisi_qp *qp) break; if (i == MAX_WAIT_COUNTS) { - dev_err(dev, "Fail to empty queue %u!\n", qp->qp_id); - return -EBUSY; + dev_err(dev, "Fail to empty queue %u!\n", qp_id); + *state = QM_STOP_QUEUE_FAIL; + return -ETIMEDOUT; } usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX); @@ -2092,6 +2081,49 @@ static int qm_drain_qp(struct hisi_qp *qp) return 0; } +/** + * qm_drain_qp() - Drain a qp. + * @qp: The qp we want to drain. + * + * If the device does not support stopping queue by sending mailbox, + * determine whether the queue is cleared by judging the tail pointers of + * sq and cq. + */ +static int qm_drain_qp(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); + u32 state = 0; + int ret; + + /* No need to judge if master OOO is blocked. */ + if (qm_check_dev_error(pf_qm)) + return 0; + + /* HW V3 supports drain qp by device */ + if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { + ret = qm_stop_qp(qp); + if (ret) { + dev_err(&qm->pdev->dev, "Failed to stop qp!\n"); + state = QM_STOP_QUEUE_FAIL; + goto set_dev_state; + } + return ret; + } + + ret = qm_wait_qp_empty(qm, &state, qp->qp_id); + if (ret) + goto set_dev_state; + + return 0; + +set_dev_state: + if (qm->debug.dev_dfx.dev_timeout) + qm->debug.dev_dfx.dev_state = state; + + return ret; +} + static int qm_stop_qp_nolock(struct hisi_qp *qp) { struct hisi_qm *qm = qp->qm; @@ -2319,7 +2351,31 @@ static int hisi_qm_uacce_start_queue(struct uacce_queue *q) static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) { - hisi_qm_stop_qp(q->priv); + struct hisi_qp *qp = q->priv; + struct hisi_qm *qm = qp->qm; + struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx; + u32 i = 0; + + hisi_qm_stop_qp(qp); + + if (!dev_dfx->dev_timeout || !dev_dfx->dev_state) + return; + + /* + * After the queue fails to be stopped, + * wait for a period of time before releasing the queue. + */ + while (++i) { + msleep(WAIT_PERIOD); + + /* Since dev_timeout maybe modified, check i >= dev_timeout */ + if (i >= dev_dfx->dev_timeout) { + dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n", + qp->qp_id, dev_dfx->dev_state); + dev_dfx->dev_state = QM_FINISH_WAIT; + break; + } + } } static int hisi_qm_is_q_updated(struct uacce_queue *q) diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 720f10874a66..2d14742ad729 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -163,6 +163,11 @@ struct qm_dev_alg { const char *alg; }; +struct qm_dev_dfx { + u32 dev_state; + u32 dev_timeout; +}; + struct dfx_diff_registers { u32 *regs; u32 reg_offset; @@ -191,6 +196,7 @@ struct qm_debug { struct dentry *debug_root; struct dentry *qm_d; struct debugfs_file files[DEBUG_FILE_NUM]; + struct qm_dev_dfx dev_dfx; unsigned int *qm_last_words; /* ACC engines recoreding last regs */ unsigned int *last_words; -- cgit v1.2.3 From 9066ac364d8659ab7c993b83c60a6182c3ec1ef9 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Wed, 7 Feb 2024 17:51:01 +0800 Subject: crypto: hisilicon/qm - change function type to void The function qm_stop_qp_nolock() always return zero, so function type is changed to void. Signed-off-by: Weili Qian Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 38 +++++++++++--------------------------- include/linux/hisi_acc_qm.h | 2 +- 2 files changed, 12 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 41dff28326f1..92f0a1d9b4a6 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2124,7 +2124,7 @@ set_dev_state: return ret; } -static int qm_stop_qp_nolock(struct hisi_qp *qp) +static void qm_stop_qp_nolock(struct hisi_qp *qp) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; @@ -2138,7 +2138,7 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) */ if (atomic_read(&qp->qp_status.flags) != QP_START) { qp->is_resetting = false; - return 0; + return; } atomic_set(&qp->qp_status.flags, QP_STOP); @@ -2155,25 +2155,19 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) qp_stop_fail_cb(qp); dev_dbg(dev, "stop queue %u!", qp->qp_id); - - return 0; } /** * hisi_qm_stop_qp() - Stop a qp in qm. * @qp: The qp we want to stop. * - * This function is reverse of hisi_qm_start_qp. Return 0 if successful. + * This function is reverse of hisi_qm_start_qp. */ -int hisi_qm_stop_qp(struct hisi_qp *qp) +void hisi_qm_stop_qp(struct hisi_qp *qp) { - int ret; - down_write(&qp->qm->qps_lock); - ret = qm_stop_qp_nolock(qp); + qm_stop_qp_nolock(qp); up_write(&qp->qm->qps_lock); - - return ret; } EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); @@ -3120,25 +3114,18 @@ static int qm_restart(struct hisi_qm *qm) } /* Stop started qps in reset flow */ -static int qm_stop_started_qp(struct hisi_qm *qm) +static void qm_stop_started_qp(struct hisi_qm *qm) { - struct device *dev = &qm->pdev->dev; struct hisi_qp *qp; - int i, ret; + int i; for (i = 0; i < qm->qp_num; i++) { qp = &qm->qp_array[i]; - if (qp && atomic_read(&qp->qp_status.flags) == QP_START) { + if (atomic_read(&qp->qp_status.flags) == QP_START) { qp->is_resetting = true; - ret = qm_stop_qp_nolock(qp); - if (ret < 0) { - dev_err(dev, "Failed to stop qp%d!\n", i); - return ret; - } + qm_stop_qp_nolock(qp); } } - - return 0; } /** @@ -3201,11 +3188,8 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) } } - ret = qm_stop_started_qp(qm); - if (ret < 0) { - dev_err(dev, "Failed to stop started qp!\n"); - goto err_unlock; - } + qm_stop_started_qp(qm); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); } diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 2d14742ad729..9d7754ad5e9b 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -531,7 +531,7 @@ void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r); int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); -int hisi_qm_stop_qp(struct hisi_qp *qp); +void hisi_qm_stop_qp(struct hisi_qp *qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg); void hisi_qm_debug_init(struct hisi_qm *qm); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); -- cgit v1.2.3 From 2444a80c1cc2c4240f60f2162abef3797c1803de Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 14 Feb 2024 08:48:28 +0000 Subject: kobject: make uevent_seqnum atomic We will soon no longer acquire uevent_sock_mutex for most kobject_uevent_net_broadcast() calls, and also while calling uevent_net_broadcast(). Make uevent_seqnum an atomic64_t to get its own protection. This fixes a race while reading /sys/kernel/uevent_seqnum. Signed-off-by: Eric Dumazet Cc: Greg Kroah-Hartman Cc: Christian Brauner Reviewed-by: Christian Brauner Link: https://lore.kernel.org/r/20240214084829.684541-2-edumazet@google.com Signed-off-by: Greg Kroah-Hartman --- include/linux/kobject.h | 2 +- kernel/ksysfs.c | 2 +- lib/kobject_uevent.c | 17 +++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kobject.h b/include/linux/kobject.h index c30affcc43b4..c8219505a79f 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -38,7 +38,7 @@ extern char uevent_helper[]; #endif /* counter to tag the uevent, read only except for the kobject core */ -extern u64 uevent_seqnum; +extern atomic64_t uevent_seqnum; /* * The actions here must match the index to the string array diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 1d4bc493b2f4..32ae7fa74a9c 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -39,7 +39,7 @@ static struct kobj_attribute _name##_attr = __ATTR_RW(_name) static ssize_t uevent_seqnum_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sysfs_emit(buf, "%llu\n", (unsigned long long)uevent_seqnum); + return sysfs_emit(buf, "%llu\n", (u64)atomic64_read(&uevent_seqnum)); } KERNEL_ATTR_RO(uevent_seqnum); diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index fb9a2f06dd1e..9cb1a7fdaeba 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -30,7 +30,7 @@ #include -u64 uevent_seqnum; +atomic64_t uevent_seqnum; #ifdef CONFIG_UEVENT_HELPER char uevent_helper[UEVENT_HELPER_PATH_LEN] = CONFIG_UEVENT_HELPER_PATH; #endif @@ -44,7 +44,7 @@ struct uevent_sock { static LIST_HEAD(uevent_sock_list); #endif -/* This lock protects uevent_seqnum and uevent_sock_list */ +/* This lock protects uevent_sock_list */ static DEFINE_MUTEX(uevent_sock_mutex); /* the strings here must match the enum in include/linux/kobject.h */ @@ -583,13 +583,13 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, break; } - mutex_lock(&uevent_sock_mutex); /* we will send an event, so request a new sequence number */ - retval = add_uevent_var(env, "SEQNUM=%llu", ++uevent_seqnum); - if (retval) { - mutex_unlock(&uevent_sock_mutex); + retval = add_uevent_var(env, "SEQNUM=%llu", + atomic64_inc_return(&uevent_seqnum)); + if (retval) goto exit; - } + + mutex_lock(&uevent_sock_mutex); retval = kobject_uevent_net_broadcast(kobj, env, action_string, devpath); mutex_unlock(&uevent_sock_mutex); @@ -688,7 +688,8 @@ static int uevent_net_broadcast(struct sock *usk, struct sk_buff *skb, int ret; /* bump and prepare sequence number */ - ret = snprintf(buf, sizeof(buf), "SEQNUM=%llu", ++uevent_seqnum); + ret = snprintf(buf, sizeof(buf), "SEQNUM=%llu", + atomic64_inc_return(&uevent_seqnum)); if (ret < 0 || (size_t)ret >= sizeof(buf)) return -ENOMEM; ret++; -- cgit v1.2.3 From 99f638dd49ca80538addec6b3733ddb5784c9373 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Tue, 30 Jan 2024 13:23:37 +0100 Subject: usb: gadget: Support already-mapped DMA SGs Add a new 'sg_was_mapped' field to the struct usb_request. This field can be used to indicate that the scatterlist associated to the USB transfer has already been mapped into the DMA space, and it does not have to be done internally. Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20240130122340.54813-2-paul@crapouillou.net Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/udc/core.c | 7 ++++++- include/linux/usb/gadget.h | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index d59f94464b87..9d4150124fdb 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -903,6 +903,11 @@ int usb_gadget_map_request_by_dev(struct device *dev, if (req->length == 0) return 0; + if (req->sg_was_mapped) { + req->num_mapped_sgs = req->num_sgs; + return 0; + } + if (req->num_sgs) { int mapped; @@ -948,7 +953,7 @@ EXPORT_SYMBOL_GPL(usb_gadget_map_request); void usb_gadget_unmap_request_by_dev(struct device *dev, struct usb_request *req, int is_in) { - if (req->length == 0) + if (req->length == 0 || req->sg_was_mapped) return; if (req->num_mapped_sgs) { diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index a771ccc038ac..c529e4e06997 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -52,6 +52,7 @@ struct usb_ep; * @short_not_ok: When reading data, makes short packets be * treated as errors (queue stops advancing till cleanup). * @dma_mapped: Indicates if request has been mapped to DMA (internal) + * @sg_was_mapped: Set if the scatterlist has been mapped before the request * @complete: Function called when request completes, so this request and * its buffer may be re-used. The function will always be called with * interrupts disabled, and it must not sleep. @@ -111,6 +112,7 @@ struct usb_request { unsigned zero:1; unsigned short_not_ok:1; unsigned dma_mapped:1; + unsigned sg_was_mapped:1; void (*complete)(struct usb_ep *ep, struct usb_request *req); -- cgit v1.2.3 From 1dae0cb79ceacbdc7495c5f83ca71e1c12a24d7c Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 28 Jan 2024 15:05:28 +0000 Subject: iio: locking: introduce __cleanup() based direct mode claiming infrastructure Allows use of: iio_device_claim_direct_scoped(return -EBUSY, indio_dev) { } to automatically call iio_device_release_direct_mode() based on scope. Typically seen in combination with local device specific locks which are already have automated cleanup options via guard(mutex)(&st->lock) and scoped_guard(). Using both together allows most error handling to be automated. Reviewed-by: Nuno Sa Link: https://lore.kernel.org/r/20240128150537.44592-2-jic23@kernel.org Signed-off-by: Jonathan Cameron --- include/linux/iio/iio.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'include/linux') diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index c5b36d2c1e73..4f89279e531c 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -9,6 +9,7 @@ #include #include +#include #include #include /* IIO TODO LIST */ @@ -638,6 +639,33 @@ int __devm_iio_device_register(struct device *dev, struct iio_dev *indio_dev, int iio_push_event(struct iio_dev *indio_dev, u64 ev_code, s64 timestamp); int iio_device_claim_direct_mode(struct iio_dev *indio_dev); void iio_device_release_direct_mode(struct iio_dev *indio_dev); + +/* + * This autocleanup logic is normally used via + * iio_device_claim_direct_scoped(). + */ +DEFINE_GUARD(iio_claim_direct, struct iio_dev *, iio_device_claim_direct_mode(_T), + iio_device_release_direct_mode(_T)) + +DEFINE_GUARD_COND(iio_claim_direct, _try, ({ + struct iio_dev *dev; + int d = iio_device_claim_direct_mode(_T); + + if (d < 0) + dev = NULL; + else + dev = _T; + dev; + })) + +/** + * iio_device_claim_direct_scoped() - Scoped call to iio_device_claim_direct. + * @fail: What to do on failure to claim device. + * @iio_dev: Pointer to the IIO devices structure + */ +#define iio_device_claim_direct_scoped(fail, iio_dev) \ + scoped_cond_guard(iio_claim_direct_try, fail, iio_dev) + int iio_device_claim_buffer_mode(struct iio_dev *indio_dev); void iio_device_release_buffer_mode(struct iio_dev *indio_dev); -- cgit v1.2.3 From 89b1b86fc77367fac470258acdf470ffe2edc8d4 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Thu, 8 Feb 2024 16:37:19 -0300 Subject: iio: core: make iio_bus_type const Now that the driver core can properly handle constant struct bus_type, move the iio_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Acked-by: Nuno Sa Link: https://lore.kernel.org/r/20240208-bus_cleanup-iio-v1-1-4a167c3b5fb3@marliere.net Signed-off-by: Jonathan Cameron --- drivers/iio/industrialio-core.c | 2 +- include/linux/iio/iio.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index e8551a1636ba..9b2877fe8689 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -42,7 +42,7 @@ static DEFINE_IDA(iio_ida); static dev_t iio_devt; #define IIO_DEV_MAX 256 -struct bus_type iio_bus_type = { +const struct bus_type iio_bus_type = { .name = "iio", }; EXPORT_SYMBOL(iio_bus_type); diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 4f89279e531c..e370a7bb3300 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -669,7 +669,7 @@ DEFINE_GUARD_COND(iio_claim_direct, _try, ({ int iio_device_claim_buffer_mode(struct iio_dev *indio_dev); void iio_device_release_buffer_mode(struct iio_dev *indio_dev); -extern struct bus_type iio_bus_type; +extern const struct bus_type iio_bus_type; /** * iio_device_put() - reference counted deallocation of struct device -- cgit v1.2.3 From 3765d426fe864e109d00aeb48f22413b896b2eb9 Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Mon, 5 Feb 2024 10:59:25 -0800 Subject: iio: hid-sensor-als: Add light color temperature support On some platforms, ambient color sensors also support light color temperature. Add support of light color temperature. Signed-off-by: Basavaraj Natikar Signed-off-by: Srinivas Pandruvada Link: https://lore.kernel.org/r/20240205185926.3030521-4-srinivas.pandruvada@linux.intel.com Signed-off-by: Jonathan Cameron --- drivers/iio/light/hid-sensor-als.c | 21 +++++++++++++++++++++ include/linux/hid-sensor-ids.h | 1 + 2 files changed, 22 insertions(+) (limited to 'include/linux') diff --git a/drivers/iio/light/hid-sensor-als.c b/drivers/iio/light/hid-sensor-als.c index bd4cb0883121..f9d0071d9fa9 100644 --- a/drivers/iio/light/hid-sensor-als.c +++ b/drivers/iio/light/hid-sensor-als.c @@ -16,6 +16,7 @@ enum { CHANNEL_SCAN_INDEX_INTENSITY, CHANNEL_SCAN_INDEX_ILLUM, + CHANNEL_SCAN_INDEX_COLOR_TEMP, CHANNEL_SCAN_INDEX_MAX }; @@ -43,6 +44,7 @@ struct als_state { static const u32 als_usage_ids[] = { HID_USAGE_SENSOR_LIGHT_ILLUM, HID_USAGE_SENSOR_LIGHT_ILLUM, + HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE, }; static const u32 als_sensitivity_addresses[] = { @@ -74,6 +76,16 @@ static const struct iio_chan_spec als_channels[] = { BIT(IIO_CHAN_INFO_HYSTERESIS_RELATIVE), .scan_index = CHANNEL_SCAN_INDEX_ILLUM, }, + { + .type = IIO_COLORTEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_OFFSET) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_HYSTERESIS) | + BIT(IIO_CHAN_INFO_HYSTERESIS_RELATIVE), + .scan_index = CHANNEL_SCAN_INDEX_COLOR_TEMP, + }, IIO_CHAN_SOFT_TIMESTAMP(CHANNEL_SCAN_INDEX_TIMESTAMP) }; @@ -112,6 +124,11 @@ static int als_read_raw(struct iio_dev *indio_dev, min = als_state->als[chan->scan_index].logical_minimum; address = HID_USAGE_SENSOR_LIGHT_ILLUM; break; + case CHANNEL_SCAN_INDEX_COLOR_TEMP: + report_id = als_state->als[chan->scan_index].report_id; + min = als_state->als[chan->scan_index].logical_minimum; + address = HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE; + break; default: report_id = -1; break; @@ -232,6 +249,10 @@ static int als_capture_sample(struct hid_sensor_hub_device *hsdev, als_state->scan.illum[CHANNEL_SCAN_INDEX_ILLUM] = sample_data; ret = 0; break; + case HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE: + als_state->scan.illum[CHANNEL_SCAN_INDEX_COLOR_TEMP] = sample_data; + ret = 0; + break; case HID_USAGE_SENSOR_TIME_TIMESTAMP: als_state->timestamp = hid_sensor_convert_timestamp(&als_state->common_attributes, *(s64 *)raw_data); diff --git a/include/linux/hid-sensor-ids.h b/include/linux/hid-sensor-ids.h index 13b1e65fbdcc..8af4fb3e0254 100644 --- a/include/linux/hid-sensor-ids.h +++ b/include/linux/hid-sensor-ids.h @@ -21,6 +21,7 @@ #define HID_USAGE_SENSOR_ALS 0x200041 #define HID_USAGE_SENSOR_DATA_LIGHT 0x2004d0 #define HID_USAGE_SENSOR_LIGHT_ILLUM 0x2004d1 +#define HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE 0x2004d2 /* PROX (200011) */ #define HID_USAGE_SENSOR_PROX 0x200011 -- cgit v1.2.3 From 2ec17b1950bb824f9a8d5f055e466d02c40eb64c Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Mon, 5 Feb 2024 10:59:26 -0800 Subject: iio: hid-sensor-als: Add light chromaticity support On some platforms, ambient color sensors also support the x and y light colors, which represent the coordinates on the CIE 1931 chromaticity diagram. Add light chromaticity x and y. Signed-off-by: Basavaraj Natikar Signed-off-by: Srinivas Pandruvada Link: https://lore.kernel.org/r/20240205185926.3030521-5-srinivas.pandruvada@linux.intel.com Signed-off-by: Jonathan Cameron --- drivers/iio/light/hid-sensor-als.c | 46 ++++++++++++++++++++++++++++++++++++++ include/linux/hid-sensor-ids.h | 3 +++ 2 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/drivers/iio/light/hid-sensor-als.c b/drivers/iio/light/hid-sensor-als.c index f9d0071d9fa9..06580dfdfab6 100644 --- a/drivers/iio/light/hid-sensor-als.c +++ b/drivers/iio/light/hid-sensor-als.c @@ -17,6 +17,8 @@ enum { CHANNEL_SCAN_INDEX_INTENSITY, CHANNEL_SCAN_INDEX_ILLUM, CHANNEL_SCAN_INDEX_COLOR_TEMP, + CHANNEL_SCAN_INDEX_CHROMATICITY_X, + CHANNEL_SCAN_INDEX_CHROMATICITY_Y, CHANNEL_SCAN_INDEX_MAX }; @@ -45,6 +47,8 @@ static const u32 als_usage_ids[] = { HID_USAGE_SENSOR_LIGHT_ILLUM, HID_USAGE_SENSOR_LIGHT_ILLUM, HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE, + HID_USAGE_SENSOR_LIGHT_CHROMATICITY_X, + HID_USAGE_SENSOR_LIGHT_CHROMATICITY_Y, }; static const u32 als_sensitivity_addresses[] = { @@ -86,6 +90,30 @@ static const struct iio_chan_spec als_channels[] = { BIT(IIO_CHAN_INFO_HYSTERESIS_RELATIVE), .scan_index = CHANNEL_SCAN_INDEX_COLOR_TEMP, }, + { + .type = IIO_CHROMATICITY, + .modified = 1, + .channel2 = IIO_MOD_X, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_OFFSET) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_HYSTERESIS) | + BIT(IIO_CHAN_INFO_HYSTERESIS_RELATIVE), + .scan_index = CHANNEL_SCAN_INDEX_CHROMATICITY_X, + }, + { + .type = IIO_CHROMATICITY, + .modified = 1, + .channel2 = IIO_MOD_Y, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_OFFSET) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_HYSTERESIS) | + BIT(IIO_CHAN_INFO_HYSTERESIS_RELATIVE), + .scan_index = CHANNEL_SCAN_INDEX_CHROMATICITY_Y, + }, IIO_CHAN_SOFT_TIMESTAMP(CHANNEL_SCAN_INDEX_TIMESTAMP) }; @@ -129,6 +157,16 @@ static int als_read_raw(struct iio_dev *indio_dev, min = als_state->als[chan->scan_index].logical_minimum; address = HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE; break; + case CHANNEL_SCAN_INDEX_CHROMATICITY_X: + report_id = als_state->als[chan->scan_index].report_id; + min = als_state->als[chan->scan_index].logical_minimum; + address = HID_USAGE_SENSOR_LIGHT_CHROMATICITY_X; + break; + case CHANNEL_SCAN_INDEX_CHROMATICITY_Y: + report_id = als_state->als[chan->scan_index].report_id; + min = als_state->als[chan->scan_index].logical_minimum; + address = HID_USAGE_SENSOR_LIGHT_CHROMATICITY_Y; + break; default: report_id = -1; break; @@ -253,6 +291,14 @@ static int als_capture_sample(struct hid_sensor_hub_device *hsdev, als_state->scan.illum[CHANNEL_SCAN_INDEX_COLOR_TEMP] = sample_data; ret = 0; break; + case HID_USAGE_SENSOR_LIGHT_CHROMATICITY_X: + als_state->scan.illum[CHANNEL_SCAN_INDEX_CHROMATICITY_X] = sample_data; + ret = 0; + break; + case HID_USAGE_SENSOR_LIGHT_CHROMATICITY_Y: + als_state->scan.illum[CHANNEL_SCAN_INDEX_CHROMATICITY_Y] = sample_data; + ret = 0; + break; case HID_USAGE_SENSOR_TIME_TIMESTAMP: als_state->timestamp = hid_sensor_convert_timestamp(&als_state->common_attributes, *(s64 *)raw_data); diff --git a/include/linux/hid-sensor-ids.h b/include/linux/hid-sensor-ids.h index 8af4fb3e0254..6730ee900ee1 100644 --- a/include/linux/hid-sensor-ids.h +++ b/include/linux/hid-sensor-ids.h @@ -22,6 +22,9 @@ #define HID_USAGE_SENSOR_DATA_LIGHT 0x2004d0 #define HID_USAGE_SENSOR_LIGHT_ILLUM 0x2004d1 #define HID_USAGE_SENSOR_LIGHT_COLOR_TEMPERATURE 0x2004d2 +#define HID_USAGE_SENSOR_LIGHT_CHROMATICITY 0x2004d3 +#define HID_USAGE_SENSOR_LIGHT_CHROMATICITY_X 0x2004d4 +#define HID_USAGE_SENSOR_LIGHT_CHROMATICITY_Y 0x2004d5 /* PROX (200011) */ #define HID_USAGE_SENSOR_PROX 0x200011 -- cgit v1.2.3 From 80e4021c25d8c1ddae0dd655ed5f6b1e938dd79b Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 14 Feb 2024 21:16:19 +0100 Subject: net: mdio: add helpers for accessing the EEE CAP2 registers This adds helpers for accessing the EEE CAP2 registers. For now only 2500baseT and 5000baseT modes are supported. Signed-off-by: Heiner Kallweit Signed-off-by: David S. Miller --- include/linux/mdio.h | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 79ceee3c8673..fd8ff310f9eb 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -439,6 +439,42 @@ static inline void mii_eee_cap1_mod_linkmode_t(unsigned long *adv, u32 val) adv, val & MDIO_EEE_10GKR); } +/** + * mii_eee_cap2_mod_linkmode_sup_t() + * @adv: target the linkmode settings + * @val: register value + * + * A function that translates value of following registers to the linkmode: + * IEEE 802.3-2022 45.2.3.11 "EEE control and capability 2" register (3.21) + */ +static inline void mii_eee_cap2_mod_linkmode_sup_t(unsigned long *adv, u32 val) +{ + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + adv, val & MDIO_EEE_2_5GT); + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + adv, val & MDIO_EEE_5GT); +} + +/** + * mii_eee_cap2_mod_linkmode_adv_t() + * @adv: target the linkmode advertisement settings + * @val: register value + * + * A function that translates value of following registers to the linkmode: + * IEEE 802.3-2022 45.2.7.16 "EEE advertisement 2" register (7.62) + * IEEE 802.3-2022 45.2.7.17 "EEE link partner ability 2" register (7.63) + * Note: Currently this function is the same as mii_eee_cap2_mod_linkmode_sup_t. + * For certain, not yet supported, modes however the bits differ. + * Therefore create separate functions already. + */ +static inline void mii_eee_cap2_mod_linkmode_adv_t(unsigned long *adv, u32 val) +{ + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + adv, val & MDIO_EEE_2_5GT); + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + adv, val & MDIO_EEE_5GT); +} + /** * linkmode_to_mii_eee_cap1_t() * @adv: the linkmode advertisement settings @@ -466,6 +502,25 @@ static inline u32 linkmode_to_mii_eee_cap1_t(unsigned long *adv) return result; } +/** + * linkmode_to_mii_eee_cap2_t() + * @adv: the linkmode advertisement settings + * + * A function that translates linkmode to value for IEEE 802.3-2022 45.2.7.16 + * "EEE advertisement 2" register (7.62) + */ +static inline u32 linkmode_to_mii_eee_cap2_t(unsigned long *adv) +{ + u32 result = 0; + + if (linkmode_test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, adv)) + result |= MDIO_EEE_2_5GT; + if (linkmode_test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, adv)) + result |= MDIO_EEE_5GT; + + return result; +} + /** * mii_10base_t1_adv_mod_linkmode_t() * @adv: linkmode advertisement settings -- cgit v1.2.3 From ef6ee3a31bdc699391f2db4eff407fdb06895809 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Wed, 14 Feb 2024 21:17:11 +0100 Subject: net: phy: add PHY_EEE_CAP2_FEATURES As a prerequisite for adding EEE CAP2 register support, complement PHY_EEE_CAP1_FEATURES with PHY_EEE_CAP2_FEATURES. For now only 2500baseT and 5000baseT modes are supported. Signed-off-by: Heiner Kallweit Signed-off-by: David S. Miller --- drivers/net/phy/phy_device.c | 11 +++++++++++ include/linux/phy.h | 2 ++ 2 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index d63dca535746..2eefee970851 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -148,6 +148,14 @@ static const int phy_eee_cap1_features_array[] = { __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_eee_cap1_features) __ro_after_init; EXPORT_SYMBOL_GPL(phy_eee_cap1_features); +static const int phy_eee_cap2_features_array[] = { + ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + ETHTOOL_LINK_MODE_5000baseT_Full_BIT, +}; + +__ETHTOOL_DECLARE_LINK_MODE_MASK(phy_eee_cap2_features) __ro_after_init; +EXPORT_SYMBOL_GPL(phy_eee_cap2_features); + static void features_init(void) { /* 10/100 half/full*/ @@ -232,6 +240,9 @@ static void features_init(void) linkmode_set_bit_array(phy_eee_cap1_features_array, ARRAY_SIZE(phy_eee_cap1_features_array), phy_eee_cap1_features); + linkmode_set_bit_array(phy_eee_cap2_features_array, + ARRAY_SIZE(phy_eee_cap2_features_array), + phy_eee_cap2_features); } diff --git a/include/linux/phy.h b/include/linux/phy.h index c2dda21b39e1..e3ab2c347a59 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -54,6 +54,7 @@ extern __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_10gbit_features) __ro_after_init; extern __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_10gbit_fec_features) __ro_after_init; extern __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_10gbit_full_features) __ro_after_init; extern __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_eee_cap1_features) __ro_after_init; +extern __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_eee_cap2_features) __ro_after_init; #define PHY_BASIC_FEATURES ((unsigned long *)&phy_basic_features) #define PHY_BASIC_T1_FEATURES ((unsigned long *)&phy_basic_t1_features) @@ -65,6 +66,7 @@ extern __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_eee_cap1_features) __ro_after_init; #define PHY_10GBIT_FEC_FEATURES ((unsigned long *)&phy_10gbit_fec_features) #define PHY_10GBIT_FULL_FEATURES ((unsigned long *)&phy_10gbit_full_features) #define PHY_EEE_CAP1_FEATURES ((unsigned long *)&phy_eee_cap1_features) +#define PHY_EEE_CAP2_FEATURES ((unsigned long *)&phy_eee_cap2_features) extern const int phy_basic_ports_array[3]; extern const int phy_fibre_port_array[1]; -- cgit v1.2.3 From 4a92857d6e8383eca6d661538bb25dc7004fd391 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 14:52:17 +0100 Subject: gpio: constify opaque pointer "data" in gpio_device_find() The opaque pointer "data" in each match function used by gpio_device_find() is a pointer to const, thus the same argument passed to gpio_device_find() can adjusted similarly. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 2 +- include/linux/gpio/driver.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 02be0ba1a402..e4dd13d81b4d 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1171,7 +1171,7 @@ EXPORT_SYMBOL_GPL(gpiochip_remove); * If the function returns non-NULL, the returned reference must be freed by * the caller using gpio_device_put(). */ -struct gpio_device *gpio_device_find(void *data, +struct gpio_device *gpio_device_find(const void *data, int (*match)(struct gpio_chip *gc, const void *data)) { diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 9d0023f83a57..9c1fbfaebaa8 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -628,7 +628,7 @@ int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, void *data, struct lock_class_key *lock_key, struct lock_class_key *request_key); -struct gpio_device *gpio_device_find(void *data, +struct gpio_device *gpio_device_find(const void *data, int (*match)(struct gpio_chip *gc, const void *data)); struct gpio_device *gpio_device_find_by_label(const char *label); -- cgit v1.2.3 From 548fcf037b3f8592e9fe41469110453a777416d6 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Thu, 15 Feb 2024 13:15:38 +0200 Subject: tty: Don't include tty_buffer.h in tty.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's no need to include linux/tty_buffer.h in linux/tty.h. Move the include into tty_buffer.c that is actually using it. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20240215111538.1920-1-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_buffer.c | 1 + include/linux/tty.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index f8883afbeeba..79f0ff94ce00 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/tty.h b/include/linux/tty.h index 8c76fd97d4ad..2b2e6f0a54d6 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3 From d87c295f599cab2ab3b3df53a9098adba4a6002b Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 30 Jan 2024 10:46:27 -0800 Subject: sysfs: Introduce a mechanism to hide static attribute_groups Add a mechanism for named attribute_groups to hide their directory at sysfs_update_group() time, or otherwise skip emitting the group directory when the group is first registered. It piggybacks on is_visible() in a similar manner as SYSFS_PREALLOC, i.e. special flags in the upper bits of the returned mode. To use it, specify a symbol prefix to DEFINE_SYSFS_GROUP_VISIBLE(), and then pass that same prefix to SYSFS_GROUP_VISIBLE() when assigning the @is_visible() callback: DEFINE_SYSFS_GROUP_VISIBLE($prefix) struct attribute_group $prefix_group = { .name = $name, .is_visible = SYSFS_GROUP_VISIBLE($prefix), }; SYSFS_GROUP_VISIBLE() expects a definition of $prefix_group_visible() and $prefix_attr_visible(), where $prefix_group_visible() just returns true / false and $prefix_attr_visible() behaves as normal. The motivation for this capability is to centralize PCI device authentication in the PCI core with a named sysfs group while keeping that group hidden for devices and platforms that do not meet the requirements. In a PCI topology, most devices will not support authentication, a small subset will support just PCI CMA (Component Measurement and Authentication), a smaller subset will support PCI CMA + PCIe IDE (Link Integrity and Encryption), and only next generation server hosts will start to include a platform TSM (TEE Security Manager). Without this capability the alternatives are: * Check if all attributes are invisible and if so, hide the directory. Beyond trouble getting this to work [1], this is an ABI change for scenarios if userspace happens to depend on group visibility absent any attributes. I.e. this new capability avoids regression since it does not retroactively apply to existing cases. * Publish an empty /sys/bus/pci/devices/$pdev/tsm/ directory for all PCI devices (i.e. for the case when TSM platform support is present, but device support is absent). Unfortunate that this will be a vestigial empty directory in the vast majority of cases. * Reintroduce usage of runtime calls to sysfs_{create,remove}_group() in the PCI core. Bjorn has already indicated that he does not want to see any growth of pci_sysfs_init() [2]. * Drop the named group and simulate a directory by prefixing all TSM-related attributes with "tsm_". Unfortunate to not use the naming capability of a sysfs group as intended. In comparison, there is a small potential for regression if for some reason an @is_visible() callback had dependencies on how many times it was called. Additionally, it is no longer an error to update a group that does not have its directory already present, and it is no longer a WARN() to remove a group that was never visible. Link: https://lore.kernel.org/all/2024012321-envious-procedure-4a58@gregkh/ [1] Link: https://lore.kernel.org/linux-pci/20231019200110.GA1410324@bhelgaas/ [2] Signed-off-by: Dan Williams Signed-off-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/2024013028-deflator-flaring-ec62@gregkh Signed-off-by: Greg Kroah-Hartman --- fs/sysfs/group.c | 45 ++++++++++++++++++++++++++++-------- include/linux/sysfs.h | 63 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 87 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c index 138676463336..ccb275cdabcb 100644 --- a/fs/sysfs/group.c +++ b/fs/sysfs/group.c @@ -31,6 +31,17 @@ static void remove_files(struct kernfs_node *parent, kernfs_remove_by_name(parent, (*bin_attr)->attr.name); } +static umode_t __first_visible(const struct attribute_group *grp, struct kobject *kobj) +{ + if (grp->attrs && grp->is_visible) + return grp->is_visible(kobj, grp->attrs[0], 0); + + if (grp->bin_attrs && grp->is_bin_visible) + return grp->is_bin_visible(kobj, grp->bin_attrs[0], 0); + + return 0; +} + static int create_files(struct kernfs_node *parent, struct kobject *kobj, kuid_t uid, kgid_t gid, const struct attribute_group *grp, int update) @@ -52,6 +63,7 @@ static int create_files(struct kernfs_node *parent, struct kobject *kobj, kernfs_remove_by_name(parent, (*attr)->name); if (grp->is_visible) { mode = grp->is_visible(kobj, *attr, i); + mode &= ~SYSFS_GROUP_INVISIBLE; if (!mode) continue; } @@ -81,6 +93,7 @@ static int create_files(struct kernfs_node *parent, struct kobject *kobj, (*bin_attr)->attr.name); if (grp->is_bin_visible) { mode = grp->is_bin_visible(kobj, *bin_attr, i); + mode &= ~SYSFS_GROUP_INVISIBLE; if (!mode) continue; } @@ -127,16 +140,31 @@ static int internal_create_group(struct kobject *kobj, int update, kobject_get_ownership(kobj, &uid, &gid); if (grp->name) { + umode_t mode = __first_visible(grp, kobj); + + if (mode & SYSFS_GROUP_INVISIBLE) + mode = 0; + else + mode = S_IRWXU | S_IRUGO | S_IXUGO; + if (update) { kn = kernfs_find_and_get(kobj->sd, grp->name); if (!kn) { - pr_warn("Can't update unknown attr grp name: %s/%s\n", - kobj->name, grp->name); - return -EINVAL; + pr_debug("attr grp %s/%s not created yet\n", + kobj->name, grp->name); + /* may have been invisible prior to this update */ + update = 0; + } else if (!mode) { + sysfs_remove_group(kobj, grp); + kernfs_put(kn); + return 0; } - } else { - kn = kernfs_create_dir_ns(kobj->sd, grp->name, - S_IRWXU | S_IRUGO | S_IXUGO, + } + + if (!update) { + if (!mode) + return 0; + kn = kernfs_create_dir_ns(kobj->sd, grp->name, mode, uid, gid, kobj, NULL); if (IS_ERR(kn)) { if (PTR_ERR(kn) == -EEXIST) @@ -279,9 +307,8 @@ void sysfs_remove_group(struct kobject *kobj, if (grp->name) { kn = kernfs_find_and_get(parent, grp->name); if (!kn) { - WARN(!kn, KERN_WARNING - "sysfs group '%s' not found for kobject '%s'\n", - grp->name, kobject_name(kobj)); + pr_debug("sysfs group '%s' not found for kobject '%s'\n", + grp->name, kobject_name(kobj)); return; } } else { diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index b717a70219f6..a42642b277dd 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -61,22 +61,32 @@ do { \ /** * struct attribute_group - data structure used to declare an attribute group. * @name: Optional: Attribute group name - * If specified, the attribute group will be created in - * a new subdirectory with this name. + * If specified, the attribute group will be created in a + * new subdirectory with this name. Additionally when a + * group is named, @is_visible and @is_bin_visible may + * return SYSFS_GROUP_INVISIBLE to control visibility of + * the directory itself. * @is_visible: Optional: Function to return permissions associated with an - * attribute of the group. Will be called repeatedly for each - * non-binary attribute in the group. Only read/write + * attribute of the group. Will be called repeatedly for + * each non-binary attribute in the group. Only read/write * permissions as well as SYSFS_PREALLOC are accepted. Must - * return 0 if an attribute is not visible. The returned value - * will replace static permissions defined in struct attribute. + * return 0 if an attribute is not visible. The returned + * value will replace static permissions defined in struct + * attribute. Use SYSFS_GROUP_VISIBLE() when assigning this + * callback to specify separate _group_visible() and + * _attr_visible() handlers. * @is_bin_visible: * Optional: Function to return permissions associated with a * binary attribute of the group. Will be called repeatedly * for each binary attribute in the group. Only read/write - * permissions as well as SYSFS_PREALLOC are accepted. Must - * return 0 if a binary attribute is not visible. The returned - * value will replace static permissions defined in - * struct bin_attribute. + * permissions as well as SYSFS_PREALLOC (and the + * visibility flags for named groups) are accepted. Must + * return 0 if a binary attribute is not visible. The + * returned value will replace static permissions defined + * in struct bin_attribute. If @is_visible is not set, Use + * SYSFS_GROUP_VISIBLE() when assigning this callback to + * specify separate _group_visible() and _attr_visible() + * handlers. * @attrs: Pointer to NULL terminated list of attributes. * @bin_attrs: Pointer to NULL terminated list of binary attributes. * Either attrs or bin_attrs or both must be provided. @@ -91,13 +101,42 @@ struct attribute_group { struct bin_attribute **bin_attrs; }; +#define SYSFS_PREALLOC 010000 +#define SYSFS_GROUP_INVISIBLE 020000 + +/* + * The first call to is_visible() in the create / update path may + * indicate visibility for the entire group + */ +#define DEFINE_SYSFS_GROUP_VISIBLE(name) \ + static inline umode_t sysfs_group_visible_##name( \ + struct kobject *kobj, struct attribute *attr, int n) \ + { \ + if (n == 0 && !name##_group_visible(kobj)) \ + return SYSFS_GROUP_INVISIBLE; \ + return name##_attr_visible(kobj, attr, n); \ + } + +/* + * Same as DEFINE_SYSFS_GROUP_VISIBLE, but for groups with only binary + * attributes + */ +#define DEFINE_SYSFS_BIN_GROUP_VISIBLE(name) \ + static inline umode_t sysfs_group_visible_##name( \ + struct kobject *kobj, struct bin_attribute *attr, int n) \ + { \ + if (n == 0 && !name##_group_visible(kobj)) \ + return SYSFS_GROUP_INVISIBLE; \ + return name##_attr_visible(kobj, attr, n); \ + } + +#define SYSFS_GROUP_VISIBLE(fn) sysfs_group_visible_##fn + /* * Use these macros to make defining attributes easier. * See include/linux/device.h for examples.. */ -#define SYSFS_PREALLOC 010000 - #define __ATTR(_name, _mode, _show, _store) { \ .attr = {.name = __stringify(_name), \ .mode = VERIFY_OCTAL_PERMISSIONS(_mode) }, \ -- cgit v1.2.3 From 05013062a89fa4c2d5913dfb81a8ae0268e0a9dc Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 14 Feb 2024 10:31:54 +0100 Subject: pwm: lpss-*: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-lpss drivers to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/b567ab5dd992e361eb884fa6c2cac11be9c7dde3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pinctrl/intel/pinctrl-intel.c | 6 +++--- drivers/pwm/pwm-lpss-pci.c | 8 ++++---- drivers/pwm/pwm-lpss-platform.c | 8 ++++---- drivers/pwm/pwm-lpss.c | 24 ++++++++++++------------ drivers/pwm/pwm-lpss.h | 1 - include/linux/platform_data/x86/pwm-lpss.h | 4 ++-- 6 files changed, 25 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index d6f29e6faab7..89bd7ce6711a 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -1492,7 +1492,7 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl, .base_unit_bits = 22, .bypass = true, }; - struct pwm_lpss_chip *pwm; + struct pwm_chip *chip; if (!(community->features & PINCTRL_FEATURE_PWM)) return 0; @@ -1500,8 +1500,8 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl, if (!IS_REACHABLE(CONFIG_PWM_LPSS)) return 0; - pwm = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info); - return PTR_ERR_OR_ZERO(pwm); + chip = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info); + return PTR_ERR_OR_ZERO(chip); } int intel_pinctrl_probe(struct platform_device *pdev, diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c index 34acfe99b74f..25045c229520 100644 --- a/drivers/pwm/pwm-lpss-pci.c +++ b/drivers/pwm/pwm-lpss-pci.c @@ -18,7 +18,7 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, const struct pci_device_id *id) { const struct pwm_lpss_boardinfo *info; - struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; int err; err = pcim_enable_device(pdev); @@ -30,9 +30,9 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, return err; info = (struct pwm_lpss_boardinfo *)id->driver_data; - lpwm = devm_pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); - if (IS_ERR(lpwm)) - return PTR_ERR(lpwm); + chip = devm_pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); + if (IS_ERR(chip)) + return PTR_ERR(chip); pm_runtime_put(&pdev->dev); pm_runtime_allow(&pdev->dev); diff --git a/drivers/pwm/pwm-lpss-platform.c b/drivers/pwm/pwm-lpss-platform.c index 5f6ee300e342..dbc9f5b17bdc 100644 --- a/drivers/pwm/pwm-lpss-platform.c +++ b/drivers/pwm/pwm-lpss-platform.c @@ -20,7 +20,7 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) { const struct pwm_lpss_boardinfo *info; - struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; void __iomem *base; info = device_get_match_data(&pdev->dev); @@ -31,9 +31,9 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - lpwm = devm_pwm_lpss_probe(&pdev->dev, base, info); - if (IS_ERR(lpwm)) - return PTR_ERR(lpwm); + chip = devm_pwm_lpss_probe(&pdev->dev, base, info); + if (IS_ERR(chip)) + return PTR_ERR(chip); /* * On Cherry Trail devices the GFX0._PS0 AML checks if the controller diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c index 394c768f5a5f..867e2bc8c601 100644 --- a/drivers/pwm/pwm-lpss.c +++ b/drivers/pwm/pwm-lpss.c @@ -68,7 +68,7 @@ EXPORT_SYMBOL_GPL(pwm_lpss_tng_info); static inline struct pwm_lpss_chip *to_lpwm(struct pwm_chip *chip) { - return container_of(chip, struct pwm_lpss_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 pwm_lpss_read(const struct pwm_device *pwm) @@ -245,10 +245,11 @@ static const struct pwm_ops pwm_lpss_ops = { .get_state = pwm_lpss_get_state, }; -struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, - const struct pwm_lpss_boardinfo *info) +struct pwm_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, + const struct pwm_lpss_boardinfo *info) { struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; unsigned long c; int i, ret; u32 ctrl; @@ -256,9 +257,10 @@ struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base if (WARN_ON(info->npwm > LPSS_MAX_PWMS)) return ERR_PTR(-ENODEV); - lpwm = devm_kzalloc(dev, sizeof(*lpwm), GFP_KERNEL); - if (!lpwm) - return ERR_PTR(-ENOMEM); + chip = devm_pwmchip_alloc(dev, info->npwm, sizeof(*lpwm)); + if (IS_ERR(chip)) + return chip; + lpwm = to_lpwm(chip); lpwm->regs = base; lpwm->info = info; @@ -267,23 +269,21 @@ struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base if (!c) return ERR_PTR(-EINVAL); - lpwm->chip.dev = dev; - lpwm->chip.ops = &pwm_lpss_ops; - lpwm->chip.npwm = info->npwm; + chip->ops = &pwm_lpss_ops; - ret = devm_pwmchip_add(dev, &lpwm->chip); + ret = devm_pwmchip_add(dev, chip); if (ret) { dev_err(dev, "failed to add PWM chip: %d\n", ret); return ERR_PTR(ret); } for (i = 0; i < lpwm->info->npwm; i++) { - ctrl = pwm_lpss_read(&lpwm->chip.pwms[i]); + ctrl = pwm_lpss_read(&chip->pwms[i]); if (ctrl & PWM_ENABLE) pm_runtime_get(dev); } - return lpwm; + return chip; } EXPORT_SYMBOL_GPL(devm_pwm_lpss_probe); diff --git a/drivers/pwm/pwm-lpss.h b/drivers/pwm/pwm-lpss.h index bf841250385f..b5267ab5193b 100644 --- a/drivers/pwm/pwm-lpss.h +++ b/drivers/pwm/pwm-lpss.h @@ -18,7 +18,6 @@ #define LPSS_MAX_PWMS 4 struct pwm_lpss_chip { - struct pwm_chip chip; void __iomem *regs; const struct pwm_lpss_boardinfo *info; }; diff --git a/include/linux/platform_data/x86/pwm-lpss.h b/include/linux/platform_data/x86/pwm-lpss.h index c852fe24fe2a..752c06b47cc8 100644 --- a/include/linux/platform_data/x86/pwm-lpss.h +++ b/include/linux/platform_data/x86/pwm-lpss.h @@ -27,7 +27,7 @@ struct pwm_lpss_boardinfo { bool other_devices_aml_touches_pwm_regs; }; -struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, - const struct pwm_lpss_boardinfo *info); +struct pwm_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, + const struct pwm_lpss_boardinfo *info); #endif /* __PLATFORM_DATA_X86_PWM_LPSS_H */ -- cgit v1.2.3 From f70405afc99b1e5a3a1e60b6c05456fde2dbe622 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 19 Feb 2024 15:41:11 +0000 Subject: locking: Add rwsem_assert_held() and rwsem_assert_held_write() Modelled after lockdep_assert_held() and lockdep_assert_held_write(), but are always active, even when lockdep is disabled. Of course, they don't test that _this_ thread is the owner, but it's sufficient to catch many bugs and doesn't incur the same performance penalty as lockdep. Acked-by: "Peter Zijlstra (Intel)" Acked-by: Waiman Long Acked-by: "Darrick J. Wong" Reviewed-by: Dave Chinner Signed-off-by: "Matthew Wilcox (Oracle)" Signed-off-by: Chandan Babu R --- include/linux/rwbase_rt.h | 9 +++++++-- include/linux/rwsem.h | 46 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rwbase_rt.h b/include/linux/rwbase_rt.h index 1d264dd08625..29c4e4f243e4 100644 --- a/include/linux/rwbase_rt.h +++ b/include/linux/rwbase_rt.h @@ -26,12 +26,17 @@ struct rwbase_rt { } while (0) -static __always_inline bool rw_base_is_locked(struct rwbase_rt *rwb) +static __always_inline bool rw_base_is_locked(const struct rwbase_rt *rwb) { return atomic_read(&rwb->readers) != READER_BIAS; } -static __always_inline bool rw_base_is_contended(struct rwbase_rt *rwb) +static inline void rw_base_assert_held_write(const struct rwbase_rt *rwb) +{ + WARN_ON(atomic_read(&rwb->readers) != WRITER_BIAS); +} + +static __always_inline bool rw_base_is_contended(const struct rwbase_rt *rwb) { return atomic_read(&rwb->readers) > 0; } diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index 9c29689ff505..4f1c18992f76 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -66,14 +66,24 @@ struct rw_semaphore { #endif }; -/* In all implementations count != 0 means locked */ +#define RWSEM_UNLOCKED_VALUE 0UL +#define RWSEM_WRITER_LOCKED (1UL << 0) +#define __RWSEM_COUNT_INIT(name) .count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE) + static inline int rwsem_is_locked(struct rw_semaphore *sem) { - return atomic_long_read(&sem->count) != 0; + return atomic_long_read(&sem->count) != RWSEM_UNLOCKED_VALUE; } -#define RWSEM_UNLOCKED_VALUE 0L -#define __RWSEM_COUNT_INIT(name) .count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE) +static inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem) +{ + WARN_ON(atomic_long_read(&sem->count) == RWSEM_UNLOCKED_VALUE); +} + +static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem) +{ + WARN_ON(!(atomic_long_read(&sem->count) & RWSEM_WRITER_LOCKED)); +} /* Common initializer macros and functions */ @@ -152,11 +162,21 @@ do { \ __init_rwsem((sem), #sem, &__key); \ } while (0) -static __always_inline int rwsem_is_locked(struct rw_semaphore *sem) +static __always_inline int rwsem_is_locked(const struct rw_semaphore *sem) { return rw_base_is_locked(&sem->rwbase); } +static inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem) +{ + WARN_ON(!rwsem_is_locked(sem)); +} + +static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem) +{ + rw_base_assert_held_write(sem); +} + static __always_inline int rwsem_is_contended(struct rw_semaphore *sem) { return rw_base_is_contended(&sem->rwbase); @@ -169,6 +189,22 @@ static __always_inline int rwsem_is_contended(struct rw_semaphore *sem) * the RT specific variant. */ +static inline void rwsem_assert_held(const struct rw_semaphore *sem) +{ + if (IS_ENABLED(CONFIG_LOCKDEP)) + lockdep_assert_held(sem); + else + rwsem_assert_held_nolockdep(sem); +} + +static inline void rwsem_assert_held_write(const struct rw_semaphore *sem) +{ + if (IS_ENABLED(CONFIG_LOCKDEP)) + lockdep_assert_held_write(sem); + else + rwsem_assert_held_write_nolockdep(sem); +} + /* * lock for reading */ -- cgit v1.2.3 From 9c446288d7b31402adb454535cb2c3cbdb55bb88 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Sat, 10 Feb 2024 21:57:16 +0100 Subject: iio: buffer-dmaengine: export buffer alloc and free functions Export iio_dmaengine_buffer_free() and iio_dmaengine_buffer_alloc(). This is in preparation of introducing IIO backends support. This will allow us to allocate a buffer and control it's lifetime from a device different from the one holding the DMA firmware properties. Effectively, in this case the struct device holding the firmware information about the DMA channels is not the same as iio_dev->dev.parent (typical case). While at it, namespace the buffer-dmaengine exports and update the current user of these buffers. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240210-iio-backend-v11-4-f5242a5fb42a@analog.com Reviewed-by: Andy Shevchenko Signed-off-by: Jonathan Cameron --- drivers/iio/adc/adi-axi-adc.c | 1 + drivers/iio/buffer/industrialio-buffer-dmaengine.c | 8 +++++--- include/linux/iio/buffer-dmaengine.h | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index c247ff1541d2..0f21d1d98b9f 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -447,3 +447,4 @@ module_platform_driver(adi_axi_adc_driver); MODULE_AUTHOR("Michael Hennerich "); MODULE_DESCRIPTION("Analog Devices Generic AXI ADC IP core driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(IIO_DMAENGINE_BUFFER); diff --git a/drivers/iio/buffer/industrialio-buffer-dmaengine.c b/drivers/iio/buffer/industrialio-buffer-dmaengine.c index 45fe7d0d42ee..a18c1da292af 100644 --- a/drivers/iio/buffer/industrialio-buffer-dmaengine.c +++ b/drivers/iio/buffer/industrialio-buffer-dmaengine.c @@ -159,7 +159,7 @@ static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = { * Once done using the buffer iio_dmaengine_buffer_free() should be used to * release it. */ -static struct iio_buffer *iio_dmaengine_buffer_alloc(struct device *dev, +struct iio_buffer *iio_dmaengine_buffer_alloc(struct device *dev, const char *channel) { struct dmaengine_buffer *dmaengine_buffer; @@ -210,6 +210,7 @@ err_free: kfree(dmaengine_buffer); return ERR_PTR(ret); } +EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_alloc, IIO_DMAENGINE_BUFFER); /** * iio_dmaengine_buffer_free() - Free dmaengine buffer @@ -217,7 +218,7 @@ err_free: * * Frees a buffer previously allocated with iio_dmaengine_buffer_alloc(). */ -static void iio_dmaengine_buffer_free(struct iio_buffer *buffer) +void iio_dmaengine_buffer_free(struct iio_buffer *buffer) { struct dmaengine_buffer *dmaengine_buffer = iio_buffer_to_dmaengine_buffer(buffer); @@ -227,6 +228,7 @@ static void iio_dmaengine_buffer_free(struct iio_buffer *buffer) iio_buffer_put(buffer); } +EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_free, IIO_DMAENGINE_BUFFER); static void __devm_iio_dmaengine_buffer_free(void *buffer) { @@ -287,7 +289,7 @@ int devm_iio_dmaengine_buffer_setup(struct device *dev, return iio_device_attach_buffer(indio_dev, buffer); } -EXPORT_SYMBOL_GPL(devm_iio_dmaengine_buffer_setup); +EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup, IIO_DMAENGINE_BUFFER); MODULE_AUTHOR("Lars-Peter Clausen "); MODULE_DESCRIPTION("DMA buffer for the IIO framework"); diff --git a/include/linux/iio/buffer-dmaengine.h b/include/linux/iio/buffer-dmaengine.h index 5c355be89814..cbb8ba957fad 100644 --- a/include/linux/iio/buffer-dmaengine.h +++ b/include/linux/iio/buffer-dmaengine.h @@ -10,6 +10,9 @@ struct iio_dev; struct device; +struct iio_buffer *iio_dmaengine_buffer_alloc(struct device *dev, + const char *channel); +void iio_dmaengine_buffer_free(struct iio_buffer *buffer); int devm_iio_dmaengine_buffer_setup(struct device *dev, struct iio_dev *indio_dev, const char *channel); -- cgit v1.2.3 From 1a97905d3e48ebe79a06d16143fbfa427c56ce5f Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Sat, 10 Feb 2024 21:57:17 +0100 Subject: iio: add the IIO backend framework This is a Framework to handle complex IIO aggregate devices. The typical architecture is to have one device as the frontend device which can be "linked" against one or multiple backend devices. All the IIO and userspace interface is expected to be registers/managed by the frontend device which will callback into the backends when needed (to get/set some configuration that it does not directly control). The basic framework interface is pretty simple: - Backends should register themselves with @devm_iio_backend_register() - Frontend devices should get backends with @devm_iio_backend_get() Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240210-iio-backend-v11-5-f5242a5fb42a@analog.com Reviewed-by: Andy Shevchenko Signed-off-by: Jonathan Cameron --- MAINTAINERS | 8 + drivers/iio/Kconfig | 9 + drivers/iio/Makefile | 1 + drivers/iio/industrialio-backend.c | 418 +++++++++++++++++++++++++++++++++++++ include/linux/iio/backend.h | 72 +++++++ 5 files changed, 508 insertions(+) create mode 100644 drivers/iio/industrialio-backend.c create mode 100644 include/linux/iio/backend.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index bc1e236f2184..6add3fde252d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10391,6 +10391,14 @@ L: linux-media@vger.kernel.org S: Maintained F: drivers/media/rc/iguanair.c +IIO BACKEND FRAMEWORK +M: Nuno Sa +R: Olivier Moysan +L: linux-iio@vger.kernel.org +S: Maintained +F: drivers/iio/industrialio-backend.c +F: include/linux/iio/backend.h + IIO DIGITAL POTENTIOMETER DAC M: Peter Rosin L: linux-iio@vger.kernel.org diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig index 52eb46ef84c1..9c351ffc7bed 100644 --- a/drivers/iio/Kconfig +++ b/drivers/iio/Kconfig @@ -71,6 +71,15 @@ config IIO_TRIGGERED_EVENT help Provides helper functions for setting up triggered events. +config IIO_BACKEND + tristate + help + Framework to handle complex IIO aggregate devices. The typical + architecture that can make use of this framework is to have one + device as the frontend device which can be "linked" against one or + multiple backend devices. The framework then makes it easy to get + and control such backend devices. + source "drivers/iio/accel/Kconfig" source "drivers/iio/adc/Kconfig" source "drivers/iio/addac/Kconfig" diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile index 9622347a1c1b..0ba0e1521ba4 100644 --- a/drivers/iio/Makefile +++ b/drivers/iio/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_IIO_GTS_HELPER) += industrialio-gts-helper.o obj-$(CONFIG_IIO_SW_DEVICE) += industrialio-sw-device.o obj-$(CONFIG_IIO_SW_TRIGGER) += industrialio-sw-trigger.o obj-$(CONFIG_IIO_TRIGGERED_EVENT) += industrialio-triggered-event.o +obj-$(CONFIG_IIO_BACKEND) += industrialio-backend.o obj-y += accel/ obj-y += adc/ diff --git a/drivers/iio/industrialio-backend.c b/drivers/iio/industrialio-backend.c new file mode 100644 index 000000000000..2fea2bbbe47f --- /dev/null +++ b/drivers/iio/industrialio-backend.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Framework to handle complex IIO aggregate devices. + * + * The typical architecture is to have one device as the frontend device which + * can be "linked" against one or multiple backend devices. All the IIO and + * userspace interface is expected to be registers/managed by the frontend + * device which will callback into the backends when needed (to get/set some + * configuration that it does not directly control). + * + * ------------------------------------------------------- + * ------------------ | ------------ ------------ ------- FPGA| + * | ADC |------------------------| | ADC CORE |---------| DMA CORE |------| RAM | | + * | (Frontend/IIO) | Serial Data (eg: LVDS) | |(backend) |---------| |------| | | + * | |------------------------| ------------ ------------ ------- | + * ------------------ ------------------------------------------------------- + * + * The framework interface is pretty simple: + * - Backends should register themselves with devm_iio_backend_register() + * - Frontend devices should get backends with devm_iio_backend_get() + * + * Also to note that the primary target for this framework are converters like + * ADC/DACs so iio_backend_ops will have some operations typical of converter + * devices. On top of that, this is "generic" for all IIO which means any kind + * of device can make use of the framework. That said, If the iio_backend_ops + * struct begins to grow out of control, we can always refactor things so that + * the industrialio-backend.c is only left with the really generic stuff. Then, + * we can build on top of it depending on the needs. + * + * Copyright (C) 2023-2024 Analog Devices Inc. + */ +#define dev_fmt(fmt) "iio-backend: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct iio_backend { + struct list_head entry; + const struct iio_backend_ops *ops; + struct device *dev; + struct module *owner; + void *priv; +}; + +/* + * Helper struct for requesting buffers. This ensures that we have all data + * that we need to free the buffer in a device managed action. + */ +struct iio_backend_buffer_pair { + struct iio_backend *back; + struct iio_buffer *buffer; +}; + +static LIST_HEAD(iio_back_list); +static DEFINE_MUTEX(iio_back_lock); + +/* + * Helper macros to call backend ops. Makes sure the option is supported. + */ +#define iio_backend_check_op(back, op) ({ \ + struct iio_backend *____back = back; \ + int ____ret = 0; \ + \ + if (!____back->ops->op) \ + ____ret = -EOPNOTSUPP; \ + \ + ____ret; \ +}) + +#define iio_backend_op_call(back, op, args...) ({ \ + struct iio_backend *__back = back; \ + int __ret; \ + \ + __ret = iio_backend_check_op(__back, op); \ + if (!__ret) \ + __ret = __back->ops->op(__back, ##args); \ + \ + __ret; \ +}) + +#define iio_backend_ptr_op_call(back, op, args...) ({ \ + struct iio_backend *__back = back; \ + void *ptr_err; \ + int __ret; \ + \ + __ret = iio_backend_check_op(__back, op); \ + if (__ret) \ + ptr_err = ERR_PTR(__ret); \ + else \ + ptr_err = __back->ops->op(__back, ##args); \ + \ + ptr_err; \ +}) + +#define iio_backend_void_op_call(back, op, args...) { \ + struct iio_backend *__back = back; \ + int __ret; \ + \ + __ret = iio_backend_check_op(__back, op); \ + if (!__ret) \ + __back->ops->op(__back, ##args); \ +} + +/** + * iio_backend_chan_enable - Enable a backend channel + * @back: Backend device + * @chan: Channel number + * + * RETURNS: + * 0 on success, negative error number on failure. + */ +int iio_backend_chan_enable(struct iio_backend *back, unsigned int chan) +{ + return iio_backend_op_call(back, chan_enable, chan); +} +EXPORT_SYMBOL_NS_GPL(iio_backend_chan_enable, IIO_BACKEND); + +/** + * iio_backend_chan_disable - Disable a backend channel + * @back: Backend device + * @chan: Channel number + * + * RETURNS: + * 0 on success, negative error number on failure. + */ +int iio_backend_chan_disable(struct iio_backend *back, unsigned int chan) +{ + return iio_backend_op_call(back, chan_disable, chan); +} +EXPORT_SYMBOL_NS_GPL(iio_backend_chan_disable, IIO_BACKEND); + +static void __iio_backend_disable(void *back) +{ + iio_backend_void_op_call(back, disable); +} + +/** + * devm_iio_backend_enable - Device managed backend enable + * @dev: Consumer device for the backend + * @back: Backend device + * + * RETURNS: + * 0 on success, negative error number on failure. + */ +int devm_iio_backend_enable(struct device *dev, struct iio_backend *back) +{ + int ret; + + ret = iio_backend_op_call(back, enable); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, __iio_backend_disable, back); +} +EXPORT_SYMBOL_NS_GPL(devm_iio_backend_enable, IIO_BACKEND); + +/** + * iio_backend_data_format_set - Configure the channel data format + * @back: Backend device + * @chan: Channel number + * @data: Data format + * + * Properly configure a channel with respect to the expected data format. A + * @struct iio_backend_data_fmt must be passed with the settings. + * + * RETURNS: + * 0 on success, negative error number on failure. + */ +int iio_backend_data_format_set(struct iio_backend *back, unsigned int chan, + const struct iio_backend_data_fmt *data) +{ + if (!data || data->type >= IIO_BACKEND_DATA_TYPE_MAX) + return -EINVAL; + + return iio_backend_op_call(back, data_format_set, chan, data); +} +EXPORT_SYMBOL_NS_GPL(iio_backend_data_format_set, IIO_BACKEND); + +static void iio_backend_free_buffer(void *arg) +{ + struct iio_backend_buffer_pair *pair = arg; + + iio_backend_void_op_call(pair->back, free_buffer, pair->buffer); +} + +/** + * devm_iio_backend_request_buffer - Device managed buffer request + * @dev: Consumer device for the backend + * @back: Backend device + * @indio_dev: IIO device + * + * Request an IIO buffer from the backend. The type of the buffer (typically + * INDIO_BUFFER_HARDWARE) is up to the backend to decide. This is because, + * normally, the backend dictates what kind of buffering we can get. + * + * The backend .free_buffer() hooks is automatically called on @dev detach. + * + * RETURNS: + * 0 on success, negative error number on failure. + */ +int devm_iio_backend_request_buffer(struct device *dev, + struct iio_backend *back, + struct iio_dev *indio_dev) +{ + struct iio_backend_buffer_pair *pair; + struct iio_buffer *buffer; + + pair = devm_kzalloc(dev, sizeof(*pair), GFP_KERNEL); + if (!pair) + return -ENOMEM; + + buffer = iio_backend_ptr_op_call(back, request_buffer, indio_dev); + if (IS_ERR(buffer)) + return PTR_ERR(buffer); + + /* weak reference should be all what we need */ + pair->back = back; + pair->buffer = buffer; + + return devm_add_action_or_reset(dev, iio_backend_free_buffer, pair); +} +EXPORT_SYMBOL_NS_GPL(devm_iio_backend_request_buffer, IIO_BACKEND); + +static void iio_backend_release(void *arg) +{ + struct iio_backend *back = arg; + + module_put(back->owner); +} + +static int __devm_iio_backend_get(struct device *dev, struct iio_backend *back) +{ + struct device_link *link; + int ret; + + /* + * Make sure the provider cannot be unloaded before the consumer module. + * Note that device_links would still guarantee that nothing is + * accessible (and breaks) but this makes it explicit that the consumer + * module must be also unloaded. + */ + if (!try_module_get(back->owner)) + return dev_err_probe(dev, -ENODEV, + "Cannot get module reference\n"); + + ret = devm_add_action_or_reset(dev, iio_backend_release, back); + if (ret) + return ret; + + link = device_link_add(dev, back->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) + return dev_err_probe(dev, -EINVAL, + "Could not link to supplier(%s)\n", + dev_name(back->dev)); + + dev_dbg(dev, "Found backend(%s) device\n", dev_name(back->dev)); + + return 0; +} + +/** + * devm_iio_backend_get - Device managed backend device get + * @dev: Consumer device for the backend + * @name: Backend name + * + * Get's the backend associated with @dev. + * + * RETURNS: + * A backend pointer, negative error pointer otherwise. + */ +struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name) +{ + struct fwnode_handle *fwnode; + struct iio_backend *back; + unsigned int index; + int ret; + + if (name) { + ret = device_property_match_string(dev, "io-backend-names", + name); + if (ret < 0) + return ERR_PTR(ret); + index = ret; + } else { + index = 0; + } + + fwnode = fwnode_find_reference(dev_fwnode(dev), "io-backends", index); + if (IS_ERR(fwnode)) { + dev_err_probe(dev, PTR_ERR(fwnode), + "Cannot get Firmware reference\n"); + return ERR_CAST(fwnode); + } + + guard(mutex)(&iio_back_lock); + list_for_each_entry(back, &iio_back_list, entry) { + if (!device_match_fwnode(back->dev, fwnode)) + continue; + + fwnode_handle_put(fwnode); + ret = __devm_iio_backend_get(dev, back); + if (ret) + return ERR_PTR(ret); + + return back; + } + + fwnode_handle_put(fwnode); + return ERR_PTR(-EPROBE_DEFER); +} +EXPORT_SYMBOL_NS_GPL(devm_iio_backend_get, IIO_BACKEND); + +/** + * __devm_iio_backend_get_from_fwnode_lookup - Device managed fwnode backend device get + * @dev: Consumer device for the backend + * @fwnode: Firmware node of the backend device + * + * Search the backend list for a device matching @fwnode. + * This API should not be used and it's only present for preventing the first + * user of this framework to break it's DT ABI. + * + * RETURNS: + * A backend pointer, negative error pointer otherwise. + */ +struct iio_backend * +__devm_iio_backend_get_from_fwnode_lookup(struct device *dev, + struct fwnode_handle *fwnode) +{ + struct iio_backend *back; + int ret; + + guard(mutex)(&iio_back_lock); + list_for_each_entry(back, &iio_back_list, entry) { + if (!device_match_fwnode(back->dev, fwnode)) + continue; + + ret = __devm_iio_backend_get(dev, back); + if (ret) + return ERR_PTR(ret); + + return back; + } + + return ERR_PTR(-EPROBE_DEFER); +} +EXPORT_SYMBOL_NS_GPL(__devm_iio_backend_get_from_fwnode_lookup, IIO_BACKEND); + +/** + * iio_backend_get_priv - Get driver private data + * @back: Backend device + */ +void *iio_backend_get_priv(const struct iio_backend *back) +{ + return back->priv; +} +EXPORT_SYMBOL_NS_GPL(iio_backend_get_priv, IIO_BACKEND); + +static void iio_backend_unregister(void *arg) +{ + struct iio_backend *back = arg; + + guard(mutex)(&iio_back_lock); + list_del(&back->entry); +} + +/** + * devm_iio_backend_register - Device managed backend device register + * @dev: Backend device being registered + * @ops: Backend ops + * @priv: Device private data + * + * @ops is mandatory. Not providing it results in -EINVAL. + * + * RETURNS: + * 0 on success, negative error number on failure. + */ +int devm_iio_backend_register(struct device *dev, + const struct iio_backend_ops *ops, void *priv) +{ + struct iio_backend *back; + + if (!ops) + return dev_err_probe(dev, -EINVAL, "No backend ops given\n"); + + /* + * Through device_links, we guarantee that a frontend device cannot be + * bound/exist if the backend driver is not around. Hence, we can bind + * the backend object lifetime with the device being passed since + * removing it will tear the frontend/consumer down. + */ + back = devm_kzalloc(dev, sizeof(*back), GFP_KERNEL); + if (!back) + return -ENOMEM; + + back->ops = ops; + back->owner = dev->driver->owner; + back->dev = dev; + back->priv = priv; + scoped_guard(mutex, &iio_back_lock) + list_add(&back->entry, &iio_back_list); + + return devm_add_action_or_reset(dev, iio_backend_unregister, back); +} +EXPORT_SYMBOL_NS_GPL(devm_iio_backend_register, IIO_BACKEND); + +MODULE_AUTHOR("Nuno Sa "); +MODULE_DESCRIPTION("Framework to handle complex IIO aggregate devices"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h new file mode 100644 index 000000000000..a6d79381866e --- /dev/null +++ b/include/linux/iio/backend.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _IIO_BACKEND_H_ +#define _IIO_BACKEND_H_ + +#include + +struct fwnode_handle; +struct iio_backend; +struct device; +struct iio_dev; + +enum iio_backend_data_type { + IIO_BACKEND_TWOS_COMPLEMENT, + IIO_BACKEND_OFFSET_BINARY, + IIO_BACKEND_DATA_TYPE_MAX +}; + +/** + * struct iio_backend_data_fmt - Backend data format + * @type: Data type. + * @sign_extend: Bool to tell if the data is sign extended. + * @enable: Enable/Disable the data format module. If disabled, + * not formatting will happen. + */ +struct iio_backend_data_fmt { + enum iio_backend_data_type type; + bool sign_extend; + bool enable; +}; + +/** + * struct iio_backend_ops - operations structure for an iio_backend + * @enable: Enable backend. + * @disable: Disable backend. + * @chan_enable: Enable one channel. + * @chan_disable: Disable one channel. + * @data_format_set: Configure the data format for a specific channel. + * @request_buffer: Request an IIO buffer. + * @free_buffer: Free an IIO buffer. + **/ +struct iio_backend_ops { + int (*enable)(struct iio_backend *back); + void (*disable)(struct iio_backend *back); + int (*chan_enable)(struct iio_backend *back, unsigned int chan); + int (*chan_disable)(struct iio_backend *back, unsigned int chan); + int (*data_format_set)(struct iio_backend *back, unsigned int chan, + const struct iio_backend_data_fmt *data); + struct iio_buffer *(*request_buffer)(struct iio_backend *back, + struct iio_dev *indio_dev); + void (*free_buffer)(struct iio_backend *back, + struct iio_buffer *buffer); +}; + +int iio_backend_chan_enable(struct iio_backend *back, unsigned int chan); +int iio_backend_chan_disable(struct iio_backend *back, unsigned int chan); +int devm_iio_backend_enable(struct device *dev, struct iio_backend *back); +int iio_backend_data_format_set(struct iio_backend *back, unsigned int chan, + const struct iio_backend_data_fmt *data); +int devm_iio_backend_request_buffer(struct device *dev, + struct iio_backend *back, + struct iio_dev *indio_dev); + +void *iio_backend_get_priv(const struct iio_backend *conv); +struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name); +struct iio_backend * +__devm_iio_backend_get_from_fwnode_lookup(struct device *dev, + struct fwnode_handle *fwnode); + +int devm_iio_backend_register(struct device *dev, + const struct iio_backend_ops *ops, void *priv); + +#endif -- cgit v1.2.3 From 794ef0e57854d794173c8ab6bcce3285032dcd95 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Sat, 10 Feb 2024 21:57:19 +0100 Subject: iio: adc: adi-axi-adc: move to backend framework Move to the IIO backend framework. Devices supported by adi-axi-adc now register themselves as backend devices. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240210-iio-backend-v11-7-f5242a5fb42a@analog.com Reviewed-by: Andy Shevchenko Signed-off-by: Jonathan Cameron --- drivers/iio/adc/Kconfig | 2 +- drivers/iio/adc/adi-axi-adc.c | 384 +++++++++--------------------------- include/linux/iio/adc/adi-axi-adc.h | 68 ------- 3 files changed, 96 insertions(+), 358 deletions(-) delete mode 100644 include/linux/iio/adc/adi-axi-adc.h (limited to 'include/linux') diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 6793faacb4dc..02a55809cb6a 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -309,7 +309,7 @@ config ADI_AXI_ADC select IIO_BUFFER_HW_CONSUMER select IIO_BUFFER_DMAENGINE select REGMAP_MMIO - depends on OF + select IIO_BACKEND help Say yes here to build support for Analog Devices Generic AXI ADC IP core. The IP core is used for interfacing with diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index 0f21d1d98b9f..4156639b3c8b 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -17,13 +18,12 @@ #include #include -#include -#include -#include -#include - #include -#include + +#include +#include +#include +#include /* * Register definitions: @@ -44,6 +44,7 @@ #define ADI_AXI_REG_CHAN_CTRL_PN_SEL_OWR BIT(10) #define ADI_AXI_REG_CHAN_CTRL_IQCOR_EN BIT(9) #define ADI_AXI_REG_CHAN_CTRL_DCFILT_EN BIT(8) +#define ADI_AXI_REG_CHAN_CTRL_FMT_MASK GENMASK(6, 4) #define ADI_AXI_REG_CHAN_CTRL_FMT_SIGNEXT BIT(6) #define ADI_AXI_REG_CHAN_CTRL_FMT_TYPE BIT(5) #define ADI_AXI_REG_CHAN_CTRL_FMT_EN BIT(4) @@ -55,286 +56,100 @@ ADI_AXI_REG_CHAN_CTRL_FMT_EN | \ ADI_AXI_REG_CHAN_CTRL_ENABLE) -struct adi_axi_adc_core_info { - unsigned int version; -}; - struct adi_axi_adc_state { - struct mutex lock; - - struct adi_axi_adc_client *client; struct regmap *regmap; -}; - -struct adi_axi_adc_client { - struct list_head entry; - struct adi_axi_adc_conv conv; - struct adi_axi_adc_state *state; struct device *dev; - const struct adi_axi_adc_core_info *info; }; -static LIST_HEAD(registered_clients); -static DEFINE_MUTEX(registered_clients_lock); - -static struct adi_axi_adc_client *conv_to_client(struct adi_axi_adc_conv *conv) -{ - return container_of(conv, struct adi_axi_adc_client, conv); -} - -void *adi_axi_adc_conv_priv(struct adi_axi_adc_conv *conv) -{ - struct adi_axi_adc_client *cl = conv_to_client(conv); - - return (char *)cl + ALIGN(sizeof(struct adi_axi_adc_client), - IIO_DMA_MINALIGN); -} -EXPORT_SYMBOL_NS_GPL(adi_axi_adc_conv_priv, IIO_ADI_AXI); - -static int adi_axi_adc_config_dma_buffer(struct device *dev, - struct iio_dev *indio_dev) -{ - const char *dma_name; - - if (!device_property_present(dev, "dmas")) - return 0; - - if (device_property_read_string(dev, "dma-names", &dma_name)) - dma_name = "rx"; - - return devm_iio_dmaengine_buffer_setup(indio_dev->dev.parent, - indio_dev, dma_name); -} - -static int adi_axi_adc_read_raw(struct iio_dev *indio_dev, - struct iio_chan_spec const *chan, - int *val, int *val2, long mask) -{ - struct adi_axi_adc_state *st = iio_priv(indio_dev); - struct adi_axi_adc_conv *conv = &st->client->conv; - - if (!conv->read_raw) - return -EOPNOTSUPP; - - return conv->read_raw(conv, chan, val, val2, mask); -} - -static int adi_axi_adc_write_raw(struct iio_dev *indio_dev, - struct iio_chan_spec const *chan, - int val, int val2, long mask) -{ - struct adi_axi_adc_state *st = iio_priv(indio_dev); - struct adi_axi_adc_conv *conv = &st->client->conv; - - if (!conv->write_raw) - return -EOPNOTSUPP; - - return conv->write_raw(conv, chan, val, val2, mask); -} - -static int adi_axi_adc_read_avail(struct iio_dev *indio_dev, - struct iio_chan_spec const *chan, - const int **vals, int *type, int *length, - long mask) -{ - struct adi_axi_adc_state *st = iio_priv(indio_dev); - struct adi_axi_adc_conv *conv = &st->client->conv; - - if (!conv->read_avail) - return -EOPNOTSUPP; - - return conv->read_avail(conv, chan, vals, type, length, mask); -} - -static int adi_axi_adc_update_scan_mode(struct iio_dev *indio_dev, - const unsigned long *scan_mask) +static int axi_adc_enable(struct iio_backend *back) { - struct adi_axi_adc_state *st = iio_priv(indio_dev); - struct adi_axi_adc_conv *conv = &st->client->conv; - unsigned int i; + struct adi_axi_adc_state *st = iio_backend_get_priv(back); int ret; - for (i = 0; i < conv->chip_info->num_channels; i++) { - if (test_bit(i, scan_mask)) - ret = regmap_set_bits(st->regmap, - ADI_AXI_REG_CHAN_CTRL(i), - ADI_AXI_REG_CHAN_CTRL_ENABLE); - else - ret = regmap_clear_bits(st->regmap, - ADI_AXI_REG_CHAN_CTRL(i), - ADI_AXI_REG_CHAN_CTRL_ENABLE); - if (ret) - return ret; - } - - return 0; -} - -static struct adi_axi_adc_conv *adi_axi_adc_conv_register(struct device *dev, - size_t sizeof_priv) -{ - struct adi_axi_adc_client *cl; - size_t alloc_size; - - alloc_size = ALIGN(sizeof(struct adi_axi_adc_client), IIO_DMA_MINALIGN); - if (sizeof_priv) - alloc_size += ALIGN(sizeof_priv, IIO_DMA_MINALIGN); - - cl = kzalloc(alloc_size, GFP_KERNEL); - if (!cl) - return ERR_PTR(-ENOMEM); - - mutex_lock(®istered_clients_lock); - - cl->dev = get_device(dev); - - list_add_tail(&cl->entry, ®istered_clients); - - mutex_unlock(®istered_clients_lock); + ret = regmap_set_bits(st->regmap, ADI_AXI_REG_RSTN, + ADI_AXI_REG_RSTN_MMCM_RSTN); + if (ret) + return ret; - return &cl->conv; + fsleep(10000); + return regmap_set_bits(st->regmap, ADI_AXI_REG_RSTN, + ADI_AXI_REG_RSTN_RSTN | ADI_AXI_REG_RSTN_MMCM_RSTN); } -static void adi_axi_adc_conv_unregister(struct adi_axi_adc_conv *conv) +static void axi_adc_disable(struct iio_backend *back) { - struct adi_axi_adc_client *cl = conv_to_client(conv); - - mutex_lock(®istered_clients_lock); - - list_del(&cl->entry); - put_device(cl->dev); + struct adi_axi_adc_state *st = iio_backend_get_priv(back); - mutex_unlock(®istered_clients_lock); - - kfree(cl); + regmap_write(st->regmap, ADI_AXI_REG_RSTN, 0); } -static void devm_adi_axi_adc_conv_release(void *conv) +static int axi_adc_data_format_set(struct iio_backend *back, unsigned int chan, + const struct iio_backend_data_fmt *data) { - adi_axi_adc_conv_unregister(conv); + struct adi_axi_adc_state *st = iio_backend_get_priv(back); + u32 val; + + if (!data->enable) + return regmap_clear_bits(st->regmap, + ADI_AXI_REG_CHAN_CTRL(chan), + ADI_AXI_REG_CHAN_CTRL_FMT_EN); + + val = FIELD_PREP(ADI_AXI_REG_CHAN_CTRL_FMT_EN, true); + if (data->sign_extend) + val |= FIELD_PREP(ADI_AXI_REG_CHAN_CTRL_FMT_SIGNEXT, true); + if (data->type == IIO_BACKEND_OFFSET_BINARY) + val |= FIELD_PREP(ADI_AXI_REG_CHAN_CTRL_FMT_TYPE, true); + + return regmap_update_bits(st->regmap, ADI_AXI_REG_CHAN_CTRL(chan), + ADI_AXI_REG_CHAN_CTRL_FMT_MASK, val); } -struct adi_axi_adc_conv *devm_adi_axi_adc_conv_register(struct device *dev, - size_t sizeof_priv) +static int axi_adc_chan_enable(struct iio_backend *back, unsigned int chan) { - struct adi_axi_adc_conv *conv; - int ret; - - conv = adi_axi_adc_conv_register(dev, sizeof_priv); - if (IS_ERR(conv)) - return conv; + struct adi_axi_adc_state *st = iio_backend_get_priv(back); - ret = devm_add_action_or_reset(dev, devm_adi_axi_adc_conv_release, - conv); - if (ret) - return ERR_PTR(ret); - - return conv; + return regmap_set_bits(st->regmap, ADI_AXI_REG_CHAN_CTRL(chan), + ADI_AXI_REG_CHAN_CTRL_ENABLE); } -EXPORT_SYMBOL_NS_GPL(devm_adi_axi_adc_conv_register, IIO_ADI_AXI); - -static const struct iio_info adi_axi_adc_info = { - .read_raw = &adi_axi_adc_read_raw, - .write_raw = &adi_axi_adc_write_raw, - .update_scan_mode = &adi_axi_adc_update_scan_mode, - .read_avail = &adi_axi_adc_read_avail, -}; - -static const struct adi_axi_adc_core_info adi_axi_adc_10_0_a_info = { - .version = ADI_AXI_PCORE_VER(10, 0, 'a'), -}; -static struct adi_axi_adc_client *adi_axi_adc_attach_client(struct device *dev) +static int axi_adc_chan_disable(struct iio_backend *back, unsigned int chan) { - const struct adi_axi_adc_core_info *info; - struct adi_axi_adc_client *cl; - struct device_node *cln; - - info = of_device_get_match_data(dev); - if (!info) - return ERR_PTR(-ENODEV); - - cln = of_parse_phandle(dev->of_node, "adi,adc-dev", 0); - if (!cln) { - dev_err(dev, "No 'adi,adc-dev' node defined\n"); - return ERR_PTR(-ENODEV); - } - - mutex_lock(®istered_clients_lock); - - list_for_each_entry(cl, ®istered_clients, entry) { - if (!cl->dev) - continue; - - if (cl->dev->of_node != cln) - continue; - - if (!try_module_get(cl->dev->driver->owner)) { - mutex_unlock(®istered_clients_lock); - of_node_put(cln); - return ERR_PTR(-ENODEV); - } - - get_device(cl->dev); - cl->info = info; - mutex_unlock(®istered_clients_lock); - of_node_put(cln); - return cl; - } + struct adi_axi_adc_state *st = iio_backend_get_priv(back); - mutex_unlock(®istered_clients_lock); - of_node_put(cln); - - return ERR_PTR(-EPROBE_DEFER); + return regmap_clear_bits(st->regmap, ADI_AXI_REG_CHAN_CTRL(chan), + ADI_AXI_REG_CHAN_CTRL_ENABLE); } -static int adi_axi_adc_setup_channels(struct device *dev, - struct adi_axi_adc_state *st) +static struct iio_buffer *axi_adc_request_buffer(struct iio_backend *back, + struct iio_dev *indio_dev) { - struct adi_axi_adc_conv *conv = &st->client->conv; - int i, ret; + struct adi_axi_adc_state *st = iio_backend_get_priv(back); + struct iio_buffer *buffer; + const char *dma_name; + int ret; - if (conv->preenable_setup) { - ret = conv->preenable_setup(conv); - if (ret) - return ret; - } + if (device_property_read_string(st->dev, "dma-names", &dma_name)) + dma_name = "rx"; - for (i = 0; i < conv->chip_info->num_channels; i++) { - ret = regmap_write(st->regmap, ADI_AXI_REG_CHAN_CTRL(i), - ADI_AXI_REG_CHAN_CTRL_DEFAULTS); - if (ret) - return ret; + buffer = iio_dmaengine_buffer_alloc(st->dev, dma_name); + if (IS_ERR(buffer)) { + dev_err(st->dev, "Could not get DMA buffer, %ld\n", + PTR_ERR(buffer)); + return ERR_CAST(buffer); } - return 0; -} - -static int axi_adc_reset(struct adi_axi_adc_state *st) -{ - int ret; - - ret = regmap_write(st->regmap, ADI_AXI_REG_RSTN, 0); - if (ret) - return ret; - - mdelay(10); - ret = regmap_write(st->regmap, ADI_AXI_REG_RSTN, - ADI_AXI_REG_RSTN_MMCM_RSTN); + indio_dev->modes |= INDIO_BUFFER_HARDWARE; + ret = iio_device_attach_buffer(indio_dev, buffer); if (ret) - return ret; + return ERR_PTR(ret); - mdelay(10); - return regmap_write(st->regmap, ADI_AXI_REG_RSTN, - ADI_AXI_REG_RSTN_RSTN | ADI_AXI_REG_RSTN_MMCM_RSTN); + return buffer; } -static void adi_axi_adc_cleanup(void *data) +static void axi_adc_free_buffer(struct iio_backend *back, + struct iio_buffer *buffer) { - struct adi_axi_adc_client *cl = data; - - put_device(cl->dev); - module_put(cl->dev->driver->owner); + iio_dmaengine_buffer_free(buffer); } static const struct regmap_config axi_adc_regmap_config = { @@ -344,45 +159,47 @@ static const struct regmap_config axi_adc_regmap_config = { .max_register = 0x0800, }; +static const struct iio_backend_ops adi_axi_adc_generic = { + .enable = axi_adc_enable, + .disable = axi_adc_disable, + .data_format_set = axi_adc_data_format_set, + .chan_enable = axi_adc_chan_enable, + .chan_disable = axi_adc_chan_disable, + .request_buffer = axi_adc_request_buffer, + .free_buffer = axi_adc_free_buffer, +}; + static int adi_axi_adc_probe(struct platform_device *pdev) { - struct adi_axi_adc_conv *conv; - struct iio_dev *indio_dev; - struct adi_axi_adc_client *cl; + const unsigned int *expected_ver; struct adi_axi_adc_state *st; void __iomem *base; unsigned int ver; int ret; - cl = adi_axi_adc_attach_client(&pdev->dev); - if (IS_ERR(cl)) - return PTR_ERR(cl); - - ret = devm_add_action_or_reset(&pdev->dev, adi_axi_adc_cleanup, cl); - if (ret) - return ret; - - indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*st)); - if (indio_dev == NULL) + st = devm_kzalloc(&pdev->dev, sizeof(*st), GFP_KERNEL); + if (!st) return -ENOMEM; - st = iio_priv(indio_dev); - st->client = cl; - cl->state = st; - mutex_init(&st->lock); - base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); + st->dev = &pdev->dev; st->regmap = devm_regmap_init_mmio(&pdev->dev, base, &axi_adc_regmap_config); if (IS_ERR(st->regmap)) return PTR_ERR(st->regmap); - conv = &st->client->conv; + expected_ver = device_get_match_data(&pdev->dev); + if (!expected_ver) + return -ENODEV; - ret = axi_adc_reset(st); + /* + * Force disable the core. Up to the frontend to enable us. And we can + * still read/write registers... + */ + ret = regmap_write(st->regmap, ADI_AXI_REG_RSTN, 0); if (ret) return ret; @@ -390,33 +207,19 @@ static int adi_axi_adc_probe(struct platform_device *pdev) if (ret) return ret; - if (cl->info->version > ver) { + if (*expected_ver > ver) { dev_err(&pdev->dev, "IP core version is too old. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n", - ADI_AXI_PCORE_VER_MAJOR(cl->info->version), - ADI_AXI_PCORE_VER_MINOR(cl->info->version), - ADI_AXI_PCORE_VER_PATCH(cl->info->version), + ADI_AXI_PCORE_VER_MAJOR(*expected_ver), + ADI_AXI_PCORE_VER_MINOR(*expected_ver), + ADI_AXI_PCORE_VER_PATCH(*expected_ver), ADI_AXI_PCORE_VER_MAJOR(ver), ADI_AXI_PCORE_VER_MINOR(ver), ADI_AXI_PCORE_VER_PATCH(ver)); return -ENODEV; } - indio_dev->info = &adi_axi_adc_info; - indio_dev->name = "adi-axi-adc"; - indio_dev->modes = INDIO_DIRECT_MODE; - indio_dev->num_channels = conv->chip_info->num_channels; - indio_dev->channels = conv->chip_info->channels; - - ret = adi_axi_adc_config_dma_buffer(&pdev->dev, indio_dev); - if (ret) - return ret; - - ret = adi_axi_adc_setup_channels(&pdev->dev, st); - if (ret) - return ret; - - ret = devm_iio_device_register(&pdev->dev, indio_dev); + ret = devm_iio_backend_register(&pdev->dev, &adi_axi_adc_generic, st); if (ret) return ret; @@ -428,6 +231,8 @@ static int adi_axi_adc_probe(struct platform_device *pdev) return 0; } +static unsigned int adi_axi_adc_10_0_a_info = ADI_AXI_PCORE_VER(10, 0, 'a'); + /* Match table for of_platform binding */ static const struct of_device_id adi_axi_adc_of_match[] = { { .compatible = "adi,axi-adc-10.0.a", .data = &adi_axi_adc_10_0_a_info }, @@ -448,3 +253,4 @@ MODULE_AUTHOR("Michael Hennerich "); MODULE_DESCRIPTION("Analog Devices Generic AXI ADC IP core driver"); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(IIO_DMAENGINE_BUFFER); +MODULE_IMPORT_NS(IIO_BACKEND); diff --git a/include/linux/iio/adc/adi-axi-adc.h b/include/linux/iio/adc/adi-axi-adc.h deleted file mode 100644 index b7904992d561..000000000000 --- a/include/linux/iio/adc/adi-axi-adc.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Analog Devices Generic AXI ADC IP core driver/library - * Link: https://wiki.analog.com/resources/fpga/docs/axi_adc_ip - * - * Copyright 2012-2020 Analog Devices Inc. - */ -#ifndef __ADI_AXI_ADC_H__ -#define __ADI_AXI_ADC_H__ - -struct device; -struct iio_chan_spec; - -/** - * struct adi_axi_adc_chip_info - Chip specific information - * @name Chip name - * @id Chip ID (usually product ID) - * @channels Channel specifications of type @struct iio_chan_spec - * @num_channels Number of @channels - * @scale_table Supported scales by the chip; tuples of 2 ints - * @num_scales Number of scales in the table - * @max_rate Maximum sampling rate supported by the device - */ -struct adi_axi_adc_chip_info { - const char *name; - unsigned int id; - - const struct iio_chan_spec *channels; - unsigned int num_channels; - - const unsigned int (*scale_table)[2]; - int num_scales; - - unsigned long max_rate; -}; - -/** - * struct adi_axi_adc_conv - data of the ADC attached to the AXI ADC - * @chip_info chip info details for the client ADC - * @preenable_setup op to run in the client before enabling the AXI ADC - * @reg_access IIO debugfs_reg_access hook for the client ADC - * @read_raw IIO read_raw hook for the client ADC - * @write_raw IIO write_raw hook for the client ADC - * @read_avail IIO read_avail hook for the client ADC - */ -struct adi_axi_adc_conv { - const struct adi_axi_adc_chip_info *chip_info; - - int (*preenable_setup)(struct adi_axi_adc_conv *conv); - int (*reg_access)(struct adi_axi_adc_conv *conv, unsigned int reg, - unsigned int writeval, unsigned int *readval); - int (*read_raw)(struct adi_axi_adc_conv *conv, - struct iio_chan_spec const *chan, - int *val, int *val2, long mask); - int (*write_raw)(struct adi_axi_adc_conv *conv, - struct iio_chan_spec const *chan, - int val, int val2, long mask); - int (*read_avail)(struct adi_axi_adc_conv *conv, - struct iio_chan_spec const *chan, - const int **val, int *type, int *length, long mask); -}; - -struct adi_axi_adc_conv *devm_adi_axi_adc_conv_register(struct device *dev, - size_t sizeof_priv); - -void *adi_axi_adc_conv_priv(struct adi_axi_adc_conv *conv); - -#endif -- cgit v1.2.3 From dc36561e1548a8ca93b34ef385da03c289ec5ac0 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Sun, 21 Jan 2024 19:09:00 +0800 Subject: firmware: arm_scmi: Implement clock get permissions ARM SCMI v3.2 introduces clock get permission command. To implement the same let us stash the values of those permissions in the scmi_clock_info. They indicate if the operation is forbidden or not. If the CLOCK_GET_PERMISSIONS command is not supported, the default permissions are set to allow the operations, otherwise they will be set according to the response of CLOCK_GET_PERMISSIONS from the SCMI platform firmware. Reviewed-by: Cristian Marussi Signed-off-by: Peng Fan Link: https://lore.kernel.org/r/20240121110901.1414856-1-peng.fan@oss.nxp.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/clock.c | 64 +++++++++++++++++++++++++++++++++++++++ include/linux/scmi_protocol.h | 3 ++ 2 files changed, 67 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c index 2e4d6479a639..959e48aba1b5 100644 --- a/drivers/firmware/arm_scmi/clock.c +++ b/drivers/firmware/arm_scmi/clock.c @@ -28,8 +28,13 @@ enum scmi_clock_protocol_cmd { CLOCK_POSSIBLE_PARENTS_GET = 0xC, CLOCK_PARENT_SET = 0xD, CLOCK_PARENT_GET = 0xE, + CLOCK_GET_PERMISSIONS = 0xF, }; +#define CLOCK_STATE_CONTROL_ALLOWED BIT(31) +#define CLOCK_PARENT_CONTROL_ALLOWED BIT(30) +#define CLOCK_RATE_CONTROL_ALLOWED BIT(29) + enum clk_state { CLK_STATE_DISABLE, CLK_STATE_ENABLE, @@ -49,6 +54,7 @@ struct scmi_msg_resp_clock_attributes { #define SUPPORTS_RATE_CHANGE_REQUESTED_NOTIF(x) ((x) & BIT(30)) #define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(29)) #define SUPPORTS_PARENT_CLOCK(x) ((x) & BIT(28)) +#define SUPPORTS_GET_PERMISSIONS(x) ((x) & BIT(1)) u8 name[SCMI_SHORT_NAME_MAX_SIZE]; __le32 clock_enable_latency; }; @@ -293,6 +299,35 @@ static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph, u3 return ret; } +static int +scmi_clock_get_permissions(const struct scmi_protocol_handle *ph, u32 clk_id, + struct scmi_clock_info *clk) +{ + struct scmi_xfer *t; + u32 perm; + int ret; + + ret = ph->xops->xfer_get_init(ph, CLOCK_GET_PERMISSIONS, + sizeof(clk_id), sizeof(perm), &t); + if (ret) + return ret; + + put_unaligned_le32(clk_id, t->tx.buf); + + ret = ph->xops->do_xfer(ph, t); + if (!ret) { + perm = get_unaligned_le32(t->rx.buf); + + clk->state_ctrl_forbidden = !(perm & CLOCK_STATE_CONTROL_ALLOWED); + clk->rate_ctrl_forbidden = !(perm & CLOCK_RATE_CONTROL_ALLOWED); + clk->parent_ctrl_forbidden = !(perm & CLOCK_PARENT_CONTROL_ALLOWED); + } + + ph->xops->xfer_put(ph, t); + + return ret; +} + static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph, u32 clk_id, struct scmi_clock_info *clk, u32 version) @@ -339,6 +374,8 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph, clk->rate_change_requested_notifications = true; if (SUPPORTS_PARENT_CLOCK(attributes)) scmi_clock_possible_parents(ph, clk_id, clk); + if (SUPPORTS_GET_PERMISSIONS(attributes)) + scmi_clock_get_permissions(ph, clk_id, clk); } return ret; @@ -511,6 +548,14 @@ static int scmi_clock_rate_set(const struct scmi_protocol_handle *ph, struct scmi_xfer *t; struct scmi_clock_set_rate *cfg; struct clock_info *ci = ph->get_priv(ph); + struct scmi_clock_info *clk; + + clk = scmi_clock_domain_lookup(ci, clk_id); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + if (clk->rate_ctrl_forbidden) + return -EACCES; ret = ph->xops->xfer_get_init(ph, CLOCK_RATE_SET, sizeof(*cfg), 0, &t); if (ret) @@ -596,6 +641,9 @@ scmi_clock_set_parent(const struct scmi_protocol_handle *ph, u32 clk_id, if (parent_id >= clk->num_parents) return -EINVAL; + if (clk->parent_ctrl_forbidden) + return -EACCES; + ret = ph->xops->xfer_get_init(ph, CLOCK_PARENT_SET, sizeof(*cfg), 0, &t); if (ret) @@ -679,6 +727,14 @@ static int scmi_clock_enable(const struct scmi_protocol_handle *ph, u32 clk_id, bool atomic) { struct clock_info *ci = ph->get_priv(ph); + struct scmi_clock_info *clk; + + clk = scmi_clock_domain_lookup(ci, clk_id); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + if (clk->state_ctrl_forbidden) + return -EACCES; return ci->clock_config_set(ph, clk_id, CLK_STATE_ENABLE, NULL_OEM_TYPE, 0, atomic); @@ -688,6 +744,14 @@ static int scmi_clock_disable(const struct scmi_protocol_handle *ph, u32 clk_id, bool atomic) { struct clock_info *ci = ph->get_priv(ph); + struct scmi_clock_info *clk; + + clk = scmi_clock_domain_lookup(ci, clk_id); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + if (clk->state_ctrl_forbidden) + return -EACCES; return ci->clock_config_set(ph, clk_id, CLK_STATE_DISABLE, NULL_OEM_TYPE, 0, atomic); diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index f2f05fb42d28..0cc40af5519a 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -47,6 +47,9 @@ struct scmi_clock_info { bool rate_discrete; bool rate_changed_notifications; bool rate_change_requested_notifications; + bool state_ctrl_forbidden; + bool rate_ctrl_forbidden; + bool parent_ctrl_forbidden; union { struct { int num_rates; -- cgit v1.2.3 From 989e8661dc45babf43070d519011dfc1e33c8875 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 11 Feb 2024 12:51:29 -0300 Subject: firmware: arm_ffa: Make ffa_bus_type const Now that the driver core can properly handle constant struct bus_type, move the ffa_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Cristian Marussi Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240211-bus_cleanup-firmware2-v1-1-1851c92c7be7@marliere.net Signed-off-by: Sudeep Holla --- drivers/firmware/arm_ffa/bus.c | 2 +- include/linux/arm_ffa.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c index 1c7940ba5539..2f557e90f2eb 100644 --- a/drivers/firmware/arm_ffa/bus.c +++ b/drivers/firmware/arm_ffa/bus.c @@ -105,7 +105,7 @@ static struct attribute *ffa_device_attributes_attrs[] = { }; ATTRIBUTE_GROUPS(ffa_device_attributes); -struct bus_type ffa_bus_type = { +const struct bus_type ffa_bus_type = { .name = "arm_ffa", .match = ffa_device_match, .probe = ffa_device_probe, diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index 3d0fde57ba90..c906f666ff5d 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -209,7 +209,7 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev) { return false; } #define module_ffa_driver(__ffa_driver) \ module_driver(__ffa_driver, ffa_register, ffa_unregister) -extern struct bus_type ffa_bus_type; +extern const struct bus_type ffa_bus_type; /* FFA transport related */ struct ffa_partition_info { -- cgit v1.2.3 From 21d2e6737c9789aa9b23c8a4131cbca8260139fd Mon Sep 17 00:00:00 2001 From: Mina Almasry Date: Wed, 14 Feb 2024 14:34:03 -0800 Subject: net: add netmem to skb_frag_t Use struct netmem* instead of page in skb_frag_t. Currently struct netmem* is always a struct page underneath, but the abstraction allows efforts to add support for skb frags not backed by pages. There is unfortunately 1 instance where the skb_frag_t is assumed to be a exactly a bio_vec in kcm. For this case, WARN_ON_ONCE and return error before doing a cast. Add skb[_frag]_fill_netmem_*() and skb_add_rx_frag_netmem() helpers so that the API can be used to create netmem skbs. Signed-off-by: Mina Almasry Acked-by: Paolo Abeni Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 100 +++++++++++++++++++++++++++++++++++-------------- net/core/skbuff.c | 34 +++++++++++++---- net/kcm/kcmsock.c | 7 ++-- 3 files changed, 102 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 696e7680656f..e3a2ed5d09ad 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -37,6 +37,7 @@ #endif #include #include +#include /** * DOC: skb checksums @@ -359,7 +360,11 @@ extern int sysctl_max_skb_frags; */ #define GSO_BY_FRAGS 0xFFFF -typedef struct bio_vec skb_frag_t; +typedef struct skb_frag { + netmem_ref netmem; + unsigned int len; + unsigned int offset; +} skb_frag_t; /** * skb_frag_size() - Returns the size of a skb fragment @@ -367,7 +372,7 @@ typedef struct bio_vec skb_frag_t; */ static inline unsigned int skb_frag_size(const skb_frag_t *frag) { - return frag->bv_len; + return frag->len; } /** @@ -377,7 +382,7 @@ static inline unsigned int skb_frag_size(const skb_frag_t *frag) */ static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size) { - frag->bv_len = size; + frag->len = size; } /** @@ -387,7 +392,7 @@ static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size) */ static inline void skb_frag_size_add(skb_frag_t *frag, int delta) { - frag->bv_len += delta; + frag->len += delta; } /** @@ -397,7 +402,7 @@ static inline void skb_frag_size_add(skb_frag_t *frag, int delta) */ static inline void skb_frag_size_sub(skb_frag_t *frag, int delta) { - frag->bv_len -= delta; + frag->len -= delta; } /** @@ -417,7 +422,7 @@ static inline bool skb_frag_must_loop(struct page *p) * skb_frag_foreach_page - loop over pages in a fragment * * @f: skb frag to operate on - * @f_off: offset from start of f->bv_page + * @f_off: offset from start of f->netmem * @f_len: length from f_off to loop over * @p: (temp var) current page * @p_off: (temp var) offset from start of current page, @@ -2429,22 +2434,37 @@ static inline unsigned int skb_pagelen(const struct sk_buff *skb) return skb_headlen(skb) + __skb_pagelen(skb); } +static inline void skb_frag_fill_netmem_desc(skb_frag_t *frag, + netmem_ref netmem, int off, + int size) +{ + frag->netmem = netmem; + frag->offset = off; + skb_frag_size_set(frag, size); +} + static inline void skb_frag_fill_page_desc(skb_frag_t *frag, struct page *page, int off, int size) { - frag->bv_page = page; - frag->bv_offset = off; - skb_frag_size_set(frag, size); + skb_frag_fill_netmem_desc(frag, page_to_netmem(page), off, size); +} + +static inline void __skb_fill_netmem_desc_noacc(struct skb_shared_info *shinfo, + int i, netmem_ref netmem, + int off, int size) +{ + skb_frag_t *frag = &shinfo->frags[i]; + + skb_frag_fill_netmem_desc(frag, netmem, off, size); } static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, int i, struct page *page, int off, int size) { - skb_frag_t *frag = &shinfo->frags[i]; - - skb_frag_fill_page_desc(frag, page, off, size); + __skb_fill_netmem_desc_noacc(shinfo, i, page_to_netmem(page), off, + size); } /** @@ -2460,10 +2480,10 @@ static inline void skb_len_add(struct sk_buff *skb, int delta) } /** - * __skb_fill_page_desc - initialise a paged fragment in an skb + * __skb_fill_netmem_desc - initialise a fragment in an skb * @skb: buffer containing fragment to be initialised - * @i: paged fragment index to initialise - * @page: the page to use for this fragment + * @i: fragment index to initialise + * @netmem: the netmem to use for this fragment * @off: the offset to the data with @page * @size: the length of the data * @@ -2472,10 +2492,12 @@ static inline void skb_len_add(struct sk_buff *skb, int delta) * * Does not take any additional reference on the fragment. */ -static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, - struct page *page, int off, int size) +static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i, + netmem_ref netmem, int off, int size) { - __skb_fill_page_desc_noacc(skb_shinfo(skb), i, page, off, size); + struct page *page = netmem_to_page(netmem); + + __skb_fill_netmem_desc_noacc(skb_shinfo(skb), i, netmem, off, size); /* Propagate page pfmemalloc to the skb if we can. The problem is * that not all callers have unique ownership of the page but rely @@ -2483,7 +2505,20 @@ static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, */ page = compound_head(page); if (page_is_pfmemalloc(page)) - skb->pfmemalloc = true; + skb->pfmemalloc = true; +} + +static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, + struct page *page, int off, int size) +{ + __skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size); +} + +static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i, + netmem_ref netmem, int off, int size) +{ + __skb_fill_netmem_desc(skb, i, netmem, off, size); + skb_shinfo(skb)->nr_frags = i + 1; } /** @@ -2503,8 +2538,7 @@ static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, static inline void skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size) { - __skb_fill_page_desc(skb, i, page, off, size); - skb_shinfo(skb)->nr_frags = i + 1; + skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size); } /** @@ -2528,8 +2562,16 @@ static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i, shinfo->nr_frags = i + 1; } -void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, - int size, unsigned int truesize); +void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem, + int off, int size, unsigned int truesize); + +static inline void skb_add_rx_frag(struct sk_buff *skb, int i, + struct page *page, int off, int size, + unsigned int truesize) +{ + skb_add_rx_frag_netmem(skb, i, page_to_netmem(page), off, size, + truesize); +} void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, unsigned int truesize); @@ -3378,7 +3420,7 @@ static inline void skb_propagate_pfmemalloc(const struct page *page, */ static inline unsigned int skb_frag_off(const skb_frag_t *frag) { - return frag->bv_offset; + return frag->offset; } /** @@ -3388,7 +3430,7 @@ static inline unsigned int skb_frag_off(const skb_frag_t *frag) */ static inline void skb_frag_off_add(skb_frag_t *frag, int delta) { - frag->bv_offset += delta; + frag->offset += delta; } /** @@ -3398,7 +3440,7 @@ static inline void skb_frag_off_add(skb_frag_t *frag, int delta) */ static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset) { - frag->bv_offset = offset; + frag->offset = offset; } /** @@ -3409,7 +3451,7 @@ static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset) static inline void skb_frag_off_copy(skb_frag_t *fragto, const skb_frag_t *fragfrom) { - fragto->bv_offset = fragfrom->bv_offset; + fragto->offset = fragfrom->offset; } /** @@ -3420,7 +3462,7 @@ static inline void skb_frag_off_copy(skb_frag_t *fragto, */ static inline struct page *skb_frag_page(const skb_frag_t *frag) { - return frag->bv_page; + return netmem_to_page(frag->netmem); } /** @@ -3528,7 +3570,7 @@ static inline void *skb_frag_address_safe(const skb_frag_t *frag) static inline void skb_frag_page_copy(skb_frag_t *fragto, const skb_frag_t *fragfrom) { - fragto->bv_page = fragfrom->bv_page; + fragto->netmem = fragfrom->netmem; } bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index b41856585c24..1434c422f76e 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -115,6 +115,24 @@ static struct kmem_cache *skb_small_head_cache __ro_after_init; int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; EXPORT_SYMBOL(sysctl_max_skb_frags); +/* kcm_write_msgs() relies on casting paged frags to bio_vec to use + * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the + * netmem is a page. + */ +static_assert(offsetof(struct bio_vec, bv_page) == + offsetof(skb_frag_t, netmem)); +static_assert(sizeof_field(struct bio_vec, bv_page) == + sizeof_field(skb_frag_t, netmem)); + +static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len)); +static_assert(sizeof_field(struct bio_vec, bv_len) == + sizeof_field(skb_frag_t, len)); + +static_assert(offsetof(struct bio_vec, bv_offset) == + offsetof(skb_frag_t, offset)); +static_assert(sizeof_field(struct bio_vec, bv_offset) == + sizeof_field(skb_frag_t, offset)); + #undef FN #define FN(reason) [SKB_DROP_REASON_##reason] = #reason, static const char * const drop_reasons[] = { @@ -845,17 +863,17 @@ skb_fail: } EXPORT_SYMBOL(__napi_alloc_skb); -void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, - int size, unsigned int truesize) +void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem, + int off, int size, unsigned int truesize) { DEBUG_NET_WARN_ON_ONCE(size > truesize); - skb_fill_page_desc(skb, i, page, off, size); + skb_fill_netmem_desc(skb, i, netmem, off, size); skb->len += size; skb->data_len += size; skb->truesize += truesize; } -EXPORT_SYMBOL(skb_add_rx_frag); +EXPORT_SYMBOL(skb_add_rx_frag_netmem); void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, unsigned int truesize) @@ -1999,10 +2017,11 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) /* skb frags point to kernel buffers */ for (i = 0; i < new_frags - 1; i++) { - __skb_fill_page_desc(skb, i, head, 0, psize); + __skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize); head = (struct page *)page_private(head); } - __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); + __skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0, + d_off); skb_shinfo(skb)->nr_frags = new_frags; release: @@ -3740,7 +3759,8 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) if (plen) { page = virt_to_head_page(from->head); offset = from->data - (unsigned char *)page_address(page); - __skb_fill_page_desc(to, 0, page, offset, plen); + __skb_fill_netmem_desc(to, 0, page_to_netmem(page), + offset, plen); get_page(page); j = 1; len -= plen; diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 1184d40167b8..73c200c5c8e4 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -627,7 +627,8 @@ retry: skb = txm->frag_skb; } - if (WARN_ON(!skb_shinfo(skb)->nr_frags)) { + if (WARN_ON(!skb_shinfo(skb)->nr_frags) || + WARN_ON_ONCE(!skb_frag_page(&skb_shinfo(skb)->frags[0]))) { ret = -EINVAL; goto out; } @@ -637,8 +638,8 @@ retry: msize += skb_frag_size(&skb_shinfo(skb)->frags[i]); iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, - skb_shinfo(skb)->frags, skb_shinfo(skb)->nr_frags, - msize); + (const struct bio_vec *)skb_shinfo(skb)->frags, + skb_shinfo(skb)->nr_frags, msize); iov_iter_advance(&msg.msg_iter, txm->frag_offset); do { -- cgit v1.2.3 From 219eee9c0d16f1b754a8b85275854ab17df0850a Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 16 Feb 2024 12:36:57 +0100 Subject: net: skbuff: add overflow debug check to pull/push helpers syzbot managed to trigger following splat: BUG: KASAN: use-after-free in __skb_flow_dissect+0x4a3b/0x5e50 Read of size 1 at addr ffff888208a4000e by task a.out/2313 [..] __skb_flow_dissect+0x4a3b/0x5e50 __skb_get_hash+0xb4/0x400 ip_tunnel_xmit+0x77e/0x26f0 ipip_tunnel_xmit+0x298/0x410 .. Analysis shows that the skb has a valid ->head, but bogus ->data pointer. skb->data gets its bogus value via the neigh layer, which does: 1556 __skb_pull(skb, skb_network_offset(skb)); ... and the skb was already dodgy at this point: skb_network_offset(skb) returns a negative value due to an earlier overflow of skb->network_header (u16). __skb_pull thus "adjusts" skb->data by a huge offset, pointing outside skb->head area. Allow debug builds to splat when we try to pull/push more than INT_MAX bytes. After this, the syzkaller reproducer yields a more precise splat before the flow dissector attempts to read off skb->data memory: WARNING: CPU: 5 PID: 2313 at include/linux/skbuff.h:2653 neigh_connected_output+0x28e/0x400 ip_finish_output2+0xb25/0xed0 iptunnel_xmit+0x4ff/0x870 ipgre_xmit+0x78e/0xbb0 Signed-off-by: Florian Westphal Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20240216113700.23013-1-fw@strlen.de Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index e3a2ed5d09ad..28c7cb7ce251 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2684,6 +2684,8 @@ static inline void skb_put_u8(struct sk_buff *skb, u8 val) void *skb_push(struct sk_buff *skb, unsigned int len); static inline void *__skb_push(struct sk_buff *skb, unsigned int len) { + DEBUG_NET_WARN_ON_ONCE(len > INT_MAX); + skb->data -= len; skb->len += len; return skb->data; @@ -2692,6 +2694,8 @@ static inline void *__skb_push(struct sk_buff *skb, unsigned int len) void *skb_pull(struct sk_buff *skb, unsigned int len); static inline void *__skb_pull(struct sk_buff *skb, unsigned int len) { + DEBUG_NET_WARN_ON_ONCE(len > INT_MAX); + skb->len -= len; if (unlikely(skb->len < skb->data_len)) { #if defined(CONFIG_DEBUG_NET) @@ -2716,6 +2720,8 @@ void *__pskb_pull_tail(struct sk_buff *skb, int delta); static inline enum skb_drop_reason pskb_may_pull_reason(struct sk_buff *skb, unsigned int len) { + DEBUG_NET_WARN_ON_ONCE(len > INT_MAX); + if (likely(len <= skb_headlen(skb))) return SKB_NOT_DROPPED_YET; -- cgit v1.2.3 From 22ffc748a6475e75e058ecb16c5afdd6b9f1885f Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 12 Feb 2024 12:32:33 +0000 Subject: firmware: arm_scmi: Report frequencies in the perf notifications Extend the perf notification report to include pre-calculated frequencies corresponding to the reported limits/levels event; such frequencies are properly computed based on the stored known OPPs information taking into consideration if the current operating mode is level indexed or not. Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240212123233.1230090-12-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/perf.c | 69 ++++++++++++++++++++++++++++++++++++++++ include/linux/scmi_protocol.h | 3 ++ 2 files changed, 72 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index e20d137a92f6..981e327e63e3 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -1055,18 +1055,47 @@ static int scmi_perf_set_notify_enabled(const struct scmi_protocol_handle *ph, return ret; } +static int +scmi_perf_xlate_opp_to_freq(struct perf_dom_info *dom, + unsigned int index, unsigned long *freq) +{ + struct scmi_opp *opp; + + if (!dom || !freq) + return -EINVAL; + + if (!dom->level_indexing_mode) { + opp = xa_load(&dom->opps_by_lvl, index); + if (!opp) + return -ENODEV; + + *freq = opp->perf * dom->mult_factor; + } else { + opp = xa_load(&dom->opps_by_idx, index); + if (!opp) + return -ENODEV; + + *freq = opp->indicative_freq * dom->mult_factor; + } + + return 0; +} + static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph, u8 evt_id, ktime_t timestamp, const void *payld, size_t payld_sz, void *report, u32 *src_id) { + int ret; void *rep = NULL; + struct perf_dom_info *dom; switch (evt_id) { case SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED: { const struct scmi_perf_limits_notify_payld *p = payld; struct scmi_perf_limits_report *r = report; + unsigned long freq_min, freq_max; if (sizeof(*p) != payld_sz) break; @@ -1076,14 +1105,36 @@ static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph, r->domain_id = le32_to_cpu(p->domain_id); r->range_max = le32_to_cpu(p->range_max); r->range_min = le32_to_cpu(p->range_min); + /* Check if the reported domain exist at all */ + dom = scmi_perf_domain_lookup(ph, r->domain_id); + if (IS_ERR(dom)) + break; + /* + * Event will be reported from this point on... + * ...even if, later, xlated frequencies were not retrieved. + */ *src_id = r->domain_id; rep = r; + + ret = scmi_perf_xlate_opp_to_freq(dom, r->range_max, &freq_max); + if (ret) + break; + + ret = scmi_perf_xlate_opp_to_freq(dom, r->range_min, &freq_min); + if (ret) + break; + + /* Report translated freqs ONLY if both available */ + r->range_max_freq = freq_max; + r->range_min_freq = freq_min; + break; } case SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED: { const struct scmi_perf_level_notify_payld *p = payld; struct scmi_perf_level_report *r = report; + unsigned long freq; if (sizeof(*p) != payld_sz) break; @@ -1091,9 +1142,27 @@ static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph, r->timestamp = timestamp; r->agent_id = le32_to_cpu(p->agent_id); r->domain_id = le32_to_cpu(p->domain_id); + /* Report translated freqs ONLY if available */ r->performance_level = le32_to_cpu(p->performance_level); + /* Check if the reported domain exist at all */ + dom = scmi_perf_domain_lookup(ph, r->domain_id); + if (IS_ERR(dom)) + break; + /* + * Event will be reported from this point on... + * ...even if, later, xlated frequencies were not retrieved. + */ *src_id = r->domain_id; rep = r; + + /* Report translated freqs ONLY if available */ + ret = scmi_perf_xlate_opp_to_freq(dom, r->performance_level, + &freq); + if (ret) + break; + + r->performance_level_freq = freq; + break; } default: diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 0cc40af5519a..9b9351e07a11 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -956,6 +956,8 @@ struct scmi_perf_limits_report { unsigned int domain_id; unsigned int range_max; unsigned int range_min; + unsigned long range_max_freq; + unsigned long range_min_freq; }; struct scmi_perf_level_report { @@ -963,6 +965,7 @@ struct scmi_perf_level_report { unsigned int agent_id; unsigned int domain_id; unsigned int performance_level; + unsigned long performance_level_freq; }; struct scmi_sensor_trip_point_report { -- cgit v1.2.3 From aaf2230036b709f979caccfbaa7a8bf671c22124 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 13 Feb 2024 11:45:25 -0300 Subject: tc: make tc_bus_type const Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the tc_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Acked-by: Maciej W. Rozycki Reviewed-by: Greg Kroah-Hartman Signed-off-by: Thomas Bogendoerfer --- drivers/tc/tc-driver.c | 2 +- include/linux/tc.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/tc/tc-driver.c b/drivers/tc/tc-driver.c index d45f2c1ff341..1c9d983a5a1f 100644 --- a/drivers/tc/tc-driver.c +++ b/drivers/tc/tc-driver.c @@ -95,7 +95,7 @@ static int tc_bus_match(struct device *dev, struct device_driver *drv) return 0; } -struct bus_type tc_bus_type = { +const struct bus_type tc_bus_type = { .name = "tc", .match = tc_bus_match, }; diff --git a/include/linux/tc.h b/include/linux/tc.h index a60639f37963..1638660abf5e 100644 --- a/include/linux/tc.h +++ b/include/linux/tc.h @@ -120,7 +120,7 @@ static inline unsigned long tc_get_speed(struct tc_bus *tbus) #ifdef CONFIG_TC -extern struct bus_type tc_bus_type; +extern const struct bus_type tc_bus_type; extern int tc_register_driver(struct tc_driver *tdrv); extern void tc_unregister_driver(struct tc_driver *tdrv); -- cgit v1.2.3 From 78b74638eb6dffd9b24bc3b121556a9039292df6 Mon Sep 17 00:00:00 2001 From: Paul Durrant Date: Thu, 15 Feb 2024 15:28:59 +0000 Subject: KVM: pfncache: add a mark-dirty helper At the moment pages are marked dirty by open-coded calls to mark_page_dirty_in_slot(), directly deferefencing the gpa and memslot from the cache. After a subsequent patch these may not always be set so add a helper now so that caller will protected from the need to know about this detail. Signed-off-by: Paul Durrant Reviewed-by: David Woodhouse Link: https://lore.kernel.org/r/20240215152916.1158-5-paul@xen.org [sean: decrease indentation, use gpa_to_gfn()] Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 +- arch/x86/kvm/xen.c | 6 +++--- include/linux/kvm_host.h | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index bf10a9073a09..f0f37c769a3a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3160,7 +3160,7 @@ static void kvm_setup_guest_pvclock(struct kvm_vcpu *v, guest_hv_clock->version = ++vcpu->hv_clock.version; - mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); + kvm_gpc_mark_dirty_in_slot(gpc); read_unlock_irqrestore(&gpc->lock, flags); trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index f3327508ae41..2d001a9c6378 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -453,11 +453,11 @@ static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic) } if (user_len2) { - mark_page_dirty_in_slot(v->kvm, gpc2->memslot, gpc2->gpa >> PAGE_SHIFT); + kvm_gpc_mark_dirty_in_slot(gpc2); read_unlock(&gpc2->lock); } - mark_page_dirty_in_slot(v->kvm, gpc1->memslot, gpc1->gpa >> PAGE_SHIFT); + kvm_gpc_mark_dirty_in_slot(gpc1); read_unlock_irqrestore(&gpc1->lock, flags); } @@ -565,7 +565,7 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) WRITE_ONCE(vi->evtchn_upcall_pending, 1); } - mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); + kvm_gpc_mark_dirty_in_slot(gpc); read_unlock_irqrestore(&gpc->lock, flags); /* For the per-vCPU lapic vector, deliver it as MSI. */ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 7e7fd25b09b3..604ae285d9a9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1795,6 +1795,16 @@ static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) return kvm_is_error_hva(hva); } +static inline void kvm_gpc_mark_dirty_in_slot(struct gfn_to_pfn_cache *gpc) +{ + lockdep_assert_held(&gpc->lock); + + if (!gpc->memslot) + return; + + mark_page_dirty_in_slot(gpc->kvm, gpc->memslot, gpa_to_gfn(gpc->gpa)); +} + enum kvm_stat_kind { KVM_STAT_VM, KVM_STAT_VCPU, -- cgit v1.2.3 From a4bff3df51472f555ab8dea05a3d2faf4abbf199 Mon Sep 17 00:00:00 2001 From: Paul Durrant Date: Thu, 15 Feb 2024 15:29:00 +0000 Subject: KVM: pfncache: remove KVM_GUEST_USES_PFN usage As noted in [1] the KVM_GUEST_USES_PFN usage flag is never set by any callers of kvm_gpc_init(), and for good reason: the implementation is incomplete/broken. And it's not clear that there will ever be a user of KVM_GUEST_USES_PFN, as coordinating vCPUs with mmu_notifier events is non-trivial. Remove KVM_GUEST_USES_PFN and all related code, e.g. dropping KVM_GUEST_USES_PFN also makes the 'vcpu' argument redundant, to avoid having to reason about broken code as __kvm_gpc_refresh() evolves. Moreover, all existing callers specify KVM_HOST_USES_PFN so the usage check in hva_to_pfn_retry() and hence the 'usage' argument to kvm_gpc_init() are also redundant. [1] https://lore.kernel.org/all/ZQiR8IpqOZrOpzHC@google.com Signed-off-by: Paul Durrant Reviewed-by: David Woodhouse Link: https://lore.kernel.org/r/20240215152916.1158-6-paul@xen.org [sean: explicitly call out that guest usage is incomplete] Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 +- arch/x86/kvm/xen.c | 14 ++++------- include/linux/kvm_host.h | 11 +-------- include/linux/kvm_types.h | 8 ------- virt/kvm/pfncache.c | 61 +++++++---------------------------------------- 5 files changed, 16 insertions(+), 80 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f0f37c769a3a..415723a28dce 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12056,7 +12056,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.regs_avail = ~0; vcpu->arch.regs_dirty = ~0; - kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN); + kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm); if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 2d001a9c6378..e90464225467 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -2108,14 +2108,10 @@ void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0); - kvm_gpc_init(&vcpu->arch.xen.runstate_cache, vcpu->kvm, NULL, - KVM_HOST_USES_PFN); - kvm_gpc_init(&vcpu->arch.xen.runstate2_cache, vcpu->kvm, NULL, - KVM_HOST_USES_PFN); - kvm_gpc_init(&vcpu->arch.xen.vcpu_info_cache, vcpu->kvm, NULL, - KVM_HOST_USES_PFN); - kvm_gpc_init(&vcpu->arch.xen.vcpu_time_info_cache, vcpu->kvm, NULL, - KVM_HOST_USES_PFN); + kvm_gpc_init(&vcpu->arch.xen.runstate_cache, vcpu->kvm); + kvm_gpc_init(&vcpu->arch.xen.runstate2_cache, vcpu->kvm); + kvm_gpc_init(&vcpu->arch.xen.vcpu_info_cache, vcpu->kvm); + kvm_gpc_init(&vcpu->arch.xen.vcpu_time_info_cache, vcpu->kvm); } void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) @@ -2158,7 +2154,7 @@ void kvm_xen_init_vm(struct kvm *kvm) { mutex_init(&kvm->arch.xen.xen_lock); idr_init(&kvm->arch.xen.evtchn_ports); - kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm, NULL, KVM_HOST_USES_PFN); + kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm); } void kvm_xen_destroy_vm(struct kvm *kvm) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 604ae285d9a9..3e1c04608c67 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1319,21 +1319,12 @@ void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); * * @gpc: struct gfn_to_pfn_cache object. * @kvm: pointer to kvm instance. - * @vcpu: vCPU to be used for marking pages dirty and to be woken on - * invalidation. - * @usage: indicates if the resulting host physical PFN is used while - * the @vcpu is IN_GUEST_MODE (in which case invalidation of - * the cache from MMU notifiers---but not for KVM memslot - * changes!---will also force @vcpu to exit the guest and - * refresh the cache); and/or if the PFN used directly - * by KVM (and thus needs a kernel virtual mapping). * * This sets up a gfn_to_pfn_cache by initializing locks and assigning the * immutable attributes. Note, the cache must be zero-allocated (or zeroed by * the caller before init). */ -void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm, - struct kvm_vcpu *vcpu, enum pfn_cache_usage usage); +void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm); /** * kvm_gpc_activate - prepare a cached kernel mapping and HPA for a given guest diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index 9d1f7835d8c1..d93f6522b2c3 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -49,12 +49,6 @@ typedef u64 hfn_t; typedef hfn_t kvm_pfn_t; -enum pfn_cache_usage { - KVM_GUEST_USES_PFN = BIT(0), - KVM_HOST_USES_PFN = BIT(1), - KVM_GUEST_AND_HOST_USE_PFN = KVM_GUEST_USES_PFN | KVM_HOST_USES_PFN, -}; - struct gfn_to_hva_cache { u64 generation; gpa_t gpa; @@ -69,13 +63,11 @@ struct gfn_to_pfn_cache { unsigned long uhva; struct kvm_memory_slot *memslot; struct kvm *kvm; - struct kvm_vcpu *vcpu; struct list_head list; rwlock_t lock; struct mutex refresh_lock; void *khva; kvm_pfn_t pfn; - enum pfn_cache_usage usage; bool active; bool valid; }; diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index f3571f44d9af..6f4b537eb25b 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -25,9 +25,7 @@ void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, unsigned long start, unsigned long end, bool may_block) { - DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS); struct gfn_to_pfn_cache *gpc; - bool evict_vcpus = false; spin_lock(&kvm->gpc_lock); list_for_each_entry(gpc, &kvm->gpc_list, list) { @@ -37,43 +35,10 @@ void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, unsigned long start, if (gpc->valid && !is_error_noslot_pfn(gpc->pfn) && gpc->uhva >= start && gpc->uhva < end) { gpc->valid = false; - - /* - * If a guest vCPU could be using the physical address, - * it needs to be forced out of guest mode. - */ - if (gpc->usage & KVM_GUEST_USES_PFN) { - if (!evict_vcpus) { - evict_vcpus = true; - bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS); - } - __set_bit(gpc->vcpu->vcpu_idx, vcpu_bitmap); - } } write_unlock_irq(&gpc->lock); } spin_unlock(&kvm->gpc_lock); - - if (evict_vcpus) { - /* - * KVM needs to ensure the vCPU is fully out of guest context - * before allowing the invalidation to continue. - */ - unsigned int req = KVM_REQ_OUTSIDE_GUEST_MODE; - bool called; - - /* - * If the OOM reaper is active, then all vCPUs should have - * been stopped already, so perform the request without - * KVM_REQUEST_WAIT and be sad if any needed to be IPI'd. - */ - if (!may_block) - req &= ~KVM_REQUEST_WAIT; - - called = kvm_make_vcpus_request_mask(kvm, req, vcpu_bitmap); - - WARN_ON_ONCE(called && !may_block); - } } bool kvm_gpc_check(struct gfn_to_pfn_cache *gpc, unsigned long len) @@ -206,16 +171,14 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) * pfn. Note, kmap() and memremap() can both sleep, so this * too must be done outside of gpc->lock! */ - if (gpc->usage & KVM_HOST_USES_PFN) { - if (new_pfn == gpc->pfn) - new_khva = old_khva; - else - new_khva = gpc_map(new_pfn); - - if (!new_khva) { - kvm_release_pfn_clean(new_pfn); - goto out_error; - } + if (new_pfn == gpc->pfn) + new_khva = old_khva; + else + new_khva = gpc_map(new_pfn); + + if (!new_khva) { + kvm_release_pfn_clean(new_pfn); + goto out_error; } write_lock_irq(&gpc->lock); @@ -346,18 +309,12 @@ int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len) return __kvm_gpc_refresh(gpc, gpc->gpa, len); } -void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm, - struct kvm_vcpu *vcpu, enum pfn_cache_usage usage) +void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm) { - WARN_ON_ONCE(!usage || (usage & KVM_GUEST_AND_HOST_USE_PFN) != usage); - WARN_ON_ONCE((usage & KVM_GUEST_USES_PFN) && !vcpu); - rwlock_init(&gpc->lock); mutex_init(&gpc->refresh_lock); gpc->kvm = kvm; - gpc->vcpu = vcpu; - gpc->usage = usage; gpc->pfn = KVM_PFN_ERR_FAULT; gpc->uhva = KVM_HVA_ERR_BAD; } -- cgit v1.2.3 From 9e7325acb3dc8df8a6370ebc18aef107aae17ef0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 15 Feb 2024 15:29:03 +0000 Subject: KVM: s390: Refactor kvm_is_error_gpa() into kvm_is_gpa_in_memslot() Rename kvm_is_error_gpa() to kvm_is_gpa_in_memslot() and invert the polarity accordingly in order to (a) free up kvm_is_error_gpa() to match with kvm_is_error_{hva,page}(), and (b) to make it more obvious that the helper is doing a memslot lookup, i.e. not simply checking for INVALID_GPA. No functional change intended. Link: https://lore.kernel.org/r/20240215152916.1158-9-paul@xen.org Signed-off-by: Sean Christopherson --- arch/s390/kvm/diag.c | 2 +- arch/s390/kvm/gaccess.c | 14 +++++++------- arch/s390/kvm/kvm-s390.c | 4 ++-- arch/s390/kvm/priv.c | 4 ++-- arch/s390/kvm/sigp.c | 2 +- include/linux/kvm_host.h | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index 3c65b8258ae6..2a32438e09ce 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c @@ -102,7 +102,7 @@ static int __diag_page_ref_service(struct kvm_vcpu *vcpu) parm.token_addr & 7 || parm.zarch != 0x8000000000000000ULL) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); - if (kvm_is_error_gpa(vcpu->kvm, parm.token_addr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, parm.token_addr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); vcpu->arch.pfault_token = parm.token_addr; diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 5bfcc50c1a68..415c99649e43 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -664,7 +664,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, case ASCE_TYPE_REGION1: { union region1_table_entry rfte; - if (kvm_is_error_gpa(vcpu->kvm, ptr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr)) return PGM_ADDRESSING; if (deref_table(vcpu->kvm, ptr, &rfte.val)) return -EFAULT; @@ -682,7 +682,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, case ASCE_TYPE_REGION2: { union region2_table_entry rste; - if (kvm_is_error_gpa(vcpu->kvm, ptr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr)) return PGM_ADDRESSING; if (deref_table(vcpu->kvm, ptr, &rste.val)) return -EFAULT; @@ -700,7 +700,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, case ASCE_TYPE_REGION3: { union region3_table_entry rtte; - if (kvm_is_error_gpa(vcpu->kvm, ptr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr)) return PGM_ADDRESSING; if (deref_table(vcpu->kvm, ptr, &rtte.val)) return -EFAULT; @@ -728,7 +728,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, case ASCE_TYPE_SEGMENT: { union segment_table_entry ste; - if (kvm_is_error_gpa(vcpu->kvm, ptr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr)) return PGM_ADDRESSING; if (deref_table(vcpu->kvm, ptr, &ste.val)) return -EFAULT; @@ -748,7 +748,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, ptr = ste.fc0.pto * (PAGE_SIZE / 2) + vaddr.px * 8; } } - if (kvm_is_error_gpa(vcpu->kvm, ptr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, ptr)) return PGM_ADDRESSING; if (deref_table(vcpu->kvm, ptr, &pte.val)) return -EFAULT; @@ -770,7 +770,7 @@ absolute_address: *prot = PROT_TYPE_IEP; return PGM_PROTECTION; } - if (kvm_is_error_gpa(vcpu->kvm, raddr.addr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, raddr.addr)) return PGM_ADDRESSING; *gpa = raddr.addr; return 0; @@ -957,7 +957,7 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, return rc; } else { gpa = kvm_s390_real_to_abs(vcpu, ga); - if (kvm_is_error_gpa(vcpu->kvm, gpa)) { + if (!kvm_is_gpa_in_memslot(vcpu->kvm, gpa)) { rc = PGM_ADDRESSING; prot = PROT_NONE; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ea63ac769889..3e5a1d7aa81a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2878,7 +2878,7 @@ static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop) srcu_idx = srcu_read_lock(&kvm->srcu); - if (kvm_is_error_gpa(kvm, mop->gaddr)) { + if (!kvm_is_gpa_in_memslot(kvm, mop->gaddr)) { r = PGM_ADDRESSING; goto out_unlock; } @@ -2940,7 +2940,7 @@ static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *m srcu_idx = srcu_read_lock(&kvm->srcu); - if (kvm_is_error_gpa(kvm, mop->gaddr)) { + if (!kvm_is_gpa_in_memslot(kvm, mop->gaddr)) { r = PGM_ADDRESSING; goto out_unlock; } diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index f875a404a0a0..1be19cc9d73c 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -149,7 +149,7 @@ static int handle_set_prefix(struct kvm_vcpu *vcpu) * first page, since address is 8k aligned and memory pieces are always * at least 1MB aligned and have at least a size of 1MB. */ - if (kvm_is_error_gpa(vcpu->kvm, address)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, address)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); kvm_s390_set_prefix(vcpu, address); @@ -464,7 +464,7 @@ static int handle_test_block(struct kvm_vcpu *vcpu) return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); addr = kvm_s390_real_to_abs(vcpu, addr); - if (kvm_is_error_gpa(vcpu->kvm, addr)) + if (!kvm_is_gpa_in_memslot(vcpu->kvm, addr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); /* * We don't expect errors on modern systems, and do not care diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c index d9696b530064..55c34cb35428 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/sigp.c @@ -172,7 +172,7 @@ static int __sigp_set_prefix(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu, * first page, since address is 8k aligned and memory pieces are always * at least 1MB aligned and have at least a size of 1MB. */ - if (kvm_is_error_gpa(vcpu->kvm, irq.u.prefix.address)) { + if (!kvm_is_gpa_in_memslot(vcpu->kvm, irq.u.prefix.address)) { *reg &= 0xffffffff00000000UL; *reg |= SIGP_STATUS_INVALID_PARAMETER; return SIGP_CC_STATUS_STORED; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3e1c04608c67..81a9d1cf91a2 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1779,11 +1779,11 @@ static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn) return (hpa_t)pfn << PAGE_SHIFT; } -static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) +static inline bool kvm_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa) { unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); - return kvm_is_error_hva(hva); + return !kvm_is_error_hva(hva); } static inline void kvm_gpc_mark_dirty_in_slot(struct gfn_to_pfn_cache *gpc) -- cgit v1.2.3 From 721f5b0dda784829b833039fbb42f420b9f86575 Mon Sep 17 00:00:00 2001 From: Paul Durrant Date: Thu, 15 Feb 2024 15:29:04 +0000 Subject: KVM: pfncache: allow a cache to be activated with a fixed (userspace) HVA Some pfncache pages may actually be overlays on guest memory that have a fixed HVA within the VMM. It's pointless to invalidate such cached mappings if the overlay is moved so allow a cache to be activated directly with the HVA to cater for such cases. A subsequent patch will make use of this facility. Signed-off-by: Paul Durrant Reviewed-by: David Woodhouse Link: https://lore.kernel.org/r/20240215152916.1158-10-paul@xen.org Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 31 +++++++++++++++ virt/kvm/pfncache.c | 98 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 101 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 81a9d1cf91a2..fa070c36f98a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -148,6 +148,11 @@ static inline bool kvm_is_error_hva(unsigned long addr) #endif +static inline bool kvm_is_error_gpa(gpa_t gpa) +{ + return gpa == INVALID_GPA; +} + #define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT)) static inline bool is_error_page(struct page *page) @@ -1344,6 +1349,22 @@ void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm); */ int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len); +/** + * kvm_gpc_activate_hva - prepare a cached kernel mapping and HPA for a given HVA. + * + * @gpc: struct gfn_to_pfn_cache object. + * @hva: userspace virtual address to map. + * @len: sanity check; the range being access must fit a single page. + * + * @return: 0 for success. + * -EINVAL for a mapping which would cross a page boundary. + * -EFAULT for an untranslatable guest physical address. + * + * The semantics of this function are the same as those of kvm_gpc_activate(). It + * merely bypasses a layer of address translation. + */ +int kvm_gpc_activate_hva(struct gfn_to_pfn_cache *gpc, unsigned long hva, unsigned long len); + /** * kvm_gpc_check - check validity of a gfn_to_pfn_cache. * @@ -1390,6 +1411,16 @@ int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len); */ void kvm_gpc_deactivate(struct gfn_to_pfn_cache *gpc); +static inline bool kvm_gpc_is_gpa_active(struct gfn_to_pfn_cache *gpc) +{ + return gpc->active && !kvm_is_error_gpa(gpc->gpa); +} + +static inline bool kvm_gpc_is_hva_active(struct gfn_to_pfn_cache *gpc) +{ + return gpc->active && kvm_is_error_gpa(gpc->gpa); +} + void kvm_sigset_activate(struct kvm_vcpu *vcpu); void kvm_sigset_deactivate(struct kvm_vcpu *vcpu); diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 97eec8ee3449..a47ca6fd75c2 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -48,7 +48,14 @@ bool kvm_gpc_check(struct gfn_to_pfn_cache *gpc, unsigned long len) if (!gpc->active) return false; - if (gpc->generation != slots->generation || kvm_is_error_hva(gpc->uhva)) + /* + * If the page was cached from a memslot, make sure the memslots have + * not been re-configured. + */ + if (!kvm_is_error_gpa(gpc->gpa) && gpc->generation != slots->generation) + return false; + + if (kvm_is_error_hva(gpc->uhva)) return false; if (offset_in_page(gpc->uhva) + len > PAGE_SIZE) @@ -209,11 +216,10 @@ out_error: return -EFAULT; } -static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa, +static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long uhva, unsigned long len) { - struct kvm_memslots *slots = kvm_memslots(gpc->kvm); - unsigned long page_offset = offset_in_page(gpa); + unsigned long page_offset; bool unmap_old = false; unsigned long old_uhva; kvm_pfn_t old_pfn; @@ -221,10 +227,16 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa, void *old_khva; int ret; + /* Either gpa or uhva must be valid, but not both */ + if (WARN_ON_ONCE(kvm_is_error_gpa(gpa) == kvm_is_error_hva(uhva))) + return -EINVAL; + /* - * If must fit within a single page. The 'len' argument is - * only to enforce that. + * The cached acces must fit within a single page. The 'len' argument + * exists only to enforce that. */ + page_offset = kvm_is_error_gpa(gpa) ? offset_in_page(uhva) : + offset_in_page(gpa); if (page_offset + len > PAGE_SIZE) return -EINVAL; @@ -246,29 +258,39 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa, old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva); old_uhva = PAGE_ALIGN_DOWN(gpc->uhva); - /* Refresh the userspace HVA if necessary */ - if (gpc->gpa != gpa || gpc->generation != slots->generation || - kvm_is_error_hva(gpc->uhva)) { - gfn_t gfn = gpa_to_gfn(gpa); - - gpc->gpa = gpa; - gpc->generation = slots->generation; - gpc->memslot = __gfn_to_memslot(slots, gfn); - gpc->uhva = gfn_to_hva_memslot(gpc->memslot, gfn); + if (kvm_is_error_gpa(gpa)) { + gpc->gpa = INVALID_GPA; + gpc->memslot = NULL; + gpc->uhva = PAGE_ALIGN_DOWN(uhva); - if (kvm_is_error_hva(gpc->uhva)) { - ret = -EFAULT; - goto out; - } - - /* - * Even if the GPA and/or the memslot generation changed, the - * HVA may still be the same. - */ if (gpc->uhva != old_uhva) hva_change = true; } else { - gpc->uhva = old_uhva; + struct kvm_memslots *slots = kvm_memslots(gpc->kvm); + + if (gpc->gpa != gpa || gpc->generation != slots->generation || + kvm_is_error_hva(gpc->uhva)) { + gfn_t gfn = gpa_to_gfn(gpa); + + gpc->gpa = gpa; + gpc->generation = slots->generation; + gpc->memslot = __gfn_to_memslot(slots, gfn); + gpc->uhva = gfn_to_hva_memslot(gpc->memslot, gfn); + + if (kvm_is_error_hva(gpc->uhva)) { + ret = -EFAULT; + goto out; + } + + /* + * Even if the GPA and/or the memslot generation changed, the + * HVA may still be the same. + */ + if (gpc->uhva != old_uhva) + hva_change = true; + } else { + gpc->uhva = old_uhva; + } } /* Note: the offset must be correct before calling hva_to_pfn_retry() */ @@ -319,7 +341,15 @@ out_unlock: int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len) { - return __kvm_gpc_refresh(gpc, gpc->gpa, len); + /* + * If the GPA is valid then ignore the HVA, as a cache can be GPA-based + * or HVA-based, not both. For GPA-based caches, the HVA will be + * recomputed during refresh if necessary. + */ + unsigned long uhva = kvm_is_error_gpa(gpc->gpa) ? gpc->uhva : + KVM_HVA_ERR_BAD; + + return __kvm_gpc_refresh(gpc, gpc->gpa, uhva, len); } void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm) @@ -329,10 +359,12 @@ void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm) gpc->kvm = kvm; gpc->pfn = KVM_PFN_ERR_FAULT; + gpc->gpa = INVALID_GPA; gpc->uhva = KVM_HVA_ERR_BAD; } -int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len) +static int __kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long uhva, + unsigned long len) { struct kvm *kvm = gpc->kvm; @@ -353,7 +385,17 @@ int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len) gpc->active = true; write_unlock_irq(&gpc->lock); } - return __kvm_gpc_refresh(gpc, gpa, len); + return __kvm_gpc_refresh(gpc, gpa, uhva, len); +} + +int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len) +{ + return __kvm_gpc_activate(gpc, gpa, KVM_HVA_ERR_BAD, len); +} + +int kvm_gpc_activate_hva(struct gfn_to_pfn_cache *gpc, unsigned long uhva, unsigned long len) +{ + return __kvm_gpc_activate(gpc, INVALID_GPA, uhva, len); } void kvm_gpc_deactivate(struct gfn_to_pfn_cache *gpc) -- cgit v1.2.3 From 87161a2b0aed9e9b614bbf6fe8697ad560ceb0cb Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Tue, 6 Feb 2024 11:21:00 -0800 Subject: f2fs: deprecate io_bits Let's deprecate an unused io_bits feature to save CPU cycles and memory. Reviewed-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/filesystems/f2fs.rst | 2 - fs/f2fs/data.c | 73 +------------------------------ fs/f2fs/f2fs.h | 25 +++-------- fs/f2fs/file.c | 2 - fs/f2fs/gc.c | 10 +---- fs/f2fs/segment.c | 9 +--- fs/f2fs/super.c | 88 +------------------------------------- include/linux/f2fs_fs.h | 6 --- 8 files changed, 10 insertions(+), 205 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index 32cbfa864f38..9ac5083dae8e 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -229,8 +229,6 @@ mode=%s Control block allocation mode which supports "adaptive" option for more randomness. Please, use these options for your experiments and we strongly recommend to re-format the filesystem after using these options. -io_bits=%u Set the bit size of write IO requests. It should be set - with "mode=lfs". usrquota Enable plain user disk quota accounting. grpquota Enable plain group disk quota accounting. prjquota Enable plain project quota accounting. diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 05158f89ef32..828c797cd47c 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -338,17 +338,6 @@ static void f2fs_write_end_io(struct bio *bio) struct page *page = bvec->bv_page; enum count_type type = WB_DATA_TYPE(page, false); - if (page_private_dummy(page)) { - clear_page_private_dummy(page); - unlock_page(page); - mempool_free(page, sbi->write_io_dummy); - - if (unlikely(bio->bi_status)) - f2fs_stop_checkpoint(sbi, true, - STOP_CP_REASON_WRITE_FAIL); - continue; - } - fscrypt_finalize_bounce_page(&page); #ifdef CONFIG_F2FS_FS_COMPRESSION @@ -522,50 +511,13 @@ void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, submit_bio(bio); } -static void f2fs_align_write_bio(struct f2fs_sb_info *sbi, struct bio *bio) -{ - unsigned int start = - (bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS) % F2FS_IO_SIZE(sbi); - - if (start == 0) - return; - - /* fill dummy pages */ - for (; start < F2FS_IO_SIZE(sbi); start++) { - struct page *page = - mempool_alloc(sbi->write_io_dummy, - GFP_NOIO | __GFP_NOFAIL); - f2fs_bug_on(sbi, !page); - - lock_page(page); - - zero_user_segment(page, 0, PAGE_SIZE); - set_page_private_dummy(page); - - if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) - f2fs_bug_on(sbi, 1); - } -} - static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type) { WARN_ON_ONCE(is_read_io(bio_op(bio))); - if (type == DATA || type == NODE) { - if (f2fs_lfs_mode(sbi) && current->plug) - blk_finish_plug(current->plug); - - if (F2FS_IO_ALIGNED(sbi)) { - f2fs_align_write_bio(sbi, bio); - /* - * In the NODE case, we lose next block address chain. - * So, we need to do checkpoint in f2fs_sync_file. - */ - if (type == NODE) - set_sbi_flag(sbi, SBI_NEED_CP); - } - } + if (f2fs_lfs_mode(sbi) && current->plug && PAGE_TYPE_ON_MAIN(type)) + blk_finish_plug(current->plug); trace_f2fs_submit_write_bio(sbi->sb, type, bio); iostat_update_submit_ctx(bio, type); @@ -794,16 +746,6 @@ static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, block_t last_blkaddr, block_t cur_blkaddr) { - if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) { - unsigned int filled_blocks = - F2FS_BYTES_TO_BLK(bio->bi_iter.bi_size); - unsigned int io_size = F2FS_IO_SIZE(sbi); - unsigned int left_vecs = bio->bi_max_vecs - bio->bi_vcnt; - - /* IOs in bio is aligned and left space of vectors is not enough */ - if (!(filled_blocks % io_size) && left_vecs < io_size) - return false; - } if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) return false; return io_type_is_mergeable(io, fio); @@ -1055,14 +997,6 @@ next: __submit_merged_bio(io); alloc_new: if (io->bio == NULL) { - if (F2FS_IO_ALIGNED(sbi) && - (fio->type == DATA || fio->type == NODE) && - fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { - dec_page_count(sbi, WB_DATA_TYPE(bio_page, - fio->compressed_page)); - fio->retry = 1; - goto skip; - } io->bio = __bio_alloc(fio, BIO_MAX_VECS); f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, bio_page->index, fio, GFP_NOIO); @@ -1092,7 +1026,6 @@ alloc_new: __submit_merged_bio(io); } #endif -skip: if (fio->in_list) goto next; out: @@ -2669,8 +2602,6 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) if (fio) { if (page_private_gcing(fio->page)) return true; - if (page_private_dummy(fio->page)) - return true; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) return true; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 543898482f8b..4c52136cbc10 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -148,7 +148,6 @@ struct f2fs_rwsem { struct f2fs_mount_info { unsigned int opt; - int write_io_size_bits; /* Write IO size bits */ block_t root_reserved_blocks; /* root reserved blocks */ kuid_t s_resuid; /* reserved blocks for uid */ kgid_t s_resgid; /* reserved blocks for gid */ @@ -1117,6 +1116,7 @@ enum count_type { * ... Only can be used with META. */ #define PAGE_TYPE_OF_BIO(type) ((type) > META ? META : (type)) +#define PAGE_TYPE_ON_MAIN(type) ((type) == DATA || (type) == NODE) enum page_type { DATA = 0, NODE = 1, /* should not change this */ @@ -1211,7 +1211,6 @@ struct f2fs_io_info { unsigned int submitted:1; /* indicate IO submission */ unsigned int in_list:1; /* indicate fio is in io_list */ unsigned int is_por:1; /* indicate IO is from recovery or not */ - unsigned int retry:1; /* need to reallocate block address */ unsigned int encrypted:1; /* indicate file is encrypted */ unsigned int post_read:1; /* require post read */ enum iostat_type io_type; /* io type */ @@ -1413,18 +1412,16 @@ static inline void f2fs_clear_bit(unsigned int nr, char *addr); * Layout A: lowest bit should be 1 * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... | * bit 0 PAGE_PRIVATE_NOT_POINTER - * bit 1 PAGE_PRIVATE_DUMMY_WRITE - * bit 2 PAGE_PRIVATE_ONGOING_MIGRATION - * bit 3 PAGE_PRIVATE_INLINE_INODE - * bit 4 PAGE_PRIVATE_REF_RESOURCE - * bit 5- f2fs private data + * bit 1 PAGE_PRIVATE_ONGOING_MIGRATION + * bit 2 PAGE_PRIVATE_INLINE_INODE + * bit 3 PAGE_PRIVATE_REF_RESOURCE + * bit 4- f2fs private data * * Layout B: lowest bit should be 0 * page.private is a wrapped pointer. */ enum { PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ - PAGE_PRIVATE_DUMMY_WRITE, /* data page for padding aligned IO */ PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */ PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ @@ -1571,7 +1568,6 @@ struct f2fs_sb_info { struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */ /* keep migration IO order for LFS mode */ struct f2fs_rwsem io_order_lock; - mempool_t *write_io_dummy; /* Dummy pages */ pgoff_t page_eio_ofs[NR_PAGE_TYPE]; /* EIO page offset */ int page_eio_cnt[NR_PAGE_TYPE]; /* EIO count */ @@ -2307,10 +2303,6 @@ static inline int inc_valid_block_count(struct f2fs_sb_info *sbi, if (!__allow_reserved_blocks(sbi, inode, true)) avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks; - if (F2FS_IO_ALIGNED(sbi)) - avail_user_block_count -= sbi->blocks_per_seg * - SM_I(sbi)->additional_reserved_segments; - if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { if (avail_user_block_count > sbi->unusable_block_count) avail_user_block_count -= sbi->unusable_block_count; @@ -2378,17 +2370,14 @@ static inline void clear_page_private_##name(struct page *page) \ PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE); PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE); PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE); PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE); PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE); PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); -PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE); static inline unsigned long get_page_private_data(struct page *page) { @@ -2644,10 +2633,6 @@ static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, if (!__allow_reserved_blocks(sbi, inode, false)) valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks; - if (F2FS_IO_ALIGNED(sbi)) - valid_block_count += sbi->blocks_per_seg * - SM_I(sbi)->additional_reserved_segments; - user_block_count = sbi->user_block_count; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) user_block_count -= sbi->unusable_block_count; diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 25b119cf3499..c6cd9474ba2d 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -822,8 +822,6 @@ static bool f2fs_force_buffered_io(struct inode *inode, int rw) */ if (f2fs_sb_has_blkzoned(sbi) && (rw == WRITE)) return true; - if (f2fs_lfs_mode(sbi) && rw == WRITE && F2FS_IO_ALIGNED(sbi)) - return true; if (is_sbi_flag_set(sbi, SBI_CP_DISABLED)) return true; diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index a079eebfb080..6899f434ad68 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1184,7 +1184,6 @@ static int ra_data_block(struct inode *inode, pgoff_t index) .op_flags = 0, .encrypted_page = NULL, .in_list = 0, - .retry = 0, }; int err; @@ -1273,7 +1272,6 @@ static int move_data_block(struct inode *inode, block_t bidx, .op_flags = 0, .encrypted_page = NULL, .in_list = 0, - .retry = 0, }; struct dnode_of_data dn; struct f2fs_summary sum; @@ -1393,18 +1391,12 @@ static int move_data_block(struct inode *inode, block_t bidx, fio.op_flags = REQ_SYNC; fio.new_blkaddr = newaddr; f2fs_submit_page_write(&fio); - if (fio.retry) { - err = -EAGAIN; - if (PageWriteback(fio.encrypted_page)) - end_page_writeback(fio.encrypted_page); - goto put_page_out; - } f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE); f2fs_update_data_blkaddr(&dn, newaddr); set_inode_flag(inode, FI_APPEND_WRITE); -put_page_out: + f2fs_put_page(fio.encrypted_page, 1); recover_block: if (err) diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 7901ede58113..e5759813276a 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3507,9 +3507,6 @@ void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, if (fio) { struct f2fs_bio_info *io; - if (F2FS_IO_ALIGNED(sbi)) - fio->retry = 0; - INIT_LIST_HEAD(&fio->list); fio->in_list = 1; io = sbi->write_io[fio->type] + fio->temp; @@ -3557,7 +3554,7 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) if (keep_order) f2fs_down_read(&fio->sbi->io_order_lock); -reallocate: + f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, &fio->new_blkaddr, sum, type, fio); if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) @@ -3565,10 +3562,6 @@ reallocate: /* writeout dirty page into bdev */ f2fs_submit_page_write(fio); - if (fio->retry) { - fio->old_blkaddr = fio->new_blkaddr; - goto reallocate; - } f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 1b718bebfaa1..f1516fd5088a 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -138,7 +138,6 @@ enum { Opt_resgid, Opt_resuid, Opt_mode, - Opt_io_size_bits, Opt_fault_injection, Opt_fault_type, Opt_lazytime, @@ -217,7 +216,6 @@ static match_table_t f2fs_tokens = { {Opt_resgid, "resgid=%u"}, {Opt_resuid, "resuid=%u"}, {Opt_mode, "mode=%s"}, - {Opt_io_size_bits, "io_bits=%u"}, {Opt_fault_injection, "fault_injection=%u"}, {Opt_fault_type, "fault_type=%u"}, {Opt_lazytime, "lazytime"}, @@ -349,46 +347,6 @@ static inline void limit_reserve_root(struct f2fs_sb_info *sbi) F2FS_OPTION(sbi).s_resgid)); } -static inline int adjust_reserved_segment(struct f2fs_sb_info *sbi) -{ - unsigned int sec_blks = sbi->blocks_per_seg * sbi->segs_per_sec; - unsigned int avg_vblocks; - unsigned int wanted_reserved_segments; - block_t avail_user_block_count; - - if (!F2FS_IO_ALIGNED(sbi)) - return 0; - - /* average valid block count in section in worst case */ - avg_vblocks = sec_blks / F2FS_IO_SIZE(sbi); - - /* - * we need enough free space when migrating one section in worst case - */ - wanted_reserved_segments = (F2FS_IO_SIZE(sbi) / avg_vblocks) * - reserved_segments(sbi); - wanted_reserved_segments -= reserved_segments(sbi); - - avail_user_block_count = sbi->user_block_count - - sbi->current_reserved_blocks - - F2FS_OPTION(sbi).root_reserved_blocks; - - if (wanted_reserved_segments * sbi->blocks_per_seg > - avail_user_block_count) { - f2fs_err(sbi, "IO align feature can't grab additional reserved segment: %u, available segments: %u", - wanted_reserved_segments, - avail_user_block_count >> sbi->log_blocks_per_seg); - return -ENOSPC; - } - - SM_I(sbi)->additional_reserved_segments = wanted_reserved_segments; - - f2fs_info(sbi, "IO align feature needs additional reserved segment: %u", - wanted_reserved_segments); - - return 0; -} - static inline void adjust_unusable_cap_perc(struct f2fs_sb_info *sbi) { if (!F2FS_OPTION(sbi).unusable_cap_perc) @@ -919,16 +877,6 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) } kfree(name); break; - case Opt_io_size_bits: - if (args->from && match_int(args, &arg)) - return -EINVAL; - if (arg <= 0 || arg > __ilog2_u32(BIO_MAX_VECS)) { - f2fs_warn(sbi, "Not support %ld, larger than %d", - BIT(arg), BIO_MAX_VECS); - return -EINVAL; - } - F2FS_OPTION(sbi).write_io_size_bits = arg; - break; #ifdef CONFIG_F2FS_FAULT_INJECTION case Opt_fault_injection: if (args->from && match_int(args, &arg)) @@ -1398,12 +1346,6 @@ default_check: } #endif - if (F2FS_IO_SIZE_BITS(sbi) && !f2fs_lfs_mode(sbi)) { - f2fs_err(sbi, "Should set mode=lfs with %luKB-sized IO", - F2FS_IO_SIZE_KB(sbi)); - return -EINVAL; - } - if (test_opt(sbi, INLINE_XATTR_SIZE)) { int min_size, max_size; @@ -1724,7 +1666,6 @@ static void f2fs_put_super(struct super_block *sb) f2fs_destroy_page_array_cache(sbi); f2fs_destroy_xattr_caches(sbi); - mempool_destroy(sbi->write_io_dummy); #ifdef CONFIG_QUOTA for (i = 0; i < MAXQUOTAS; i++) kfree(F2FS_OPTION(sbi).s_qf_names[i]); @@ -2084,9 +2025,6 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) F2FS_OPTION(sbi).s_resuid), from_kgid_munged(&init_user_ns, F2FS_OPTION(sbi).s_resgid)); - if (F2FS_IO_SIZE_BITS(sbi)) - seq_printf(seq, ",io_bits=%u", - F2FS_OPTION(sbi).write_io_size_bits); #ifdef CONFIG_F2FS_FAULT_INJECTION if (test_opt(sbi, FAULT_INJECTION)) { seq_printf(seq, ",fault_injection=%u", @@ -2338,7 +2276,6 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) bool no_read_extent_cache = !test_opt(sbi, READ_EXTENT_CACHE); bool no_age_extent_cache = !test_opt(sbi, AGE_EXTENT_CACHE); bool enable_checkpoint = !test_opt(sbi, DISABLE_CHECKPOINT); - bool no_io_align = !F2FS_IO_ALIGNED(sbi); bool no_atgc = !test_opt(sbi, ATGC); bool no_discard = !test_opt(sbi, DISCARD); bool no_compress_cache = !test_opt(sbi, COMPRESS_CACHE); @@ -2446,12 +2383,6 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) goto restore_opts; } - if (no_io_align == !!F2FS_IO_ALIGNED(sbi)) { - err = -EINVAL; - f2fs_warn(sbi, "switch io_bits option is not allowed"); - goto restore_opts; - } - if (no_compress_cache == !!test_opt(sbi, COMPRESS_CACHE)) { err = -EINVAL; f2fs_warn(sbi, "switch compress_cache option is not allowed"); @@ -4314,8 +4245,6 @@ static int f2fs_scan_devices(struct f2fs_sb_info *sbi) FDEV(i).total_segments, FDEV(i).start_blk, FDEV(i).end_blk); } - f2fs_info(sbi, - "IO Block Size: %8ld KB", F2FS_IO_SIZE_KB(sbi)); return 0; } @@ -4528,19 +4457,10 @@ try_onemore: if (err) goto free_iostat; - if (F2FS_IO_ALIGNED(sbi)) { - sbi->write_io_dummy = - mempool_create_page_pool(2 * (F2FS_IO_SIZE(sbi) - 1), 0); - if (!sbi->write_io_dummy) { - err = -ENOMEM; - goto free_percpu; - } - } - /* init per sbi slab cache */ err = f2fs_init_xattr_caches(sbi); if (err) - goto free_io_dummy; + goto free_percpu; err = f2fs_init_page_array_cache(sbi); if (err) goto free_xattr_cache; @@ -4628,10 +4548,6 @@ try_onemore: goto free_nm; } - err = adjust_reserved_segment(sbi); - if (err) - goto free_nm; - /* For write statistics */ sbi->sectors_written_start = f2fs_get_sectors_written(sbi); @@ -4862,8 +4778,6 @@ free_page_array_cache: f2fs_destroy_page_array_cache(sbi); free_xattr_cache: f2fs_destroy_xattr_caches(sbi); -free_io_dummy: - mempool_destroy(sbi->write_io_dummy); free_percpu: destroy_percpu_info(sbi); free_iostat: diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 053137a0fe45..9b69c50255b2 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -40,12 +40,6 @@ #define F2FS_ENC_UTF8_12_1 1 -#define F2FS_IO_SIZE(sbi) BIT(F2FS_OPTION(sbi).write_io_size_bits) /* Blocks */ -#define F2FS_IO_SIZE_KB(sbi) BIT(F2FS_OPTION(sbi).write_io_size_bits + 2) /* KB */ -#define F2FS_IO_SIZE_BITS(sbi) (F2FS_OPTION(sbi).write_io_size_bits) /* power of 2 */ -#define F2FS_IO_SIZE_MASK(sbi) (F2FS_IO_SIZE(sbi) - 1) -#define F2FS_IO_ALIGNED(sbi) (F2FS_IO_SIZE(sbi) > 1) - /* This flag is used by node and meta inodes, and by recovery */ #define GFP_F2FS_ZERO (GFP_NOFS | __GFP_ZERO) -- cgit v1.2.3 From 6e5f0f6383b4896c7e9b943d84b136149d0f45e9 Mon Sep 17 00:00:00 2001 From: Hongyu Jin Date: Wed, 24 Jan 2024 13:35:53 +0800 Subject: dm io: Support IO priority Some IO will dispatch from kworker with different io_context settings than the submitting task, we may need to specify a priority to avoid losing priority. Add IO priority parameter to dm_io() and update all callers. Co-developed-by: Yibin Ding Signed-off-by: Yibin Ding Signed-off-by: Hongyu Jin Reviewed-by: Eric Biggers Reviewed-by: Mikulas Patocka Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 6 +++--- drivers/md/dm-integrity.c | 12 ++++++------ drivers/md/dm-io.c | 23 +++++++++++++---------- drivers/md/dm-kcopyd.c | 4 ++-- drivers/md/dm-log.c | 4 ++-- drivers/md/dm-raid1.c | 6 +++--- drivers/md/dm-snap-persistent.c | 4 ++-- drivers/md/dm-verity-target.c | 2 +- drivers/md/dm-writecache.c | 8 ++++---- include/linux/dm-io.h | 3 ++- 10 files changed, 38 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 13c65b7e1ed6..f5541b8f6320 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1315,7 +1315,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL); + r = dm_io(&io_req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -2167,7 +2167,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c) if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; - return dm_io(&io_req, 1, &io_reg, NULL); + return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); @@ -2191,7 +2191,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; /* discards are optional */ - return dm_io(&io_req, 1, &io_reg, NULL); + return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 44a1dcc06dd7..c3c666937112 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -555,7 +555,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) } } - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) return r; @@ -1073,7 +1073,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? "reading journal" : "writing journal", r); @@ -1190,7 +1190,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u io_loc.sector = target; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); fn(-1UL, data); @@ -1519,7 +1519,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat fr.io_reg.count = 0, fr.ic = ic; init_completion(&fr.comp); - r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL); + r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); BUG_ON(r); } @@ -1728,7 +1728,7 @@ static void integrity_recheck(struct dm_integrity_io *dio) io_loc.sector = sector; io_loc.count = ic->sectors_per_block; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dio->bi_status = errno_to_blk_status(r); goto free_ret; @@ -2808,7 +2808,7 @@ next_chunk: io_loc.sector = get_data_sector(ic, area, offset); io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, "reading data", r); goto err; diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index f053ce245814..7409490259d1 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -305,7 +305,7 @@ static void km_dp_init(struct dpages *dp, void *data) */ static void do_region(const blk_opf_t opf, unsigned int region, struct dm_io_region *where, struct dpages *dp, - struct io *io) + struct io *io, unsigned short ioprio) { struct bio *bio; struct page *page; @@ -354,6 +354,7 @@ static void do_region(const blk_opf_t opf, unsigned int region, &io->client->bios); bio->bi_iter.bi_sector = where->sector + (where->count - remaining); bio->bi_end_io = endio; + bio->bi_ioprio = ioprio; store_io_and_region_in_bio(bio, io, region); if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) { @@ -383,7 +384,7 @@ static void do_region(const blk_opf_t opf, unsigned int region, static void dispatch_io(blk_opf_t opf, unsigned int num_regions, struct dm_io_region *where, struct dpages *dp, - struct io *io, int sync) + struct io *io, int sync, unsigned short ioprio) { int i; struct dpages old_pages = *dp; @@ -400,7 +401,7 @@ static void dispatch_io(blk_opf_t opf, unsigned int num_regions, for (i = 0; i < num_regions; i++) { *dp = old_pages; if (where[i].count || (opf & REQ_PREFLUSH)) - do_region(opf, i, where + i, dp, io); + do_region(opf, i, where + i, dp, io, ioprio); } /* @@ -425,7 +426,7 @@ static void sync_io_complete(unsigned long error, void *context) static int sync_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, struct dpages *dp, - unsigned long *error_bits) + unsigned long *error_bits, unsigned short ioprio) { struct io *io; struct sync_io sio; @@ -447,7 +448,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(opf, num_regions, where, dp, io, 1); + dispatch_io(opf, num_regions, where, dp, io, 1, ioprio); wait_for_completion_io(&sio.wait); @@ -459,7 +460,8 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, static int async_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, - struct dpages *dp, io_notify_fn fn, void *context) + struct dpages *dp, io_notify_fn fn, void *context, + unsigned short ioprio) { struct io *io; @@ -479,7 +481,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(opf, num_regions, where, dp, io, 0); + dispatch_io(opf, num_regions, where, dp, io, 0, ioprio); return 0; } @@ -521,7 +523,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, } int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *where, unsigned long *sync_error_bits) + struct dm_io_region *where, unsigned long *sync_error_bits, + unsigned short ioprio) { int r; struct dpages dp; @@ -532,11 +535,11 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_opf, &dp, sync_error_bits); + io_req->bi_opf, &dp, sync_error_bits, ioprio); return async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp, io_req->notify.fn, - io_req->notify.context); + io_req->notify.context, ioprio); } EXPORT_SYMBOL(dm_io); diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 36bcfdccae04..6ea75436a433 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job) io_job_start(job->kc->throttle); if (job->op == REQ_OP_READ) - r = dm_io(&io_req, 1, &job->source, NULL); + r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT); else - r = dm_io(&io_req, job->num_dests, job->dests, NULL); + r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT); return r; } diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index f9f84236dfcd..f7f9c2100937 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op) { lc->io_req.bi_opf = op; - return dm_io(&lc->io_req, 1, &lc->header_location, NULL); + return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT); } static int flush_header(struct log_c *lc) @@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc) lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - return dm_io(&lc->io_req, 1, &null_location, NULL); + return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT); } static int read_header(struct log_c *log) diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index ddcb2bc4a617..9511dae5b556 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -278,7 +278,7 @@ static int mirror_flush(struct dm_target *ti) } error_bits = -1; - dm_io(&io_req, ms->nr_mirrors, io, &error_bits); + dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT); if (unlikely(error_bits != 0)) { for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error_bits)) @@ -554,7 +554,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) map_region(&io, m, bio); bio_set_m(bio, m); - BUG_ON(dm_io(&io_req, 1, &io, NULL)); + BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT)); } static inline int region_in_sync(struct mirror_set *ms, region_t region, @@ -681,7 +681,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) */ bio_set_m(bio, get_default_mirror(ms)); - BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL)); + BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT)); } static void do_writes(struct mirror_set *ms, struct bio_list *writes) diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index 15649921f2a9..568d10842b1f 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work) { struct mdata_req *req = container_of(work, struct mdata_req, work); - req->result = dm_io(req->io_req, 1, req->where, NULL); + req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT); } /* @@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf, struct mdata_req req; if (!metadata) - return dm_io(&io_req, 1, &where, NULL); + return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT); req.where = &where; req.io_req = &io_req; diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 06c8b637849c..f55fbbe5d22f 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -511,7 +511,7 @@ static int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, io_loc.bdev = v->data_dev->bdev; io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) goto free_ret; diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index b463c28c39ad..01ab141bc51e 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -531,7 +531,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) req.notify.context = &endio; /* writing via async dm-io (implied by notify.fn above) won't return an error */ - (void) dm_io(&req, 1, ®ion, NULL); + (void) dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); i = j; } @@ -568,7 +568,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) req.notify.fn = NULL; req.notify.context = NULL; - r = dm_io(&req, 1, ®ion, NULL); + r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error writing superblock"); } @@ -596,7 +596,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) req.client = wc->dm_io; req.notify.fn = NULL; - r = dm_io(&req, 1, ®ion, NULL); + r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error flushing metadata: %d", r); } @@ -990,7 +990,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors req.client = wc->dm_io; req.notify.fn = NULL; - return dm_io(&req, 1, ®ion, NULL); + return dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); } static void writecache_resume(struct dm_target *ti) diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index 7595142f3fc5..7b2968612b7e 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -80,7 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client); * error occurred doing io to the corresponding region. */ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *region, unsigned int long *sync_error_bits); + struct dm_io_region *region, unsigned int long *sync_error_bits, + unsigned short ioprio); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_IO_H */ -- cgit v1.2.3 From e9b2238e47cb66521381c0bf9234e979afa19d77 Mon Sep 17 00:00:00 2001 From: Hongyu Jin Date: Wed, 24 Jan 2024 13:35:54 +0800 Subject: dm bufio: Support IO priority Some IO will dispatch from kworker with different io_context settings than the submitting task, we may need to specify a priority to avoid losing priority. Add dm_bufio_read_with_ioprio() and dm_bufio_prefetch_with_ioprio() for use by bufio users to pass an ioprio other than IOPRIO_DEFAULT. Co-developed-by: Yibin Ding Signed-off-by: Yibin Ding Signed-off-by: Hongyu Jin Reviewed-by: Eric Biggers Reviewed-by: Mikulas Patocka [snitzer: introduced _with_ioprio() wrappers to reduce churn] Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 68 ++++++++++++++++++++++++++++++++++-------------- include/linux/dm-bufio.h | 7 +++++ 2 files changed, 56 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index f5541b8f6320..dad75b8d0b55 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1292,7 +1292,8 @@ static void dmio_complete(unsigned long error, void *context) } static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, - unsigned int n_sectors, unsigned int offset) + unsigned int n_sectors, unsigned int offset, + unsigned short ioprio) { int r; struct dm_io_request io_req = { @@ -1315,7 +1316,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, ®ion, NULL, ioprio); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -1331,7 +1332,8 @@ static void bio_complete(struct bio *bio) } static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, - unsigned int n_sectors, unsigned int offset) + unsigned int n_sectors, unsigned int offset, + unsigned short ioprio) { struct bio *bio; char *ptr; @@ -1339,13 +1341,14 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN); if (!bio) { - use_dmio(b, op, sector, n_sectors, offset); + use_dmio(b, op, sector, n_sectors, offset, ioprio); return; } bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); bio->bi_iter.bi_sector = sector; bio->bi_end_io = bio_complete; bio->bi_private = b; + bio->bi_ioprio = ioprio; ptr = (char *)b->data + offset; len = n_sectors << SECTOR_SHIFT; @@ -1368,7 +1371,7 @@ static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block return sector; } -static void submit_io(struct dm_buffer *b, enum req_op op, +static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio, void (*end_io)(struct dm_buffer *, blk_status_t)) { unsigned int n_sectors; @@ -1398,9 +1401,9 @@ static void submit_io(struct dm_buffer *b, enum req_op op, } if (b->data_mode != DATA_MODE_VMALLOC) - use_bio(b, op, sector, n_sectors, offset); + use_bio(b, op, sector, n_sectors, offset, ioprio); else - use_dmio(b, op, sector, n_sectors, offset); + use_dmio(b, op, sector, n_sectors, offset, ioprio); } /* @@ -1456,7 +1459,7 @@ static void __write_dirty_buffer(struct dm_buffer *b, b->write_end = b->dirty_end; if (!write_list) - submit_io(b, REQ_OP_WRITE, write_endio); + submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); else list_add_tail(&b->write_list, write_list); } @@ -1470,7 +1473,7 @@ static void __flush_write_list(struct list_head *write_list) struct dm_buffer *b = list_entry(write_list->next, struct dm_buffer, write_list); list_del(&b->write_list); - submit_io(b, REQ_OP_WRITE, write_endio); + submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); cond_resched(); } blk_finish_plug(&plug); @@ -1852,7 +1855,8 @@ static void read_endio(struct dm_buffer *b, blk_status_t status) * and uses dm_bufio_mark_buffer_dirty to write new data back). */ static void *new_read(struct dm_bufio_client *c, sector_t block, - enum new_flag nf, struct dm_buffer **bp) + enum new_flag nf, struct dm_buffer **bp, + unsigned short ioprio) { int need_submit = 0; struct dm_buffer *b; @@ -1905,7 +1909,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, return NULL; if (need_submit) - submit_io(b, REQ_OP_READ, read_endio); + submit_io(b, REQ_OP_READ, ioprio, read_endio); if (nf != NF_GET) /* we already tested this condition above */ wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); @@ -1926,32 +1930,46 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp) { - return new_read(c, block, NF_GET, bp); + return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_get); -void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, - struct dm_buffer **bp) +static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio) { if (WARN_ON_ONCE(dm_bufio_in_request())) return ERR_PTR(-EINVAL); - return new_read(c, block, NF_READ, bp); + return new_read(c, block, NF_READ, bp, ioprio); +} + +void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp) +{ + return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_read); +void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio) +{ + return __dm_bufio_read(c, block, bp, ioprio); +} +EXPORT_SYMBOL_GPL(dm_bufio_read_with_ioprio); + void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp) { if (WARN_ON_ONCE(dm_bufio_in_request())) return ERR_PTR(-EINVAL); - return new_read(c, block, NF_FRESH, bp); + return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_new); -void dm_bufio_prefetch(struct dm_bufio_client *c, - sector_t block, unsigned int n_blocks) +static void __dm_bufio_prefetch(struct dm_bufio_client *c, + sector_t block, unsigned int n_blocks, + unsigned short ioprio) { struct blk_plug plug; @@ -1987,7 +2005,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, dm_bufio_unlock(c); if (need_submit) - submit_io(b, REQ_OP_READ, read_endio); + submit_io(b, REQ_OP_READ, ioprio, read_endio); dm_bufio_release(b); cond_resched(); @@ -2002,8 +2020,20 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, flush_plug: blk_finish_plug(&plug); } + +void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks) +{ + return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT); +} EXPORT_SYMBOL_GPL(dm_bufio_prefetch); +void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block, + unsigned int n_blocks, unsigned short ioprio) +{ + return __dm_bufio_prefetch(c, block, n_blocks, ioprio); +} +EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio); + void dm_bufio_release(struct dm_buffer *b) { struct dm_bufio_client *c = b->c; diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h index 75e7d8cbb532..d1503b815a78 100644 --- a/include/linux/dm-bufio.h +++ b/include/linux/dm-bufio.h @@ -64,6 +64,9 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start); void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp); +void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio); + /* * Like dm_bufio_read, but return buffer from cache, don't read * it. If the buffer is not in the cache, return NULL. @@ -86,6 +89,10 @@ void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks); +void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, + sector_t block, unsigned int n_blocks, + unsigned short ioprio); + /* * Release a reference obtained with dm_bufio_{read,get,new}. The data * pointer and dm_buffer pointer is no longer valid after this call. -- cgit v1.2.3 From 2f3bfa8e30b5b4864a200be0dc2fb55d8e4b35e4 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 17 Feb 2024 11:03:06 +0100 Subject: net: wan: framer: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20240217100306.86740-1-krzysztof.kozlowski@linaro.org Signed-off-by: Jakub Kicinski --- drivers/net/wan/framer/framer-core.c | 9 +++++---- include/linux/framer/framer-provider.h | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wan/framer/framer-core.c b/drivers/net/wan/framer/framer-core.c index c04dc88bda6c..33b358b99f70 100644 --- a/drivers/net/wan/framer/framer-core.c +++ b/drivers/net/wan/framer/framer-core.c @@ -384,7 +384,7 @@ static struct framer_provider *framer_provider_of_lookup(const struct device_nod return ERR_PTR(-EPROBE_DEFER); } -static struct framer *framer_of_get_from_provider(struct of_phandle_args *args) +static struct framer *framer_of_get_from_provider(const struct of_phandle_args *args) { struct framer_provider *framer_provider; struct framer *framer; @@ -735,7 +735,8 @@ EXPORT_SYMBOL_GPL(devm_framer_create); * should provide a custom of_xlate function that reads the *args* and returns * the appropriate framer. */ -struct framer *framer_provider_simple_of_xlate(struct device *dev, struct of_phandle_args *args) +struct framer *framer_provider_simple_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct class_dev_iter iter; struct framer *framer; @@ -768,7 +769,7 @@ EXPORT_SYMBOL_GPL(framer_provider_simple_of_xlate); struct framer_provider * __framer_provider_of_register(struct device *dev, struct module *owner, struct framer *(*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { struct framer_provider *framer_provider; @@ -830,7 +831,7 @@ static void devm_framer_provider_of_unregister(struct device *dev, void *res) struct framer_provider * __devm_framer_provider_of_register(struct device *dev, struct module *owner, struct framer *(*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { struct framer_provider **ptr, *framer_provider; diff --git a/include/linux/framer/framer-provider.h b/include/linux/framer/framer-provider.h index 782cd5fc83d5..f6fd2dd92591 100644 --- a/include/linux/framer/framer-provider.h +++ b/include/linux/framer/framer-provider.h @@ -93,7 +93,7 @@ struct framer_provider { struct module *owner; struct list_head list; struct framer * (*of_xlate)(struct device *dev, - struct of_phandle_args *args); + const struct of_phandle_args *args); }; static inline void framer_set_drvdata(struct framer *framer, void *data) @@ -118,19 +118,19 @@ struct framer *devm_framer_create(struct device *dev, struct device_node *node, const struct framer_ops *ops); struct framer *framer_provider_simple_of_xlate(struct device *dev, - struct of_phandle_args *args); + const struct of_phandle_args *args); struct framer_provider * __framer_provider_of_register(struct device *dev, struct module *owner, struct framer *(*of_xlate)(struct device *dev, - struct of_phandle_args *args)); + const struct of_phandle_args *args)); void framer_provider_of_unregister(struct framer_provider *framer_provider); struct framer_provider * __devm_framer_provider_of_register(struct device *dev, struct module *owner, struct framer *(*of_xlate)(struct device *dev, - struct of_phandle_args *args)); + const struct of_phandle_args *args)); void framer_notify_status_change(struct framer *framer); @@ -154,7 +154,7 @@ static inline struct framer *devm_framer_create(struct device *dev, struct devic } static inline struct framer *framer_provider_simple_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { return ERR_PTR(-ENOSYS); } @@ -162,7 +162,7 @@ static inline struct framer *framer_provider_simple_of_xlate(struct device *dev, static inline struct framer_provider * __framer_provider_of_register(struct device *dev, struct module *owner, struct framer *(*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { return ERR_PTR(-ENOSYS); } @@ -174,7 +174,7 @@ void framer_provider_of_unregister(struct framer_provider *framer_provider) static inline struct framer_provider * __devm_framer_provider_of_register(struct device *dev, struct module *owner, struct framer *(*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { return ERR_PTR(-ENOSYS); } -- cgit v1.2.3 From 557f8c582a9ba8abe6aa0fd734b6f342af106b26 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 18 Jan 2024 15:06:05 -0800 Subject: ubsan: Reintroduce signed overflow sanitizer In order to mitigate unexpected signed wrap-around[1], bring back the signed integer overflow sanitizer. It was removed in commit 6aaa31aeb9cf ("ubsan: remove overflow checks") because it was effectively a no-op when combined with -fno-strict-overflow (which correctly changes signed overflow from being "undefined" to being explicitly "wrap around"). Compilers are adjusting their sanitizers to trap wrap-around and to detecting common code patterns that should not be instrumented (e.g. "var + offset < var"). Prepare for this and explicitly rename the option from "OVERFLOW" to "WRAP" to more accurately describe the behavior. To annotate intentional wrap-around arithmetic, the helpers wrapping_add/sub/mul_wrap() can be used for individual statements. At the function level, the __signed_wrap attribute can be used to mark an entire function as expecting its signed arithmetic to wrap around. For a single object file the Makefile can use "UBSAN_SIGNED_WRAP_target.o := n" to mark it as wrapping, and for an entire directory, "UBSAN_SIGNED_WRAP := n" can be used. Additionally keep these disabled under CONFIG_COMPILE_TEST for now. Link: https://github.com/KSPP/linux/issues/26 [1] Cc: Miguel Ojeda Cc: Nathan Chancellor Cc: Peter Zijlstra Cc: Hao Luo Reviewed-by: Marco Elver Reviewed-by: Justin Stitt Signed-off-by: Kees Cook --- include/linux/compiler_types.h | 9 +++++- lib/Kconfig.ubsan | 15 +++++++++- lib/test_ubsan.c | 37 +++++++++++++++++++++++ lib/ubsan.c | 68 ++++++++++++++++++++++++++++++++++++++++++ lib/ubsan.h | 4 +++ scripts/Makefile.lib | 3 ++ scripts/Makefile.ubsan | 3 ++ 7 files changed, 137 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 6f1ca49306d2..ee9d272008a5 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -282,11 +282,18 @@ struct ftrace_likely_data { #define __no_sanitize_or_inline __always_inline #endif +/* Do not trap wrapping arithmetic within an annotated function. */ +#ifdef CONFIG_UBSAN_SIGNED_WRAP +# define __signed_wrap __attribute__((no_sanitize("signed-integer-overflow"))) +#else +# define __signed_wrap +#endif + /* Section for code which can't be instrumented at all */ #define __noinstr_section(section) \ noinline notrace __attribute((__section__(section))) \ __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage \ - __no_sanitize_memory + __no_sanitize_memory __signed_wrap #define noinstr __noinstr_section(".noinstr.text") diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 56d7653f4941..48a67058f84e 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -87,7 +87,6 @@ config UBSAN_LOCAL_BOUNDS config UBSAN_SHIFT bool "Perform checking for bit-shift overflows" - default UBSAN depends on $(cc-option,-fsanitize=shift) help This option enables -fsanitize=shift which checks for bit-shift @@ -116,6 +115,20 @@ config UBSAN_UNREACHABLE This option enables -fsanitize=unreachable which checks for control flow reaching an expected-to-be-unreachable position. +config UBSAN_SIGNED_WRAP + bool "Perform checking for signed arithmetic wrap-around" + default UBSAN + depends on !COMPILE_TEST + depends on $(cc-option,-fsanitize=signed-integer-overflow) + help + This option enables -fsanitize=signed-integer-overflow which checks + for wrap-around of any arithmetic operations with signed integers. + This currently performs nearly no instrumentation due to the + kernel's use of -fno-strict-overflow which converts all would-be + arithmetic undefined behavior into wrap-around arithmetic. Future + sanitizer versions will allow for wrap-around checking (rather than + exclusively undefined behavior). + config UBSAN_BOOL bool "Perform checking for non-boolean values used as boolean" default UBSAN diff --git a/lib/test_ubsan.c b/lib/test_ubsan.c index f4ee2484d4b5..276c12140ee2 100644 --- a/lib/test_ubsan.c +++ b/lib/test_ubsan.c @@ -11,6 +11,39 @@ typedef void(*test_ubsan_fp)(void); #config, IS_ENABLED(config) ? "y" : "n"); \ } while (0) +static void test_ubsan_add_overflow(void) +{ + volatile int val = INT_MAX; + + UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + val += 2; +} + +static void test_ubsan_sub_overflow(void) +{ + volatile int val = INT_MIN; + volatile int val2 = 2; + + UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + val -= val2; +} + +static void test_ubsan_mul_overflow(void) +{ + volatile int val = INT_MAX / 2; + + UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + val *= 3; +} + +static void test_ubsan_negate_overflow(void) +{ + volatile int val = INT_MIN; + + UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + val = -val; +} + static void test_ubsan_divrem_overflow(void) { volatile int val = 16; @@ -90,6 +123,10 @@ static void test_ubsan_misaligned_access(void) } static const test_ubsan_fp test_ubsan_array[] = { + test_ubsan_add_overflow, + test_ubsan_sub_overflow, + test_ubsan_mul_overflow, + test_ubsan_negate_overflow, test_ubsan_shift_out_of_bounds, test_ubsan_out_of_bounds, test_ubsan_load_invalid_value, diff --git a/lib/ubsan.c b/lib/ubsan.c index df4f8d1354bb..5fc107f61934 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -222,6 +222,74 @@ static void ubsan_epilogue(void) check_panic_on_warn("UBSAN"); } +static void handle_overflow(struct overflow_data *data, void *lhs, + void *rhs, char op) +{ + + struct type_descriptor *type = data->type; + char lhs_val_str[VALUE_LENGTH]; + char rhs_val_str[VALUE_LENGTH]; + + if (suppress_report(&data->location)) + return; + + ubsan_prologue(&data->location, type_is_signed(type) ? + "signed-integer-overflow" : + "unsigned-integer-overflow"); + + val_to_string(lhs_val_str, sizeof(lhs_val_str), type, lhs); + val_to_string(rhs_val_str, sizeof(rhs_val_str), type, rhs); + pr_err("%s %c %s cannot be represented in type %s\n", + lhs_val_str, + op, + rhs_val_str, + type->type_name); + + ubsan_epilogue(); +} + +void __ubsan_handle_add_overflow(void *data, + void *lhs, void *rhs) +{ + + handle_overflow(data, lhs, rhs, '+'); +} +EXPORT_SYMBOL(__ubsan_handle_add_overflow); + +void __ubsan_handle_sub_overflow(void *data, + void *lhs, void *rhs) +{ + handle_overflow(data, lhs, rhs, '-'); +} +EXPORT_SYMBOL(__ubsan_handle_sub_overflow); + +void __ubsan_handle_mul_overflow(void *data, + void *lhs, void *rhs) +{ + handle_overflow(data, lhs, rhs, '*'); +} +EXPORT_SYMBOL(__ubsan_handle_mul_overflow); + +void __ubsan_handle_negate_overflow(void *_data, void *old_val) +{ + struct overflow_data *data = _data; + char old_val_str[VALUE_LENGTH]; + + if (suppress_report(&data->location)) + return; + + ubsan_prologue(&data->location, "negation-overflow"); + + val_to_string(old_val_str, sizeof(old_val_str), data->type, old_val); + + pr_err("negation of %s cannot be represented in type %s:\n", + old_val_str, data->type->type_name); + + ubsan_epilogue(); +} +EXPORT_SYMBOL(__ubsan_handle_negate_overflow); + + void __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs) { struct overflow_data *data = _data; diff --git a/lib/ubsan.h b/lib/ubsan.h index 5d99ab81913b..0abbbac8700d 100644 --- a/lib/ubsan.h +++ b/lib/ubsan.h @@ -124,6 +124,10 @@ typedef s64 s_max; typedef u64 u_max; #endif +void __ubsan_handle_add_overflow(void *data, void *lhs, void *rhs); +void __ubsan_handle_sub_overflow(void *data, void *lhs, void *rhs); +void __ubsan_handle_mul_overflow(void *data, void *lhs, void *rhs); +void __ubsan_handle_negate_overflow(void *_data, void *old_val); void __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs); void __ubsan_handle_type_mismatch(struct type_mismatch_data *data, void *ptr); void __ubsan_handle_type_mismatch_v1(void *_data, void *ptr); diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 52efc520ae4f..b4a248c20654 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -177,6 +177,9 @@ ifeq ($(CONFIG_UBSAN),y) _c_flags += $(if $(patsubst n%,, \ $(UBSAN_SANITIZE_$(basetarget).o)$(UBSAN_SANITIZE)y), \ $(CFLAGS_UBSAN)) +_c_flags += $(if $(patsubst n%,, \ + $(UBSAN_SIGNED_WRAP_$(basetarget).o)$(UBSAN_SANITIZE_$(basetarget).o)$(UBSAN_SIGNED_WRAP)$(UBSAN_SANITIZE)y), \ + $(CFLAGS_UBSAN_SIGNED_WRAP)) endif ifeq ($(CONFIG_KCOV),y) diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index 7cf42231042b..b2d3b273b802 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -13,3 +13,6 @@ ubsan-cflags-$(CONFIG_UBSAN_ENUM) += -fsanitize=enum ubsan-cflags-$(CONFIG_UBSAN_TRAP) += $(call cc-option,-fsanitize-trap=undefined,-fsanitize-undefined-trap-on-error) export CFLAGS_UBSAN := $(ubsan-cflags-y) + +ubsan-signed-wrap-cflags-$(CONFIG_UBSAN_SIGNED_WRAP) += -fsanitize=signed-integer-overflow +export CFLAGS_UBSAN_SIGNED_WRAP := $(ubsan-signed-wrap-cflags-y) -- cgit v1.2.3 From f478898e0aa74a759fcf629a3ee8b040467b8533 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 2 Feb 2024 03:18:14 -0800 Subject: string: Redefine strscpy_pad() as a macro In preparation for making strscpy_pad()'s 3rd argument optional, redefine it as a macro. This also has the benefit of allowing greater FORITFY introspection, as it couldn't see into the strscpy() nor the memset() within strscpy_pad(). Cc: Andy Shevchenko Cc: Andrew Morton Cc: Reviewed-by: Justin Stitt Signed-off-by: Kees Cook --- include/linux/string.h | 33 +++++++++++++++++++++++++++++++-- lib/string_helpers.c | 34 ---------------------------------- 2 files changed, 31 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/include/linux/string.h b/include/linux/string.h index ab148d8dbfc1..78b28004c5ba 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -70,8 +70,37 @@ extern char * strncpy(char *,const char *, __kernel_size_t); ssize_t strscpy(char *, const char *, size_t); #endif -/* Wraps calls to strscpy()/memset(), no arch specific code required */ -ssize_t strscpy_pad(char *dest, const char *src, size_t count); +/** + * strscpy_pad() - Copy a C-string into a sized buffer + * @dest: Where to copy the string to + * @src: Where to copy the string from + * @count: Size of destination buffer + * + * Copy the string, or as much of it as fits, into the dest buffer. The + * behavior is undefined if the string buffers overlap. The destination + * buffer is always %NUL terminated, unless it's zero-sized. + * + * If the source string is shorter than the destination buffer, the + * remaining bytes in the buffer will be filled with %NUL bytes. + * + * For full explanation of why you may want to consider using the + * 'strscpy' functions please see the function docstring for strscpy(). + * + * Returns: + * * The number of characters copied (not including the trailing %NULs) + * * -E2BIG if count is 0 or @src was truncated. + */ +#define strscpy_pad(dest, src, count) ({ \ + char *__dst = (dest); \ + const char *__src = (src); \ + const size_t __count = (count); \ + ssize_t __wrote; \ + \ + __wrote = strscpy(__dst, __src, __count); \ + if (__wrote >= 0 && __wrote < __count) \ + memset(__dst + __wrote + 1, 0, __count - __wrote - 1); \ + __wrote; \ +}) #ifndef __HAVE_ARCH_STRCAT extern char * strcat(char *, const char *); diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 7713f73e66b0..606c3099013f 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -825,40 +825,6 @@ char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n) } EXPORT_SYMBOL_GPL(devm_kasprintf_strarray); -/** - * strscpy_pad() - Copy a C-string into a sized buffer - * @dest: Where to copy the string to - * @src: Where to copy the string from - * @count: Size of destination buffer - * - * Copy the string, or as much of it as fits, into the dest buffer. The - * behavior is undefined if the string buffers overlap. The destination - * buffer is always %NUL terminated, unless it's zero-sized. - * - * If the source string is shorter than the destination buffer, zeros - * the tail of the destination buffer. - * - * For full explanation of why you may want to consider using the - * 'strscpy' functions please see the function docstring for strscpy(). - * - * Returns: - * * The number of characters copied (not including the trailing %NUL) - * * -E2BIG if count is 0 or @src was truncated. - */ -ssize_t strscpy_pad(char *dest, const char *src, size_t count) -{ - ssize_t written; - - written = strscpy(dest, src, count); - if (written < 0 || written == count - 1) - return written; - - memset(dest + written + 1, 0, count - written - 1); - - return written; -} -EXPORT_SYMBOL(strscpy_pad); - /** * skip_spaces - Removes leading whitespace from @str. * @str: The string to be stripped. -- cgit v1.2.3 From e6584c3964f2ff76a9fb5a701e4a59997b35e547 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 20 Sep 2023 12:38:14 -0700 Subject: string: Allow 2-argument strscpy() Using sizeof(dst) for the "size" argument in strscpy() is the overwhelmingly common case. Instead of requiring this everywhere, allow a 2-argument version to be used that will use the sizeof() internally. There are other functions in the kernel with optional arguments[1], so this isn't unprecedented, and improves readability. Update and relocate the kern-doc for strscpy() too, and drop __HAVE_ARCH_STRSCPY as it is unused. Adjust ARCH=um build to notice the changed export name, as it doesn't do full header includes for the string helpers. This could additionally let us save a few hundred lines of code: 1177 files changed, 2455 insertions(+), 3026 deletions(-) with a treewide cleanup using Coccinelle: @needless_arg@ expression DST, SRC; @@ strscpy(DST, SRC -, sizeof(DST) ) Link: https://elixir.bootlin.com/linux/v6.7/source/include/linux/pci.h#L1517 [1] Reviewed-by: Justin Stitt Cc: Andy Shevchenko Cc: linux-hardening@vger.kernel.org Signed-off-by: Kees Cook --- arch/um/include/shared/user.h | 3 ++- include/linux/fortify-string.h | 22 ++-------------------- include/linux/string.h | 38 +++++++++++++++++++++++++++++++++++--- lib/string.c | 6 ++---- 4 files changed, 41 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/arch/um/include/shared/user.h b/arch/um/include/shared/user.h index 981e11d8e025..9568cc04cbb7 100644 --- a/arch/um/include/shared/user.h +++ b/arch/um/include/shared/user.h @@ -51,7 +51,8 @@ static inline int printk(const char *fmt, ...) extern int in_aton(char *str); extern size_t strlcat(char *, const char *, size_t); -extern size_t strscpy(char *, const char *, size_t); +extern size_t sized_strscpy(char *, const char *, size_t); +#define strscpy(dst, src, size) sized_strscpy(dst, src, size) /* Copied from linux/compiler-gcc.h since we can't include it directly */ #define barrier() __asm__ __volatile__("": : :"memory") diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 89a6888f2f9e..06b3aaa63724 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -215,26 +215,8 @@ __kernel_size_t __fortify_strlen(const char * const POS p) } /* Defined after fortified strnlen() to reuse it. */ -extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(strscpy); -/** - * strscpy - Copy a C-string into a sized buffer - * - * @p: Where to copy the string to - * @q: Where to copy the string from - * @size: Size of destination buffer - * - * Copy the source string @q, or as much of it as fits, into the destination - * @p buffer. The behavior is undefined if the string buffers overlap. The - * destination @p buffer is always NUL terminated, unless it's zero-sized. - * - * Preferred to strncpy() since it always returns a valid string, and - * doesn't unnecessarily force the tail of the destination buffer to be - * zero padded. If padding is desired please use strscpy_pad(). - * - * Returns the number of characters copied in @p (not including the - * trailing %NUL) or -E2BIG if @size is 0 or the copy of @q was truncated. - */ -__FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, size_t size) +extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(sized_strscpy); +__FORTIFY_INLINE ssize_t sized_strscpy(char * const POS p, const char * const POS q, size_t size) { /* Use string size rather than possible enclosing struct size. */ const size_t p_size = __member_size(p); diff --git a/include/linux/string.h b/include/linux/string.h index 78b28004c5ba..0d66bf9407fd 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -2,6 +2,7 @@ #ifndef _LINUX_STRING_H_ #define _LINUX_STRING_H_ +#include #include #include /* for inline */ #include /* for size_t */ @@ -66,9 +67,40 @@ extern char * strcpy(char *,const char *); #ifndef __HAVE_ARCH_STRNCPY extern char * strncpy(char *,const char *, __kernel_size_t); #endif -#ifndef __HAVE_ARCH_STRSCPY -ssize_t strscpy(char *, const char *, size_t); -#endif +ssize_t sized_strscpy(char *, const char *, size_t); + +/* + * The 2 argument style can only be used when dst is an array with a + * known size. + */ +#define __strscpy0(dst, src, ...) \ + sized_strscpy(dst, src, sizeof(dst) + __must_be_array(dst)) +#define __strscpy1(dst, src, size) sized_strscpy(dst, src, size) + +/** + * strscpy - Copy a C-string into a sized buffer + * @dst: Where to copy the string to + * @src: Where to copy the string from + * @...: Size of destination buffer (optional) + * + * Copy the source string @src, or as much of it as fits, into the + * destination @dst buffer. The behavior is undefined if the string + * buffers overlap. The destination @dst buffer is always NUL terminated, + * unless it's zero-sized. + * + * The size argument @... is only required when @dst is not an array, or + * when the copy needs to be smaller than sizeof(@dst). + * + * Preferred to strncpy() since it always returns a valid string, and + * doesn't unnecessarily force the tail of the destination buffer to be + * zero padded. If padding is desired please use strscpy_pad(). + * + * Returns the number of characters copied in @dst (not including the + * trailing %NUL) or -E2BIG if @size is 0 or the copy from @src was + * truncated. + */ +#define strscpy(dst, src, ...) \ + CONCATENATE(__strscpy, COUNT_ARGS(__VA_ARGS__))(dst, src, __VA_ARGS__) /** * strscpy_pad() - Copy a C-string into a sized buffer diff --git a/lib/string.c b/lib/string.c index f791559102f6..966da44bfc86 100644 --- a/lib/string.c +++ b/lib/string.c @@ -104,8 +104,7 @@ char *strncpy(char *dest, const char *src, size_t count) EXPORT_SYMBOL(strncpy); #endif -#ifndef __HAVE_ARCH_STRSCPY -ssize_t strscpy(char *dest, const char *src, size_t count) +ssize_t sized_strscpy(char *dest, const char *src, size_t count) { const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; size_t max = count; @@ -171,8 +170,7 @@ ssize_t strscpy(char *dest, const char *src, size_t count) return -E2BIG; } -EXPORT_SYMBOL(strscpy); -#endif +EXPORT_SYMBOL(sized_strscpy); /** * stpcpy - copy a string from src to dest returning a pointer to the new end -- cgit v1.2.3 From 8366d124ec937c3815212c00daf00b687eb27969 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 2 Feb 2024 03:40:23 -0800 Subject: string: Allow 2-argument strscpy_pad() Similar to strscpy(), update strscpy_pad()'s 3rd argument to be optional when the destination is a compile-time known size array. Cc: Andy Shevchenko Cc: Reviewed-by: Justin Stitt Signed-off-by: Kees Cook --- include/linux/string.h | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/string.h b/include/linux/string.h index 0d66bf9407fd..96e6b1af86b5 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -77,6 +77,10 @@ ssize_t sized_strscpy(char *, const char *, size_t); sized_strscpy(dst, src, sizeof(dst) + __must_be_array(dst)) #define __strscpy1(dst, src, size) sized_strscpy(dst, src, size) +#define __strscpy_pad0(dst, src, ...) \ + sized_strscpy_pad(dst, src, sizeof(dst) + __must_be_array(dst)) +#define __strscpy_pad1(dst, src, size) sized_strscpy_pad(dst, src, size) + /** * strscpy - Copy a C-string into a sized buffer * @dst: Where to copy the string to @@ -102,11 +106,23 @@ ssize_t sized_strscpy(char *, const char *, size_t); #define strscpy(dst, src, ...) \ CONCATENATE(__strscpy, COUNT_ARGS(__VA_ARGS__))(dst, src, __VA_ARGS__) +#define sized_strscpy_pad(dest, src, count) ({ \ + char *__dst = (dest); \ + const char *__src = (src); \ + const size_t __count = (count); \ + ssize_t __wrote; \ + \ + __wrote = sized_strscpy(__dst, __src, __count); \ + if (__wrote >= 0 && __wrote < __count) \ + memset(__dst + __wrote + 1, 0, __count - __wrote - 1); \ + __wrote; \ +}) + /** * strscpy_pad() - Copy a C-string into a sized buffer - * @dest: Where to copy the string to + * @dst: Where to copy the string to * @src: Where to copy the string from - * @count: Size of destination buffer + * @...: Size of destination buffer * * Copy the string, or as much of it as fits, into the dest buffer. The * behavior is undefined if the string buffers overlap. The destination @@ -122,17 +138,8 @@ ssize_t sized_strscpy(char *, const char *, size_t); * * The number of characters copied (not including the trailing %NULs) * * -E2BIG if count is 0 or @src was truncated. */ -#define strscpy_pad(dest, src, count) ({ \ - char *__dst = (dest); \ - const char *__src = (src); \ - const size_t __count = (count); \ - ssize_t __wrote; \ - \ - __wrote = strscpy(__dst, __src, __count); \ - if (__wrote >= 0 && __wrote < __count) \ - memset(__dst + __wrote + 1, 0, __count - __wrote - 1); \ - __wrote; \ -}) +#define strscpy_pad(dst, src, ...) \ + CONCATENATE(__strscpy_pad, COUNT_ARGS(__VA_ARGS__))(dst, src, __VA_ARGS__) #ifndef __HAVE_ARCH_STRCAT extern char * strcat(char *, const char *); -- cgit v1.2.3 From f0397e27d1204a7e6581d140c2de7fd11383c6ba Mon Sep 17 00:00:00 2001 From: Jeffrey Hugo Date: Mon, 19 Feb 2024 11:07:48 -0700 Subject: Revert "bus: mhi: core: Add support for reading MHI info from device" This reverts commit 3316ab2b45f6bf4797d8d65b22fda3cc13318890. The MHI spec owner pointed out that the SOC_HW_VERSION register is part of the BHIe segment, and only valid on devices which implement BHIe. Only a small subset of MHI devices implement BHIe so blindly accessing the register for all devices is not correct. Also, since the BHIe segment offset is not used when accessing the register, any implementation which moves the BHIe segment will result in accessing some other register. We've seen that accessing this register on AIC100 which does not support BHIe can result in initialization failures. We could try to put checks into the code to address these issues, but in the roughly 4 years this functionality has existed, no one has used it. Easier to drop this dead code and address the issues if anyone comes up with a real world use for it. Signed-off-by: Jeffrey Hugo Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20240219180748.1591527-1-quic_jhugo@quicinc.com Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/host/init.c | 12 ------------ drivers/bus/mhi/host/internal.h | 6 ------ include/linux/mhi.h | 17 ----------------- 3 files changed, 35 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c index 944da46e6f11..44f934981de8 100644 --- a/drivers/bus/mhi/host/init.c +++ b/drivers/bus/mhi/host/init.c @@ -914,7 +914,6 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan; struct mhi_cmd *mhi_cmd; struct mhi_device *mhi_dev; - u32 soc_info; int ret, i; if (!mhi_cntrl || !mhi_cntrl->cntrl_dev || !mhi_cntrl->regs || @@ -989,17 +988,6 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, mhi_cntrl->unmap_single = mhi_unmap_single_no_bb; } - /* Read the MHI device info */ - ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, - SOC_HW_VERSION_OFFS, &soc_info); - if (ret) - goto err_destroy_wq; - - mhi_cntrl->family_number = FIELD_GET(SOC_HW_VERSION_FAM_NUM_BMSK, soc_info); - mhi_cntrl->device_number = FIELD_GET(SOC_HW_VERSION_DEV_NUM_BMSK, soc_info); - mhi_cntrl->major_version = FIELD_GET(SOC_HW_VERSION_MAJOR_VER_BMSK, soc_info); - mhi_cntrl->minor_version = FIELD_GET(SOC_HW_VERSION_MINOR_VER_BMSK, soc_info); - mhi_cntrl->index = ida_alloc(&mhi_controller_ida, GFP_KERNEL); if (mhi_cntrl->index < 0) { ret = mhi_cntrl->index; diff --git a/drivers/bus/mhi/host/internal.h b/drivers/bus/mhi/host/internal.h index 091244cf17c6..5fe49311b8eb 100644 --- a/drivers/bus/mhi/host/internal.h +++ b/drivers/bus/mhi/host/internal.h @@ -15,12 +15,6 @@ extern struct bus_type mhi_bus_type; #define MHI_SOC_RESET_REQ_OFFSET 0xb0 #define MHI_SOC_RESET_REQ BIT(0) -#define SOC_HW_VERSION_OFFS 0x224 -#define SOC_HW_VERSION_FAM_NUM_BMSK GENMASK(31, 28) -#define SOC_HW_VERSION_DEV_NUM_BMSK GENMASK(27, 16) -#define SOC_HW_VERSION_MAJOR_VER_BMSK GENMASK(15, 8) -#define SOC_HW_VERSION_MINOR_VER_BMSK GENMASK(7, 0) - struct mhi_ctxt { struct mhi_event_ctxt *er_ctxt; struct mhi_chan_ctxt *chan_ctxt; diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 474d32cb0520..77b8c0a26674 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -320,10 +320,6 @@ struct mhi_controller_config { * @hw_ev_rings: Number of hardware event rings * @sw_ev_rings: Number of software event rings * @nr_irqs: Number of IRQ allocated by bus master (required) - * @family_number: MHI controller family number - * @device_number: MHI controller device number - * @major_version: MHI controller major revision number - * @minor_version: MHI controller minor revision number * @serial_number: MHI controller serial number obtained from BHI * @mhi_event: MHI event ring configurations table * @mhi_cmd: MHI command ring configurations table @@ -368,15 +364,6 @@ struct mhi_controller_config { * Fields marked as (required) need to be populated by the controller driver * before calling mhi_register_controller(). For the fields marked as (optional) * they can be populated depending on the usecase. - * - * The following fields are present for the purpose of implementing any device - * specific quirks or customizations for specific MHI revisions used in device - * by the controller drivers. The MHI stack will just populate these fields - * during mhi_register_controller(): - * family_number - * device_number - * major_version - * minor_version */ struct mhi_controller { struct device *cntrl_dev; @@ -407,10 +394,6 @@ struct mhi_controller { u32 hw_ev_rings; u32 sw_ev_rings; u32 nr_irqs; - u32 family_number; - u32 device_number; - u32 major_version; - u32 minor_version; u32 serial_number; struct mhi_event *mhi_event; -- cgit v1.2.3 From b64e74e95aa6491b31477e9002aab1d8df3995bf Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 19 Feb 2024 07:27:09 +0100 Subject: mm: move mapping_set_update out of mapping_set_update is only used inside mm/. Move mapping_set_update to mm/internal.h and turn it into an inline function instead of a macro. Signed-off-by: Christoph Hellwig Reviewed-by: "Matthew Wilcox (Oracle)" Signed-off-by: Chandan Babu R --- include/linux/swap.h | 10 ---------- mm/filemap.c | 9 +++++++++ mm/internal.h | 4 ++++ mm/workingset.c | 1 + 4 files changed, 14 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 4db00ddad261..755fc64ba48d 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -350,16 +350,6 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); void workingset_refault(struct folio *folio, void *shadow); void workingset_activation(struct folio *folio); -/* Only track the nodes of mappings with shadow entries */ -void workingset_update_node(struct xa_node *node); -extern struct list_lru shadow_nodes; -#define mapping_set_update(xas, mapping) do { \ - if (!dax_mapping(mapping) && !shmem_mapping(mapping)) { \ - xas_set_update(xas, workingset_update_node); \ - xas_set_lru(xas, &shadow_nodes); \ - } \ -} while (0) - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; diff --git a/mm/filemap.c b/mm/filemap.c index 750e779c23db..6c8b089f00d2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -124,6 +124,15 @@ * ->private_lock (zap_pte_range->block_dirty_folio) */ +static void mapping_set_update(struct xa_state *xas, + struct address_space *mapping) +{ + if (dax_mapping(mapping) || shmem_mapping(mapping)) + return; + xas_set_update(xas, workingset_update_node); + xas_set_lru(xas, &shadow_nodes); +} + static void page_cache_delete(struct address_space *mapping, struct folio *folio, void *shadow) { diff --git a/mm/internal.h b/mm/internal.h index f309a010d50f..4398f572485f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1266,4 +1266,8 @@ static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry, } #endif /* CONFIG_SHRINKER_DEBUG */ +/* Only track the nodes of mappings with shadow entries */ +void workingset_update_node(struct xa_node *node); +extern struct list_lru shadow_nodes; + #endif /* __MM_INTERNAL_H */ diff --git a/mm/workingset.c b/mm/workingset.c index 226012974328..f2a0ecaf708d 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -16,6 +16,7 @@ #include #include #include +#include "internal.h" /* * Double CLOCK lists -- cgit v1.2.3 From aefacb2041f77784059b86c5fd151066859ad19a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 19 Feb 2024 07:27:10 +0100 Subject: shmem: move shmem_mapping out of line shmem_aops really should not be exported to the world. Move shmem_mapping and export it as internal for the one semi-legitimate modular user in udmabuf. This effectively reverts commit 30e6a51dbb05 ("mm/shmem.c: make shmem_mapping() inline"). which added a bogus shmem_aops non-GPL export for no reason whatsoever as there as no shmem_mapping call outside of core MM code at that point. Signed-off-by: Christoph Hellwig Reviewed-by: "Matthew Wilcox (Oracle)" Signed-off-by: Chandan Babu R --- include/linux/shmem_fs.h | 6 +----- mm/shmem.c | 11 ++++++++--- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 2caa6b86106a..6b96a87e4bc8 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -97,11 +97,7 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM -extern const struct address_space_operations shmem_aops; -static inline bool shmem_mapping(struct address_space *mapping) -{ - return mapping->a_ops == &shmem_aops; -} +bool shmem_mapping(struct address_space *mapping); #else static inline bool shmem_mapping(struct address_space *mapping) { diff --git a/mm/shmem.c b/mm/shmem.c index d7c84ff62186..f607b0cab7e4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -254,7 +254,7 @@ static void shmem_inode_unacct_blocks(struct inode *inode, long pages) } static const struct super_operations shmem_ops; -const struct address_space_operations shmem_aops; +static const struct address_space_operations shmem_aops; static const struct file_operations shmem_file_operations; static const struct inode_operations shmem_inode_operations; static const struct inode_operations shmem_dir_inode_operations; @@ -263,6 +263,12 @@ static const struct vm_operations_struct shmem_vm_ops; static const struct vm_operations_struct shmem_anon_vm_ops; static struct file_system_type shmem_fs_type; +bool shmem_mapping(struct address_space *mapping) +{ + return mapping->a_ops == &shmem_aops; +} +EXPORT_SYMBOL_GPL(shmem_mapping); + bool vma_is_anon_shmem(struct vm_area_struct *vma) { return vma->vm_ops == &shmem_anon_vm_ops; @@ -4466,7 +4472,7 @@ static int shmem_error_remove_folio(struct address_space *mapping, return 0; } -const struct address_space_operations shmem_aops = { +static const struct address_space_operations shmem_aops = { .writepage = shmem_writepage, .dirty_folio = noop_dirty_folio, #ifdef CONFIG_TMPFS @@ -4478,7 +4484,6 @@ const struct address_space_operations shmem_aops = { #endif .error_remove_folio = shmem_error_remove_folio, }; -EXPORT_SYMBOL(shmem_aops); static const struct file_operations shmem_file_operations = { .mmap = shmem_mmap, -- cgit v1.2.3 From 26ea8511c849f9fea325bcdbd8b41ea031a53afe Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 29 Jan 2024 12:52:11 +0100 Subject: of: Add of_phandle_args_equal() helper Add a helper comparing two "struct of_phandle_args" to avoid reinventing the wheel. Reviewed-by: Philipp Zabel Acked-by: Rob Herring Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240129115216.96479-2-krzysztof.kozlowski@linaro.org Signed-off-by: Philipp Zabel --- include/linux/of.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/of.h b/include/linux/of.h index 6a9ddf20e79a..85bcc05b278d 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -1065,6 +1065,22 @@ static inline int of_parse_phandle_with_optional_args(const struct device_node * 0, index, out_args); } +/** + * of_phandle_args_equal() - Compare two of_phandle_args + * @a1: First of_phandle_args to compare + * @a2: Second of_phandle_args to compare + * + * Return: True if a1 and a2 are the same (same node pointer, same phandle + * args), false otherwise. + */ +static inline bool of_phandle_args_equal(const struct of_phandle_args *a1, + const struct of_phandle_args *a2) +{ + return a1->np == a2->np && + a1->args_count == a2->args_count && + !memcmp(a1->args, a2->args, sizeof(a1->args[0]) * a1->args_count); +} + /** * of_property_count_u8_elems - Count the number of u8 elements in a property * -- cgit v1.2.3 From 0f28982835c29752cdb657f1f8316b2ea42c407a Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 29 Jan 2024 12:52:12 +0100 Subject: cpufreq: do not open-code of_phandle_args_equal() Use newly added of_phandle_args_equal() helper to compare two of_phandle_args. Acked-by: Viresh Kumar Reviewed-by: Philipp Zabel Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240129115216.96479-3-krzysztof.kozlowski@linaro.org Signed-off-by: Philipp Zabel --- include/linux/cpufreq.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index afda5f24d3dd..3cd06dafb04b 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -1149,8 +1149,7 @@ static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_ if (ret < 0) continue; - if (pargs->np == args.np && pargs->args_count == args.args_count && - !memcmp(pargs->args, args.args, sizeof(args.args[0]) * args.args_count)) + if (of_phandle_args_equal(pargs, &args)) cpumask_set_cpu(cpu, cpumask); of_node_put(args.np); -- cgit v1.2.3 From c721f189e89c0d4db119d7bb2b46768d0fb5f6b1 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 29 Jan 2024 12:52:14 +0100 Subject: reset: Instantiate reset GPIO controller for shared reset-gpios Devices sharing a reset GPIO could use the reset framework for coordinated handling of that shared GPIO line. We have several cases of such needs, at least for Devicetree-based platforms. If Devicetree-based device requests a reset line, while "resets" Devicetree property is missing but there is a "reset-gpios" one, instantiate a new "reset-gpio" platform device which will handle such reset line. This allows seamless handling of such shared reset-gpios without need of changing Devicetree binding [1]. To avoid creating multiple "reset-gpio" platform devices, store the Devicetree "reset-gpios" GPIO specifiers used for new devices on a linked list. Later such Devicetree GPIO specifier (phandle to GPIO controller, GPIO number and GPIO flags) is used to check if reset controller for given GPIO was already registered. If two devices have conflicting "reset-gpios" property, e.g. with different ACTIVE_xxx flags, this would allow to spawn two separate "reset-gpio" devices, where the second would fail probing on busy GPIO request. Link: https://lore.kernel.org/all/YXi5CUCEi7YmNxXM@robh.at.kernel.org/ [1] Cc: Bartosz Golaszewski Cc: Chris Packham Cc: Sean Anderson Reviewed-by: Philipp Zabel Signed-off-by: Krzysztof Kozlowski Acked-by: Bartosz Golaszewski Acked-by: Linus Walleij Link: https://lore.kernel.org/r/20240129115216.96479-5-krzysztof.kozlowski@linaro.org Signed-off-by: Philipp Zabel --- drivers/reset/core.c | 224 ++++++++++++++++++++++++++++++++++++--- include/linux/reset-controller.h | 4 + 2 files changed, 215 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/reset/core.c b/drivers/reset/core.c index 4d5a78d3c085..dba74e857be6 100644 --- a/drivers/reset/core.c +++ b/drivers/reset/core.c @@ -5,14 +5,19 @@ * Copyright 2013 Philipp Zabel, Pengutronix */ #include +#include #include #include #include #include #include +#include +#include +#include #include #include #include +#include #include #include #include @@ -23,6 +28,11 @@ static LIST_HEAD(reset_controller_list); static DEFINE_MUTEX(reset_lookup_mutex); static LIST_HEAD(reset_lookup_list); +/* Protects reset_gpio_lookup_list */ +static DEFINE_MUTEX(reset_gpio_lookup_mutex); +static LIST_HEAD(reset_gpio_lookup_list); +static DEFINE_IDA(reset_gpio_ida); + /** * struct reset_control - a reset control * @rcdev: a pointer to the reset controller device @@ -63,6 +73,16 @@ struct reset_control_array { struct reset_control *rstc[] __counted_by(num_rstcs); }; +/** + * struct reset_gpio_lookup - lookup key for ad-hoc created reset-gpio devices + * @of_args: phandle to the reset controller with all the args like GPIO number + * @list: list entry for the reset_gpio_lookup_list + */ +struct reset_gpio_lookup { + struct of_phandle_args of_args; + struct list_head list; +}; + static const char *rcdev_name(struct reset_controller_dev *rcdev) { if (rcdev->dev) @@ -71,6 +91,9 @@ static const char *rcdev_name(struct reset_controller_dev *rcdev) if (rcdev->of_node) return rcdev->of_node->full_name; + if (rcdev->of_args) + return rcdev->of_args->np->full_name; + return NULL; } @@ -99,6 +122,9 @@ static int of_reset_simple_xlate(struct reset_controller_dev *rcdev, */ int reset_controller_register(struct reset_controller_dev *rcdev) { + if (rcdev->of_node && rcdev->of_args) + return -EINVAL; + if (!rcdev->of_xlate) { rcdev->of_reset_n_cells = 1; rcdev->of_xlate = of_reset_simple_xlate; @@ -813,12 +839,171 @@ static void __reset_control_put_internal(struct reset_control *rstc) kref_put(&rstc->refcnt, __reset_control_release); } +static int __reset_add_reset_gpio_lookup(int id, struct device_node *np, + unsigned int gpio, + unsigned int of_flags) +{ + const struct fwnode_handle *fwnode = of_fwnode_handle(np); + unsigned int lookup_flags; + const char *label_tmp; + + /* + * Later we map GPIO flags between OF and Linux, however not all + * constants from include/dt-bindings/gpio/gpio.h and + * include/linux/gpio/machine.h match each other. + */ + if (of_flags > GPIO_ACTIVE_LOW) { + pr_err("reset-gpio code does not support GPIO flags %u for GPIO %u\n", + of_flags, gpio); + return -EINVAL; + } + + struct gpio_device *gdev __free(gpio_device_put) = gpio_device_find_by_fwnode(fwnode); + if (!gdev) + return -EPROBE_DEFER; + + label_tmp = gpio_device_get_label(gdev); + if (!label_tmp) + return -EINVAL; + + char *label __free(kfree) = kstrdup(label_tmp, GFP_KERNEL); + if (!label) + return -ENOMEM; + + /* Size: one lookup entry plus sentinel */ + struct gpiod_lookup_table *lookup __free(kfree) = kzalloc(struct_size(lookup, table, 2), + GFP_KERNEL); + if (!lookup) + return -ENOMEM; + + lookup->dev_id = kasprintf(GFP_KERNEL, "reset-gpio.%d", id); + if (!lookup->dev_id) + return -ENOMEM; + + lookup_flags = GPIO_PERSISTENT; + lookup_flags |= of_flags & GPIO_ACTIVE_LOW; + lookup->table[0] = GPIO_LOOKUP(no_free_ptr(label), gpio, "reset", + lookup_flags); + + /* Not freed on success, because it is persisent subsystem data. */ + gpiod_add_lookup_table(no_free_ptr(lookup)); + + return 0; +} + +/* + * @args: phandle to the GPIO provider with all the args like GPIO number + */ +static int __reset_add_reset_gpio_device(const struct of_phandle_args *args) +{ + struct reset_gpio_lookup *rgpio_dev; + struct platform_device *pdev; + int id, ret; + + /* + * Currently only #gpio-cells=2 is supported with the meaning of: + * args[0]: GPIO number + * args[1]: GPIO flags + * TODO: Handle other cases. + */ + if (args->args_count != 2) + return -ENOENT; + + /* + * Registering reset-gpio device might cause immediate + * bind, resulting in its probe() registering new reset controller thus + * taking reset_list_mutex lock via reset_controller_register(). + */ + lockdep_assert_not_held(&reset_list_mutex); + + mutex_lock(&reset_gpio_lookup_mutex); + + list_for_each_entry(rgpio_dev, &reset_gpio_lookup_list, list) { + if (args->np == rgpio_dev->of_args.np) { + if (of_phandle_args_equal(args, &rgpio_dev->of_args)) + goto out; /* Already on the list, done */ + } + } + + id = ida_alloc(&reset_gpio_ida, GFP_KERNEL); + if (id < 0) { + ret = id; + goto err_unlock; + } + + /* Not freed on success, because it is persisent subsystem data. */ + rgpio_dev = kzalloc(sizeof(*rgpio_dev), GFP_KERNEL); + if (!rgpio_dev) { + ret = -ENOMEM; + goto err_ida_free; + } + + ret = __reset_add_reset_gpio_lookup(id, args->np, args->args[0], + args->args[1]); + if (ret < 0) + goto err_kfree; + + rgpio_dev->of_args = *args; + /* + * We keep the device_node reference, but of_args.np is put at the end + * of __of_reset_control_get(), so get it one more time. + * Hold reference as long as rgpio_dev memory is valid. + */ + of_node_get(rgpio_dev->of_args.np); + pdev = platform_device_register_data(NULL, "reset-gpio", id, + &rgpio_dev->of_args, + sizeof(rgpio_dev->of_args)); + ret = PTR_ERR_OR_ZERO(pdev); + if (ret) + goto err_put; + + list_add(&rgpio_dev->list, &reset_gpio_lookup_list); + +out: + mutex_unlock(&reset_gpio_lookup_mutex); + + return 0; + +err_put: + of_node_put(rgpio_dev->of_args.np); +err_kfree: + kfree(rgpio_dev); +err_ida_free: + ida_free(&reset_gpio_ida, id); +err_unlock: + mutex_unlock(&reset_gpio_lookup_mutex); + + return ret; +} + +static struct reset_controller_dev *__reset_find_rcdev(const struct of_phandle_args *args, + bool gpio_fallback) +{ + struct reset_controller_dev *rcdev; + + lockdep_assert_held(&reset_list_mutex); + + list_for_each_entry(rcdev, &reset_controller_list, list) { + if (gpio_fallback) { + if (rcdev->of_args && of_phandle_args_equal(args, + rcdev->of_args)) + return rcdev; + } else { + if (args->np == rcdev->of_node) + return rcdev; + } + } + + return NULL; +} + struct reset_control * __of_reset_control_get(struct device_node *node, const char *id, int index, bool shared, bool optional, bool acquired) { + bool gpio_fallback = false; struct reset_control *rstc; - struct reset_controller_dev *r, *rcdev; + struct reset_controller_dev *rcdev; struct of_phandle_args args; int rstc_id; int ret; @@ -839,39 +1024,52 @@ __of_reset_control_get(struct device_node *node, const char *id, int index, index, &args); if (ret == -EINVAL) return ERR_PTR(ret); - if (ret) - return optional ? NULL : ERR_PTR(ret); + if (ret) { + if (!IS_ENABLED(CONFIG_RESET_GPIO)) + return optional ? NULL : ERR_PTR(ret); - mutex_lock(&reset_list_mutex); - rcdev = NULL; - list_for_each_entry(r, &reset_controller_list, list) { - if (args.np == r->of_node) { - rcdev = r; - break; + /* + * There can be only one reset-gpio for regular devices, so + * don't bother with the "reset-gpios" phandle index. + */ + ret = of_parse_phandle_with_args(node, "reset-gpios", "#gpio-cells", + 0, &args); + if (ret) + return optional ? NULL : ERR_PTR(ret); + + gpio_fallback = true; + + ret = __reset_add_reset_gpio_device(&args); + if (ret) { + rstc = ERR_PTR(ret); + goto out_put; } } + mutex_lock(&reset_list_mutex); + rcdev = __reset_find_rcdev(&args, gpio_fallback); if (!rcdev) { rstc = ERR_PTR(-EPROBE_DEFER); - goto out; + goto out_unlock; } if (WARN_ON(args.args_count != rcdev->of_reset_n_cells)) { rstc = ERR_PTR(-EINVAL); - goto out; + goto out_unlock; } rstc_id = rcdev->of_xlate(rcdev, &args); if (rstc_id < 0) { rstc = ERR_PTR(rstc_id); - goto out; + goto out_unlock; } /* reset_list_mutex also protects the rcdev's reset_control list */ rstc = __reset_control_get_internal(rcdev, rstc_id, shared, acquired); -out: +out_unlock: mutex_unlock(&reset_list_mutex); +out_put: of_node_put(args.np); return rstc; diff --git a/include/linux/reset-controller.h b/include/linux/reset-controller.h index 0fa4f60e1186..357df16ede32 100644 --- a/include/linux/reset-controller.h +++ b/include/linux/reset-controller.h @@ -60,6 +60,9 @@ struct reset_control_lookup { * @reset_control_head: head of internal list of requested reset controls * @dev: corresponding driver model device struct * @of_node: corresponding device tree node as phandle target + * @of_args: for reset-gpios controllers: corresponding phandle args with + * of_node and GPIO number complementing of_node; either this or + * of_node should be present * @of_reset_n_cells: number of cells in reset line specifiers * @of_xlate: translation function to translate from specifier as found in the * device tree to id as given to the reset control ops, defaults @@ -73,6 +76,7 @@ struct reset_controller_dev { struct list_head reset_control_head; struct device *dev; struct device_node *of_node; + const struct of_phandle_args *of_args; int of_reset_n_cells; int (*of_xlate)(struct reset_controller_dev *rcdev, const struct of_phandle_args *reset_spec); -- cgit v1.2.3 From 3f801968889459ecae1eab524b039676e6eaa319 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Wed, 14 Feb 2024 14:41:02 +0100 Subject: netfilter: move nf_reinject into nfnetlink_queue modules No need to keep this in the core, move it to the nfnetlink_queue module. nf_reroute is moved too, there were no other callers. Signed-off-by: Florian Westphal --- include/linux/netfilter.h | 1 - include/net/netfilter/nf_queue.h | 1 - net/netfilter/nf_queue.c | 106 ----------------------------- net/netfilter/nfnetlink_queue.c | 142 +++++++++++++++++++++++++++++++++++++++ net/netfilter/utils.c | 37 ---------- 5 files changed, 142 insertions(+), 145 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index 80900d910992..ffb5e0297eb5 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -370,7 +370,6 @@ __sum16 nf_checksum_partial(struct sk_buff *skb, unsigned int hook, u_int8_t protocol, unsigned short family); int nf_route(struct net *net, struct dst_entry **dst, struct flowi *fl, bool strict, unsigned short family); -int nf_reroute(struct sk_buff *skb, struct nf_queue_entry *entry); #include diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index c81021ab07aa..4aeffddb7586 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -35,7 +35,6 @@ struct nf_queue_handler { void nf_register_queue_handler(const struct nf_queue_handler *qh); void nf_unregister_queue_handler(void); -void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict); bool nf_queue_entry_get_refs(struct nf_queue_entry *entry); void nf_queue_entry_free(struct nf_queue_entry *entry); diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c index e2f334f70281..7f12e56e6e52 100644 --- a/net/netfilter/nf_queue.c +++ b/net/netfilter/nf_queue.c @@ -248,109 +248,3 @@ int nf_queue(struct sk_buff *skb, struct nf_hook_state *state, return 0; } EXPORT_SYMBOL_GPL(nf_queue); - -static unsigned int nf_iterate(struct sk_buff *skb, - struct nf_hook_state *state, - const struct nf_hook_entries *hooks, - unsigned int *index) -{ - const struct nf_hook_entry *hook; - unsigned int verdict, i = *index; - - while (i < hooks->num_hook_entries) { - hook = &hooks->hooks[i]; -repeat: - verdict = nf_hook_entry_hookfn(hook, skb, state); - if (verdict != NF_ACCEPT) { - *index = i; - if (verdict != NF_REPEAT) - return verdict; - goto repeat; - } - i++; - } - - *index = i; - return NF_ACCEPT; -} - -static struct nf_hook_entries *nf_hook_entries_head(const struct net *net, u8 pf, u8 hooknum) -{ - switch (pf) { -#ifdef CONFIG_NETFILTER_FAMILY_BRIDGE - case NFPROTO_BRIDGE: - return rcu_dereference(net->nf.hooks_bridge[hooknum]); -#endif - case NFPROTO_IPV4: - return rcu_dereference(net->nf.hooks_ipv4[hooknum]); - case NFPROTO_IPV6: - return rcu_dereference(net->nf.hooks_ipv6[hooknum]); - default: - WARN_ON_ONCE(1); - return NULL; - } - - return NULL; -} - -/* Caller must hold rcu read-side lock */ -void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) -{ - const struct nf_hook_entry *hook_entry; - const struct nf_hook_entries *hooks; - struct sk_buff *skb = entry->skb; - const struct net *net; - unsigned int i; - int err; - u8 pf; - - net = entry->state.net; - pf = entry->state.pf; - - hooks = nf_hook_entries_head(net, pf, entry->state.hook); - - i = entry->hook_index; - if (WARN_ON_ONCE(!hooks || i >= hooks->num_hook_entries)) { - kfree_skb(skb); - nf_queue_entry_free(entry); - return; - } - - hook_entry = &hooks->hooks[i]; - - /* Continue traversal iff userspace said ok... */ - if (verdict == NF_REPEAT) - verdict = nf_hook_entry_hookfn(hook_entry, skb, &entry->state); - - if (verdict == NF_ACCEPT) { - if (nf_reroute(skb, entry) < 0) - verdict = NF_DROP; - } - - if (verdict == NF_ACCEPT) { -next_hook: - ++i; - verdict = nf_iterate(skb, &entry->state, hooks, &i); - } - - switch (verdict & NF_VERDICT_MASK) { - case NF_ACCEPT: - case NF_STOP: - local_bh_disable(); - entry->state.okfn(entry->state.net, entry->state.sk, skb); - local_bh_enable(); - break; - case NF_QUEUE: - err = nf_queue(skb, &entry->state, i, verdict); - if (err == 1) - goto next_hook; - break; - case NF_STOLEN: - break; - default: - kfree_skb(skb); - } - - nf_queue_entry_free(entry); -} -EXPORT_SYMBOL(nf_reinject); diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 5cf38fc0a366..00f4bd21c59b 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -225,6 +225,148 @@ find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id) return entry; } +static unsigned int nf_iterate(struct sk_buff *skb, + struct nf_hook_state *state, + const struct nf_hook_entries *hooks, + unsigned int *index) +{ + const struct nf_hook_entry *hook; + unsigned int verdict, i = *index; + + while (i < hooks->num_hook_entries) { + hook = &hooks->hooks[i]; +repeat: + verdict = nf_hook_entry_hookfn(hook, skb, state); + if (verdict != NF_ACCEPT) { + *index = i; + if (verdict != NF_REPEAT) + return verdict; + goto repeat; + } + i++; + } + + *index = i; + return NF_ACCEPT; +} + +static struct nf_hook_entries *nf_hook_entries_head(const struct net *net, u8 pf, u8 hooknum) +{ + switch (pf) { +#ifdef CONFIG_NETFILTER_FAMILY_BRIDGE + case NFPROTO_BRIDGE: + return rcu_dereference(net->nf.hooks_bridge[hooknum]); +#endif + case NFPROTO_IPV4: + return rcu_dereference(net->nf.hooks_ipv4[hooknum]); + case NFPROTO_IPV6: + return rcu_dereference(net->nf.hooks_ipv6[hooknum]); + default: + WARN_ON_ONCE(1); + return NULL; + } + + return NULL; +} + +static int nf_ip_reroute(struct sk_buff *skb, const struct nf_queue_entry *entry) +{ +#ifdef CONFIG_INET + const struct ip_rt_info *rt_info = nf_queue_entry_reroute(entry); + + if (entry->state.hook == NF_INET_LOCAL_OUT) { + const struct iphdr *iph = ip_hdr(skb); + + if (!(iph->tos == rt_info->tos && + skb->mark == rt_info->mark && + iph->daddr == rt_info->daddr && + iph->saddr == rt_info->saddr)) + return ip_route_me_harder(entry->state.net, entry->state.sk, + skb, RTN_UNSPEC); + } +#endif + return 0; +} + +static int nf_reroute(struct sk_buff *skb, struct nf_queue_entry *entry) +{ + const struct nf_ipv6_ops *v6ops; + int ret = 0; + + switch (entry->state.pf) { + case AF_INET: + ret = nf_ip_reroute(skb, entry); + break; + case AF_INET6: + v6ops = rcu_dereference(nf_ipv6_ops); + if (v6ops) + ret = v6ops->reroute(skb, entry); + break; + } + return ret; +} + +/* caller must hold rcu read-side lock */ +static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) +{ + const struct nf_hook_entry *hook_entry; + const struct nf_hook_entries *hooks; + struct sk_buff *skb = entry->skb; + const struct net *net; + unsigned int i; + int err; + u8 pf; + + net = entry->state.net; + pf = entry->state.pf; + + hooks = nf_hook_entries_head(net, pf, entry->state.hook); + + i = entry->hook_index; + if (WARN_ON_ONCE(!hooks || i >= hooks->num_hook_entries)) { + kfree_skb_reason(skb, SKB_DROP_REASON_NETFILTER_DROP); + nf_queue_entry_free(entry); + return; + } + + hook_entry = &hooks->hooks[i]; + + /* Continue traversal iff userspace said ok... */ + if (verdict == NF_REPEAT) + verdict = nf_hook_entry_hookfn(hook_entry, skb, &entry->state); + + if (verdict == NF_ACCEPT) { + if (nf_reroute(skb, entry) < 0) + verdict = NF_DROP; + } + + if (verdict == NF_ACCEPT) { +next_hook: + ++i; + verdict = nf_iterate(skb, &entry->state, hooks, &i); + } + + switch (verdict & NF_VERDICT_MASK) { + case NF_ACCEPT: + case NF_STOP: + local_bh_disable(); + entry->state.okfn(entry->state.net, entry->state.sk, skb); + local_bh_enable(); + break; + case NF_QUEUE: + err = nf_queue(skb, &entry->state, i, verdict); + if (err == 1) + goto next_hook; + break; + case NF_STOLEN: + break; + default: + kfree_skb(skb); + } + + nf_queue_entry_free(entry); +} + static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) { const struct nf_ct_hook *ct_hook; diff --git a/net/netfilter/utils.c b/net/netfilter/utils.c index acef4155f0da..008419db815a 100644 --- a/net/netfilter/utils.c +++ b/net/netfilter/utils.c @@ -179,43 +179,6 @@ int nf_route(struct net *net, struct dst_entry **dst, struct flowi *fl, } EXPORT_SYMBOL_GPL(nf_route); -static int nf_ip_reroute(struct sk_buff *skb, const struct nf_queue_entry *entry) -{ -#ifdef CONFIG_INET - const struct ip_rt_info *rt_info = nf_queue_entry_reroute(entry); - - if (entry->state.hook == NF_INET_LOCAL_OUT) { - const struct iphdr *iph = ip_hdr(skb); - - if (!(iph->tos == rt_info->tos && - skb->mark == rt_info->mark && - iph->daddr == rt_info->daddr && - iph->saddr == rt_info->saddr)) - return ip_route_me_harder(entry->state.net, entry->state.sk, - skb, RTN_UNSPEC); - } -#endif - return 0; -} - -int nf_reroute(struct sk_buff *skb, struct nf_queue_entry *entry) -{ - const struct nf_ipv6_ops *v6ops; - int ret = 0; - - switch (entry->state.pf) { - case AF_INET: - ret = nf_ip_reroute(skb, entry); - break; - case AF_INET6: - v6ops = rcu_dereference(nf_ipv6_ops); - if (v6ops) - ret = v6ops->reroute(skb, entry); - break; - } - return ret; -} - /* Only get and check the lengths, not do any hop-by-hop stuff. */ int nf_ip6_check_hbh_len(struct sk_buff *skb, u32 *plen) { -- cgit v1.2.3 From 78b88ef392c1ccc189c74cf73c179cf59d23a258 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Mon, 19 Feb 2024 17:45:48 +0000 Subject: net: wan: framer: remove children from struct framer_ops kdoc Remove documentation of non-existent children field from the Kernel doc for struct framer_ops. Introduced by 82c944d05b1a ("net: wan: Add framer framework support") Signed-off-by: Simon Horman Acked-by: Herve Codina Signed-off-by: David S. Miller --- include/linux/framer/framer-provider.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/framer/framer-provider.h b/include/linux/framer/framer-provider.h index f6fd2dd92591..9724d4b44b9c 100644 --- a/include/linux/framer/framer-provider.h +++ b/include/linux/framer/framer-provider.h @@ -83,7 +83,6 @@ struct framer_ops { /** * struct framer_provider - represents the framer provider * @dev: framer provider device - * @children: can be used to override the default (dev->of_node) child node * @owner: the module owner having of_xlate * @list: to maintain a linked list of framer providers * @of_xlate: function pointer to obtain framer instance from framer pointer -- cgit v1.2.3 From ee975351cf0c2a11cdf97eae58265c126cb32850 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Mon, 19 Feb 2024 12:40:51 -0800 Subject: net: mdio: mdio-bcm-unimac: Manage clock around I/O accesses Up until now we have managed not to have the mdio-bcm-unimac manage its clock except during probe and suspend/resume. This works most of the time, except where it does not. With a fully modular build, we can get into a situation whereby the GENET driver is fully registered, and so is the mdio-bcm-unimac driver, however the Ethernet PHY driver is not yet, because it depends on a resource that is not yet available (e.g.: GPIO provider). In that state, the network device is not usable yet, and so to conserve power, the GENET driver will have turned off its "main" clock which feeds its MDIO controller. When the PHY driver finally probes however, we make an access to the PHY registers to e.g.: disable interrupts, and this causes a bus error within the MDIO controller space because the MDIO controller clock(s) are turned off. To remedy that, we manage the clock around all of the I/O accesses to the hardware which are done exclusively during read, write and clock divider configuration. This ensures that the register space is accessible, and this also ensures that there are not unnecessarily elevated reference counts keeping the clocks active when the network device is administratively turned off. It would be the case with the previous way of managing the clock. Reviewed-by: Jacob Keller Signed-off-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/mdio/mdio-bcm-unimac.c | 93 ++++++++++++++------------- include/linux/platform_data/mdio-bcm-unimac.h | 3 + 2 files changed, 53 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/mdio/mdio-bcm-unimac.c b/drivers/net/mdio/mdio-bcm-unimac.c index 68f8ee0ec8ba..6fe08427fdd4 100644 --- a/drivers/net/mdio/mdio-bcm-unimac.c +++ b/drivers/net/mdio/mdio-bcm-unimac.c @@ -94,6 +94,10 @@ static int unimac_mdio_read(struct mii_bus *bus, int phy_id, int reg) int ret; u32 cmd; + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + /* Prepare the read operation */ cmd = MDIO_RD | (phy_id << MDIO_PMD_SHIFT) | (reg << MDIO_REG_SHIFT); unimac_mdio_writel(priv, cmd, MDIO_CMD); @@ -103,7 +107,7 @@ static int unimac_mdio_read(struct mii_bus *bus, int phy_id, int reg) ret = priv->wait_func(priv->wait_func_data); if (ret) - return ret; + goto out; cmd = unimac_mdio_readl(priv, MDIO_CMD); @@ -112,10 +116,15 @@ static int unimac_mdio_read(struct mii_bus *bus, int phy_id, int reg) * that condition here and ignore the MDIO controller read failure * indication. */ - if (!(bus->phy_ignore_ta_mask & 1 << phy_id) && (cmd & MDIO_READ_FAIL)) - return -EIO; + if (!(bus->phy_ignore_ta_mask & 1 << phy_id) && (cmd & MDIO_READ_FAIL)) { + ret = -EIO; + goto out; + } - return cmd & 0xffff; + ret = cmd & 0xffff; +out: + clk_disable_unprepare(priv->clk); + return ret; } static int unimac_mdio_write(struct mii_bus *bus, int phy_id, @@ -123,6 +132,11 @@ static int unimac_mdio_write(struct mii_bus *bus, int phy_id, { struct unimac_mdio_priv *priv = bus->priv; u32 cmd; + int ret; + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; /* Prepare the write operation */ cmd = MDIO_WR | (phy_id << MDIO_PMD_SHIFT) | @@ -131,7 +145,10 @@ static int unimac_mdio_write(struct mii_bus *bus, int phy_id, unimac_mdio_start(priv); - return priv->wait_func(priv->wait_func_data); + ret = priv->wait_func(priv->wait_func_data); + clk_disable_unprepare(priv->clk); + + return ret; } /* Workaround for integrated BCM7xxx Gigabit PHYs which have a problem with @@ -178,14 +195,19 @@ static int unimac_mdio_reset(struct mii_bus *bus) return 0; } -static void unimac_mdio_clk_set(struct unimac_mdio_priv *priv) +static int unimac_mdio_clk_set(struct unimac_mdio_priv *priv) { unsigned long rate; u32 reg, div; + int ret; /* Keep the hardware default values */ if (!priv->clk_freq) - return; + return 0; + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; if (!priv->clk) rate = 250000000; @@ -195,7 +217,8 @@ static void unimac_mdio_clk_set(struct unimac_mdio_priv *priv) div = (rate / (2 * priv->clk_freq)) - 1; if (div & ~MDIO_CLK_DIV_MASK) { pr_warn("Incorrect MDIO clock frequency, ignoring\n"); - return; + ret = 0; + goto out; } /* The MDIO clock is the reference clock (typically 250Mhz) divided by @@ -205,6 +228,9 @@ static void unimac_mdio_clk_set(struct unimac_mdio_priv *priv) reg &= ~(MDIO_CLK_DIV_MASK << MDIO_CLK_DIV_SHIFT); reg |= div << MDIO_CLK_DIV_SHIFT; unimac_mdio_writel(priv, reg, MDIO_CFG); +out: + clk_disable_unprepare(priv->clk); + return ret; } static int unimac_mdio_probe(struct platform_device *pdev) @@ -235,24 +261,12 @@ static int unimac_mdio_probe(struct platform_device *pdev) return -ENOMEM; } - priv->clk = devm_clk_get_optional(&pdev->dev, NULL); - if (IS_ERR(priv->clk)) - return PTR_ERR(priv->clk); - - ret = clk_prepare_enable(priv->clk); - if (ret) - return ret; - if (of_property_read_u32(np, "clock-frequency", &priv->clk_freq)) priv->clk_freq = 0; - unimac_mdio_clk_set(priv); - priv->mii_bus = mdiobus_alloc(); - if (!priv->mii_bus) { - ret = -ENOMEM; - goto out_clk_disable; - } + if (!priv->mii_bus) + return -ENOMEM; bus = priv->mii_bus; bus->priv = priv; @@ -261,17 +275,29 @@ static int unimac_mdio_probe(struct platform_device *pdev) priv->wait_func = pdata->wait_func; priv->wait_func_data = pdata->wait_func_data; bus->phy_mask = ~pdata->phy_mask; + priv->clk = pdata->clk; } else { bus->name = "unimac MII bus"; priv->wait_func_data = priv; priv->wait_func = unimac_mdio_poll; + priv->clk = devm_clk_get_optional(&pdev->dev, NULL); + } + + if (IS_ERR(priv->clk)) { + ret = PTR_ERR(priv->clk); + goto out_mdio_free; } + bus->parent = &pdev->dev; bus->read = unimac_mdio_read; bus->write = unimac_mdio_write; bus->reset = unimac_mdio_reset; snprintf(bus->id, MII_BUS_ID_SIZE, "%s-%d", pdev->name, pdev->id); + ret = unimac_mdio_clk_set(priv); + if (ret) + goto out_mdio_free; + ret = of_mdiobus_register(bus, np); if (ret) { dev_err(&pdev->dev, "MDIO bus registration failed\n"); @@ -286,8 +312,6 @@ static int unimac_mdio_probe(struct platform_device *pdev) out_mdio_free: mdiobus_free(bus); -out_clk_disable: - clk_disable_unprepare(priv->clk); return ret; } @@ -297,34 +321,17 @@ static void unimac_mdio_remove(struct platform_device *pdev) mdiobus_unregister(priv->mii_bus); mdiobus_free(priv->mii_bus); - clk_disable_unprepare(priv->clk); -} - -static int __maybe_unused unimac_mdio_suspend(struct device *d) -{ - struct unimac_mdio_priv *priv = dev_get_drvdata(d); - - clk_disable_unprepare(priv->clk); - - return 0; } static int __maybe_unused unimac_mdio_resume(struct device *d) { struct unimac_mdio_priv *priv = dev_get_drvdata(d); - int ret; - ret = clk_prepare_enable(priv->clk); - if (ret) - return ret; - - unimac_mdio_clk_set(priv); - - return 0; + return unimac_mdio_clk_set(priv); } static SIMPLE_DEV_PM_OPS(unimac_mdio_pm_ops, - unimac_mdio_suspend, unimac_mdio_resume); + NULL, unimac_mdio_resume); static const struct of_device_id unimac_mdio_ids[] = { { .compatible = "brcm,asp-v2.1-mdio", }, diff --git a/include/linux/platform_data/mdio-bcm-unimac.h b/include/linux/platform_data/mdio-bcm-unimac.h index 8a5f9f0b2c52..724e1f57b81f 100644 --- a/include/linux/platform_data/mdio-bcm-unimac.h +++ b/include/linux/platform_data/mdio-bcm-unimac.h @@ -1,11 +1,14 @@ #ifndef __MDIO_BCM_UNIMAC_PDATA_H #define __MDIO_BCM_UNIMAC_PDATA_H +struct clk; + struct unimac_mdio_pdata { u32 phy_mask; int (*wait_func)(void *data); void *wait_func_data; const char *bus_name; + struct clk *clk; }; #define UNIMAC_MDIO_DRV_NAME "unimac-mdio" -- cgit v1.2.3 From f79ab5d2bced9bd7c0ce86d2aa5b70d053001bb4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 15 Feb 2024 17:41:36 +0200 Subject: wifi: cfg80211: Add KHZ_PER_GHZ to units.h and reuse The KHZ_PER_GHZ might be used by others (with the name aligned with similar constants). Define it in units.h and convert wireless to use it. Signed-off-by: Andy Shevchenko Reviewed-by: Simon Horman Link: https://msgid.link/20240215154136.630029-1-andriy.shevchenko@linux.intel.com Signed-off-by: Johannes Berg --- include/linux/units.h | 5 ++++- net/wireless/reg.c | 7 +++---- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/units.h b/include/linux/units.h index 45110daaf8d3..00e15de33eca 100644 --- a/include/linux/units.h +++ b/include/linux/units.h @@ -24,10 +24,13 @@ #define NANOHZ_PER_HZ 1000000000UL #define MICROHZ_PER_HZ 1000000UL #define MILLIHZ_PER_HZ 1000UL + #define HZ_PER_KHZ 1000UL -#define KHZ_PER_MHZ 1000UL #define HZ_PER_MHZ 1000000UL +#define KHZ_PER_MHZ 1000UL +#define KHZ_PER_GHZ 1000000UL + #define MILLIWATT_PER_WATT 1000UL #define MICROWATT_PER_MILLIWATT 1000UL #define MICROWATT_PER_WATT 1000000UL diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 50cadbad485f..753f8e9aa4b1 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -57,6 +57,8 @@ #include #include #include +#include + #include #include "core.h" #include "reg.h" @@ -1289,20 +1291,17 @@ static bool is_valid_rd(const struct ieee80211_regdomain *rd) static bool freq_in_rule_band(const struct ieee80211_freq_range *freq_range, u32 freq_khz) { -#define ONE_GHZ_IN_KHZ 1000000 /* * From 802.11ad: directional multi-gigabit (DMG): * Pertaining to operation in a frequency band containing a channel * with the Channel starting frequency above 45 GHz. */ - u32 limit = freq_khz > 45 * ONE_GHZ_IN_KHZ ? - 20 * ONE_GHZ_IN_KHZ : 2 * ONE_GHZ_IN_KHZ; + u32 limit = freq_khz > 45 * KHZ_PER_GHZ ? 20 * KHZ_PER_GHZ : 2 * KHZ_PER_GHZ; if (abs(freq_khz - freq_range->start_freq_khz) <= limit) return true; if (abs(freq_khz - freq_range->end_freq_khz) <= limit) return true; return false; -#undef ONE_GHZ_IN_KHZ } /* -- cgit v1.2.3 From 6bd14aee0bd25525ab229acd9bfe536dd8642364 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 16 Feb 2024 13:54:31 +0200 Subject: wifi: mac80211: align ieee80211_mle_get_bss_param_ch_cnt() Align the prototype of ieee80211_mle_get_bss_param_ch_cnt() to also take a u8 * like the other functions, and make it return -1 when the field isn't found, so that mac80211 can check that instead of explicitly open-coding the check. Signed-off-by: Johannes Berg Signed-off-by: Miri Korenblit Link: https://msgid.link/20240216135047.583309181bc3.Ia61cb0b4fc034d5ac8fcfaf6f6fb2e115fadafe7@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 11 ++++++----- net/mac80211/mlme.c | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index e4322238f273..303c75459897 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4990,17 +4990,18 @@ static inline int ieee80211_mle_get_link_id(const u8 *data) /** * ieee80211_mle_get_bss_param_ch_cnt - returns the BSS parameter change count - * @mle: the basic multi link element + * @data: pointer to the basic multi link element * * The element is assumed to be of the correct type (BASIC) and big enough, * this must be checked using ieee80211_mle_type_ok(). * * If the BSS parameter change count value can't be found (the presence bit - * for it is clear), 0 will be returned. + * for it is clear), -1 will be returned. */ -static inline u8 -ieee80211_mle_get_bss_param_ch_cnt(const struct ieee80211_multi_link_elem *mle) +static inline int +ieee80211_mle_get_bss_param_ch_cnt(const u8 *data) { + const struct ieee80211_multi_link_elem *mle = (const void *)data; u16 control = le16_to_cpu(mle->control); const u8 *common = mle->variable; @@ -5008,7 +5009,7 @@ ieee80211_mle_get_bss_param_ch_cnt(const struct ieee80211_multi_link_elem *mle) common += sizeof(struct ieee80211_mle_basic_common_info); if (!(control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)) - return 0; + return -1; if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) common += 1; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 87ffc19770b8..5f2e9f5e1779 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -4202,13 +4202,14 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, */ assoc_data->link[link_id].status = WLAN_STATUS_SUCCESS; if (elems->ml_basic) { - if (!(elems->ml_basic->control & - cpu_to_le16(IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT))) { + int bss_param_ch_cnt = + ieee80211_mle_get_bss_param_ch_cnt((const void *)elems->ml_basic); + + if (bss_param_ch_cnt < 0) { ret = false; goto out; } - link->u.mgd.bss_param_ch_cnt = - ieee80211_mle_get_bss_param_ch_cnt(elems->ml_basic); + link->u.mgd.bss_param_ch_cnt = bss_param_ch_cnt; } } else if (elems->parse_error & IEEE80211_PARSE_ERR_DUP_NEST_ML_BASIC || !elems->prof || -- cgit v1.2.3 From 894dd84e49ec114a2dde7b312ae4cada40d15bdb Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 16 Feb 2024 13:54:32 +0200 Subject: wifi: cfg80211: use ML element parsing helpers Use the existing ML element parsing helpers and add a new one for this (ieee80211_mle_get_mld_id). Signed-off-by: Johannes Berg Signed-off-by: Miri Korenblit Link: https://msgid.link/20240216135047.4da47b1f035b.I437a5570ac456449facb0b147851ef24a1e473c2@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 38 ++++++++++++++++++++++++++++++++++++++ net/wireless/scan.c | 41 ++++++++--------------------------------- 2 files changed, 46 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 303c75459897..3385a2cc5b09 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -5115,6 +5115,44 @@ static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data) return get_unaligned_le16(common); } +/** + * ieee80211_mle_get_mld_id - returns the MLD ID + * @data: pointer to the multi link element + * + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + * + * If the MLD ID is not present, 0 will be returned. + */ +static inline u8 ieee80211_mle_get_mld_id(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + /* + * common points now at the beginning of + * ieee80211_mle_basic_common_info + */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_MLD_ID)) + return 0; + + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP) + common += 2; + + return *common; +} + /** * ieee80211_mle_size_ok - validate multi-link element size * @data: pointer to the element data diff --git a/net/wireless/scan.c b/net/wireless/scan.c index e46dfc71c497..7cf36b8d3ae7 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -2833,17 +2833,16 @@ cfg80211_parse_ml_elem_sta_data(struct wiphy *wiphy, struct cfg80211_bss *bss; u8 mld_id, reporter_link_id, bss_change_count; u16 seen_links = 0; - const u8 *pos; u8 i; - if (!ieee80211_mle_size_ok(elem->data + 1, elem->datalen - 1)) + if (!ieee80211_mle_type_ok(elem->data + 1, + IEEE80211_ML_CONTROL_TYPE_BASIC, + elem->datalen - 1)) return; - ml_elem = (void *)elem->data + 1; + ml_elem = (void *)(elem->data + 1); control = le16_to_cpu(ml_elem->control); - if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) != - IEEE80211_ML_CONTROL_TYPE_BASIC) - return; + ml_common_len = ml_elem->variable[0]; /* Must be present when transmitted by an AP (in a probe response) */ if (!(control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) || @@ -2851,24 +2850,8 @@ cfg80211_parse_ml_elem_sta_data(struct wiphy *wiphy, !(control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP)) return; - ml_common_len = ml_elem->variable[0]; - - /* length + MLD MAC address */ - pos = ml_elem->variable + 1 + 6; - - reporter_link_id = pos[0]; - pos += 1; - - bss_change_count = pos[0]; - pos += 1; - - if (u16_get_bits(control, IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)) - pos += 2; - if (u16_get_bits(control, IEEE80211_MLC_BASIC_PRES_EML_CAPA)) - pos += 2; - - /* MLD capabilities and operations */ - pos += 2; + reporter_link_id = ieee80211_mle_get_link_id(elem->data + 1); + bss_change_count = ieee80211_mle_get_bss_param_ch_cnt(elem->data + 1); /* * The MLD ID of the reporting AP is always zero. It is set if the AP @@ -2876,15 +2859,7 @@ cfg80211_parse_ml_elem_sta_data(struct wiphy *wiphy, * relating to a nontransmitted BSS (matching the Multi-BSSID Index, * Draft P802.11be_D3.2, 35.3.4.2) */ - if (u16_get_bits(control, IEEE80211_MLC_BASIC_PRES_MLD_ID)) { - mld_id = *pos; - pos += 1; - } else { - mld_id = 0; - } - - /* Extended MLD capabilities and operations */ - pos += 2; + mld_id = ieee80211_mle_get_mld_id(elem->data + 1); /* Fully defrag the ML element for sta information/profile iteration */ mle = cfg80211_defrag_mle(elem, tx_data->ie, tx_data->ielen, gfp); -- cgit v1.2.3 From a3c70a3cf11eb4b6409afc2cce1a3747e1dfe96f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 20 Feb 2024 15:50:01 -0800 Subject: bpf: Shrink size of struct bpf_map/bpf_array. Back in 2018 the commit be95a845cc44 ("bpf: avoid false sharing of map refcount with max_entries") added ____cacheline_aligned to "struct bpf_map" to make sure that fields like refcnt don't share a cache line with max_entries that is used to bounds check map access. That was done to make spectre style attacks harder. The main mitigation is done via code similar to array_index_nospec(), of course. This was an additional precaution. It increased the size of "struct bpf_map" a little, but it's affect on all other maps (like array) is significant, since "struct bpf_map" is typically the first member in other map types. Undo this ____cacheline_aligned tag. Instead move freeze_mutex field around, so that refcnt and max_entries are still in different cache lines. The main effect is seen in sizeof(struct bpf_array) that reduces from 320 to 248 bytes. BEFORE: struct bpf_map { const struct bpf_map_ops * ops; /* 0 8 */ ... char name[16]; /* 96 16 */ /* XXX 16 bytes hole, try to pack */ /* --- cacheline 2 boundary (128 bytes) --- */ atomic64_t refcnt __attribute__((__aligned__(64))); /* 128 8 */ ... /* size: 256, cachelines: 4, members: 30 */ /* sum members: 232, holes: 1, sum holes: 16 */ /* padding: 8 */ /* paddings: 1, sum paddings: 2 */ } __attribute__((__aligned__(64))); struct bpf_array { struct bpf_map map; /* 0 256 */ ... /* size: 320, cachelines: 5, members: 5 */ /* padding: 48 */ /* paddings: 1, sum paddings: 8 */ } __attribute__((__aligned__(64))); AFTER: struct bpf_map { /* size: 232, cachelines: 4, members: 30 */ /* paddings: 1, sum paddings: 2 */ /* last cacheline: 40 bytes */ }; struct bpf_array { /* size: 248, cachelines: 4, members: 5 */ /* last cacheline: 56 bytes */ }; Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20240220235001.57411-1-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c7aa99b44dbd..814dc913a968 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -251,10 +251,7 @@ struct bpf_list_node_kern { } __attribute__((aligned(8))); struct bpf_map { - /* The first two cachelines with read-mostly members of which some - * are also accessed in fast-path (e.g. ops, max_entries). - */ - const struct bpf_map_ops *ops ____cacheline_aligned; + const struct bpf_map_ops *ops; struct bpf_map *inner_map_meta; #ifdef CONFIG_SECURITY void *security; @@ -276,17 +273,14 @@ struct bpf_map { struct obj_cgroup *objcg; #endif char name[BPF_OBJ_NAME_LEN]; - /* The 3rd and 4th cacheline with misc members to avoid false sharing - * particularly with refcounting. - */ - atomic64_t refcnt ____cacheline_aligned; + struct mutex freeze_mutex; + atomic64_t refcnt; atomic64_t usercnt; /* rcu is used before freeing and work is only used during freeing */ union { struct work_struct work; struct rcu_head rcu; }; - struct mutex freeze_mutex; atomic64_t writecnt; /* 'Ownership' of program-containing map is claimed by the first program * that is going to use this map or by the first program which FD is -- cgit v1.2.3 From ac0c530619cefa68fba816dabbcf6f4ffbf60c3d Mon Sep 17 00:00:00 2001 From: Dimitri Fedrau Date: Sun, 18 Feb 2024 08:57:39 +0100 Subject: net: phy: Support 100/1000BT1 linkmode advertisements Extend helper functions mii_t1_adv_m_mod_linkmode_t and linkmode_adv_to_mii_t1_adv_m_t to support 100BT1 and 1000BT1 linkmode advertisements. Reviewed-by: Andrew Lunn Signed-off-by: Dimitri Fedrau Link: https://lore.kernel.org/r/20240218075753.18067-3-dima.fedrau@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/mdio.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mdio.h b/include/linux/mdio.h index fd8ff310f9eb..68f8d2e970d4 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -373,6 +373,10 @@ static inline void mii_t1_adv_m_mod_linkmode_t(unsigned long *advertising, u32 l { linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT1L_Full_BIT, advertising, lpa & MDIO_AN_T1_ADV_M_B10L); + linkmode_mod_bit(ETHTOOL_LINK_MODE_100baseT1_Full_BIT, + advertising, lpa & MDIO_AN_T1_ADV_M_100BT1); + linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT1_Full_BIT, + advertising, lpa & MDIO_AN_T1_ADV_M_1000BT1); } /** @@ -409,6 +413,10 @@ static inline u32 linkmode_adv_to_mii_t1_adv_m_t(unsigned long *advertising) if (linkmode_test_bit(ETHTOOL_LINK_MODE_10baseT1L_Full_BIT, advertising)) result |= MDIO_AN_T1_ADV_M_B10L; + if (linkmode_test_bit(ETHTOOL_LINK_MODE_100baseT1_Full_BIT, advertising)) + result |= MDIO_AN_T1_ADV_M_100BT1; + if (linkmode_test_bit(ETHTOOL_LINK_MODE_1000baseT1_Full_BIT, advertising)) + result |= MDIO_AN_T1_ADV_M_1000BT1; return result; } -- cgit v1.2.3 From e57e4c7f6cc943be3346f938361334bb3634db3d Mon Sep 17 00:00:00 2001 From: Dimitri Fedrau Date: Sun, 18 Feb 2024 08:57:42 +0100 Subject: net: phy: marvell-88q2xxx: add driver for the Marvell 88Q2220 PHY Add a driver for the Marvell 88Q2220. This driver allows to detect the link, switch between 100BASE-T1 and 1000BASE-T1 and switch between master and slave mode. Autonegotiation is supported. Reviewed-by: Andrew Lunn Tested-by: Gregor Herburger Signed-off-by: Dimitri Fedrau Link: https://lore.kernel.org/r/20240218075753.18067-6-dima.fedrau@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/phy/marvell-88q2xxx.c | 210 ++++++++++++++++++++++++++++++++++++-- include/linux/marvell_phy.h | 1 + 2 files changed, 205 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c index dcebb4643aff..9829facde253 100644 --- a/drivers/net/phy/marvell-88q2xxx.c +++ b/drivers/net/phy/marvell-88q2xxx.c @@ -1,11 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 /* * Marvell 88Q2XXX automotive 100BASE-T1/1000BASE-T1 PHY driver + * + * Derived from Marvell Q222x API + * + * Copyright (C) 2024 Liebherr-Electronics and Drives GmbH */ #include #include #include +#define PHY_ID_88Q2220_REVB0 (MARVELL_PHY_ID_88Q2220 | 0x1) + #define MDIO_MMD_AN_MV_STAT 32769 #define MDIO_MMD_AN_MV_STAT_ANEG 0x0100 #define MDIO_MMD_AN_MV_STAT_LOCAL_RX 0x1000 @@ -13,6 +19,11 @@ #define MDIO_MMD_AN_MV_STAT_LOCAL_MASTER 0x4000 #define MDIO_MMD_AN_MV_STAT_MS_CONF_FAULT 0x8000 +#define MDIO_MMD_AN_MV_STAT2 32794 +#define MDIO_MMD_AN_MV_STAT2_AN_RESOLVED 0x0800 +#define MDIO_MMD_AN_MV_STAT2_100BT1 0x2000 +#define MDIO_MMD_AN_MV_STAT2_1000BT1 0x4000 + #define MDIO_MMD_PCS_MV_100BT1_STAT1 33032 #define MDIO_MMD_PCS_MV_100BT1_STAT1_IDLE_ERROR 0x00ff #define MDIO_MMD_PCS_MV_100BT1_STAT1_JABBER 0x0100 @@ -29,6 +40,42 @@ #define MDIO_MMD_PCS_MV_RX_STAT 33328 +struct mmd_val { + int devad; + u32 regnum; + u16 val; +}; + +static const struct mmd_val mv88q222x_revb0_init_seq0[] = { + { MDIO_MMD_PCS, 0x8033, 0x6801 }, + { MDIO_MMD_AN, MDIO_AN_T1_CTRL, 0x0 }, + { MDIO_MMD_PMAPMD, MDIO_CTRL1, + MDIO_CTRL1_LPOWER | MDIO_PMA_CTRL1_SPEED1000 }, + { MDIO_MMD_PCS, 0xfe1b, 0x48 }, + { MDIO_MMD_PCS, 0xffe4, 0x6b6 }, + { MDIO_MMD_PMAPMD, MDIO_CTRL1, 0x0 }, + { MDIO_MMD_PCS, MDIO_CTRL1, 0x0 }, +}; + +static const struct mmd_val mv88q222x_revb0_init_seq1[] = { + { MDIO_MMD_PCS, 0xfe79, 0x0 }, + { MDIO_MMD_PCS, 0xfe07, 0x125a }, + { MDIO_MMD_PCS, 0xfe09, 0x1288 }, + { MDIO_MMD_PCS, 0xfe08, 0x2588 }, + { MDIO_MMD_PCS, 0xfe11, 0x1105 }, + { MDIO_MMD_PCS, 0xfe72, 0x042c }, + { MDIO_MMD_PCS, 0xfbba, 0xcb2 }, + { MDIO_MMD_PCS, 0xfbbb, 0xc4a }, + { MDIO_MMD_AN, 0x8032, 0x2020 }, + { MDIO_MMD_AN, 0x8031, 0xa28 }, + { MDIO_MMD_AN, 0x8031, 0xc28 }, + { MDIO_MMD_PCS, 0xffdb, 0xfc10 }, + { MDIO_MMD_PCS, 0xfe1b, 0x58 }, + { MDIO_MMD_PCS, 0xfe79, 0x4 }, + { MDIO_MMD_PCS, 0xfe5f, 0xe8 }, + { MDIO_MMD_PCS, 0xfe05, 0x755c }, +}; + static int mv88q2xxx_soft_reset(struct phy_device *phydev) { int ret; @@ -125,24 +172,90 @@ out: static int mv88q2xxx_read_link(struct phy_device *phydev) { - int ret; - /* The 88Q2XXX PHYs do not have the PMA/PMD status register available, * therefore we need to read the link status from the vendor specific * registers depending on the speed. */ + if (phydev->speed == SPEED_1000) - ret = mv88q2xxx_read_link_gbit(phydev); + return mv88q2xxx_read_link_gbit(phydev); + else if (phydev->speed == SPEED_100) + return mv88q2xxx_read_link_100m(phydev); + + phydev->link = false; + return 0; +} + +static int mv88q2xxx_read_master_slave_state(struct phy_device *phydev) +{ + int ret; + + phydev->master_slave_state = MASTER_SLAVE_STATE_UNKNOWN; + ret = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_MMD_AN_MV_STAT); + if (ret < 0) + return ret; + + if (ret & MDIO_MMD_AN_MV_STAT_LOCAL_MASTER) + phydev->master_slave_state = MASTER_SLAVE_STATE_MASTER; else - ret = mv88q2xxx_read_link_100m(phydev); + phydev->master_slave_state = MASTER_SLAVE_STATE_SLAVE; - return ret; + return 0; +} + +static int mv88q2xxx_read_aneg_speed(struct phy_device *phydev) +{ + int ret; + + phydev->speed = SPEED_UNKNOWN; + ret = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_MMD_AN_MV_STAT2); + if (ret < 0) + return ret; + + if (!(ret & MDIO_MMD_AN_MV_STAT2_AN_RESOLVED)) + return 0; + + if (ret & MDIO_MMD_AN_MV_STAT2_100BT1) + phydev->speed = SPEED_100; + else if (ret & MDIO_MMD_AN_MV_STAT2_1000BT1) + phydev->speed = SPEED_1000; + + return 0; } static int mv88q2xxx_read_status(struct phy_device *phydev) { int ret; + if (phydev->autoneg == AUTONEG_ENABLE) { + /* We have to get the negotiated speed first, otherwise we are + * not able to read the link. + */ + ret = mv88q2xxx_read_aneg_speed(phydev); + if (ret < 0) + return ret; + + ret = mv88q2xxx_read_link(phydev); + if (ret < 0) + return ret; + + ret = genphy_c45_read_lpa(phydev); + if (ret < 0) + return ret; + + ret = genphy_c45_baset1_read_status(phydev); + if (ret < 0) + return ret; + + ret = mv88q2xxx_read_master_slave_state(phydev); + if (ret < 0) + return ret; + + phy_resolve_aneg_linkmode(phydev); + + return 0; + } + ret = mv88q2xxx_read_link(phydev); if (ret < 0) return ret; @@ -171,7 +284,9 @@ static int mv88q2xxx_get_features(struct phy_device *phydev) * sequence provided by Marvell. Disable it for now until a proper * workaround is found or a new PHY revision is released. */ - linkmode_clear_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, phydev->supported); + if (phydev->drv->phy_id == MARVELL_PHY_ID_88Q2110) + linkmode_clear_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + phydev->supported); return 0; } @@ -241,6 +356,75 @@ static int mv88q2xxx_get_sqi_max(struct phy_device *phydev) return 15; } +static int mv88q222x_soft_reset(struct phy_device *phydev) +{ + int ret; + + /* Enable RESET of DCL */ + if (phydev->autoneg == AUTONEG_ENABLE || phydev->speed == SPEED_1000) { + ret = phy_write_mmd(phydev, MDIO_MMD_PCS, 0xfe1b, 0x48); + if (ret < 0) + return ret; + } + + ret = phy_write_mmd(phydev, MDIO_MMD_PCS, MDIO_PCS_1000BT1_CTRL, + MDIO_PCS_1000BT1_CTRL_RESET); + if (ret < 0) + return ret; + + ret = phy_write_mmd(phydev, MDIO_MMD_PCS, 0xffe4, 0xc); + if (ret < 0) + return ret; + + /* Disable RESET of DCL */ + if (phydev->autoneg == AUTONEG_ENABLE || phydev->speed == SPEED_1000) + return phy_write_mmd(phydev, MDIO_MMD_PCS, 0xfe1b, 0x58); + + return 0; +} + +static int mv88q222x_config_aneg(struct phy_device *phydev) +{ + int ret; + + ret = genphy_c45_config_aneg(phydev); + if (ret) + return ret; + + return mv88q222x_soft_reset(phydev); +} + +static int mv88q222x_revb0_config_init(struct phy_device *phydev) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(mv88q222x_revb0_init_seq0); i++) { + ret = phy_write_mmd(phydev, mv88q222x_revb0_init_seq0[i].devad, + mv88q222x_revb0_init_seq0[i].regnum, + mv88q222x_revb0_init_seq0[i].val); + if (ret < 0) + return ret; + } + + usleep_range(5000, 10000); + + for (i = 0; i < ARRAY_SIZE(mv88q222x_revb0_init_seq1); i++) { + ret = phy_write_mmd(phydev, mv88q222x_revb0_init_seq1[i].devad, + mv88q222x_revb0_init_seq1[i].regnum, + mv88q222x_revb0_init_seq1[i].val); + if (ret < 0) + return ret; + } + + /* The 88Q2XXX PHYs do have the extended ability register available, but + * register MDIO_PMA_EXTABLE where they should signalize it does not + * work according to specification. Therefore, we force it here. + */ + phydev->pma_extable = MDIO_PMA_EXTABLE_BT1; + + return 0; +} + static struct phy_driver mv88q2xxx_driver[] = { { .phy_id = MARVELL_PHY_ID_88Q2110, @@ -255,12 +439,26 @@ static struct phy_driver mv88q2xxx_driver[] = { .get_sqi = mv88q2xxx_get_sqi, .get_sqi_max = mv88q2xxx_get_sqi_max, }, + { + PHY_ID_MATCH_EXACT(PHY_ID_88Q2220_REVB0), + .name = "mv88q2220", + .get_features = mv88q2xxx_get_features, + .config_aneg = mv88q222x_config_aneg, + .aneg_done = genphy_c45_aneg_done, + .config_init = mv88q222x_revb0_config_init, + .read_status = mv88q2xxx_read_status, + .soft_reset = mv88q222x_soft_reset, + .set_loopback = genphy_c45_loopback, + .get_sqi = mv88q2xxx_get_sqi, + .get_sqi_max = mv88q2xxx_get_sqi_max, + }, }; module_phy_driver(mv88q2xxx_driver); static struct mdio_device_id __maybe_unused mv88q2xxx_tbl[] = { { MARVELL_PHY_ID_88Q2110, MARVELL_PHY_ID_MASK }, + { PHY_ID_MATCH_EXACT(PHY_ID_88Q2220_REVB0), }, { /*sentinel*/ } }; MODULE_DEVICE_TABLE(mdio, mv88q2xxx_tbl); diff --git a/include/linux/marvell_phy.h b/include/linux/marvell_phy.h index 9b54c4f0677f..693eba9869e4 100644 --- a/include/linux/marvell_phy.h +++ b/include/linux/marvell_phy.h @@ -26,6 +26,7 @@ #define MARVELL_PHY_ID_88E2110 0x002b09b0 #define MARVELL_PHY_ID_88X2222 0x01410f10 #define MARVELL_PHY_ID_88Q2110 0x002b0980 +#define MARVELL_PHY_ID_88Q2220 0x002b0b20 /* Marvel 88E1111 in Finisar SFP module with modified PHY ID */ #define MARVELL_PHY_ID_88E1111_FINISAR 0x01ff0cc0 -- cgit v1.2.3 From c5f1e2d1890935a734c302b9b8579748222b8e1e Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Mon, 8 Jan 2024 14:27:43 +0100 Subject: mm/memory_hotplug: introduce MEM_PREPARE_ONLINE/MEM_FINISH_OFFLINE notifiers Patch series "implement "memmap on memory" feature on s390". This series provides "memmap on memory" support on s390 platform. "memmap on memory" allows struct pages array to be allocated from the hotplugged memory range instead of allocating it from main system memory. s390 currently preallocates struct pages array for all potentially possible memory, which ensures memory onlining always succeeds, but with the cost of significant memory consumption from the available system memory during boottime. In certain extreme configuration, this could lead to ipl failure. "memmap on memory" ensures struct pages array are populated from self contained hotplugged memory range instead of depleting the available system memory and this could eliminate ipl failure on s390 platform. On other platforms, system might go OOM when the physically hotplugged memory depletes the available memory before it is onlined. Hence, "memmap on memory" feature was introduced as described in commit a08a2ae34613 ("mm,memory_hotplug: allocate memmap from the added memory range"). Unlike other architectures, s390 memory blocks are not physically accessible until it is online. To make it physically accessible two new memory notifiers MEM_PREPARE_ONLINE / MEM_FINISH_OFFLINE are added and this notifier lets the hypervisor inform that the memory should be made physically accessible. This allows for "memmap on memory" initialization during memory hotplug onlining phase, which is performed before calling MEM_GOING_ONLINE notifier. Patch 1 introduces MEM_PREPARE_ONLINE/MEM_FINISH_OFFLINE memory notifiers to prepare the transition of memory to and from a physically accessible state. New mhp_flag MHP_OFFLINE_INACCESSIBLE is introduced to ensure altmap cannot be written when adding memory - before it is set online. This enhancement is crucial for implementing the "memmap on memory" feature for s390 in a subsequent patch. Patches 2 allocates vmemmap pages from self-contained memory range for s390. It allocates memory map (struct pages array) from the hotplugged memory range, rather than using system memory by passing altmap to vmemmap functions. Patch 3 removes unhandled memory notifier types on s390. Patch 4 implements MEM_PREPARE_ONLINE/MEM_FINISH_OFFLINE memory notifiers on s390. MEM_PREPARE_ONLINE memory notifier makes memory block physical accessible via sclp assign command. The notifier ensures self-contained memory maps are accessible and hence enabling the "memmap on memory" on s390. MEM_FINISH_OFFLINE memory notifier shifts the memory block to an inaccessible state via sclp unassign command. Patch 5 finally enables MHP_MEMMAP_ON_MEMORY on s390. This patch (of 5): Introduce MEM_PREPARE_ONLINE/MEM_FINISH_OFFLINE memory notifiers to prepare the transition of memory to and from a physically accessible state. This enhancement is crucial for implementing the "memmap on memory" feature for s390 in a subsequent patch. Platforms such as x86 can support physical memory hotplug via ACPI. When there is physical memory hotplug, ACPI event leads to the memory addition with the following callchain: acpi_memory_device_add() -> acpi_memory_enable_device() -> __add_memory() After this, the hotplugged memory is physically accessible, and altmap support prepared, before the "memmap on memory" initialization in memory_block_online() is called. On s390, memory hotplug works in a different way. The available hotplug memory has to be defined upfront in the hypervisor, but it is made physically accessible only when the user sets it online via sysfs, currently in the MEM_GOING_ONLINE notifier. This is too late and "memmap on memory" initialization is performed before calling MEM_GOING_ONLINE notifier. During the memory hotplug addition phase, altmap support is prepared and during the memory onlining phase s390 requires memory to be physically accessible and then subsequently initiate the "memmap on memory" initialization process. The memory provider will handle new MEM_PREPARE_ONLINE / MEM_FINISH_OFFLINE notifications and make the memory accessible. The mhp_flag MHP_OFFLINE_INACCESSIBLE is introduced and is relevant when used along with MHP_MEMMAP_ON_MEMORY, because the altmap cannot be written (e.g., poisoned) when adding memory -- before it is set online. This allows for adding memory with an altmap that is not currently made available by a hypervisor. When onlining that memory, the hypervisor can be instructed to make that memory accessible via the new notifiers and the onlining phase will not require any memory allocations, which is helpful in low-memory situations. All architectures ignore unknown memory notifiers. Therefore, the introduction of these new notifiers does not result in any functional modifications across architectures. Link: https://lkml.kernel.org/r/20240108132747.3238763-1-sumanthk@linux.ibm.com Link: https://lkml.kernel.org/r/20240108132747.3238763-2-sumanthk@linux.ibm.com Signed-off-by: Sumanth Korikkar Suggested-by: Gerald Schaefer Suggested-by: David Hildenbrand Acked-by: David Hildenbrand Cc: Alexander Gordeev Cc: Aneesh Kumar K.V Cc: Anshuman Khandual Cc: Heiko Carstens Cc: Michal Hocko Cc: Oscar Salvador Cc: Vasily Gorbik Signed-off-by: Andrew Morton --- drivers/base/memory.c | 23 ++++++++++++++++++++++- include/linux/memory.h | 9 +++++++++ include/linux/memory_hotplug.h | 18 +++++++++++++++++- include/linux/memremap.h | 1 + mm/memory_hotplug.c | 17 ++++++++++++++--- mm/sparse.c | 3 ++- 6 files changed, 65 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 14f964a7719b..c0436f46cfb7 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -188,6 +188,7 @@ static int memory_block_online(struct memory_block *mem) unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; unsigned long nr_vmemmap_pages = 0; + struct memory_notify arg; struct zone *zone; int ret; @@ -207,9 +208,19 @@ static int memory_block_online(struct memory_block *mem) if (mem->altmap) nr_vmemmap_pages = mem->altmap->free; + arg.altmap_start_pfn = start_pfn; + arg.altmap_nr_pages = nr_vmemmap_pages; + arg.start_pfn = start_pfn + nr_vmemmap_pages; + arg.nr_pages = nr_pages - nr_vmemmap_pages; mem_hotplug_begin(); + ret = memory_notify(MEM_PREPARE_ONLINE, &arg); + ret = notifier_to_errno(ret); + if (ret) + goto out_notifier; + if (nr_vmemmap_pages) { - ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); + ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, + zone, mem->altmap->inaccessible); if (ret) goto out; } @@ -231,7 +242,11 @@ static int memory_block_online(struct memory_block *mem) nr_vmemmap_pages); mem->zone = zone; + mem_hotplug_done(); + return ret; out: + memory_notify(MEM_FINISH_OFFLINE, &arg); +out_notifier: mem_hotplug_done(); return ret; } @@ -244,6 +259,7 @@ static int memory_block_offline(struct memory_block *mem) unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; unsigned long nr_vmemmap_pages = 0; + struct memory_notify arg; int ret; if (!mem->zone) @@ -275,6 +291,11 @@ static int memory_block_offline(struct memory_block *mem) mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); mem->zone = NULL; + arg.altmap_start_pfn = start_pfn; + arg.altmap_nr_pages = nr_vmemmap_pages; + arg.start_pfn = start_pfn + nr_vmemmap_pages; + arg.nr_pages = nr_pages - nr_vmemmap_pages; + memory_notify(MEM_FINISH_OFFLINE, &arg); out: mem_hotplug_done(); return ret; diff --git a/include/linux/memory.h b/include/linux/memory.h index f53cfdaaaa41..939a16bd5cea 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -96,8 +96,17 @@ int set_memory_block_size_order(unsigned int order); #define MEM_GOING_ONLINE (1<<3) #define MEM_CANCEL_ONLINE (1<<4) #define MEM_CANCEL_OFFLINE (1<<5) +#define MEM_PREPARE_ONLINE (1<<6) +#define MEM_FINISH_OFFLINE (1<<7) struct memory_notify { + /* + * The altmap_start_pfn and altmap_nr_pages fields are designated for + * specifying the altmap range and are exclusively intended for use in + * MEM_PREPARE_ONLINE/MEM_FINISH_OFFLINE notifiers. + */ + unsigned long altmap_start_pfn; + unsigned long altmap_nr_pages; unsigned long start_pfn; unsigned long nr_pages; int status_change_nid_normal; diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7d2076583494..ee00015575aa 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -106,6 +106,22 @@ typedef int __bitwise mhp_t; * implies the node id (nid). */ #define MHP_NID_IS_MGID ((__force mhp_t)BIT(2)) +/* + * The hotplugged memory is completely inaccessible while the memory is + * offline. The memory provider will handle MEM_PREPARE_ONLINE / + * MEM_FINISH_OFFLINE notifications and make the memory accessible. + * + * This flag is only relevant when used along with MHP_MEMMAP_ON_MEMORY, + * because the altmap cannot be written (e.g., poisoned) when adding + * memory -- before it is set online. + * + * This allows for adding memory with an altmap that is not currently + * made available by a hypervisor. When onlining that memory, the + * hypervisor can be instructed to make that memory available, and + * the onlining phase will not require any memory allocations, which is + * helpful in low-memory situations. + */ +#define MHP_OFFLINE_INACCESSIBLE ((__force mhp_t)BIT(3)) /* * Extended parameters for memory hotplug: @@ -154,7 +170,7 @@ extern void adjust_present_page_count(struct page *page, long nr_pages); /* VM interface that may be used by firmware interface */ extern int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, - struct zone *zone); + struct zone *zone, bool mhp_off_inaccessible); extern void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages); extern int online_pages(unsigned long pfn, unsigned long nr_pages, struct zone *zone, struct memory_group *group); diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 744c830f4b13..9837f3e6fb95 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -25,6 +25,7 @@ struct vmem_altmap { unsigned long free; unsigned long align; unsigned long alloc; + bool inaccessible; }; /* diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 21890994c1d3..707027f69150 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1087,7 +1087,7 @@ void adjust_present_page_count(struct page *page, struct memory_group *group, } int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, - struct zone *zone) + struct zone *zone, bool mhp_off_inaccessible) { unsigned long end_pfn = pfn + nr_pages; int ret, i; @@ -1096,6 +1096,15 @@ int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, if (ret) return ret; + /* + * Memory block is accessible at this stage and hence poison the struct + * pages now. If the memory block is accessible during memory hotplug + * addition phase, then page poisining is already performed in + * sparse_add_section(). + */ + if (mhp_off_inaccessible) + page_init_poison(pfn_to_page(pfn), sizeof(struct page) * nr_pages); + move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE); for (i = 0; i < nr_pages; i++) @@ -1415,7 +1424,7 @@ static void __ref remove_memory_blocks_and_altmaps(u64 start, u64 size) } static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group, - u64 start, u64 size) + u64 start, u64 size, mhp_t mhp_flags) { unsigned long memblock_size = memory_block_size_bytes(); u64 cur_start; @@ -1431,6 +1440,8 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group, }; mhp_altmap.free = memory_block_memmap_on_memory_pages(); + if (mhp_flags & MHP_OFFLINE_INACCESSIBLE) + mhp_altmap.inaccessible = true; params.altmap = kmemdup(&mhp_altmap, sizeof(struct vmem_altmap), GFP_KERNEL); if (!params.altmap) { @@ -1516,7 +1527,7 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) */ if ((mhp_flags & MHP_MEMMAP_ON_MEMORY) && mhp_supports_memmap_on_memory(memory_block_size_bytes())) { - ret = create_altmaps_and_memory_blocks(nid, group, start, size); + ret = create_altmaps_and_memory_blocks(nid, group, start, size, mhp_flags); if (ret) goto error; } else { diff --git a/mm/sparse.c b/mm/sparse.c index 338cf946dee8..aed0951b87fa 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -908,7 +908,8 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn, * Poison uninitialized struct pages in order to catch invalid flags * combinations. */ - page_init_poison(memmap, sizeof(struct page) * nr_pages); + if (!altmap || !altmap->inaccessible) + page_init_poison(memmap, sizeof(struct page) * nr_pages); ms = __nr_to_section(section_nr); set_section_nid(section_nr, nid); -- cgit v1.2.3 From a02b8bfe9a60ecb97bab7ba1a10a513bf78a7866 Mon Sep 17 00:00:00 2001 From: Haifeng Xu Date: Thu, 28 Dec 2023 06:27:15 +0000 Subject: mm: list_lru: remove unused macro list_lru_init_key() list_lru_init_key() isn't used by anyone, remove it to clean up. Link: https://lkml.kernel.org/r/20231228062715.338672-2-haifeng.xu@shopee.com Signed-off-by: Haifeng Xu Acked-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Shakeel Butt Signed-off-by: Andrew Morton --- include/linux/list_lru.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h index 7675a48a0701..c679e6b293c4 100644 --- a/include/linux/list_lru.h +++ b/include/linux/list_lru.h @@ -62,8 +62,6 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, #define list_lru_init(lru) \ __list_lru_init((lru), false, NULL, NULL) -#define list_lru_init_key(lru, key) \ - __list_lru_init((lru), false, (key), NULL) #define list_lru_init_memcg(lru, shrinker) \ __list_lru_init((lru), true, NULL, shrinker) -- cgit v1.2.3 From 5662400a9ac03f38ef3b84e4ff9a640a4604bef9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 11 Jan 2024 15:24:20 +0000 Subject: mm: add pfn_swap_entry_folio() Patch series "mm: convert mm counter to take a folio", v3. Make sure all mm_counter() and mm_counter_file() callers have a folio, then convert mm counter functions to take a folio, which saves some compound_head() calls. This patch (of 10): Thanks to the compound_head() hidden inside PageLocked(), this saves a call to compound_head() over calling page_folio(pfn_swap_entry_to_page()) Link: https://lkml.kernel.org/r/20240111152429.3374566-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240111152429.3374566-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Kefeng Wang Signed-off-by: Andrew Morton --- include/linux/swapops.h | 13 +++++++++++++ mm/filemap.c | 2 +- mm/huge_memory.c | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swapops.h b/include/linux/swapops.h index bff1e8d97de0..48b700ba1d18 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -468,6 +468,19 @@ static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) return p; } +static inline struct folio *pfn_swap_entry_folio(swp_entry_t entry) +{ + struct folio *folio = pfn_folio(swp_offset_pfn(entry)); + + /* + * Any use of migration entries may only occur while the + * corresponding folio is locked + */ + BUG_ON(is_migration_entry(entry) && !folio_test_locked(folio)); + + return folio; +} + /* * A pfn swap entry is a special type of swap entry that always has a pfn stored * in the swap offset. They are used to represent unaddressable device memory diff --git a/mm/filemap.c b/mm/filemap.c index 0d7e20edf46f..142864338ca4 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1354,7 +1354,7 @@ void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) unsigned long pflags; bool in_thrashing; wait_queue_head_t *q; - struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); + struct folio *folio = pfn_swap_entry_folio(entry); q = folio_waitqueue(folio); if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 94c958f7ebb5..5468b2f97cbf 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2045,7 +2045,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION if (is_swap_pmd(*pmd)) { swp_entry_t entry = pmd_to_swp_entry(*pmd); - struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); + struct folio *folio = pfn_swap_entry_folio(entry); pmd_t newpmd; VM_BUG_ON(!is_pmd_migration_entry(*pmd)); -- cgit v1.2.3 From a23f517b0e1554467b0eb3bc1ebcb4d626217302 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 11 Jan 2024 15:24:28 +0000 Subject: mm: convert mm_counter() to take a folio Now all callers of mm_counter() have a folio, convert mm_counter() to take a folio. Saves a call to compound_head() hidden inside PageAnon(). Link: https://lkml.kernel.org/r/20240111152429.3374566-10-willy@infradead.org Signed-off-by: Kefeng Wang Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Signed-off-by: Andrew Morton --- arch/s390/mm/pgtable.c | 2 +- include/linux/mm.h | 6 +++--- mm/memory.c | 10 +++++----- mm/rmap.c | 8 ++++---- mm/userfaultfd.c | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 7e5dd4b17664..b71432b15d66 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -723,7 +723,7 @@ static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) else if (is_migration_entry(entry)) { struct folio *folio = pfn_swap_entry_folio(entry); - dec_mm_counter(mm, mm_counter(&folio->page)); + dec_mm_counter(mm, mm_counter(folio)); } free_swap_and_cache(entry); } diff --git a/include/linux/mm.h b/include/linux/mm.h index f5a97dec5169..22e597b36b38 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2603,11 +2603,11 @@ static inline int mm_counter_file(struct page *page) return MM_FILEPAGES; } -static inline int mm_counter(struct page *page) +static inline int mm_counter(struct folio *folio) { - if (PageAnon(page)) + if (folio_test_anon(folio)) return MM_ANONPAGES; - return mm_counter_file(page); + return mm_counter_file(&folio->page); } static inline unsigned long get_mm_rss(struct mm_struct *mm) diff --git a/mm/memory.c b/mm/memory.c index 4bed82009ea7..87ef98099847 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -808,7 +808,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, } else if (is_migration_entry(entry)) { folio = pfn_swap_entry_folio(entry); - rss[mm_counter(&folio->page)]++; + rss[mm_counter(folio)]++; if (!is_readable_migration_entry(entry) && is_cow_mapping(vm_flags)) { @@ -840,7 +840,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, * keep things as they are. */ folio_get(folio); - rss[mm_counter(page)]++; + rss[mm_counter(folio)]++; /* Cannot fail as these pages cannot get pinned. */ folio_try_dup_anon_rmap_pte(folio, page, src_vma); @@ -1476,7 +1476,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (pte_young(ptent) && likely(vma_has_recency(vma))) folio_mark_accessed(folio); } - rss[mm_counter(page)]--; + rss[mm_counter(folio)]--; if (!delay_rmap) { folio_remove_rmap_pte(folio, page, vma); if (unlikely(page_mapcount(page) < 0)) @@ -1504,7 +1504,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, * see zap_install_uffd_wp_if_needed(). */ WARN_ON_ONCE(!vma_is_anonymous(vma)); - rss[mm_counter(page)]--; + rss[mm_counter(folio)]--; if (is_device_private_entry(entry)) folio_remove_rmap_pte(folio, page, vma); folio_put(folio); @@ -1519,7 +1519,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, folio = pfn_swap_entry_folio(entry); if (!should_zap_folio(details, folio)) continue; - rss[mm_counter(&folio->page)]--; + rss[mm_counter(folio)]--; } else if (pte_marker_entry_uffd_wp(entry)) { /* * For anon: always drop the marker; for file: only diff --git a/mm/rmap.c b/mm/rmap.c index f5d43edad529..4648cf1d8178 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1780,7 +1780,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz); } else { - dec_mm_counter(mm, mm_counter(&folio->page)); + dec_mm_counter(mm, mm_counter(folio)); set_pte_at(mm, address, pvmw.pte, pteval); } @@ -1795,7 +1795,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * migration) will not expect userfaults on already * copied pages. */ - dec_mm_counter(mm, mm_counter(&folio->page)); + dec_mm_counter(mm, mm_counter(folio)); } else if (folio_test_anon(folio)) { swp_entry_t entry = page_swap_entry(subpage); pte_t swp_pte; @@ -2181,7 +2181,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz); } else { - dec_mm_counter(mm, mm_counter(&folio->page)); + dec_mm_counter(mm, mm_counter(folio)); set_pte_at(mm, address, pvmw.pte, pteval); } @@ -2196,7 +2196,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * migration) will not expect userfaults on already * copied pages. */ - dec_mm_counter(mm, mm_counter(&folio->page)); + dec_mm_counter(mm, mm_counter(folio)); } else { swp_entry_t entry; pte_t swp_pte; diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 7cf7d4384259..ae80c3714829 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -124,7 +124,7 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, * Must happen after rmap, as mm_counter() checks mapping (via * PageAnon()), which is set by __page_set_anon_rmap(). */ - inc_mm_counter(dst_mm, mm_counter(page)); + inc_mm_counter(dst_mm, mm_counter(folio)); set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); -- cgit v1.2.3 From 6b27cc6c66abf0f0b091a95ca1ad4e0fc68c11fd Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 11 Jan 2024 15:24:29 +0000 Subject: mm: convert mm_counter_file() to take a folio Now all callers of mm_counter_file() have a folio, convert mm_counter_file() to take a folio. Saves a call to compound_head() hidden inside PageSwapBacked(). Link: https://lkml.kernel.org/r/20240111152429.3374566-11-willy@infradead.org Signed-off-by: Kefeng Wang Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/mm.h | 8 ++++---- kernel/events/uprobes.c | 2 +- mm/huge_memory.c | 4 ++-- mm/khugepaged.c | 4 ++-- mm/memory.c | 10 +++++----- mm/rmap.c | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 22e597b36b38..ac6b71cbdffb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2595,10 +2595,10 @@ static inline void dec_mm_counter(struct mm_struct *mm, int member) mm_trace_rss_stat(mm, member); } -/* Optimized variant when page is already known not to be PageAnon */ -static inline int mm_counter_file(struct page *page) +/* Optimized variant when folio is already known not to be anon */ +static inline int mm_counter_file(struct folio *folio) { - if (PageSwapBacked(page)) + if (folio_test_swapbacked(folio)) return MM_SHMEMPAGES; return MM_FILEPAGES; } @@ -2607,7 +2607,7 @@ static inline int mm_counter(struct folio *folio) { if (folio_test_anon(folio)) return MM_ANONPAGES; - return mm_counter_file(&folio->page); + return mm_counter_file(folio); } static inline unsigned long get_mm_rss(struct mm_struct *mm) diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 929e98c62965..e4834d23e1d1 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -188,7 +188,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, dec_mm_counter(mm, MM_ANONPAGES); if (!folio_test_anon(old_folio)) { - dec_mm_counter(mm, mm_counter_file(old_page)); + dec_mm_counter(mm, mm_counter_file(old_folio)); inc_mm_counter(mm, MM_ANONPAGES); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7a28a7db08ea..f005f0424735 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1931,7 +1931,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, } else { if (arch_needs_pgtable_deposit()) zap_deposited_table(tlb->mm, pmd); - add_mm_counter(tlb->mm, mm_counter_file(&folio->page), + add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PMD_NR); } @@ -2456,7 +2456,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio_remove_rmap_pmd(folio, page, vma); folio_put(folio); } - add_mm_counter(mm, mm_counter_file(&folio->page), -HPAGE_PMD_NR); + add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); return; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 2b219acb528e..fe43fbc44525 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1634,7 +1634,7 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, /* step 3: set proper refcount and mm_counters. */ if (nr_ptes) { folio_ref_sub(folio, nr_ptes); - add_mm_counter(mm, mm_counter_file(&folio->page), -nr_ptes); + add_mm_counter(mm, mm_counter_file(folio), -nr_ptes); } /* step 4: remove empty page table */ @@ -1665,7 +1665,7 @@ abort: if (nr_ptes) { flush_tlb_mm(mm); folio_ref_sub(folio, nr_ptes); - add_mm_counter(mm, mm_counter_file(&folio->page), -nr_ptes); + add_mm_counter(mm, mm_counter_file(folio), -nr_ptes); } if (start_pte) pte_unmap_unlock(start_pte, ptl); diff --git a/mm/memory.c b/mm/memory.c index 87ef98099847..5e608edfe330 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -966,7 +966,7 @@ copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (page) { folio_get(folio); folio_dup_file_rmap_pte(folio, page); - rss[mm_counter_file(page)]++; + rss[mm_counter_file(folio)]++; } /* @@ -1873,7 +1873,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, return -EBUSY; /* Ok, finally just insert the thing.. */ folio_get(folio); - inc_mm_counter(vma->vm_mm, mm_counter_file(page)); + inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); folio_add_file_rmap_pte(folio, page, vma); set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot)); return 0; @@ -3178,7 +3178,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { if (old_folio) { if (!folio_test_anon(old_folio)) { - dec_mm_counter(mm, mm_counter_file(&old_folio->page)); + dec_mm_counter(mm, mm_counter_file(old_folio)); inc_mm_counter(mm, MM_ANONPAGES); } } else { @@ -4483,7 +4483,7 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) if (write) entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); - add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR); + add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); folio_add_file_rmap_pmd(folio, page, vma); /* @@ -4546,7 +4546,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, folio_add_new_anon_rmap(folio, vma, addr); folio_add_lru_vma(folio, vma); } else { - add_mm_counter(vma->vm_mm, mm_counter_file(page), nr); + add_mm_counter(vma->vm_mm, mm_counter_file(folio), nr); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); diff --git a/mm/rmap.c b/mm/rmap.c index 4648cf1d8178..1cf2bffa48ed 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1903,7 +1903,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * * See Documentation/mm/mmu_notifier.rst */ - dec_mm_counter(mm, mm_counter_file(&folio->page)); + dec_mm_counter(mm, mm_counter_file(folio)); } discard: if (unlikely(folio_test_hugetlb(folio))) -- cgit v1.2.3 From f796feabb9f5b1e5c48780a7a0023ab4b82336dd Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Tue, 20 Feb 2024 12:00:01 +0100 Subject: udp: add local "peek offset enabled" flag We want to re-organize the struct sock layout. The sk_peek_off field location is problematic, as most protocols want it in the RX read area, while UDP wants it on a cacheline different from sk_receive_queue. Create a local (inside udp_sock) copy of the 'peek offset is enabled' flag and place it inside the same cacheline of reader_queue. Check such flag before reading sk_peek_off. This will save potential false sharing and cache misses in the fast-path. Tested under UDP flood with small packets. The struct sock layout update causes a 4% performance drop, and this patch restores completely the original tput. Signed-off-by: Paolo Abeni Reviewed-by: Eric Dumazet Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/67ab679c15fbf49fa05b3ffe05d91c47ab84f147.1708426665.git.pabeni@redhat.com Signed-off-by: Jakub Kicinski --- include/linux/udp.h | 10 ++++++++++ net/ipv4/af_inet.c | 2 +- net/ipv4/udp.c | 2 +- net/ipv6/af_inet6.c | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/udp.h b/include/linux/udp.h index d04188714dca..3748e82b627b 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -92,6 +92,9 @@ struct udp_sock { /* This fields follows rcvbuf value, and is touched by udp_recvmsg */ int forward_threshold; + + /* Cache friendly copy of sk->sk_peek_off >= 0 */ + bool peeking_with_offset; }; #define udp_test_bit(nr, sk) \ @@ -109,6 +112,13 @@ struct udp_sock { #define udp_sk(ptr) container_of_const(ptr, struct udp_sock, inet.sk) +static inline int udp_set_peek_off(struct sock *sk, int val) +{ + sk_set_peek_off(sk, val); + WRITE_ONCE(udp_sk(sk)->peeking_with_offset, val >= 0); + return 0; +} + static inline void udp_set_no_check6_tx(struct sock *sk, bool val) { udp_assign_bit(NO_CHECK6_TX, sk, val); diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index ad278009e469..5daebdcbca32 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1103,7 +1103,7 @@ const struct proto_ops inet_dgram_ops = { .recvmsg = inet_recvmsg, .mmap = sock_no_mmap, .splice_eof = inet_splice_eof, - .set_peek_off = sk_set_peek_off, + .set_peek_off = udp_set_peek_off, #ifdef CONFIG_COMPAT .compat_ioctl = inet_compat_ioctl, #endif diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index f631b0a21af4..38cce7cc51f6 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1589,7 +1589,7 @@ int udp_init_sock(struct sock *sk) void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) { - if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { + if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset))) { bool slow = lock_sock_fast(sk); sk_peek_offset_bwd(sk, len); diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 959bfd9f6344..b90d46533cdc 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -736,7 +736,7 @@ const struct proto_ops inet6_dgram_ops = { .recvmsg = inet6_recvmsg, /* retpoline's sake */ .read_skb = udp_read_skb, .mmap = sock_no_mmap, - .set_peek_off = sk_set_peek_off, + .set_peek_off = udp_set_peek_off, #ifdef CONFIG_COMPAT .compat_ioctl = inet6_compat_ioctl, #endif -- cgit v1.2.3 From 265b07df758a998f60cf5b5aec6bd72ca676655e Mon Sep 17 00:00:00 2001 From: Shradha Todi Date: Tue, 20 Feb 2024 14:10:45 +0530 Subject: clk: Provide managed helper to get and enable bulk clocks Provide a managed devm_clk_bulk* wrapper to get and enable all bulk clocks in order to simplify drivers that keeps all clocks enabled for the time of driver operation. Suggested-by: Marek Szyprowski Reviewed-by: Alim Akhtar Reviewed-by: Manivannan Sadhasivam Signed-off-by: Shradha Todi Link: https://lore.kernel.org/r/20240220084046.23786-2-shradha.t@samsung.com Signed-off-by: Stephen Boyd --- drivers/clk/clk-devres.c | 40 ++++++++++++++++++++++++++++++++++++++++ include/linux/clk.h | 22 ++++++++++++++++++++++ 2 files changed, 62 insertions(+) (limited to 'include/linux') diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index 737aa70e2cb3..90e6078fb6e1 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -182,6 +182,46 @@ int __must_check devm_clk_bulk_get_all(struct device *dev, } EXPORT_SYMBOL_GPL(devm_clk_bulk_get_all); +static void devm_clk_bulk_release_all_enable(struct device *dev, void *res) +{ + struct clk_bulk_devres *devres = res; + + clk_bulk_disable_unprepare(devres->num_clks, devres->clks); + clk_bulk_put_all(devres->num_clks, devres->clks); +} + +int __must_check devm_clk_bulk_get_all_enable(struct device *dev, + struct clk_bulk_data **clks) +{ + struct clk_bulk_devres *devres; + int ret; + + devres = devres_alloc(devm_clk_bulk_release_all_enable, + sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + ret = clk_bulk_get_all(dev, &devres->clks); + if (ret > 0) { + *clks = devres->clks; + devres->num_clks = ret; + } else { + devres_free(devres); + return ret; + } + + ret = clk_bulk_prepare_enable(devres->num_clks, *clks); + if (!ret) { + devres_add(dev, devres); + } else { + clk_bulk_put_all(devres->num_clks, devres->clks); + devres_free(devres); + } + + return ret; +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_all_enable); + static int devm_clk_match(struct device *dev, void *res, void *data) { struct clk **c = res; diff --git a/include/linux/clk.h b/include/linux/clk.h index 06f1b292f8a0..0f44d3863de2 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -478,6 +478,22 @@ int __must_check devm_clk_bulk_get_optional(struct device *dev, int num_clks, int __must_check devm_clk_bulk_get_all(struct device *dev, struct clk_bulk_data **clks); +/** + * devm_clk_bulk_get_all_enable - Get and enable all clocks of the consumer (managed) + * @dev: device for clock "consumer" + * @clks: pointer to the clk_bulk_data table of consumer + * + * Returns success (0) or negative errno. + * + * This helper function allows drivers to get all clocks of the + * consumer and enables them in one operation with management. + * The clks will automatically be disabled and freed when the device + * is unbound. + */ + +int __must_check devm_clk_bulk_get_all_enable(struct device *dev, + struct clk_bulk_data **clks); + /** * devm_clk_get - lookup and obtain a managed reference to a clock producer. * @dev: device for clock "consumer" @@ -968,6 +984,12 @@ static inline int __must_check devm_clk_bulk_get_all(struct device *dev, return 0; } +static inline int __must_check devm_clk_bulk_get_all_enable(struct device *dev, + struct clk_bulk_data **clks) +{ + return 0; +} + static inline struct clk *devm_get_clk_from_child(struct device *dev, struct device_node *np, const char *con_id) { -- cgit v1.2.3 From ff773fd2199960ffab0caae07451fe0f12b05bb8 Mon Sep 17 00:00:00 2001 From: Théo Lebrun Date: Wed, 21 Feb 2024 19:22:09 +0100 Subject: clk: fixed-factor: add optional accuracy support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed factor clock reports the parent clock accuracy. Add flags and acc fields to `struct clk_fixed_factor` to support setting a fixed accuracy. The default if no flag is set is not changed: use the parent clock accuracy. Signed-off-by: Théo Lebrun Link: https://lore.kernel.org/r/20240221-mbly-clk-v7-1-31d4ce3630c3@bootlin.com Signed-off-by: Stephen Boyd --- drivers/clk/clk-fixed-factor.c | 28 +++++++++++++++++++++------- include/linux/clk-provider.h | 12 +++++++++++- 2 files changed, 32 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c index b3e66202b942..bc2644a9bd7d 100644 --- a/drivers/clk/clk-fixed-factor.c +++ b/drivers/clk/clk-fixed-factor.c @@ -57,10 +57,22 @@ static int clk_factor_set_rate(struct clk_hw *hw, unsigned long rate, return 0; } +static unsigned long clk_factor_recalc_accuracy(struct clk_hw *hw, + unsigned long parent_accuracy) +{ + struct clk_fixed_factor *fix = to_clk_fixed_factor(hw); + + if (fix->flags & CLK_FIXED_FACTOR_FIXED_ACCURACY) + return fix->acc; + + return parent_accuracy; +} + const struct clk_ops clk_fixed_factor_ops = { .round_rate = clk_factor_round_rate, .set_rate = clk_factor_set_rate, .recalc_rate = clk_factor_recalc_rate, + .recalc_accuracy = clk_factor_recalc_accuracy, }; EXPORT_SYMBOL_GPL(clk_fixed_factor_ops); @@ -81,7 +93,7 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, const char *name, const char *parent_name, const struct clk_hw *parent_hw, int index, unsigned long flags, unsigned int mult, unsigned int div, - bool devm) + unsigned long acc, unsigned int fixflags, bool devm) { struct clk_fixed_factor *fix; struct clk_init_data init = { }; @@ -105,6 +117,8 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, fix->mult = mult; fix->div = div; fix->hw.init = &init; + fix->acc = acc; + fix->flags = fixflags; init.name = name; init.ops = &clk_fixed_factor_ops; @@ -152,7 +166,7 @@ struct clk_hw *devm_clk_hw_register_fixed_factor_index(struct device *dev, unsigned int mult, unsigned int div) { return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, NULL, index, - flags, mult, div, true); + flags, mult, div, 0, 0, true); } EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor_index); @@ -174,7 +188,7 @@ struct clk_hw *devm_clk_hw_register_fixed_factor_parent_hw(struct device *dev, unsigned long flags, unsigned int mult, unsigned int div) { return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, parent_hw, - -1, flags, mult, div, true); + -1, flags, mult, div, 0, 0, true); } EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor_parent_hw); @@ -184,7 +198,7 @@ struct clk_hw *clk_hw_register_fixed_factor_parent_hw(struct device *dev, { return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, parent_hw, -1, flags, mult, div, - false); + 0, 0, false); } EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor_parent_hw); @@ -193,7 +207,7 @@ struct clk_hw *clk_hw_register_fixed_factor(struct device *dev, unsigned int mult, unsigned int div) { return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, NULL, -1, - flags, mult, div, false); + flags, mult, div, 0, 0, false); } EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor); @@ -240,7 +254,7 @@ struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev, unsigned int mult, unsigned int div) { return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, NULL, -1, - flags, mult, div, true); + flags, mult, div, 0, 0, true); } EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor); @@ -267,7 +281,7 @@ static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node) of_property_read_string(node, "clock-output-names", &clk_name); hw = __clk_hw_register_fixed_factor(NULL, node, clk_name, NULL, NULL, 0, - 0, mult, div, false); + 0, mult, div, 0, 0, false); if (IS_ERR(hw)) { /* * Clear OF_POPULATED flag so that clock registration can be diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 1293c38ddb7f..7ddc952c8c67 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -1084,18 +1084,28 @@ void of_fixed_factor_clk_setup(struct device_node *node); * @hw: handle between common and hardware-specific interfaces * @mult: multiplier * @div: divider + * @acc: fixed accuracy in ppb + * @flags: behavior modifying flags * * Clock with a fixed multiplier and divider. The output frequency is the * parent clock rate divided by div and multiplied by mult. - * Implements .recalc_rate, .set_rate and .round_rate + * Implements .recalc_rate, .set_rate, .round_rate and .recalc_accuracy + * + * Flags: + * * CLK_FIXED_FACTOR_FIXED_ACCURACY - Use the value in @acc instead of the + * parent clk accuracy. */ struct clk_fixed_factor { struct clk_hw hw; unsigned int mult; unsigned int div; + unsigned long acc; + unsigned int flags; }; +#define CLK_FIXED_FACTOR_FIXED_ACCURACY BIT(0) + #define to_clk_fixed_factor(_hw) container_of(_hw, struct clk_fixed_factor, hw) extern const struct clk_ops clk_fixed_factor_ops; -- cgit v1.2.3 From ae156a3633d377d43990eb539f8a007c0c2bf769 Mon Sep 17 00:00:00 2001 From: Théo Lebrun Date: Wed, 21 Feb 2024 19:22:10 +0100 Subject: clk: fixed-factor: add fwname-based constructor functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add four functions to register clk_hw based on the fw_name field in clk_parent_data, ie the value in the DT property `clock-names`. There are variants for devm or not and passing an accuracy or not passing one: - clk_hw_register_fixed_factor_fwname - clk_hw_register_fixed_factor_with_accuracy_fwname - devm_clk_hw_register_fixed_factor_fwname - devm_clk_hw_register_fixed_factor_with_accuracy_fwname The `struct clk_parent_data` init is extracted from __clk_hw_register_fixed_factor to each calling function. It is required to allow each function to pass whatever field they want, not only index. Signed-off-by: Théo Lebrun Link: https://lore.kernel.org/r/20240221-mbly-clk-v7-2-31d4ce3630c3@bootlin.com Signed-off-by: Stephen Boyd --- drivers/clk/clk-fixed-factor.c | 85 +++++++++++++++++++++++++++++++++++------- include/linux/clk-provider.h | 14 +++++++ 2 files changed, 85 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c index bc2644a9bd7d..fe0500a1af3e 100644 --- a/drivers/clk/clk-fixed-factor.c +++ b/drivers/clk/clk-fixed-factor.c @@ -91,13 +91,12 @@ static void devm_clk_hw_register_fixed_factor_release(struct device *dev, void * static struct clk_hw * __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, const char *name, const char *parent_name, - const struct clk_hw *parent_hw, int index, + const struct clk_hw *parent_hw, const struct clk_parent_data *pdata, unsigned long flags, unsigned int mult, unsigned int div, unsigned long acc, unsigned int fixflags, bool devm) { struct clk_fixed_factor *fix; struct clk_init_data init = { }; - struct clk_parent_data pdata = { .index = index }; struct clk_hw *hw; int ret; @@ -128,7 +127,7 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, else if (parent_hw) init.parent_hws = &parent_hw; else - init.parent_data = &pdata; + init.parent_data = pdata; init.num_parents = 1; hw = &fix->hw; @@ -165,7 +164,9 @@ struct clk_hw *devm_clk_hw_register_fixed_factor_index(struct device *dev, const char *name, unsigned int index, unsigned long flags, unsigned int mult, unsigned int div) { - return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, NULL, index, + const struct clk_parent_data pdata = { .index = index }; + + return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, NULL, &pdata, flags, mult, div, 0, 0, true); } EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor_index); @@ -187,8 +188,10 @@ struct clk_hw *devm_clk_hw_register_fixed_factor_parent_hw(struct device *dev, const char *name, const struct clk_hw *parent_hw, unsigned long flags, unsigned int mult, unsigned int div) { + const struct clk_parent_data pdata = { .index = -1 }; + return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, parent_hw, - -1, flags, mult, div, 0, 0, true); + &pdata, flags, mult, div, 0, 0, true); } EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor_parent_hw); @@ -196,9 +199,10 @@ struct clk_hw *clk_hw_register_fixed_factor_parent_hw(struct device *dev, const char *name, const struct clk_hw *parent_hw, unsigned long flags, unsigned int mult, unsigned int div) { - return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, - parent_hw, -1, flags, mult, div, - 0, 0, false); + const struct clk_parent_data pdata = { .index = -1 }; + + return __clk_hw_register_fixed_factor(dev, NULL, name, NULL, parent_hw, + &pdata, flags, mult, div, 0, 0, false); } EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor_parent_hw); @@ -206,11 +210,37 @@ struct clk_hw *clk_hw_register_fixed_factor(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned int mult, unsigned int div) { - return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, NULL, -1, - flags, mult, div, 0, 0, false); + const struct clk_parent_data pdata = { .index = -1 }; + + return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, NULL, + &pdata, flags, mult, div, 0, 0, false); } EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor); +struct clk_hw *clk_hw_register_fixed_factor_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div) +{ + const struct clk_parent_data pdata = { .index = -1, .fw_name = fw_name }; + + return __clk_hw_register_fixed_factor(dev, np, name, NULL, NULL, + &pdata, flags, mult, div, 0, 0, false); +} +EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor_fwname); + +struct clk_hw *clk_hw_register_fixed_factor_with_accuracy_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div, + unsigned long acc) +{ + const struct clk_parent_data pdata = { .index = -1, .fw_name = fw_name }; + + return __clk_hw_register_fixed_factor(dev, np, name, NULL, NULL, + &pdata, flags, mult, div, acc, + CLK_FIXED_FACTOR_FIXED_ACCURACY, false); +} +EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor_with_accuracy_fwname); + struct clk *clk_register_fixed_factor(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned int mult, unsigned int div) @@ -253,16 +283,43 @@ struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned int mult, unsigned int div) { - return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, NULL, -1, - flags, mult, div, 0, 0, true); + const struct clk_parent_data pdata = { .index = -1 }; + + return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, NULL, + &pdata, flags, mult, div, 0, 0, true); } EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor); +struct clk_hw *devm_clk_hw_register_fixed_factor_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div) +{ + const struct clk_parent_data pdata = { .index = -1, .fw_name = fw_name }; + + return __clk_hw_register_fixed_factor(dev, np, name, NULL, NULL, + &pdata, flags, mult, div, 0, 0, true); +} +EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor_fwname); + +struct clk_hw *devm_clk_hw_register_fixed_factor_with_accuracy_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div, + unsigned long acc) +{ + const struct clk_parent_data pdata = { .index = -1, .fw_name = fw_name }; + + return __clk_hw_register_fixed_factor(dev, np, name, NULL, NULL, + &pdata, flags, mult, div, acc, + CLK_FIXED_FACTOR_FIXED_ACCURACY, true); +} +EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor_with_accuracy_fwname); + #ifdef CONFIG_OF static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node) { struct clk_hw *hw; const char *clk_name = node->name; + const struct clk_parent_data pdata = { .index = 0 }; u32 div, mult; int ret; @@ -280,8 +337,8 @@ static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node) of_property_read_string(node, "clock-output-names", &clk_name); - hw = __clk_hw_register_fixed_factor(NULL, node, clk_name, NULL, NULL, 0, - 0, mult, div, 0, 0, false); + hw = __clk_hw_register_fixed_factor(NULL, node, clk_name, NULL, NULL, + &pdata, 0, mult, div, 0, 0, false); if (IS_ERR(hw)) { /* * Clear OF_POPULATED flag so that clock registration can be diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 7ddc952c8c67..4a537260f655 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -1116,10 +1116,24 @@ void clk_unregister_fixed_factor(struct clk *clk); struct clk_hw *clk_hw_register_fixed_factor(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned int mult, unsigned int div); +struct clk_hw *clk_hw_register_fixed_factor_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div); +struct clk_hw *clk_hw_register_fixed_factor_with_accuracy_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div, + unsigned long acc); void clk_hw_unregister_fixed_factor(struct clk_hw *hw); struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned int mult, unsigned int div); +struct clk_hw *devm_clk_hw_register_fixed_factor_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div); +struct clk_hw *devm_clk_hw_register_fixed_factor_with_accuracy_fwname(struct device *dev, + struct device_node *np, const char *name, const char *fw_name, + unsigned long flags, unsigned int mult, unsigned int div, + unsigned long acc); struct clk_hw *devm_clk_hw_register_fixed_factor_index(struct device *dev, const char *name, unsigned int index, unsigned long flags, unsigned int mult, unsigned int div); -- cgit v1.2.3 From e4ad2b0130eff1cc72f93ea7fd184b0e420f0736 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Wed, 14 Feb 2024 18:30:02 +0000 Subject: firmware: arm_scmi: Add clock check for extended config support SCMI v3.2 added support to set/get clock custom OEM types; such support is conditionally present, though, depending on an extended config attribute bit possibly advertised by the platform server on a per-domain base. Add a check to verify if OEM types are supported before allowing any kind of OEM-specific get/set operation. Also add a check around all the new v3.2 clock features. Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240214183006.3403207-4-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/clock.c | 33 +++++++++++++++++++++++++++------ include/linux/scmi_protocol.h | 1 + 2 files changed, 28 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c index 85eda5db40ba..add350bf2a7f 100644 --- a/drivers/firmware/arm_scmi/clock.c +++ b/drivers/firmware/arm_scmi/clock.c @@ -54,6 +54,7 @@ struct scmi_msg_resp_clock_attributes { #define SUPPORTS_RATE_CHANGE_REQUESTED_NOTIF(x) ((x) & BIT(30)) #define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(29)) #define SUPPORTS_PARENT_CLOCK(x) ((x) & BIT(28)) +#define SUPPORTS_EXTENDED_CONFIG(x) ((x) & BIT(27)) #define SUPPORTS_GET_PERMISSIONS(x) ((x) & BIT(1)) u8 name[SCMI_SHORT_NAME_MAX_SIZE]; __le32 clock_enable_latency; @@ -388,10 +389,14 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph, if (cinfo->notify_rate_change_requested_cmd && SUPPORTS_RATE_CHANGE_REQUESTED_NOTIF(attributes)) clk->rate_change_requested_notifications = true; - if (SUPPORTS_PARENT_CLOCK(attributes)) - scmi_clock_possible_parents(ph, clk_id, clk); - if (SUPPORTS_GET_PERMISSIONS(attributes)) - scmi_clock_get_permissions(ph, clk_id, clk); + if (PROTOCOL_REV_MAJOR(version) >= 0x3) { + if (SUPPORTS_PARENT_CLOCK(attributes)) + scmi_clock_possible_parents(ph, clk_id, clk); + if (SUPPORTS_GET_PERMISSIONS(attributes)) + scmi_clock_get_permissions(ph, clk_id, clk); + if (SUPPORTS_EXTENDED_CONFIG(attributes)) + clk->extended_config = true; + } } return ret; @@ -700,7 +705,7 @@ scmi_clock_get_parent(const struct scmi_protocol_handle *ph, u32 clk_id, return ret; } -/* For SCMI clock v2.1 and onwards */ +/* For SCMI clock v3.0 and onwards */ static int scmi_clock_config_set_v2(const struct scmi_protocol_handle *ph, u32 clk_id, enum clk_state state, u8 oem_type, u32 oem_val, @@ -773,7 +778,7 @@ static int scmi_clock_disable(const struct scmi_protocol_handle *ph, u32 clk_id, NULL_OEM_TYPE, 0, atomic); } -/* For SCMI clock v2.1 and onwards */ +/* For SCMI clock v3.0 and onwards */ static int scmi_clock_config_get_v2(const struct scmi_protocol_handle *ph, u32 clk_id, u8 oem_type, u32 *attributes, bool *enabled, @@ -860,6 +865,14 @@ static int scmi_clock_config_oem_set(const struct scmi_protocol_handle *ph, bool atomic) { struct clock_info *ci = ph->get_priv(ph); + struct scmi_clock_info *clk; + + clk = scmi_clock_domain_lookup(ci, clk_id); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + if (!clk->extended_config) + return -EOPNOTSUPP; return ci->clock_config_set(ph, clk_id, CLK_STATE_UNCHANGED, oem_type, oem_val, atomic); @@ -870,6 +883,14 @@ static int scmi_clock_config_oem_get(const struct scmi_protocol_handle *ph, u32 *attributes, bool atomic) { struct clock_info *ci = ph->get_priv(ph); + struct scmi_clock_info *clk; + + clk = scmi_clock_domain_lookup(ci, clk_id); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + if (!clk->extended_config) + return -EOPNOTSUPP; return ci->clock_config_get(ph, clk_id, oem_type, attributes, NULL, oem_val, atomic); diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 9b9351e07a11..46a61173c91c 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -50,6 +50,7 @@ struct scmi_clock_info { bool state_ctrl_forbidden; bool rate_ctrl_forbidden; bool parent_ctrl_forbidden; + bool extended_config; union { struct { int num_rates; -- cgit v1.2.3 From 62092c428fb528fcd117a580216915af04df450e Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Wed, 14 Feb 2024 18:30:03 +0000 Subject: firmware: arm_scmi: Add standard clock OEM definitions Add a common enum to define the standard clock OEM types defined by the SCMI specification, so as to enable the configuration of such extended configuration properties with the existent clock protocol operations. Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240214183006.3403207-5-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/clock.c | 32 +++++++++++++++++++------------- include/linux/scmi_protocol.h | 14 +++++++++++--- 2 files changed, 30 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c index add350bf2a7f..95393af655ed 100644 --- a/drivers/firmware/arm_scmi/clock.c +++ b/drivers/firmware/arm_scmi/clock.c @@ -165,10 +165,12 @@ struct clock_info { struct scmi_clock_info *clk; int (*clock_config_set)(const struct scmi_protocol_handle *ph, u32 clk_id, enum clk_state state, - u8 oem_type, u32 oem_val, bool atomic); + enum scmi_clock_oem_config oem_type, + u32 oem_val, bool atomic); int (*clock_config_get)(const struct scmi_protocol_handle *ph, - u32 clk_id, u8 oem_type, u32 *attributes, - bool *enabled, u32 *oem_val, bool atomic); + u32 clk_id, enum scmi_clock_oem_config oem_type, + u32 *attributes, bool *enabled, u32 *oem_val, + bool atomic); }; static enum scmi_clock_protocol_cmd evt_2_cmd[] = { @@ -618,7 +620,8 @@ static int scmi_clock_rate_set(const struct scmi_protocol_handle *ph, static int scmi_clock_config_set(const struct scmi_protocol_handle *ph, u32 clk_id, - enum clk_state state, u8 __unused0, u32 __unused1, + enum clk_state state, + enum scmi_clock_oem_config __unused0, u32 __unused1, bool atomic) { int ret; @@ -708,7 +711,8 @@ scmi_clock_get_parent(const struct scmi_protocol_handle *ph, u32 clk_id, /* For SCMI clock v3.0 and onwards */ static int scmi_clock_config_set_v2(const struct scmi_protocol_handle *ph, u32 clk_id, - enum clk_state state, u8 oem_type, u32 oem_val, + enum clk_state state, + enum scmi_clock_oem_config oem_type, u32 oem_val, bool atomic) { int ret; @@ -781,8 +785,8 @@ static int scmi_clock_disable(const struct scmi_protocol_handle *ph, u32 clk_id, /* For SCMI clock v3.0 and onwards */ static int scmi_clock_config_get_v2(const struct scmi_protocol_handle *ph, u32 clk_id, - u8 oem_type, u32 *attributes, bool *enabled, - u32 *oem_val, bool atomic) + enum scmi_clock_oem_config oem_type, u32 *attributes, + bool *enabled, u32 *oem_val, bool atomic) { int ret; u32 flags; @@ -823,8 +827,8 @@ scmi_clock_config_get_v2(const struct scmi_protocol_handle *ph, u32 clk_id, static int scmi_clock_config_get(const struct scmi_protocol_handle *ph, u32 clk_id, - u8 oem_type, u32 *attributes, bool *enabled, - u32 *oem_val, bool atomic) + enum scmi_clock_oem_config oem_type, u32 *attributes, + bool *enabled, u32 *oem_val, bool atomic) { int ret; struct scmi_xfer *t; @@ -861,8 +865,9 @@ static int scmi_clock_state_get(const struct scmi_protocol_handle *ph, } static int scmi_clock_config_oem_set(const struct scmi_protocol_handle *ph, - u32 clk_id, u8 oem_type, u32 oem_val, - bool atomic) + u32 clk_id, + enum scmi_clock_oem_config oem_type, + u32 oem_val, bool atomic) { struct clock_info *ci = ph->get_priv(ph); struct scmi_clock_info *clk; @@ -879,8 +884,9 @@ static int scmi_clock_config_oem_set(const struct scmi_protocol_handle *ph, } static int scmi_clock_config_oem_get(const struct scmi_protocol_handle *ph, - u32 clk_id, u8 oem_type, u32 *oem_val, - u32 *attributes, bool atomic) + u32 clk_id, + enum scmi_clock_oem_config oem_type, + u32 *oem_val, u32 *attributes, bool atomic) { struct clock_info *ci = ph->get_priv(ph); struct scmi_clock_info *clk; diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 46a61173c91c..2ee94ff0320c 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -76,6 +76,13 @@ struct scmi_handle; struct scmi_device; struct scmi_protocol_handle; +enum scmi_clock_oem_config { + SCMI_CLOCK_CFG_DUTY_CYCLE = 0x1, + SCMI_CLOCK_CFG_PHASE, + SCMI_CLOCK_CFG_OEM_START = 0x80, + SCMI_CLOCK_CFG_OEM_END = 0xFF, +}; + /** * struct scmi_clk_proto_ops - represents the various operations provided * by SCMI Clock Protocol @@ -108,10 +115,11 @@ struct scmi_clk_proto_ops { int (*state_get)(const struct scmi_protocol_handle *ph, u32 clk_id, bool *enabled, bool atomic); int (*config_oem_get)(const struct scmi_protocol_handle *ph, u32 clk_id, - u8 oem_type, u32 *oem_val, u32 *attributes, - bool atomic); + enum scmi_clock_oem_config oem_type, + u32 *oem_val, u32 *attributes, bool atomic); int (*config_oem_set)(const struct scmi_protocol_handle *ph, u32 clk_id, - u8 oem_type, u32 oem_val, bool atomic); + enum scmi_clock_oem_config oem_type, + u32 oem_val, bool atomic); int (*parent_get)(const struct scmi_protocol_handle *ph, u32 clk_id, u32 *parent_id); int (*parent_set)(const struct scmi_protocol_handle *ph, u32 clk_id, u32 parent_id); }; -- cgit v1.2.3 From bb29fd7760ae39905127afd31fc83294625ff704 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Fri, 19 Jan 2024 11:22:22 +0000 Subject: mm/zswap: make sure each swapfile always have zswap rb-tree Patch series "mm/zswap: optimize the scalability of zswap rb-tree", v2. When testing the zswap performance by using kernel build -j32 in a tmpfs directory, I found the scalability of zswap rb-tree is not good, which is protected by the only spinlock. That would cause heavy lock contention if multiple tasks zswap_store/load concurrently. So a simple solution is to split the only one zswap rb-tree into multiple rb-trees, each corresponds to SWAP_ADDRESS_SPACE_PAGES (64M). This idea is from the commit 4b3ef9daa4fc ("mm/swap: split swap cache into 64MB trunks"). Although this method can't solve the spinlock contention completely, it can mitigate much of that contention. Below is the results of kernel build in tmpfs with zswap shrinker enabled: linux-next zswap-lock-optimize real 1m9.181s 1m3.820s user 17m44.036s 17m40.100s sys 7m37.297s 4m54.622s So there are clearly improvements. And it's complementary with the ongoing zswap xarray conversion by Chris. Anyway, I think we can also merge this first, it's complementary IMHO. So I just refresh and resend this for further discussion. This patch (of 2): Not all zswap interfaces can handle the absence of the zswap rb-tree, actually only zswap_store() has handled it for now. To make things simple, we make sure each swapfile always have the zswap rb-tree prepared before being enabled and used. The preparation is unlikely to fail in practice, this patch just make it explicit. Link: https://lkml.kernel.org/r/20240117-b4-zswap-lock-optimize-v2-0-b5cc55479090@bytedance.com Link: https://lkml.kernel.org/r/20240117-b4-zswap-lock-optimize-v2-1-b5cc55479090@bytedance.com Signed-off-by: Chengming Zhou Acked-by: Nhat Pham Acked-by: Johannes Weiner Acked-by: Yosry Ahmed Cc: Chris Li Signed-off-by: Andrew Morton --- include/linux/zswap.h | 7 +++++-- mm/swapfile.c | 10 +++++++--- mm/zswap.c | 8 +++----- 3 files changed, 15 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/zswap.h b/include/linux/zswap.h index 0b709f5bc65f..eca388229d9a 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -30,7 +30,7 @@ struct zswap_lruvec_state { bool zswap_store(struct folio *folio); bool zswap_load(struct folio *folio); void zswap_invalidate(int type, pgoff_t offset); -void zswap_swapon(int type); +int zswap_swapon(int type); void zswap_swapoff(int type); void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg); void zswap_lruvec_state_init(struct lruvec *lruvec); @@ -51,7 +51,10 @@ static inline bool zswap_load(struct folio *folio) } static inline void zswap_invalidate(int type, pgoff_t offset) {} -static inline void zswap_swapon(int type) {} +static inline int zswap_swapon(int type) +{ + return 0; +} static inline void zswap_swapoff(int type) {} static inline void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) {} static inline void zswap_lruvec_state_init(struct lruvec *lruvec) {} diff --git a/mm/swapfile.c b/mm/swapfile.c index 746aa9da5302..b3a83c5dcbb8 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2348,8 +2348,6 @@ static void enable_swap_info(struct swap_info_struct *p, int prio, unsigned char *swap_map, struct swap_cluster_info *cluster_info) { - zswap_swapon(p->type); - spin_lock(&swap_lock); spin_lock(&p->lock); setup_swap_info(p, prio, swap_map, cluster_info); @@ -3167,6 +3165,10 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) if (error) goto bad_swap_unlock_inode; + error = zswap_swapon(p->type); + if (error) + goto free_swap_address_space; + /* * Flush any pending IO and dirty mappings before we start using this * swap device. @@ -3175,7 +3177,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = inode_drain_writes(inode); if (error) { inode->i_flags &= ~S_SWAPFILE; - goto free_swap_address_space; + goto free_swap_zswap; } mutex_lock(&swapon_mutex); @@ -3199,6 +3201,8 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = 0; goto out; +free_swap_zswap: + zswap_swapoff(p->type); free_swap_address_space: exit_swap_address_space(p->type); bad_swap_unlock_inode: diff --git a/mm/zswap.c b/mm/zswap.c index 2c3d77c6fe72..5a40a7b4bae8 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1518,9 +1518,6 @@ bool zswap_store(struct folio *folio) if (folio_test_large(folio)) return false; - if (!tree) - return false; - /* * If this is a duplicate, it must be removed before attempting to store * it, otherwise, if the store fails the old page won't be removed from @@ -1775,19 +1772,20 @@ void zswap_invalidate(int type, pgoff_t offset) spin_unlock(&tree->lock); } -void zswap_swapon(int type) +int zswap_swapon(int type) { struct zswap_tree *tree; tree = kzalloc(sizeof(*tree), GFP_KERNEL); if (!tree) { pr_err("alloc failed, zswap disabled for swap type %d\n", type); - return; + return -ENOMEM; } tree->rbroot = RB_ROOT; spin_lock_init(&tree->lock); zswap_trees[type] = tree; + return 0; } void zswap_swapoff(int type) -- cgit v1.2.3 From 44c7c734a5132fc02f5584c7207f1d0c483f3ccd Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Fri, 19 Jan 2024 11:22:23 +0000 Subject: mm/zswap: split zswap rb-tree Each swapfile has one rb-tree to search the mapping of swp_entry_t to zswap_entry, that use a spinlock to protect, which can cause heavy lock contention if multiple tasks zswap_store/load concurrently. Optimize the scalability problem by splitting the zswap rb-tree into multiple rb-trees, each corresponds to SWAP_ADDRESS_SPACE_PAGES (64M), just like we did in the swap cache address_space splitting. Although this method can't solve the spinlock contention completely, it can mitigate much of that contention. Below is the results of kernel build in tmpfs with zswap shrinker enabled: linux-next zswap-lock-optimize real 1m9.181s 1m3.820s user 17m44.036s 17m40.100s sys 7m37.297s 4m54.622s So there are clearly improvements. Link: https://lkml.kernel.org/r/20240117-b4-zswap-lock-optimize-v2-2-b5cc55479090@bytedance.com Signed-off-by: Chengming Zhou Acked-by: Johannes Weiner Acked-by: Nhat Pham Acked-by: Yosry Ahmed Cc: Chris Li Signed-off-by: Andrew Morton --- include/linux/zswap.h | 4 +-- mm/swapfile.c | 2 +- mm/zswap.c | 71 +++++++++++++++++++++++++++++++++------------------ 3 files changed, 49 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/zswap.h b/include/linux/zswap.h index eca388229d9a..91895ce1fdbc 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -30,7 +30,7 @@ struct zswap_lruvec_state { bool zswap_store(struct folio *folio); bool zswap_load(struct folio *folio); void zswap_invalidate(int type, pgoff_t offset); -int zswap_swapon(int type); +int zswap_swapon(int type, unsigned long nr_pages); void zswap_swapoff(int type); void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg); void zswap_lruvec_state_init(struct lruvec *lruvec); @@ -51,7 +51,7 @@ static inline bool zswap_load(struct folio *folio) } static inline void zswap_invalidate(int type, pgoff_t offset) {} -static inline int zswap_swapon(int type) +static inline int zswap_swapon(int type, unsigned long nr_pages) { return 0; } diff --git a/mm/swapfile.c b/mm/swapfile.c index b3a83c5dcbb8..0c6dde8b8604 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -3165,7 +3165,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) if (error) goto bad_swap_unlock_inode; - error = zswap_swapon(p->type); + error = zswap_swapon(p->type, maxpages); if (error) goto free_swap_address_space; diff --git a/mm/zswap.c b/mm/zswap.c index 5a40a7b4bae8..464179d43399 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -239,6 +239,7 @@ struct zswap_tree { }; static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; +static unsigned int nr_zswap_trees[MAX_SWAPFILES]; /* RCU-protected iteration */ static LIST_HEAD(zswap_pools); @@ -265,6 +266,12 @@ static bool zswap_has_pool; * helpers and fwd declarations **********************************/ +static inline struct zswap_tree *swap_zswap_tree(swp_entry_t swp) +{ + return &zswap_trees[swp_type(swp)][swp_offset(swp) + >> SWAP_ADDRESS_SPACE_SHIFT]; +} + #define zswap_pool_debug(msg, p) \ pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ zpool_get_type((p)->zpools[0])) @@ -864,7 +871,7 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o * until the entry is verified to still be alive in the tree. */ swpoffset = swp_offset(entry->swpentry); - tree = zswap_trees[swp_type(entry->swpentry)]; + tree = swap_zswap_tree(entry->swpentry); list_lru_isolate(l, item); /* * It's safe to drop the lock here because we return either @@ -1493,10 +1500,9 @@ static void zswap_fill_page(void *ptr, unsigned long value) bool zswap_store(struct folio *folio) { swp_entry_t swp = folio->swap; - int type = swp_type(swp); pgoff_t offset = swp_offset(swp); struct page *page = &folio->page; - struct zswap_tree *tree = zswap_trees[type]; + struct zswap_tree *tree = swap_zswap_tree(swp); struct zswap_entry *entry, *dupentry; struct scatterlist input, output; struct crypto_acomp_ctx *acomp_ctx; @@ -1569,7 +1575,7 @@ bool zswap_store(struct folio *folio) src = kmap_local_page(page); if (zswap_is_page_same_filled(src, &value)) { kunmap_local(src); - entry->swpentry = swp_entry(type, offset); + entry->swpentry = swp; entry->length = 0; entry->value = value; atomic_inc(&zswap_same_filled_pages); @@ -1651,7 +1657,7 @@ bool zswap_store(struct folio *folio) mutex_unlock(&acomp_ctx->mutex); /* populate entry */ - entry->swpentry = swp_entry(type, offset); + entry->swpentry = swp; entry->handle = handle; entry->length = dlen; @@ -1711,10 +1717,9 @@ shrink: bool zswap_load(struct folio *folio) { swp_entry_t swp = folio->swap; - int type = swp_type(swp); pgoff_t offset = swp_offset(swp); struct page *page = &folio->page; - struct zswap_tree *tree = zswap_trees[type]; + struct zswap_tree *tree = swap_zswap_tree(swp); struct zswap_entry *entry; u8 *dst; @@ -1757,7 +1762,7 @@ bool zswap_load(struct folio *folio) void zswap_invalidate(int type, pgoff_t offset) { - struct zswap_tree *tree = zswap_trees[type]; + struct zswap_tree *tree = swap_zswap_tree(swp_entry(type, offset)); struct zswap_entry *entry; /* find */ @@ -1772,37 +1777,53 @@ void zswap_invalidate(int type, pgoff_t offset) spin_unlock(&tree->lock); } -int zswap_swapon(int type) +int zswap_swapon(int type, unsigned long nr_pages) { - struct zswap_tree *tree; + struct zswap_tree *trees, *tree; + unsigned int nr, i; - tree = kzalloc(sizeof(*tree), GFP_KERNEL); - if (!tree) { + nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); + trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL); + if (!trees) { pr_err("alloc failed, zswap disabled for swap type %d\n", type); return -ENOMEM; } - tree->rbroot = RB_ROOT; - spin_lock_init(&tree->lock); - zswap_trees[type] = tree; + for (i = 0; i < nr; i++) { + tree = trees + i; + tree->rbroot = RB_ROOT; + spin_lock_init(&tree->lock); + } + + nr_zswap_trees[type] = nr; + zswap_trees[type] = trees; return 0; } void zswap_swapoff(int type) { - struct zswap_tree *tree = zswap_trees[type]; - struct zswap_entry *entry, *n; + struct zswap_tree *trees = zswap_trees[type]; + unsigned int i; - if (!tree) + if (!trees) return; - /* walk the tree and free everything */ - spin_lock(&tree->lock); - rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) - zswap_free_entry(entry); - tree->rbroot = RB_ROOT; - spin_unlock(&tree->lock); - kfree(tree); + for (i = 0; i < nr_zswap_trees[type]; i++) { + struct zswap_tree *tree = trees + i; + struct zswap_entry *entry, *n; + + /* walk the tree and free everything */ + spin_lock(&tree->lock); + rbtree_postorder_for_each_entry_safe(entry, n, + &tree->rbroot, + rbnode) + zswap_free_entry(entry); + tree->rbroot = RB_ROOT; + spin_unlock(&tree->lock); + } + + kvfree(trees); + nr_zswap_trees[type] = 0; zswap_trees[type] = NULL; } -- cgit v1.2.3 From 42d9358252e5d055223487d9f653c2a2ac859a2a Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Wed, 24 Jan 2024 12:03:49 -0800 Subject: mm/memory_hotplug: export mhp_supports_memmap_on_memory() In preparation for adding sysfs ABI to toggle memmap_on_memory semantics for drivers adding memory, export the mhp_supports_memmap_on_memory() helper. This allows drivers to check if memmap_on_memory support is available before trying to request it, and display an appropriate message if it isn't available. As part of this, remove the size argument to this - with recent updates to allow memmap_on_memory for larger ranges, and the internal splitting of altmaps into respective memory blocks, the size argument is meaningless. [akpm@linux-foundation.org: fix build] Link: https://lkml.kernel.org/r/20240124-vv-dax_abi-v7-4-20d16cb8d23d@intel.com Signed-off-by: Vishal Verma Acked-by: David Hildenbrand Suggested-by: David Hildenbrand Cc: Greg Kroah-Hartman Cc: Jonathan Cameron Cc: Li Zhijian Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Oscar Salvador Cc: Dan Williams Cc: Dave Jiang Cc: Dave Hansen Cc: Huang Ying Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 6 ++++++ mm/memory_hotplug.c | 17 ++++++----------- 2 files changed, 12 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index ee00015575aa..7a9ff464608d 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -137,6 +137,7 @@ struct mhp_params { bool mhp_range_allowed(u64 start, u64 size, bool need_mapping); struct range mhp_get_pluggable_range(bool need_mapping); +bool mhp_supports_memmap_on_memory(void); /* * Zone resizing functions @@ -278,6 +279,11 @@ static inline bool movable_node_is_enabled(void) return false; } +static inline bool mhp_supports_memmap_on_memory(void) +{ + return false; +} + static inline void pgdat_kswapd_lock(pg_data_t *pgdat) {} static inline void pgdat_kswapd_unlock(pg_data_t *pgdat) {} static inline void pgdat_kswapd_lock_init(pg_data_t *pgdat) {} diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 707027f69150..a444e2d7dd2b 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1337,7 +1337,7 @@ static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size) } #endif -static bool mhp_supports_memmap_on_memory(unsigned long size) +bool mhp_supports_memmap_on_memory(void) { unsigned long vmemmap_size = memory_block_memmap_size(); unsigned long memmap_pages = memory_block_memmap_on_memory_pages(); @@ -1346,17 +1346,11 @@ static bool mhp_supports_memmap_on_memory(unsigned long size) * Besides having arch support and the feature enabled at runtime, we * need a few more assumptions to hold true: * - * a) We span a single memory block: memory onlining/offlinin;g happens - * in memory block granularity. We don't want the vmemmap of online - * memory blocks to reside on offline memory blocks. In the future, - * we might want to support variable-sized memory blocks to make the - * feature more versatile. - * - * b) The vmemmap pages span complete PMDs: We don't want vmemmap code + * a) The vmemmap pages span complete PMDs: We don't want vmemmap code * to populate memory from the altmap for unrelated parts (i.e., * other memory blocks) * - * c) The vmemmap pages (and thereby the pages that will be exposed to + * b) The vmemmap pages (and thereby the pages that will be exposed to * the buddy) have to cover full pageblocks: memory onlining/offlining * code requires applicable ranges to be page-aligned, for example, to * set the migratetypes properly. @@ -1368,7 +1362,7 @@ static bool mhp_supports_memmap_on_memory(unsigned long size) * altmap as an alternative source of memory, and we do not exactly * populate a single PMD. */ - if (!mhp_memmap_on_memory() || size != memory_block_size_bytes()) + if (!mhp_memmap_on_memory()) return false; /* @@ -1391,6 +1385,7 @@ static bool mhp_supports_memmap_on_memory(unsigned long size) return arch_supports_memmap_on_memory(vmemmap_size); } +EXPORT_SYMBOL_GPL(mhp_supports_memmap_on_memory); static void __ref remove_memory_blocks_and_altmaps(u64 start, u64 size) { @@ -1526,7 +1521,7 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) * Self hosted memmap array */ if ((mhp_flags & MHP_MEMMAP_ON_MEMORY) && - mhp_supports_memmap_on_memory(memory_block_size_bytes())) { + mhp_supports_memmap_on_memory()) { ret = create_altmaps_and_memory_blocks(nid, group, start, size, mhp_flags); if (ret) goto error; -- cgit v1.2.3 From 9af47276ed83cc346263e56243756543a2a33c9d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Jan 2024 18:12:15 +0000 Subject: highmem: add kernel-doc for memcpy_*_folio() This was inadvertently skipped when adding the new functions. Link: https://lkml.kernel.org/r/20240124181217.1761674-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/highmem.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 451c1dff0e87..00341b56d291 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -439,6 +439,13 @@ static inline void memzero_page(struct page *page, size_t offset, size_t len) kunmap_local(addr); } +/** + * memcpy_from_folio - Copy a range of bytes from a folio. + * @to: The memory to copy to. + * @folio: The folio to read from. + * @offset: The first byte in the folio to read. + * @len: The number of bytes to copy. + */ static inline void memcpy_from_folio(char *to, struct folio *folio, size_t offset, size_t len) { @@ -460,6 +467,13 @@ static inline void memcpy_from_folio(char *to, struct folio *folio, } while (len > 0); } +/** + * memcpy_to_folio - Copy a range of bytes to a folio. + * @folio: The folio to write to. + * @offset: The first byte in the folio to store to. + * @from: The memory to copy from. + * @len: The number of bytes to copy. + */ static inline void memcpy_to_folio(struct folio *folio, size_t offset, const char *from, size_t len) { -- cgit v1.2.3 From 5cec4eb7fad6fb1e9a3dd8403b558d1eff7490ff Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Fri, 26 Jan 2024 16:19:44 +0800 Subject: mm and cache_info: remove unnecessary CPU cache info update For each CPU hotplug event, we will update per-CPU data slice size and corresponding PCP configuration for every online CPU to make the implementation simple. But, Kyle reported that this takes tens seconds during boot on a machine with 34 zones and 3840 CPUs. So, in this patch, for each CPU hotplug event, we only update per-CPU data slice size and corresponding PCP configuration for the CPUs that share caches with the hotplugged CPU. With the patch, the system boot time reduces 67 seconds on the machine. Link: https://lkml.kernel.org/r/20240126081944.414520-1-ying.huang@intel.com Fixes: 362d37a106dd ("mm, pcp: reduce lock contention for draining high-order pages") Signed-off-by: "Huang, Ying" Originally-by: Kyle Meyer Reported-and-tested-by: Kyle Meyer Cc: Sudeep Holla Cc: Mel Gorman Signed-off-by: Andrew Morton --- drivers/base/cacheinfo.c | 50 ++++++++++++++++++++++++++++++++++++++++++------ include/linux/gfp.h | 2 +- mm/page_alloc.c | 39 +++++++++++++++++-------------------- 3 files changed, 63 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index f1e79263fe61..23b8cba4a2a3 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -898,6 +898,37 @@ err: return rc; } +static unsigned int cpu_map_shared_cache(bool online, unsigned int cpu, + cpumask_t **map) +{ + struct cacheinfo *llc, *sib_llc; + unsigned int sibling; + + if (!last_level_cache_is_valid(cpu)) + return 0; + + llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); + + if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED) + return 0; + + if (online) { + *map = &llc->shared_cpu_map; + return cpumask_weight(*map); + } + + /* shared_cpu_map of offlined CPU will be cleared, so use sibling map */ + for_each_cpu(sibling, &llc->shared_cpu_map) { + if (sibling == cpu || !last_level_cache_is_valid(sibling)) + continue; + sib_llc = per_cpu_cacheinfo_idx(sibling, cache_leaves(sibling) - 1); + *map = &sib_llc->shared_cpu_map; + return cpumask_weight(*map); + } + + return 0; +} + /* * Calculate the size of the per-CPU data cache slice. This can be * used to estimate the size of the data cache slice that can be used @@ -929,28 +960,31 @@ static void update_per_cpu_data_slice_size_cpu(unsigned int cpu) ci->per_cpu_data_slice_size = llc->size / nr_shared; } -static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu) +static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu, + cpumask_t *cpu_map) { unsigned int icpu; - for_each_online_cpu(icpu) { + for_each_cpu(icpu, cpu_map) { if (!cpu_online && icpu == cpu) continue; update_per_cpu_data_slice_size_cpu(icpu); + setup_pcp_cacheinfo(icpu); } } static int cacheinfo_cpu_online(unsigned int cpu) { int rc = detect_cache_attributes(cpu); + cpumask_t *cpu_map; if (rc) return rc; rc = cache_add_dev(cpu); if (rc) goto err; - update_per_cpu_data_slice_size(true, cpu); - setup_pcp_cacheinfo(); + if (cpu_map_shared_cache(true, cpu, &cpu_map)) + update_per_cpu_data_slice_size(true, cpu, cpu_map); return 0; err: free_cache_attributes(cpu); @@ -959,12 +993,16 @@ err: static int cacheinfo_cpu_pre_down(unsigned int cpu) { + cpumask_t *cpu_map; + unsigned int nr_shared; + + nr_shared = cpu_map_shared_cache(false, cpu, &cpu_map); if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map)) cpu_cache_sysfs_exit(cpu); free_cache_attributes(cpu); - update_per_cpu_data_slice_size(false, cpu); - setup_pcp_cacheinfo(); + if (nr_shared > 1) + update_per_cpu_data_slice_size(false, cpu, cpu_map); return 0; } diff --git a/include/linux/gfp.h b/include/linux/gfp.h index de292a007138..09e22091f1b0 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -334,7 +334,7 @@ void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); void page_alloc_init_late(void); -void setup_pcp_cacheinfo(void); +void setup_pcp_cacheinfo(unsigned int cpu); /* * gfp_allowed_mask is set to GFP_BOOT_MASK during early boot to restrict what diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 150d4f23b010..9faca05d124e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5572,37 +5572,34 @@ static void zone_pcp_update(struct zone *zone, int cpu_online) mutex_unlock(&pcp_batch_high_lock); } -static void zone_pcp_update_cacheinfo(struct zone *zone) +static void zone_pcp_update_cacheinfo(struct zone *zone, unsigned int cpu) { - int cpu; struct per_cpu_pages *pcp; struct cpu_cacheinfo *cci; - for_each_online_cpu(cpu) { - pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); - cci = get_cpu_cacheinfo(cpu); - /* - * If data cache slice of CPU is large enough, "pcp->batch" - * pages can be preserved in PCP before draining PCP for - * consecutive high-order pages freeing without allocation. - * This can reduce zone lock contention without hurting - * cache-hot pages sharing. - */ - spin_lock(&pcp->lock); - if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch) - pcp->flags |= PCPF_FREE_HIGH_BATCH; - else - pcp->flags &= ~PCPF_FREE_HIGH_BATCH; - spin_unlock(&pcp->lock); - } + pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); + cci = get_cpu_cacheinfo(cpu); + /* + * If data cache slice of CPU is large enough, "pcp->batch" + * pages can be preserved in PCP before draining PCP for + * consecutive high-order pages freeing without allocation. + * This can reduce zone lock contention without hurting + * cache-hot pages sharing. + */ + spin_lock(&pcp->lock); + if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch) + pcp->flags |= PCPF_FREE_HIGH_BATCH; + else + pcp->flags &= ~PCPF_FREE_HIGH_BATCH; + spin_unlock(&pcp->lock); } -void setup_pcp_cacheinfo(void) +void setup_pcp_cacheinfo(unsigned int cpu) { struct zone *zone; for_each_populated_zone(zone) - zone_pcp_update_cacheinfo(zone); + zone_pcp_update_cacheinfo(zone, cpu); } /* -- cgit v1.2.3 From 7dbbc8f57d4ba3c369b692db9fd2c9653abf0bb5 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Fri, 26 Jan 2024 08:06:43 +0000 Subject: x86/mm: delete unused cpu argument to leave_mm() The argument is unused since commit 3d28ebceaffa ("x86/mm: Rework lazy TLB to track the actual loaded mm"), delete it. Link: https://lkml.kernel.org/r/20240126080644.1714297-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Cc: Andy Lutomirski Cc: Borislav Petkov (AMD) Cc: Dave Hansen Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/include/asm/mmu.h | 2 +- arch/x86/kernel/alternative.c | 2 +- arch/x86/mm/tlb.c | 2 +- arch/x86/xen/mmu_pv.c | 2 +- drivers/cpuidle/cpuidle.c | 2 +- include/linux/mmu_context.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index 0da5c227f490..ce4677b8b735 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -75,7 +75,7 @@ typedef struct { .lock = __MUTEX_INITIALIZER(mm.context.lock), \ } -void leave_mm(int cpu); +void leave_mm(void); #define leave_mm leave_mm #endif /* _ASM_X86_MMU_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 1d85cb7071cb..21108d8e6f6b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1805,7 +1805,7 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) * restoring the previous mm. */ if (this_cpu_read(cpu_tlbstate_shared.is_lazy)) - leave_mm(smp_processor_id()); + leave_mm(); temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm); switch_mm_irqs_off(NULL, mm, current); diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 5768d386efab..80b0caa82a91 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -299,7 +299,7 @@ static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, unsigned long lam, write_cr3(new_mm_cr3); } -void leave_mm(int cpu) +void leave_mm(void) { struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm); diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 72af496a160c..218773cfb009 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -913,7 +913,7 @@ static void drop_mm_ref_this_cpu(void *info) struct mm_struct *mm = info; if (this_cpu_read(cpu_tlbstate.loaded_mm) == mm) - leave_mm(smp_processor_id()); + leave_mm(); /* * If this cpu still has a stale cr3 reference, then make sure diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 737a026ef58a..02e40fd7d948 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -237,7 +237,7 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, } if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED) - leave_mm(dev->cpu); + leave_mm(); /* Take note of the planned idle state. */ sched_idle_set_state(target_state); diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index f2b7a3f04099..bbaec80c78c5 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -11,7 +11,7 @@ #endif #ifndef leave_mm -static inline void leave_mm(int cpu) { } +static inline void leave_mm(void) { } #endif /* -- cgit v1.2.3 From 3f798aa6121ab3eb572f96ab2d8558894d979a4c Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Sun, 28 Jan 2024 13:28:51 +0000 Subject: mm/list_lru: remove list_lru_putback() Since the only user zswap_lru_putback() has gone, remove list_lru_putback() too. Link: https://lkml.kernel.org/r/20240126-zswap-writeback-race-v2-3-b10479847099@bytedance.com Signed-off-by: Chengming Zhou Acked-by: Yosry Ahmed Cc: Chris Li Cc: Johannes Weiner Cc: Nhat Pham Signed-off-by: Andrew Morton --- include/linux/list_lru.h | 16 ---------------- mm/list_lru.c | 14 -------------- mm/zswap.c | 2 +- 3 files changed, 1 insertion(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h index c679e6b293c4..f2882a820690 100644 --- a/include/linux/list_lru.h +++ b/include/linux/list_lru.h @@ -168,22 +168,6 @@ static inline unsigned long list_lru_count(struct list_lru *lru) void list_lru_isolate(struct list_lru_one *list, struct list_head *item); void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item, struct list_head *head); -/** - * list_lru_putback: undo list_lru_isolate - * @lru: the lru pointer. - * @item: the item to put back. - * @nid: the node id of the sublist to put the item back to. - * @memcg: the cgroup of the sublist to put the item back to. - * - * Put back an isolated item into its original LRU. Note that unlike - * list_lru_add, this does not increment the node LRU count (as - * list_lru_isolate does not originally decrement this count). - * - * Since we might have dropped the LRU lock in between, recompute list_lru_one - * from the node's id and memcg. - */ -void list_lru_putback(struct list_lru *lru, struct list_head *item, int nid, - struct mem_cgroup *memcg); typedef enum lru_status (*list_lru_walk_cb)(struct list_head *item, struct list_lru_one *list, spinlock_t *lock, void *cb_arg); diff --git a/mm/list_lru.c b/mm/list_lru.c index 158781d1d3c2..61f3b6b1134f 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -162,20 +162,6 @@ void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item, } EXPORT_SYMBOL_GPL(list_lru_isolate_move); -void list_lru_putback(struct list_lru *lru, struct list_head *item, int nid, - struct mem_cgroup *memcg) -{ - struct list_lru_one *list = - list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); - - if (list_empty(item)) { - list_add_tail(item, &list->list); - if (!list->nr_items++) - set_shrinker_bit(memcg, nid, lru_shrinker_id(lru)); - } -} -EXPORT_SYMBOL_GPL(list_lru_putback); - unsigned long list_lru_count_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg) { diff --git a/mm/zswap.c b/mm/zswap.c index ddc8f930d343..2d7f594e6d07 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -410,7 +410,7 @@ static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry) * 2. list_lru_add() is called after memcg->kmemcg_id is updated. The * new entry will be added directly to memcg's parent's list_lru. * - * Similar reasoning holds for list_lru_del() and list_lru_putback(). + * Similar reasoning holds for list_lru_del(). */ rcu_read_lock(); memcg = mem_cgroup_from_entry(entry); -- cgit v1.2.3 From fa3bea4e1f8202d787709b7e3654eb0a99aed758 Mon Sep 17 00:00:00 2001 From: Gregory Price Date: Fri, 2 Feb 2024 12:02:37 -0500 Subject: mm/mempolicy: introduce MPOL_WEIGHTED_INTERLEAVE for weighted interleaving When a system has multiple NUMA nodes and it becomes bandwidth hungry, using the current MPOL_INTERLEAVE could be an wise option. However, if those NUMA nodes consist of different types of memory such as socket-attached DRAM and CXL/PCIe attached DRAM, the round-robin based interleave policy does not optimally distribute data to make use of their different bandwidth characteristics. Instead, interleave is more effective when the allocation policy follows each NUMA nodes' bandwidth weight rather than a simple 1:1 distribution. This patch introduces a new memory policy, MPOL_WEIGHTED_INTERLEAVE, enabling weighted interleave between NUMA nodes. Weighted interleave allows for proportional distribution of memory across multiple numa nodes, preferably apportioned to match the bandwidth of each node. For example, if a system has 1 CPU node (0), and 2 memory nodes (0,1), with bandwidth of (100GB/s, 50GB/s) respectively, the appropriate weight distribution is (2:1). Weights for each node can be assigned via the new sysfs extension: /sys/kernel/mm/mempolicy/weighted_interleave/ For now, the default value of all nodes will be `1`, which matches the behavior of standard 1:1 round-robin interleave. An extension will be added in the future to allow default values to be registered at kernel and device bringup time. The policy allocates a number of pages equal to the set weights. For example, if the weights are (2,1), then 2 pages will be allocated on node0 for every 1 page allocated on node1. The new flag MPOL_WEIGHTED_INTERLEAVE can be used in set_mempolicy(2) and mbind(2). Some high level notes about the pieces of weighted interleave: current->il_prev: Tracks the node previously allocated from. current->il_weight: The active weight of the current node (current->il_prev) When this reaches 0, current->il_prev is set to the next node and current->il_weight is set to the next weight. weighted_interleave_nodes: Counts the number of allocations as they occur, and applies the weight for the current node. When the weight reaches 0, switch to the next node. Operates only on task->mempolicy. weighted_interleave_nid: Gets the total weight of the nodemask as well as each individual node weight, then calculates the node based on the given index. Operates on VMA policies. bulk_array_weighted_interleave: Gets the total weight of the nodemask as well as each individual node weight, then calculates the number of "interleave rounds" as well as any delta ("partial round"). Calculates the number of pages for each node and allocates them. If a node was scheduled for interleave via interleave_nodes, the current weight will be allocated first. Operates only on the task->mempolicy. One piece of complexity is the interaction between a recent refactor which split the logic to acquire the "ilx" (interleave index) of an allocation and the actually application of the interleave. If a call to alloc_pages_mpol() were made with a weighted-interleave policy and ilx set to NO_INTERLEAVE_INDEX, weighted_interleave_nodes() would operate on a VMA policy - violating the description above. An inspection of all callers of alloc_pages_mpol() shows that all external callers set ilx to `0`, an index value, or will call get_vma_policy() to acquire the ilx. For example, mm/shmem.c may call into alloc_pages_mpol. The call stacks all set (pgoff_t ilx) or end up in `get_vma_policy()`. This enforces the `weighted_interleave_nodes()` and `weighted_interleave_nid()` policy requirements (task/vma respectively). Link: https://lkml.kernel.org/r/20240202170238.90004-4-gregory.price@memverge.com Suggested-by: Hasan Al Maruf Signed-off-by: Gregory Price Co-developed-by: Rakie Kim Signed-off-by: Rakie Kim Co-developed-by: Honggyu Kim Signed-off-by: Honggyu Kim Co-developed-by: Hyeongtak Ji Signed-off-by: Hyeongtak Ji Co-developed-by: Srinivasulu Thanneeru Signed-off-by: Srinivasulu Thanneeru Co-developed-by: Ravi Jonnalagadda Signed-off-by: Ravi Jonnalagadda Reviewed-by: "Huang, Ying" Cc: Dan Williams Cc: Johannes Weiner Cc: Jonathan Corbet Cc: Michal Hocko Signed-off-by: Andrew Morton --- .../admin-guide/mm/numa_memory_policy.rst | 9 + include/linux/sched.h | 1 + include/uapi/linux/mempolicy.h | 1 + mm/mempolicy.c | 218 ++++++++++++++++++++- 4 files changed, 225 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/mm/numa_memory_policy.rst b/Documentation/admin-guide/mm/numa_memory_policy.rst index eca38fa81e0f..a70f20ce1ffb 100644 --- a/Documentation/admin-guide/mm/numa_memory_policy.rst +++ b/Documentation/admin-guide/mm/numa_memory_policy.rst @@ -250,6 +250,15 @@ MPOL_PREFERRED_MANY can fall back to all existing numa nodes. This is effectively MPOL_PREFERRED allowed for a mask rather than a single node. +MPOL_WEIGHTED_INTERLEAVE + This mode operates the same as MPOL_INTERLEAVE, except that + interleaving behavior is executed based on weights set in + /sys/kernel/mm/mempolicy/weighted_interleave/ + + Weighted interleave allocates pages on nodes according to a + weight. For example if nodes [0,1] are weighted [5,2], 5 pages + will be allocated on node0 for every 2 pages allocated on node1. + NUMA memory policy supports the following optional mode flags: MPOL_F_STATIC_NODES diff --git a/include/linux/sched.h b/include/linux/sched.h index ffe8f618ab86..b9ce285d8c9c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1259,6 +1259,7 @@ struct task_struct { /* Protected by alloc_lock: */ struct mempolicy *mempolicy; short il_prev; + u8 il_weight; short pref_node_fork; #endif #ifdef CONFIG_NUMA_BALANCING diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index a8963f7ef4c2..1f9bb10d1a47 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -23,6 +23,7 @@ enum { MPOL_INTERLEAVE, MPOL_LOCAL, MPOL_PREFERRED_MANY, + MPOL_WEIGHTED_INTERLEAVE, MPOL_MAX, /* always last member of enum */ }; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 1bdc7d0d1b0b..a8db92c23697 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -19,6 +19,13 @@ * for anonymous memory. For process policy an process counter * is used. * + * weighted interleave + * Allocate memory interleaved over a set of nodes based on + * a set of weights (per-node), with normal fallback if it + * fails. Otherwise operates the same as interleave. + * Example: nodeset(0,1) & weights (2,1) - 2 pages allocated + * on node 0 for every 1 page allocated on node 1. + * * bind Only allocate memory on a specific set of nodes, * no fallback. * FIXME: memory is allocated starting with the first node @@ -441,6 +448,10 @@ static const struct mempolicy_operations mpol_ops[MPOL_MAX] = { .create = mpol_new_nodemask, .rebind = mpol_rebind_preferred, }, + [MPOL_WEIGHTED_INTERLEAVE] = { + .create = mpol_new_nodemask, + .rebind = mpol_rebind_nodemask, + }, }; static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist, @@ -858,8 +869,11 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, old = current->mempolicy; current->mempolicy = new; - if (new && new->mode == MPOL_INTERLEAVE) + if (new && (new->mode == MPOL_INTERLEAVE || + new->mode == MPOL_WEIGHTED_INTERLEAVE)) { current->il_prev = MAX_NUMNODES-1; + current->il_weight = 0; + } task_unlock(current); mpol_put(old); ret = 0; @@ -884,6 +898,7 @@ static void get_policy_nodemask(struct mempolicy *pol, nodemask_t *nodes) case MPOL_INTERLEAVE: case MPOL_PREFERRED: case MPOL_PREFERRED_MANY: + case MPOL_WEIGHTED_INTERLEAVE: *nodes = pol->nodes; break; case MPOL_LOCAL: @@ -968,6 +983,13 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, } else if (pol == current->mempolicy && pol->mode == MPOL_INTERLEAVE) { *policy = next_node_in(current->il_prev, pol->nodes); + } else if (pol == current->mempolicy && + pol->mode == MPOL_WEIGHTED_INTERLEAVE) { + if (current->il_weight) + *policy = current->il_prev; + else + *policy = next_node_in(current->il_prev, + pol->nodes); } else { err = -EINVAL; goto out; @@ -1332,7 +1354,8 @@ static long do_mbind(unsigned long start, unsigned long len, * VMAs, the nodes will still be interleaved from the targeted * nodemask, but one by one may be selected differently. */ - if (new->mode == MPOL_INTERLEAVE) { + if (new->mode == MPOL_INTERLEAVE || + new->mode == MPOL_WEIGHTED_INTERLEAVE) { struct page *page; unsigned int order; unsigned long addr = -EFAULT; @@ -1780,7 +1803,8 @@ struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, * @vma: virtual memory area whose policy is sought * @addr: address in @vma for shared policy lookup * @order: 0, or appropriate huge_page_order for interleaving - * @ilx: interleave index (output), for use only when MPOL_INTERLEAVE + * @ilx: interleave index (output), for use only when MPOL_INTERLEAVE or + * MPOL_WEIGHTED_INTERLEAVE * * Returns effective policy for a VMA at specified address. * Falls back to current->mempolicy or system default policy, as necessary. @@ -1797,7 +1821,8 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, pol = __get_vma_policy(vma, addr, ilx); if (!pol) pol = get_task_policy(current); - if (pol->mode == MPOL_INTERLEAVE) { + if (pol->mode == MPOL_INTERLEAVE || + pol->mode == MPOL_WEIGHTED_INTERLEAVE) { *ilx += vma->vm_pgoff >> order; *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); } @@ -1847,6 +1872,22 @@ bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) return zone >= dynamic_policy_zone; } +static unsigned int weighted_interleave_nodes(struct mempolicy *policy) +{ + unsigned int node = current->il_prev; + + if (!current->il_weight || !node_isset(node, policy->nodes)) { + node = next_node_in(node, policy->nodes); + /* can only happen if nodemask is being rebound */ + if (node == MAX_NUMNODES) + return node; + current->il_prev = node; + current->il_weight = get_il_weight(node); + } + current->il_weight--; + return node; +} + /* Do dynamic interleaving for a process */ static unsigned int interleave_nodes(struct mempolicy *policy) { @@ -1881,6 +1922,9 @@ unsigned int mempolicy_slab_node(void) case MPOL_INTERLEAVE: return interleave_nodes(policy); + case MPOL_WEIGHTED_INTERLEAVE: + return weighted_interleave_nodes(policy); + case MPOL_BIND: case MPOL_PREFERRED_MANY: { @@ -1919,6 +1963,45 @@ static unsigned int read_once_policy_nodemask(struct mempolicy *pol, return nodes_weight(*mask); } +static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx) +{ + nodemask_t nodemask; + unsigned int target, nr_nodes; + u8 *table; + unsigned int weight_total = 0; + u8 weight; + int nid; + + nr_nodes = read_once_policy_nodemask(pol, &nodemask); + if (!nr_nodes) + return numa_node_id(); + + rcu_read_lock(); + table = rcu_dereference(iw_table); + /* calculate the total weight */ + for_each_node_mask(nid, nodemask) { + /* detect system default usage */ + weight = table ? table[nid] : 1; + weight = weight ? weight : 1; + weight_total += weight; + } + + /* Calculate the node offset based on totals */ + target = ilx % weight_total; + nid = first_node(nodemask); + while (target) { + /* detect system default usage */ + weight = table ? table[nid] : 1; + weight = weight ? weight : 1; + if (target < weight) + break; + target -= weight; + nid = next_node_in(nid, nodemask); + } + rcu_read_unlock(); + return nid; +} + /* * Do static interleaving for interleave index @ilx. Returns the ilx'th * node in pol->nodes (starting from ilx=0), wrapping around if ilx @@ -1979,6 +2062,11 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol, *nid = (ilx == NO_INTERLEAVE_INDEX) ? interleave_nodes(pol) : interleave_nid(pol, ilx); break; + case MPOL_WEIGHTED_INTERLEAVE: + *nid = (ilx == NO_INTERLEAVE_INDEX) ? + weighted_interleave_nodes(pol) : + weighted_interleave_nid(pol, ilx); + break; } return nodemask; @@ -2040,6 +2128,7 @@ bool init_nodemask_of_mempolicy(nodemask_t *mask) case MPOL_PREFERRED_MANY: case MPOL_BIND: case MPOL_INTERLEAVE: + case MPOL_WEIGHTED_INTERLEAVE: *mask = mempolicy->nodes; break; @@ -2140,6 +2229,7 @@ struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, * node in its nodemask, we allocate the standard way. */ if (pol->mode != MPOL_INTERLEAVE && + pol->mode != MPOL_WEIGHTED_INTERLEAVE && (!nodemask || node_isset(nid, *nodemask))) { /* * First, try to allocate THP only on local node, but @@ -2275,6 +2365,114 @@ static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp, return total_allocated; } +static unsigned long alloc_pages_bulk_array_weighted_interleave(gfp_t gfp, + struct mempolicy *pol, unsigned long nr_pages, + struct page **page_array) +{ + struct task_struct *me = current; + unsigned long total_allocated = 0; + unsigned long nr_allocated = 0; + unsigned long rounds; + unsigned long node_pages, delta; + u8 *table, *weights, weight; + unsigned int weight_total = 0; + unsigned long rem_pages = nr_pages; + nodemask_t nodes; + int nnodes, node; + int resume_node = MAX_NUMNODES - 1; + u8 resume_weight = 0; + int prev_node; + int i; + + if (!nr_pages) + return 0; + + nnodes = read_once_policy_nodemask(pol, &nodes); + if (!nnodes) + return 0; + + /* Continue allocating from most recent node and adjust the nr_pages */ + node = me->il_prev; + weight = me->il_weight; + if (weight && node_isset(node, nodes)) { + node_pages = min(rem_pages, weight); + nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, + NULL, page_array); + page_array += nr_allocated; + total_allocated += nr_allocated; + /* if that's all the pages, no need to interleave */ + if (rem_pages <= weight) { + me->il_weight -= rem_pages; + return total_allocated; + } + /* Otherwise we adjust remaining pages, continue from there */ + rem_pages -= weight; + } + /* clear active weight in case of an allocation failure */ + me->il_weight = 0; + prev_node = node; + + /* create a local copy of node weights to operate on outside rcu */ + weights = kzalloc(nr_node_ids, GFP_KERNEL); + if (!weights) + return total_allocated; + + rcu_read_lock(); + table = rcu_dereference(iw_table); + if (table) + memcpy(weights, table, nr_node_ids); + rcu_read_unlock(); + + /* calculate total, detect system default usage */ + for_each_node_mask(node, nodes) { + if (!weights[node]) + weights[node] = 1; + weight_total += weights[node]; + } + + /* + * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls. + * Track which node weighted interleave should resume from. + * + * if (rounds > 0) and (delta == 0), resume_node will always be + * the node following prev_node and its weight. + */ + rounds = rem_pages / weight_total; + delta = rem_pages % weight_total; + resume_node = next_node_in(prev_node, nodes); + resume_weight = weights[resume_node]; + for (i = 0; i < nnodes; i++) { + node = next_node_in(prev_node, nodes); + weight = weights[node]; + node_pages = weight * rounds; + /* If a delta exists, add this node's portion of the delta */ + if (delta > weight) { + node_pages += weight; + delta -= weight; + } else if (delta) { + /* when delta is depleted, resume from that node */ + node_pages += delta; + resume_node = node; + resume_weight = weight - delta; + delta = 0; + } + /* node_pages can be 0 if an allocation fails and rounds == 0 */ + if (!node_pages) + break; + nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, + NULL, page_array); + page_array += nr_allocated; + total_allocated += nr_allocated; + if (total_allocated == nr_pages) + break; + prev_node = node; + } + me->il_prev = resume_node; + me->il_weight = resume_weight; + kfree(weights); + return total_allocated; +} + static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid, struct mempolicy *pol, unsigned long nr_pages, struct page **page_array) @@ -2315,6 +2513,10 @@ unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp, return alloc_pages_bulk_array_interleave(gfp, pol, nr_pages, page_array); + if (pol->mode == MPOL_WEIGHTED_INTERLEAVE) + return alloc_pages_bulk_array_weighted_interleave( + gfp, pol, nr_pages, page_array); + if (pol->mode == MPOL_PREFERRED_MANY) return alloc_pages_bulk_array_preferred_many(gfp, numa_node_id(), pol, nr_pages, page_array); @@ -2390,6 +2592,7 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b) case MPOL_INTERLEAVE: case MPOL_PREFERRED: case MPOL_PREFERRED_MANY: + case MPOL_WEIGHTED_INTERLEAVE: return !!nodes_equal(a->nodes, b->nodes); case MPOL_LOCAL: return true; @@ -2526,6 +2729,10 @@ int mpol_misplaced(struct folio *folio, struct vm_area_struct *vma, polnid = interleave_nid(pol, ilx); break; + case MPOL_WEIGHTED_INTERLEAVE: + polnid = weighted_interleave_nid(pol, ilx); + break; + case MPOL_PREFERRED: if (node_isset(curnid, pol->nodes)) goto out; @@ -2900,6 +3107,7 @@ static const char * const policy_modes[] = [MPOL_PREFERRED] = "prefer", [MPOL_BIND] = "bind", [MPOL_INTERLEAVE] = "interleave", + [MPOL_WEIGHTED_INTERLEAVE] = "weighted interleave", [MPOL_LOCAL] = "local", [MPOL_PREFERRED_MANY] = "prefer (many)", }; @@ -2959,6 +3167,7 @@ int mpol_parse_str(char *str, struct mempolicy **mpol) } break; case MPOL_INTERLEAVE: + case MPOL_WEIGHTED_INTERLEAVE: /* * Default to online nodes with memory if no nodelist */ @@ -3069,6 +3278,7 @@ void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) case MPOL_PREFERRED_MANY: case MPOL_BIND: case MPOL_INTERLEAVE: + case MPOL_WEIGHTED_INTERLEAVE: nodes = pol->nodes; break; default: -- cgit v1.2.3 From a5e8131a0329673f70faee2e9ffb02e8a5bb3c89 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 30 Jan 2024 11:34:33 +0100 Subject: arm64, powerpc, riscv, s390, x86: ptdump: refactor CONFIG_DEBUG_WX All architectures using the core ptdump functionality also implement CONFIG_DEBUG_WX, and they all do it more or less the same way, with a function called debug_checkwx() that is called by mark_rodata_ro(), which is a substitute to ptdump_check_wx() when CONFIG_DEBUG_WX is set and a no-op otherwise. Refactor by centrally defining debug_checkwx() in linux/ptdump.h and call debug_checkwx() immediately after calling mark_rodata_ro() instead of calling it at the end of every mark_rodata_ro(). On x86_32, mark_rodata_ro() first checks __supported_pte_mask has _PAGE_NX before calling debug_checkwx(). Now the check is inside the callee ptdump_walk_pgd_level_checkwx(). On powerpc_64, mark_rodata_ro() bails out early before calling ptdump_check_wx() when the MMU doesn't have KERNEL_RO feature. The check is now also done in ptdump_check_wx() as it is called outside mark_rodata_ro(). Link: https://lkml.kernel.org/r/a59b102d7964261d31ead0316a9f18628e4e7a8e.1706610398.git.christophe.leroy@csgroup.eu Signed-off-by: Christophe Leroy Reviewed-by: Alexandre Ghiti Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: "Aneesh Kumar K.V (IBM)" Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: Gerald Schaefer Cc: Greg KH Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Kees Cook Cc: Mark Rutland Cc: Michael Ellerman Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Phong Tran Cc: Russell King Cc: Steven Price Cc: Sven Schnelle Cc: Thomas Gleixner Cc: Vasily Gorbik Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/include/asm/ptdump.h | 7 ------- arch/arm64/mm/mmu.c | 2 -- arch/powerpc/mm/mmu_decl.h | 6 ------ arch/powerpc/mm/pgtable_32.c | 4 ---- arch/powerpc/mm/pgtable_64.c | 3 --- arch/powerpc/mm/ptdump/ptdump.c | 3 +++ arch/riscv/include/asm/ptdump.h | 22 ---------------------- arch/riscv/mm/init.c | 3 --- arch/riscv/mm/ptdump.c | 1 - arch/s390/include/asm/ptdump.h | 14 -------------- arch/s390/mm/dump_pagetables.c | 1 - arch/s390/mm/init.c | 2 -- arch/x86/include/asm/pgtable.h | 3 +-- arch/x86/mm/dump_pagetables.c | 3 +++ arch/x86/mm/init_32.c | 2 -- arch/x86/mm/init_64.c | 2 -- include/linux/ptdump.h | 7 +++++++ init/main.c | 2 ++ 18 files changed, 16 insertions(+), 71 deletions(-) delete mode 100644 arch/riscv/include/asm/ptdump.h delete mode 100644 arch/s390/include/asm/ptdump.h (limited to 'include/linux') diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index 581caac525b0..5b1701c76d1c 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -29,13 +29,6 @@ void __init ptdump_debugfs_register(struct ptdump_info *info, const char *name); static inline void ptdump_debugfs_register(struct ptdump_info *info, const char *name) { } #endif -void ptdump_check_wx(void); #endif /* CONFIG_PTDUMP_CORE */ -#ifdef CONFIG_DEBUG_WX -#define debug_checkwx() ptdump_check_wx() -#else -#define debug_checkwx() do { } while (0) -#endif - #endif /* __ASM_PTDUMP_H */ diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1ac7467d34c9..3a27d887f7dd 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -632,8 +632,6 @@ void mark_rodata_ro(void) section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, section_size, PAGE_KERNEL_RO); - - debug_checkwx(); } static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h index 72341b9fb552..90dcc2844056 100644 --- a/arch/powerpc/mm/mmu_decl.h +++ b/arch/powerpc/mm/mmu_decl.h @@ -171,12 +171,6 @@ static inline void mmu_mark_rodata_ro(void) { } void __init mmu_mapin_immr(void); #endif -#ifdef CONFIG_DEBUG_WX -void ptdump_check_wx(void); -#else -static inline void ptdump_check_wx(void) { } -#endif - static inline bool debug_pagealloc_enabled_or_kfence(void) { return IS_ENABLED(CONFIG_KFENCE) || debug_pagealloc_enabled(); diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index 5c02fd08d61e..12498017da8e 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -153,7 +153,6 @@ void mark_rodata_ro(void) if (v_block_mapped((unsigned long)_stext + 1)) { mmu_mark_rodata_ro(); - ptdump_check_wx(); return; } @@ -166,9 +165,6 @@ void mark_rodata_ro(void) PFN_DOWN((unsigned long)_stext); set_memory_ro((unsigned long)_stext, numpages); - - // mark_initmem_nx() should have already run by now - ptdump_check_wx(); } #endif diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 5ac1fd30341b..1b366526f4f2 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -150,9 +150,6 @@ void mark_rodata_ro(void) radix__mark_rodata_ro(); else hash__mark_rodata_ro(); - - // mark_initmem_nx() should have already run by now - ptdump_check_wx(); } void mark_initmem_nx(void) diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c index 2313053fe679..620d4917ebe8 100644 --- a/arch/powerpc/mm/ptdump/ptdump.c +++ b/arch/powerpc/mm/ptdump/ptdump.c @@ -343,6 +343,9 @@ void ptdump_check_wx(void) } }; + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !mmu_has_feature(MMU_FTR_KERNEL_RO)) + return; + ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); if (st.wx_pages) diff --git a/arch/riscv/include/asm/ptdump.h b/arch/riscv/include/asm/ptdump.h deleted file mode 100644 index 3c9ea6dd5af7..000000000000 --- a/arch/riscv/include/asm/ptdump.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2019 SiFive - */ - -#ifndef _ASM_RISCV_PTDUMP_H -#define _ASM_RISCV_PTDUMP_H - -void ptdump_check_wx(void); - -#ifdef CONFIG_DEBUG_WX -static inline void debug_checkwx(void) -{ - ptdump_check_wx(); -} -#else -static inline void debug_checkwx(void) -{ -} -#endif - -#endif /* _ASM_RISCV_PTDUMP_H */ diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fa34cf55037b..a4f218cfb845 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -723,8 +722,6 @@ void mark_rodata_ro(void) if (IS_ENABLED(CONFIG_64BIT)) set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data), set_memory_ro); - - debug_checkwx(); } #else static __init pgprot_t pgprot_from_va(uintptr_t va) diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 657c27bc07a7..075265603313 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -9,7 +9,6 @@ #include #include -#include #include #include diff --git a/arch/s390/include/asm/ptdump.h b/arch/s390/include/asm/ptdump.h deleted file mode 100644 index f960b2896606..000000000000 --- a/arch/s390/include/asm/ptdump.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef _ASM_S390_PTDUMP_H -#define _ASM_S390_PTDUMP_H - -void ptdump_check_wx(void); - -static inline void debug_checkwx(void) -{ - if (IS_ENABLED(CONFIG_DEBUG_WX)) - ptdump_check_wx(); -} - -#endif /* _ASM_S390_PTDUMP_H */ diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index d37a8f607b71..8dcb4e0c71bd 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 8d9a60ccb777..f6391442c0c2 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -109,7 +108,6 @@ void mark_rodata_ro(void) __set_memory_ro(__start_ro_after_init, __end_ro_after_init); pr_info("Write protected read-only-after-init data: %luk\n", size >> 10); - debug_checkwx(); } int set_memory_encrypted(unsigned long vaddr, int numpages) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 9d077bca6a10..6c979028e521 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -32,6 +32,7 @@ void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm); void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm, bool user); void ptdump_walk_pgd_level_checkwx(void); +#define ptdump_check_wx ptdump_walk_pgd_level_checkwx void ptdump_walk_user_pgd_level_checkwx(void); /* @@ -41,10 +42,8 @@ void ptdump_walk_user_pgd_level_checkwx(void); #define pgprot_decrypted(prot) __pgprot(cc_mkdec(pgprot_val(prot))) #ifdef CONFIG_DEBUG_WX -#define debug_checkwx() ptdump_walk_pgd_level_checkwx() #define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx() #else -#define debug_checkwx() do { } while (0) #define debug_checkwx_user() do { } while (0) #endif diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c index e1b599ecbbc2..0008524eebe9 100644 --- a/arch/x86/mm/dump_pagetables.c +++ b/arch/x86/mm/dump_pagetables.c @@ -433,6 +433,9 @@ void ptdump_walk_user_pgd_level_checkwx(void) void ptdump_walk_pgd_level_checkwx(void) { + if (!(__supported_pte_mask & _PAGE_NX)) + return; + ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false); } diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index b63403d7179d..5c736b707cae 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -800,6 +800,4 @@ void mark_rodata_ro(void) set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); #endif mark_nxdata_nx(); - if (__supported_pte_mask & _PAGE_NX) - debug_checkwx(); } diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index a0dffaca6d2b..ebdbcae48011 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1412,8 +1412,6 @@ void mark_rodata_ro(void) (void *)text_end, (void *)rodata_start); free_kernel_image_pages("unused kernel image (rodata/data gap)", (void *)rodata_end, (void *)_sdata); - - debug_checkwx(); } /* diff --git a/include/linux/ptdump.h b/include/linux/ptdump.h index 2a3a95586425..c10513739bf9 100644 --- a/include/linux/ptdump.h +++ b/include/linux/ptdump.h @@ -19,5 +19,12 @@ struct ptdump_state { }; void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd); +void ptdump_check_wx(void); + +static inline void debug_checkwx(void) +{ + if (IS_ENABLED(CONFIG_DEBUG_WX)) + ptdump_check_wx(); +} #endif /* _LINUX_PTDUMP_H */ diff --git a/init/main.c b/init/main.c index e24b0780fdff..749a9f8d2c9b 100644 --- a/init/main.c +++ b/init/main.c @@ -99,6 +99,7 @@ #include #include #include +#include #include #include @@ -1408,6 +1409,7 @@ static void mark_readonly(void) */ rcu_barrier(); mark_rodata_ro(); + debug_checkwx(); rodata_test(); } else pr_info("Kernel memory protection disabled.\n"); -- cgit v1.2.3 From 6cdc82db0c044d36137dd98f33e8aa0b8742987f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 30 Jan 2024 11:34:35 +0100 Subject: mm: ptdump: have ptdump_check_wx() return bool Have ptdump_check_wx() return true when the check is successful or false otherwise. [akpm@linux-foundation.org: fix a couple of build issues (x86_64 allmodconfig)] Link: https://lkml.kernel.org/r/7943149fe955458cb7b57cd483bf41a3aad94684.1706610398.git.christophe.leroy@csgroup.eu Signed-off-by: Christophe Leroy Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Aneesh Kumar K.V (IBM)" Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: Gerald Schaefer Cc: Greg KH Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Kees Cook Cc: Mark Rutland Cc: Michael Ellerman Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Phong Tran Cc: Russell King Cc: Steven Price Cc: Sven Schnelle Cc: Thomas Gleixner Cc: Vasily Gorbik Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/mm/ptdump.c | 11 ++++++++--- arch/powerpc/mm/ptdump/ptdump.c | 13 +++++++++---- arch/riscv/mm/ptdump.c | 11 ++++++++--- arch/s390/mm/dump_pagetables.c | 13 +++++++++---- arch/x86/include/asm/pgtable.h | 2 +- arch/x86/mm/dump_pagetables.c | 23 ++++++++++++++--------- include/linux/ptdump.h | 5 ++++- 7 files changed, 53 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index e305b6593c4e..696822f75582 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -345,7 +345,7 @@ static struct ptdump_info kernel_ptdump_info = { .base_addr = PAGE_OFFSET, }; -void ptdump_check_wx(void) +bool ptdump_check_wx(void) { struct pg_state st = { .seq = NULL, @@ -366,11 +366,16 @@ void ptdump_check_wx(void) ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); - if (st.wx_pages || st.uxn_pages) + if (st.wx_pages || st.uxn_pages) { pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n", st.wx_pages, st.uxn_pages); - else + + return false; + } else { pr_info("Checked W+X mappings: passed, no W+X pages found\n"); + + return true; + } } static int __init ptdump_init(void) diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c index b835c80371cd..9dc239967b77 100644 --- a/arch/powerpc/mm/ptdump/ptdump.c +++ b/arch/powerpc/mm/ptdump/ptdump.c @@ -327,7 +327,7 @@ static void __init build_pgtable_complete_mask(void) pg_level[i].mask |= pg_level[i].flag[j].mask; } -void ptdump_check_wx(void) +bool ptdump_check_wx(void) { struct pg_state st = { .seq = NULL, @@ -344,15 +344,20 @@ void ptdump_check_wx(void) }; if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !mmu_has_feature(MMU_FTR_KERNEL_RO)) - return; + return true; ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); - if (st.wx_pages) + if (st.wx_pages) { pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found\n", st.wx_pages); - else + + return false; + } else { pr_info("Checked W+X mappings: passed, no W+X pages found\n"); + + return true; + } } static int __init ptdump_init(void) diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 075265603313..1289cc6d3700 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -335,7 +335,7 @@ static void ptdump_walk(struct seq_file *s, struct ptd_mm_info *pinfo) ptdump_walk_pgd(&st.ptdump, pinfo->mm, NULL); } -void ptdump_check_wx(void) +bool ptdump_check_wx(void) { struct pg_state st = { .seq = NULL, @@ -356,11 +356,16 @@ void ptdump_check_wx(void) ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); - if (st.wx_pages) + if (st.wx_pages) { pr_warn("Checked W+X mappings: failed, %lu W+X pages found\n", st.wx_pages); - else + + return false; + } else { pr_info("Checked W+X mappings: passed, no W+X pages found\n"); + + return true; + } } static int ptdump_show(struct seq_file *m, void *v) diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 99da5a5602a8..ffd07ed7b4af 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -192,7 +192,7 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, } } -void ptdump_check_wx(void) +bool ptdump_check_wx(void) { struct pg_state st = { .ptdump = { @@ -215,14 +215,19 @@ void ptdump_check_wx(void) }; if (!MACHINE_HAS_NX) - return; + return true; ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); - if (st.wx_pages) + if (st.wx_pages) { pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found\n", st.wx_pages); - else + + return false; + } else { pr_info("Checked W+X mappings: passed, no %sW+X pages found\n", (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear)) ? "unexpected " : ""); + + return true; + } } #ifdef CONFIG_PTDUMP_DEBUGFS diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 6c979028e521..b50b2ef63672 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -31,7 +31,7 @@ struct seq_file; void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm); void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm, bool user); -void ptdump_walk_pgd_level_checkwx(void); +bool ptdump_walk_pgd_level_checkwx(void); #define ptdump_check_wx ptdump_walk_pgd_level_checkwx void ptdump_walk_user_pgd_level_checkwx(void); diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c index 0008524eebe9..35b2cfd47914 100644 --- a/arch/x86/mm/dump_pagetables.c +++ b/arch/x86/mm/dump_pagetables.c @@ -362,9 +362,9 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, } } -static void ptdump_walk_pgd_level_core(struct seq_file *m, - struct mm_struct *mm, pgd_t *pgd, - bool checkwx, bool dmesg) +bool ptdump_walk_pgd_level_core(struct seq_file *m, + struct mm_struct *mm, pgd_t *pgd, + bool checkwx, bool dmesg) { const struct ptdump_range ptdump_ranges[] = { #ifdef CONFIG_X86_64 @@ -391,12 +391,17 @@ static void ptdump_walk_pgd_level_core(struct seq_file *m, ptdump_walk_pgd(&st.ptdump, mm, pgd); if (!checkwx) - return; - if (st.wx_pages) + return true; + if (st.wx_pages) { pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n", st.wx_pages); - else + + return false; + } else { pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n"); + + return true; + } } void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm) @@ -431,12 +436,12 @@ void ptdump_walk_user_pgd_level_checkwx(void) #endif } -void ptdump_walk_pgd_level_checkwx(void) +bool ptdump_walk_pgd_level_checkwx(void) { if (!(__supported_pte_mask & _PAGE_NX)) - return; + return true; - ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false); + return ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false); } static int __init pt_dump_init(void) diff --git a/include/linux/ptdump.h b/include/linux/ptdump.h index c10513739bf9..8dbd51ea8626 100644 --- a/include/linux/ptdump.h +++ b/include/linux/ptdump.h @@ -18,8 +18,11 @@ struct ptdump_state { const struct ptdump_range *range; }; +bool ptdump_walk_pgd_level_core(struct seq_file *m, + struct mm_struct *mm, pgd_t *pgd, + bool checkwx, bool dmesg); void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd); -void ptdump_check_wx(void); +bool ptdump_check_wx(void); static inline void debug_checkwx(void) { -- cgit v1.2.3 From 6cdfa1d5d5d8285108495c33588c48cdda81b647 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 29 Jan 2024 13:46:42 +0100 Subject: mm/pgtable: make pte_next_pfn() independent of set_ptes() Let's provide pte_next_pfn(), independently of set_ptes(). This allows for using the generic pte_next_pfn() version in some arch-specific set_ptes() implementations, and prepares for reusing pte_next_pfn() in other context. Link: https://lkml.kernel.org/r/20240129124649.189745-9-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Christophe Leroy Tested-by: Ryan Roberts Reviewed-by: Mike Rapoport (IBM) Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Aneesh Kumar K.V Cc: Catalin Marinas Cc: Christian Borntraeger Cc: David S. Miller Cc: Dinh Nguyen Cc: Gerald Schaefer Cc: Heiko Carstens Cc: Matthew Wilcox Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Russell King (Oracle) Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index f6d0e3513948..351cd9dc7194 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -212,7 +212,6 @@ static inline int pmd_dirty(pmd_t pmd) #define arch_flush_lazy_mmu_mode() do {} while (0) #endif -#ifndef set_ptes #ifndef pte_next_pfn static inline pte_t pte_next_pfn(pte_t pte) @@ -221,6 +220,7 @@ static inline pte_t pte_next_pfn(pte_t pte) } #endif +#ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. * @mm: Address space to map the pages into. -- cgit v1.2.3 From f8d937761d65c87e9987b88ea7beb7bddc333a0e Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 29 Jan 2024 13:46:47 +0100 Subject: mm/memory: optimize fork() with PTE-mapped THP Let's implement PTE batching when consecutive (present) PTEs map consecutive pages of the same large folio, and all other PTE bits besides the PFNs are equal. We will optimize folio_pte_batch() separately, to ignore selected PTE bits. This patch is based on work by Ryan Roberts. Use __always_inline for __copy_present_ptes() and keep the handling for single PTEs completely separate from the multi-PTE case: we really want the compiler to optimize for the single-PTE case with small folios, to not degrade performance. Note that PTE batching will never exceed a single page table and will always stay within VMA boundaries. Further, processing PTE-mapped THP that maybe pinned and have PageAnonExclusive set on at least one subpage should work as expected, but there is room for improvement: We will repeatedly (1) detect a PTE batch (2) detect that we have to copy a page (3) fall back and allocate a single page to copy a single page. For now we won't care as pinned pages are a corner case, and we should rather look into maintaining only a single PageAnonExclusive bit for large folios. Link: https://lkml.kernel.org/r/20240129124649.189745-14-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Ryan Roberts Reviewed-by: Mike Rapoport (IBM) Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Aneesh Kumar K.V Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Christophe Leroy Cc: David S. Miller Cc: Dinh Nguyen Cc: Gerald Schaefer Cc: Heiko Carstens Cc: Matthew Wilcox Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Russell King (Oracle) Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 31 ++++++++++++++ mm/memory.c | 112 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 124 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 351cd9dc7194..aab227e12493 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -650,6 +650,37 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres } #endif +#ifndef wrprotect_ptes +/** + * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same + * folio. + * @mm: Address space the pages are mapped into. + * @addr: Address the first page is mapped at. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to write-protect. + * + * May be overridden by the architecture; otherwise, implemented as a simple + * loop over ptep_set_wrprotect(). + * + * Note that PTE bits in the PTE range besides the PFN can differ. For example, + * some PTEs might be write-protected. + * + * Context: The caller holds the page table lock. The PTEs map consecutive + * pages that belong to the same folio. The PTEs are all in the same PMD. + */ +static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr) +{ + for (;;) { + ptep_set_wrprotect(mm, addr, ptep); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} +#endif + /* * On some architectures hardware does not set page access bit when accessing * memory page, it is responsibility of software setting this bit. It brings diff --git a/mm/memory.c b/mm/memory.c index ca888431680c..a7eb2301a1d1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -930,15 +930,15 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return 0; } -static inline void __copy_present_pte(struct vm_area_struct *dst_vma, +static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, - pte_t pte, unsigned long addr) + pte_t pte, unsigned long addr, int nr) { struct mm_struct *src_mm = src_vma->vm_mm; /* If it's a COW mapping, write protect it both processes. */ if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) { - ptep_set_wrprotect(src_mm, addr, src_pte); + wrprotect_ptes(src_mm, addr, src_pte, nr); pte = pte_wrprotect(pte); } @@ -950,26 +950,93 @@ static inline void __copy_present_pte(struct vm_area_struct *dst_vma, if (!userfaultfd_wp(dst_vma)) pte = pte_clear_uffd_wp(pte); - set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); + set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr); +} + +/* + * Detect a PTE batch: consecutive (present) PTEs that map consecutive + * pages of the same folio. + * + * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN. + */ +static inline int folio_pte_batch(struct folio *folio, unsigned long addr, + pte_t *start_ptep, pte_t pte, int max_nr) +{ + unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio); + const pte_t *end_ptep = start_ptep + max_nr; + pte_t expected_pte = pte_next_pfn(pte); + pte_t *ptep = start_ptep + 1; + + VM_WARN_ON_FOLIO(!pte_present(pte), folio); + + while (ptep != end_ptep) { + pte = ptep_get(ptep); + + if (!pte_same(pte, expected_pte)) + break; + + /* + * Stop immediately once we reached the end of the folio. In + * corner cases the next PFN might fall into a different + * folio. + */ + if (pte_pfn(pte) == folio_end_pfn) + break; + + expected_pte = pte_next_pfn(expected_pte); + ptep++; + } + + return ptep - start_ptep; } /* - * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page - * is required to copy this pte. + * Copy one present PTE, trying to batch-process subsequent PTEs that map + * consecutive pages of the same folio by copying them as well. + * + * Returns -EAGAIN if one preallocated page is required to copy the next PTE. + * Otherwise, returns the number of copied PTEs (at least 1). */ static inline int -copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, +copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, - int *rss, struct folio **prealloc) + int max_nr, int *rss, struct folio **prealloc) { struct page *page; struct folio *folio; + int err, nr; page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte; folio = page_folio(page); + + /* + * If we likely have to copy, just don't bother with batching. Make + * sure that the common "small folio" case is as fast as possible + * by keeping the batching logic separate. + */ + if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) { + nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr); + folio_ref_add(folio, nr); + if (folio_test_anon(folio)) { + if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, + nr, src_vma))) { + folio_ref_sub(folio, nr); + return -EAGAIN; + } + rss[MM_ANONPAGES] += nr; + VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); + } else { + folio_dup_file_rmap_ptes(folio, page, nr); + rss[mm_counter_file(folio)] += nr; + } + __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, + addr, nr); + return nr; + } + folio_get(folio); if (folio_test_anon(folio)) { /* @@ -981,8 +1048,9 @@ copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, src_vma))) { /* Page may be pinned, we have to copy. */ folio_put(folio); - return copy_present_page(dst_vma, src_vma, dst_pte, src_pte, - addr, rss, prealloc, page); + err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, + addr, rss, prealloc, page); + return err ? err : 1; } rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); @@ -992,8 +1060,8 @@ copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } copy_pte: - __copy_present_pte(dst_vma, src_vma, dst_pte, src_pte, pte, addr); - return 0; + __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1); + return 1; } static inline struct folio *folio_prealloc(struct mm_struct *src_mm, @@ -1030,10 +1098,11 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *src_pte, *dst_pte; pte_t ptent; spinlock_t *src_ptl, *dst_ptl; - int progress, ret = 0; + int progress, max_nr, ret = 0; int rss[NR_MM_COUNTERS]; swp_entry_t entry = (swp_entry_t){0}; struct folio *prealloc = NULL; + int nr; again: progress = 0; @@ -1064,6 +1133,8 @@ again: arch_enter_lazy_mmu_mode(); do { + nr = 1; + /* * We are holding two locks at this point - either of them * could generate latencies in another task on another CPU. @@ -1102,9 +1173,10 @@ again: */ WARN_ON_ONCE(ret != -ENOENT); } - /* copy_present_pte() will clear `*prealloc' if consumed */ - ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte, - ptent, addr, rss, &prealloc); + /* copy_present_ptes() will clear `*prealloc' if consumed */ + max_nr = (end - addr) / PAGE_SIZE; + ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, + ptent, addr, max_nr, rss, &prealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. @@ -1121,8 +1193,10 @@ again: folio_put(prealloc); prealloc = NULL; } - progress += 8; - } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); + nr = ret; + progress += 8 * nr; + } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, + addr != end); arch_leave_lazy_mmu_mode(); pte_unmap_unlock(orig_src_pte, src_ptl); @@ -1143,7 +1217,7 @@ again: prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; - } else if (ret) { + } else if (ret < 0) { VM_WARN_ON_ONCE(1); } -- cgit v1.2.3 From 09dacb7875395b8761cde921fff767a7cd3ab862 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Fri, 2 Feb 2024 22:23:18 +0100 Subject: mm: reduce dependencies on "page_counter.h" does not need . is enough to get LONG_MAX. Files that include page_counter.h are limited. They have been compile tested or checked. $ git grep page_counter\.h include/linux/hugetlb_cgroup.h: struct page_counter hugepage[HUGE_MAX_HSTATE]; --> all files that include it have been compile tested include/linux/memcontrol.h:#include --> has been added, to be safe include/net/sock.h:#include --> already include mm/hugetlb_cgroup.c:#include mm/memcontrol.c:#include mm/page_counter.c:#include --> compile tested Link: https://lkml.kernel.org/r/adfdbe21c4d06400d7bd802868762deb85cae8b6.1706908921.git.christophe.jaillet@wanadoo.fr Signed-off-by: Christophe JAILLET Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 1 + include/linux/page_counter.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 20ff87f8e001..4e4caeaea404 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index c141ea9a95ef..8cd858d912c4 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include struct page_counter { -- cgit v1.2.3 From 73307523c9bbc4e3b35f0058cdbc15e32bd83c52 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 5 Feb 2024 10:49:29 +0530 Subject: mm/cma: make MAX_CMA_AREAS = CONFIG_CMA_AREAS There is no real difference between the global area, and other additionally configured CMA areas via CONFIG_CMA_AREAS that always defaults without user input. This makes MAX_CMA_AREAS same as CONFIG_CMA_AREAS, also incrementing its default values, thus maintaining current default for MAX_CMA_AREAS both for UMA and NUMA systems. Link: https://lkml.kernel.org/r/20240205051929.298559-1-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Signed-off-by: Andrew Morton --- include/linux/cma.h | 6 +----- mm/Kconfig | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cma.h b/include/linux/cma.h index 63873b93deaa..9db877506ea8 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -6,12 +6,8 @@ #include #include -/* - * There is always at least global CMA area and a few optional - * areas configured in kernel .config. - */ #ifdef CONFIG_CMA_AREAS -#define MAX_CMA_AREAS (1 + CONFIG_CMA_AREAS) +#define MAX_CMA_AREAS CONFIG_CMA_AREAS #endif #define CMA_MAX_NAME 64 diff --git a/mm/Kconfig b/mm/Kconfig index 35fa9940e61f..88ba99d84ac3 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -917,14 +917,14 @@ config CMA_SYSFS config CMA_AREAS int "Maximum count of the CMA areas" depends on CMA - default 19 if NUMA - default 7 + default 20 if NUMA + default 8 help CMA allows to create CMA areas for particular purpose, mainly, used as device private area. This parameter sets the maximum number of CMA area in the system. - If unsure, leave the default value "7" in UMA and "19" in NUMA. + If unsure, leave the default value "8" in UMA and "20" in NUMA. config MEM_SOFT_DIRTY bool "Track memory changes" -- cgit v1.2.3 From 0827a1fb143fae588cb6f5b9a97c405d6c2ddec9 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Sun, 4 Feb 2024 03:06:00 +0000 Subject: mm/zswap: invalidate zswap entry when swap entry free During testing I found there are some times the zswap_writeback_entry() return -ENOMEM, which is not we expected: bpftrace -e 'kr:zswap_writeback_entry {@[(int32)retval]=count()}' @[-12]: 1563 @[0]: 277221 The reason is that __read_swap_cache_async() return NULL because swapcache_prepare() failed. The reason is that we won't invalidate zswap entry when swap entry freed to the per-cpu pool, these zswap entries are still on the zswap tree and lru list. This patch moves the invalidation ahead to when swap entry freed to the per-cpu pool, since there is no any benefit to leave trashy zswap entry on the tree and lru list. With this patch: bpftrace -e 'kr:zswap_writeback_entry {@[(int32)retval]=count()}' @[0]: 259744 Note: large folio can't have zswap entry for now, so don't bother to add zswap entry invalidation in the large folio swap free path. Link: https://lkml.kernel.org/r/20240201-b4-zswap-invalidate-entry-v2-2-99d4084260a0@bytedance.com Signed-off-by: Chengming Zhou Reviewed-by: Nhat Pham Acked-by: Johannes Weiner Acked-by: Yosry Ahmed Signed-off-by: Andrew Morton --- include/linux/zswap.h | 4 ++-- mm/swap_slots.c | 3 +++ mm/swapfile.c | 1 - mm/zswap.c | 5 +++-- 4 files changed, 8 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/zswap.h b/include/linux/zswap.h index 91895ce1fdbc..341aea490070 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -29,7 +29,7 @@ struct zswap_lruvec_state { bool zswap_store(struct folio *folio); bool zswap_load(struct folio *folio); -void zswap_invalidate(int type, pgoff_t offset); +void zswap_invalidate(swp_entry_t swp); int zswap_swapon(int type, unsigned long nr_pages); void zswap_swapoff(int type); void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg); @@ -50,7 +50,7 @@ static inline bool zswap_load(struct folio *folio) return false; } -static inline void zswap_invalidate(int type, pgoff_t offset) {} +static inline void zswap_invalidate(swp_entry_t swp) {} static inline int zswap_swapon(int type, unsigned long nr_pages) { return 0; diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 0bec1f705f8e..90973ce7881d 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -273,6 +273,9 @@ void free_swap_slot(swp_entry_t entry) { struct swap_slots_cache *cache; + /* Large folio swap slot is not covered. */ + zswap_invalidate(entry); + cache = raw_cpu_ptr(&swp_slots); if (likely(use_swap_slot_cache && cache->slots_ret)) { spin_lock_irq(&cache->free_lock); diff --git a/mm/swapfile.c b/mm/swapfile.c index a8edaf4e5b8a..d1bd8d1e17bd 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -744,7 +744,6 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset, swap_slot_free_notify = NULL; while (offset <= end) { arch_swap_invalidate_page(si->type, offset); - zswap_invalidate(si->type, offset); if (swap_slot_free_notify) swap_slot_free_notify(si->bdev, offset); offset++; diff --git a/mm/zswap.c b/mm/zswap.c index 35da20d3617f..ef41a7bd81f2 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1739,9 +1739,10 @@ bool zswap_load(struct folio *folio) return true; } -void zswap_invalidate(int type, pgoff_t offset) +void zswap_invalidate(swp_entry_t swp) { - struct zswap_tree *tree = swap_zswap_tree(swp_entry(type, offset)); + pgoff_t offset = swp_offset(swp); + struct zswap_tree *tree = swap_zswap_tree(swp); struct zswap_entry *entry; spin_lock(&tree->lock); -- cgit v1.2.3 From b49547ade38a63ff39c9fbc53fb38622cb63854a Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Sun, 4 Feb 2024 03:06:01 +0000 Subject: mm/zswap: stop lru list shrinking when encounter warm region When the shrinker encounter an existing folio in swap cache, it means we are shrinking into the warmer region. We should terminate shrinking if we're in the dynamic shrinker context. This patch add LRU_STOP to support this, to avoid overshrinking. Link: https://lkml.kernel.org/r/20240201-b4-zswap-invalidate-entry-v2-3-99d4084260a0@bytedance.com Signed-off-by: Chengming Zhou Acked-by: Johannes Weiner Acked-by: Nhat Pham Reviewed-by: Yosry Ahmed Signed-off-by: Andrew Morton --- include/linux/list_lru.h | 2 ++ mm/list_lru.c | 3 +++ mm/zswap.c | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h index f2882a820690..792b67ceb631 100644 --- a/include/linux/list_lru.h +++ b/include/linux/list_lru.h @@ -24,6 +24,8 @@ enum lru_status { LRU_SKIP, /* item cannot be locked, skip */ LRU_RETRY, /* item not freeable. May drop the lock internally, but has to return locked. */ + LRU_STOP, /* stop lru list walking. May drop the lock + internally, but has to return locked. */ }; struct list_lru_one { diff --git a/mm/list_lru.c b/mm/list_lru.c index 61f3b6b1134f..3fd64736bc45 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -243,6 +243,9 @@ restart: */ assert_spin_locked(&nlru->lock); goto restart; + case LRU_STOP: + assert_spin_locked(&nlru->lock); + goto out; default: BUG(); } diff --git a/mm/zswap.c b/mm/zswap.c index ef41a7bd81f2..f8a4ac389118 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1315,8 +1315,10 @@ static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_o * into the warmer region. We should terminate shrinking (if we're in the dynamic * shrinker context). */ - if (writeback_result == -EEXIST && encountered_page_in_swapcache) + if (writeback_result == -EEXIST && encountered_page_in_swapcache) { + ret = LRU_STOP; *encountered_page_in_swapcache = true; + } } else { zswap_written_back_pages++; } -- cgit v1.2.3 From 0c32c9f7a58e7736b27f9d6766e9f21d34a26eff Mon Sep 17 00:00:00 2001 From: John Groves Date: Mon, 5 Feb 2024 18:57:37 -0600 Subject: memremap.h: correct an error in a comment It tried to send me off to memory_hotplug.h for an enum that is a few lines above... Link: https://lkml.kernel.org/r/dba0f5f01162d6fa16e4da2a9fede7f97080e92d.1707179960.git.john@groves.net Signed-off-by: John Groves Reviewed-by: Dan Williams Cc: Christoph Hellwig Signed-off-by: Andrew Morton --- include/linux/memremap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 9837f3e6fb95..3f7143ade32c 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -109,7 +109,7 @@ struct dev_pagemap_ops { * @altmap: pre-allocated/reserved memory for vmemmap allocations * @ref: reference count that pins the devm_memremap_pages() mapping * @done: completion for @ref - * @type: memory type: see MEMORY_* in memory_hotplug.h + * @type: memory type: see MEMORY_* above in memremap.h * @flags: PGMAP_* flags to specify defailed behavior * @vmemmap_shift: structural definition of how the vmemmap page metadata * is populated, specifically the metadata page order. -- cgit v1.2.3 From cfb837e8433179995709578ca5b4741adcd54ec7 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 12 Feb 2024 19:29:51 +0100 Subject: mm: document memalloc_noreclaim_save() and memalloc_pin_save() The memalloc_noreclaim_save() function currently has no documentation comment, so the implications of its usage are not obvious. Namely that it not only prevents entering reclaim (as the name suggests), but also allows using all memory reserves and thus should be only used in contexts that are allocating memory to free memory. This may lead to new improper usages being added. Thus add a documenting comment, based on the description of __GFP_MEMALLOC. While at it, also document memalloc_pin_save() so that all the memalloc_ scopes are documented. For those already documented, add missing Return: descriptions, and mark Context: description per kernel-docs style guide. In the comments describing the relevant PF_MEMALLOC flags, refer to their scope setting functions. [vbabka@suse.cz: fix issues that Mike pointed out] Link: https://lkml.kernel.org/r/20240215095827.13756-2-vbabka@suse.cz Link: https://lkml.kernel.org/r/20240212182950.32730-2-vbabka@suse.cz Signed-off-by: Vlastimil Babka Reviewed-by: Mike Rapoport (IBM) Acked-by: Michal Hocko Cc: Kent Overstreet Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Pasha Tatashin Signed-off-by: Andrew Morton --- include/linux/sched.h | 9 ++++---- include/linux/sched/mm.h | 55 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index b9ce285d8c9c..998861865b84 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1624,15 +1624,15 @@ extern struct pid *cad_pid; #define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */ #define PF_DUMPCORE 0x00000200 /* Dumped core */ #define PF_SIGNALED 0x00000400 /* Killed by a signal */ -#define PF_MEMALLOC 0x00000800 /* Allocating memory */ +#define PF_MEMALLOC 0x00000800 /* Allocating memory to free memory. See memalloc_noreclaim_save() */ #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */ #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ #define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ #define PF__HOLE__00010000 0x00010000 #define PF_KSWAPD 0x00020000 /* I am kswapd */ -#define PF_MEMALLOC_NOFS 0x00040000 /* All allocation requests will inherit GFP_NOFS */ -#define PF_MEMALLOC_NOIO 0x00080000 /* All allocation requests will inherit GFP_NOIO */ +#define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ +#define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to, * I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ @@ -1642,7 +1642,8 @@ extern struct pid *cad_pid; #define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ -#define PF_MEMALLOC_PIN 0x10000000 /* Allocation context constrained to zones which allow long term pinning. */ +#define PF_MEMALLOC_PIN 0x10000000 /* Allocations constrained to zones which allow long term pinning. + * See memalloc_pin_save() */ #define PF__HOLE__20000000 0x20000000 #define PF__HOLE__40000000 0x40000000 #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */ diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 9a19f1b42f64..7a4066d22883 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -315,7 +315,8 @@ static inline void might_alloc(gfp_t gfp_mask) * point of view. Use memalloc_noio_restore to end the scope with flags * returned by this function. * - * This function is safe to be used from any context. + * Context: This function is safe to be used from any context. + * Return: The saved flags to be passed to memalloc_noio_restore. */ static inline unsigned int memalloc_noio_save(void) { @@ -346,7 +347,8 @@ static inline void memalloc_noio_restore(unsigned int flags) * point of view. Use memalloc_nofs_restore to end the scope with flags * returned by this function. * - * This function is safe to be used from any context. + * Context: This function is safe to be used from any context. + * Return: The saved flags to be passed to memalloc_nofs_restore. */ static inline unsigned int memalloc_nofs_save(void) { @@ -368,6 +370,29 @@ static inline void memalloc_nofs_restore(unsigned int flags) current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; } +/** + * memalloc_noreclaim_save - Marks implicit __GFP_MEMALLOC scope. + * + * This function marks the beginning of the __GFP_MEMALLOC allocation scope. + * All further allocations will implicitly add the __GFP_MEMALLOC flag, which + * prevents entering reclaim and allows access to all memory reserves. This + * should only be used when the caller guarantees the allocation will allow more + * memory to be freed very shortly, i.e. it needs to allocate some memory in + * the process of freeing memory, and cannot reclaim due to potential recursion. + * + * Users of this scope have to be extremely careful to not deplete the reserves + * completely and implement a throttling mechanism which controls the + * consumption of the reserve based on the amount of freed memory. Usage of a + * pre-allocated pool (e.g. mempool) should be always considered before using + * this scope. + * + * Individual allocations under the scope can opt out using __GFP_NOMEMALLOC + * + * Context: This function should not be used in an interrupt context as that one + * does not give PF_MEMALLOC access to reserves. + * See __gfp_pfmemalloc_flags(). + * Return: The saved flags to be passed to memalloc_noreclaim_restore. + */ static inline unsigned int memalloc_noreclaim_save(void) { unsigned int flags = current->flags & PF_MEMALLOC; @@ -375,11 +400,29 @@ static inline unsigned int memalloc_noreclaim_save(void) return flags; } +/** + * memalloc_noreclaim_restore - Ends the implicit __GFP_MEMALLOC scope. + * @flags: Flags to restore. + * + * Ends the implicit __GFP_MEMALLOC scope started by memalloc_noreclaim_save + * function. Always make sure that the given flags is the return value from the + * pairing memalloc_noreclaim_save call. + */ static inline void memalloc_noreclaim_restore(unsigned int flags) { current->flags = (current->flags & ~PF_MEMALLOC) | flags; } +/** + * memalloc_pin_save - Marks implicit ~__GFP_MOVABLE scope. + * + * This function marks the beginning of the ~__GFP_MOVABLE allocation scope. + * All further allocations will implicitly remove the __GFP_MOVABLE flag, which + * will constraint the allocations to zones that allow long term pinning, i.e. + * not ZONE_MOVABLE zones. + * + * Return: The saved flags to be passed to memalloc_pin_restore. + */ static inline unsigned int memalloc_pin_save(void) { unsigned int flags = current->flags & PF_MEMALLOC_PIN; @@ -388,6 +431,14 @@ static inline unsigned int memalloc_pin_save(void) return flags; } +/** + * memalloc_pin_restore - Ends the implicit ~__GFP_MOVABLE scope. + * @flags: Flags to restore. + * + * Ends the implicit ~__GFP_MOVABLE scope started by memalloc_pin_save function. + * Always make sure that the given flags is the return value from the pairing + * memalloc_pin_save call. + */ static inline void memalloc_pin_restore(unsigned int flags) { current->flags = (current->flags & ~PF_MEMALLOC_PIN) | flags; -- cgit v1.2.3 From cc25bbe10a86a7fe3ec8468fb01e579e6216d7e1 Mon Sep 17 00:00:00 2001 From: Kinsey Ho Date: Wed, 14 Feb 2024 06:05:37 +0000 Subject: mm/mglru: improve struct lru_gen_mm_walk Rename max_seq to seq in struct lru_gen_mm_walk to keep consistent with struct lru_gen_mm_state. Note that seq is not always up to date with max_seq from lru_gen_folio. No functional changes. Link: https://lkml.kernel.org/r/20240214060538.3524462-5-kinseyho@google.com Signed-off-by: Kinsey Ho Cc: Aneesh Kumar K.V Cc: Donet Tom Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 6 +++--- mm/vmscan.c | 50 ++++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a497f189d988..633812a1d220 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -464,7 +464,7 @@ enum { #define NR_BLOOM_FILTERS 2 struct lru_gen_mm_state { - /* set to max_seq after each iteration */ + /* synced with max_seq after each iteration */ unsigned long seq; /* where the current iteration continues after */ struct list_head *head; @@ -479,8 +479,8 @@ struct lru_gen_mm_state { struct lru_gen_mm_walk { /* the lruvec under reclaim */ struct lruvec *lruvec; - /* unstable max_seq from lru_gen_folio */ - unsigned long max_seq; + /* max_seq from lru_gen_folio: can be out of date */ + unsigned long seq; /* the next address within an mm to scan */ unsigned long next_addr; /* to batch promoted pages */ diff --git a/mm/vmscan.c b/mm/vmscan.c index 1bec586d854e..577e7f9be7c6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2888,7 +2888,7 @@ static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); - hist = lru_hist_from_seq(walk->max_seq); + hist = lru_hist_from_seq(walk->seq); for (i = 0; i < NR_MM_STATS; i++) { WRITE_ONCE(mm_state->stats[hist][i], @@ -2897,7 +2897,7 @@ static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) } if (NR_HIST_GENS > 1 && last) { - hist = lru_hist_from_seq(walk->max_seq + 1); + hist = lru_hist_from_seq(walk->seq + 1); for (i = 0; i < NR_MM_STATS; i++) WRITE_ONCE(mm_state->stats[hist][i], 0); @@ -2926,9 +2926,9 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite */ spin_lock(&mm_list->lock); - VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); + VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq); - if (walk->max_seq <= mm_state->seq) + if (walk->seq <= mm_state->seq) goto done; if (!mm_state->head) @@ -2958,7 +2958,7 @@ done: spin_unlock(&mm_list->lock); if (mm && first) - reset_bloom_filter(mm_state, walk->max_seq + 1); + reset_bloom_filter(mm_state, walk->seq + 1); if (*iter) mmput_async(*iter); @@ -2968,7 +2968,7 @@ done: return last; } -static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) +static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq) { bool success = false; struct mem_cgroup *memcg = lruvec_memcg(lruvec); @@ -2977,9 +2977,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) spin_lock(&mm_list->lock); - VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); + VM_WARN_ON_ONCE(mm_state->seq + 1 < seq); - if (max_seq > mm_state->seq) { + if (seq > mm_state->seq) { mm_state->head = NULL; mm_state->tail = NULL; WRITE_ONCE(mm_state->seq, mm_state->seq + 1); @@ -3330,7 +3330,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct lru_gen_mm_walk *walk = args->private; struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + DEFINE_MAX_SEQ(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(max_seq); pte = pte_offset_map_nolock(args->mm, pmd, start & PMD_MASK, &ptl); if (!pte) @@ -3397,7 +3398,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct lru_gen_mm_walk *walk = args->private; struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + DEFINE_MAX_SEQ(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(max_seq); VM_WARN_ON_ONCE(pud_leaf(*pud)); @@ -3528,7 +3530,7 @@ restart: walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); } - if (!walk->force_scan && !test_bloom_filter(mm_state, walk->max_seq, pmd + i)) + if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) continue; walk->mm_stats[MM_NONLEAF_FOUND]++; @@ -3539,7 +3541,7 @@ restart: walk->mm_stats[MM_NONLEAF_ADDED]++; /* carry over to the next generation */ - update_bloom_filter(mm_state, walk->max_seq + 1, pmd + i); + update_bloom_filter(mm_state, walk->seq + 1, pmd + i); } walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); @@ -3610,7 +3612,7 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) err = -EBUSY; /* another thread might have called inc_max_seq() */ - if (walk->max_seq != max_seq) + if (walk->seq != max_seq) break; /* folio_update_gen() requires stable folio_memcg() */ @@ -3747,7 +3749,7 @@ next: return success; } -static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, +static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, bool can_swap, bool force_scan) { bool success; @@ -3755,14 +3757,14 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, int type, zone; struct lru_gen_folio *lrugen = &lruvec->lrugen; restart: - if (max_seq < READ_ONCE(lrugen->max_seq)) + if (seq < READ_ONCE(lrugen->max_seq)) return false; spin_lock_irq(&lruvec->lru_lock); VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); - success = max_seq == lrugen->max_seq; + success = seq == lrugen->max_seq; if (!success) goto unlock; @@ -3815,7 +3817,7 @@ unlock: return success; } -static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, +static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq, bool can_swap, bool force_scan) { bool success; @@ -3824,13 +3826,13 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, struct lru_gen_folio *lrugen = &lruvec->lrugen; struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); - VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); + VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq)); if (!mm_state) - return inc_max_seq(lruvec, max_seq, can_swap, force_scan); + return inc_max_seq(lruvec, seq, can_swap, force_scan); /* see the comment in iterate_mm_list() */ - if (max_seq <= READ_ONCE(mm_state->seq)) + if (seq <= READ_ONCE(mm_state->seq)) return false; /* @@ -3840,18 +3842,18 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, * is less efficient, but it avoids bursty page faults. */ if (!should_walk_mmu()) { - success = iterate_mm_list_nowalk(lruvec, max_seq); + success = iterate_mm_list_nowalk(lruvec, seq); goto done; } walk = set_mm_walk(NULL, true); if (!walk) { - success = iterate_mm_list_nowalk(lruvec, max_seq); + success = iterate_mm_list_nowalk(lruvec, seq); goto done; } walk->lruvec = lruvec; - walk->max_seq = max_seq; + walk->seq = seq; walk->can_swap = can_swap; walk->force_scan = force_scan; @@ -3862,7 +3864,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, } while (mm); done: if (success) { - success = inc_max_seq(lruvec, max_seq, can_swap, force_scan); + success = inc_max_seq(lruvec, seq, can_swap, force_scan); WARN_ON_ONCE(!success); } -- cgit v1.2.3 From 1cbcb564f5b67cee2fc2f78132b9733118a79c6d Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Mon, 5 Feb 2024 14:48:24 +0200 Subject: net/mlx5: Add the IFC related bits for query tracker Add the IFC related bits for query tracker. Signed-off-by: Yishai Hadas Reviewed-by: Kevin Tian Acked-by: Leon Romanovsky Link: https://lore.kernel.org/r/20240205124828.232701-2-yishaih@nvidia.com Signed-off-by: Alex Williamson --- include/linux/mlx5/mlx5_ifc.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index c726f90ab752..0e513e372bf0 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -12672,6 +12672,11 @@ struct mlx5_ifc_modify_page_track_obj_in_bits { struct mlx5_ifc_page_track_bits obj_context; }; +struct mlx5_ifc_query_page_track_obj_out_bits { + struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr; + struct mlx5_ifc_page_track_bits obj_context; +}; + struct mlx5_ifc_msecq_reg_bits { u8 reserved_at_0[0x20]; -- cgit v1.2.3 From 4de676d494cd8fb2b4c65e58c19ebbdb36673957 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Tue, 20 Feb 2024 17:20:53 +0530 Subject: vfio/pci: rename and export do_io_rw() do_io_rw() is used to read/write to the device MMIO. The grace hopper VFIO PCI variant driver require this functionality to read/write to its memory. Rename this as vfio_pci_core functions and export as GPL. Reviewed-by: Kevin Tian Reviewed-by: Yishai Hadas Signed-off-by: Ankit Agrawal Link: https://lore.kernel.org/r/20240220115055.23546-2-ankita@nvidia.com Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci_rdwr.c | 16 +++++++++------- include/linux/vfio_pci_core.h | 5 ++++- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 07fea08ea8a2..03b8f7ada1ac 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -96,10 +96,10 @@ VFIO_IOREAD(32) * reads with -1. This is intended for handling MSI-X vector tables and * leftover space for ROM BARs. */ -static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, - void __iomem *io, char __user *buf, - loff_t off, size_t count, size_t x_start, - size_t x_end, bool iswrite) +ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, + void __iomem *io, char __user *buf, + loff_t off, size_t count, size_t x_start, + size_t x_end, bool iswrite) { ssize_t done = 0; int ret; @@ -201,6 +201,7 @@ static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, return done; } +EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw); int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar) { @@ -279,8 +280,8 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, x_end = vdev->msix_offset + vdev->msix_size; } - done = do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos, - count, x_start, x_end, iswrite); + done = vfio_pci_core_do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos, + count, x_start, x_end, iswrite); if (done >= 0) *ppos += done; @@ -348,7 +349,8 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, * probing, so we don't currently worry about access in relation * to the memory enable bit in the command register. */ - done = do_io_rw(vdev, false, iomem, buf, off, count, 0, 0, iswrite); + done = vfio_pci_core_do_io_rw(vdev, false, iomem, buf, off, count, + 0, 0, iswrite); vga_put(vdev->pdev, rsrc); diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 85e84b92751b..cf9480a31f3e 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -130,7 +130,10 @@ void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar); pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, pci_channel_state_t state); - +ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, + void __iomem *io, char __user *buf, + loff_t off, size_t count, size_t x_start, + size_t x_end, bool iswrite); #define VFIO_IOWRITE_DECLATION(size) \ int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io); -- cgit v1.2.3 From 30e920e1debb437e5aea7a4ccdab61634354297a Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Tue, 20 Feb 2024 17:20:54 +0530 Subject: vfio/pci: rename and export range_intersect_range range_intersect_range determines an overlap between two ranges. If an overlap, the helper function returns the overlapping offset and size. The VFIO PCI variant driver emulates the PCI config space BAR offset registers. These offset may be accessed for read/write with a variety of lengths including sub-word sizes from sub-word offsets. The driver makes use of this helper function to read/write the targeted part of the emulated register. Make this a vfio_pci_core function, rename and export as GPL. Also update references in virtio driver. Reviewed-by: Kevin Tian Reviewed-by: Yishai Hadas Signed-off-by: Ankit Agrawal Link: https://lore.kernel.org/r/20240220115055.23546-3-ankita@nvidia.com Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci_config.c | 42 ++++++++++++++++++++++ drivers/vfio/pci/virtio/main.c | 72 ++++++++++++++------------------------ include/linux/vfio_pci_core.h | 5 +++ 3 files changed, 73 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index 7e2e62ab0869..97422aafaa7b 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -1966,3 +1966,45 @@ ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, char __user *buf, return done; } + +/** + * vfio_pci_core_range_intersect_range() - Determine overlap between a buffer + * and register offset ranges. + * @buf_start: start offset of the buffer + * @buf_cnt: number of buffer bytes + * @reg_start: start register offset + * @reg_cnt: number of register bytes + * @buf_offset: start offset of overlap in the buffer + * @intersect_count: number of overlapping bytes + * @register_offset: start offset of overlap in register + * + * Returns: true if there is overlap, false if not. + * The overlap start and size is returned through function args. + */ +bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, + loff_t reg_start, size_t reg_cnt, + loff_t *buf_offset, + size_t *intersect_count, + size_t *register_offset) +{ + if (buf_start <= reg_start && + buf_start + buf_cnt > reg_start) { + *buf_offset = reg_start - buf_start; + *intersect_count = min_t(size_t, reg_cnt, + buf_start + buf_cnt - reg_start); + *register_offset = 0; + return true; + } + + if (buf_start > reg_start && + buf_start < reg_start + reg_cnt) { + *buf_offset = 0; + *intersect_count = min_t(size_t, buf_cnt, + reg_start + reg_cnt - buf_start); + *register_offset = buf_start - reg_start; + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(vfio_pci_core_range_intersect_range); diff --git a/drivers/vfio/pci/virtio/main.c b/drivers/vfio/pci/virtio/main.c index d5af683837d3..b5d3a8c5bbc9 100644 --- a/drivers/vfio/pci/virtio/main.c +++ b/drivers/vfio/pci/virtio/main.c @@ -132,33 +132,6 @@ end: return ret ? ret : count; } -static bool range_intersect_range(loff_t range1_start, size_t count1, - loff_t range2_start, size_t count2, - loff_t *start_offset, - size_t *intersect_count, - size_t *register_offset) -{ - if (range1_start <= range2_start && - range1_start + count1 > range2_start) { - *start_offset = range2_start - range1_start; - *intersect_count = min_t(size_t, count2, - range1_start + count1 - range2_start); - *register_offset = 0; - return true; - } - - if (range1_start > range2_start && - range1_start < range2_start + count2) { - *start_offset = 0; - *intersect_count = min_t(size_t, count1, - range2_start + count2 - range1_start); - *register_offset = range1_start - range2_start; - return true; - } - - return false; -} - static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos) @@ -178,16 +151,18 @@ static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev, if (ret < 0) return ret; - if (range_intersect_range(pos, count, PCI_DEVICE_ID, sizeof(val16), - ©_offset, ©_count, ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_DEVICE_ID, + sizeof(val16), ©_offset, + ©_count, ®ister_offset)) { val16 = cpu_to_le16(VIRTIO_TRANS_ID_NET); if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, copy_count)) return -EFAULT; } if ((le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO) && - range_intersect_range(pos, count, PCI_COMMAND, sizeof(val16), - ©_offset, ©_count, ®ister_offset)) { + vfio_pci_core_range_intersect_range(pos, count, PCI_COMMAND, + sizeof(val16), ©_offset, + ©_count, ®ister_offset)) { if (copy_from_user((void *)&val16 + register_offset, buf + copy_offset, copy_count)) return -EFAULT; @@ -197,16 +172,18 @@ static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev, return -EFAULT; } - if (range_intersect_range(pos, count, PCI_REVISION_ID, sizeof(val8), - ©_offset, ©_count, ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_REVISION_ID, + sizeof(val8), ©_offset, + ©_count, ®ister_offset)) { /* Transional needs to have revision 0 */ val8 = 0; if (copy_to_user(buf + copy_offset, &val8, copy_count)) return -EFAULT; } - if (range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, sizeof(val32), - ©_offset, ©_count, ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, + sizeof(val32), ©_offset, + ©_count, ®ister_offset)) { u32 bar_mask = ~(virtvdev->bar0_virtual_buf_size - 1); u32 pci_base_addr_0 = le32_to_cpu(virtvdev->pci_base_addr_0); @@ -215,8 +192,9 @@ static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev, return -EFAULT; } - if (range_intersect_range(pos, count, PCI_SUBSYSTEM_ID, sizeof(val16), - ©_offset, ©_count, ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_SUBSYSTEM_ID, + sizeof(val16), ©_offset, + ©_count, ®ister_offset)) { /* * Transitional devices use the PCI subsystem device id as * virtio device id, same as legacy driver always did. @@ -227,8 +205,9 @@ static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev, return -EFAULT; } - if (range_intersect_range(pos, count, PCI_SUBSYSTEM_VENDOR_ID, sizeof(val16), - ©_offset, ©_count, ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_SUBSYSTEM_VENDOR_ID, + sizeof(val16), ©_offset, + ©_count, ®ister_offset)) { val16 = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET); if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, copy_count)) @@ -270,19 +249,20 @@ static ssize_t virtiovf_pci_write_config(struct vfio_device *core_vdev, loff_t copy_offset; size_t copy_count; - if (range_intersect_range(pos, count, PCI_COMMAND, sizeof(virtvdev->pci_cmd), - ©_offset, ©_count, - ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_COMMAND, + sizeof(virtvdev->pci_cmd), + ©_offset, ©_count, + ®ister_offset)) { if (copy_from_user((void *)&virtvdev->pci_cmd + register_offset, buf + copy_offset, copy_count)) return -EFAULT; } - if (range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, - sizeof(virtvdev->pci_base_addr_0), - ©_offset, ©_count, - ®ister_offset)) { + if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, + sizeof(virtvdev->pci_base_addr_0), + ©_offset, ©_count, + ®ister_offset)) { if (copy_from_user((void *)&virtvdev->pci_base_addr_0 + register_offset, buf + copy_offset, copy_count)) diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index cf9480a31f3e..a2c8b8bba711 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -134,6 +134,11 @@ ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, void __iomem *io, char __user *buf, loff_t off, size_t count, size_t x_start, size_t x_end, bool iswrite); +bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, + loff_t reg_start, size_t reg_cnt, + loff_t *buf_offset, + size_t *intersect_count, + size_t *register_offset); #define VFIO_IOWRITE_DECLATION(size) \ int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io); -- cgit v1.2.3 From da510964c095cb5e070800ef38752c453d2aa71d Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 14 Feb 2024 21:44:31 +0100 Subject: mm/mmu_gather: define ENCODED_PAGE_FLAG_DELAY_RMAP Nowadays, encoded pages are only used in mmu_gather handling. Let's update the documentation, and define ENCODED_PAGE_BIT_DELAY_RMAP. While at it, rename ENCODE_PAGE_BITS to ENCODED_PAGE_BITS. If encoded page pointers would ever be used in other context again, we'd likely want to change the defines to reflect their context (e.g., ENCODED_PAGE_FLAG_MMU_GATHER_DELAY_RMAP). For now, let's keep it simple. This is a preparation for using the remaining spare bit to indicate that the next item in an array of encoded pages is a "nr_pages" argument and not an encoded page. Link: https://lkml.kernel.org/r/20240214204435.167852-7-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Ryan Roberts Cc: Alexander Gordeev Cc: Aneesh Kumar K.V Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Christophe Leroy Cc: Heiko Carstens Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: Michal Hocko Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Peter Zijlstra (Intel) Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Will Deacon Cc: Yin Fengwei Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 17 +++++++++++------ mm/mmu_gather.c | 5 +++-- 2 files changed, 14 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 8b611e13153e..1b89eec0d6df 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -210,8 +210,8 @@ struct page { * * An 'encoded_page' pointer is a pointer to a regular 'struct page', but * with the low bits of the pointer indicating extra context-dependent - * information. Not super-common, but happens in mmu_gather and mlock - * handling, and this acts as a type system check on that use. + * information. Only used in mmu_gather handling, and this acts as a type + * system check on that use. * * We only really have two guaranteed bits in general, although you could * play with 'struct page' alignment (see CONFIG_HAVE_ALIGNED_STRUCT_PAGE) @@ -220,21 +220,26 @@ struct page { * Use the supplied helper functions to endcode/decode the pointer and bits. */ struct encoded_page; -#define ENCODE_PAGE_BITS 3ul + +#define ENCODED_PAGE_BITS 3ul + +/* Perform rmap removal after we have flushed the TLB. */ +#define ENCODED_PAGE_BIT_DELAY_RMAP 1ul + static __always_inline struct encoded_page *encode_page(struct page *page, unsigned long flags) { - BUILD_BUG_ON(flags > ENCODE_PAGE_BITS); + BUILD_BUG_ON(flags > ENCODED_PAGE_BITS); return (struct encoded_page *)(flags | (unsigned long)page); } static inline unsigned long encoded_page_flags(struct encoded_page *page) { - return ENCODE_PAGE_BITS & (unsigned long)page; + return ENCODED_PAGE_BITS & (unsigned long)page; } static inline struct page *encoded_page_ptr(struct encoded_page *page) { - return (struct page *)(~ENCODE_PAGE_BITS & (unsigned long)page); + return (struct page *)(~ENCODED_PAGE_BITS & (unsigned long)page); } /* diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index ac733d81b112..6540c99c6758 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -53,7 +53,7 @@ static void tlb_flush_rmap_batch(struct mmu_gather_batch *batch, struct vm_area_ for (int i = 0; i < batch->nr; i++) { struct encoded_page *enc = batch->encoded_pages[i]; - if (encoded_page_flags(enc)) { + if (encoded_page_flags(enc) & ENCODED_PAGE_BIT_DELAY_RMAP) { struct page *page = encoded_page_ptr(enc); folio_remove_rmap_pte(page_folio(page), page, vma); } @@ -119,6 +119,7 @@ static void tlb_batch_list_free(struct mmu_gather *tlb) bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, bool delay_rmap, int page_size) { + int flags = delay_rmap ? ENCODED_PAGE_BIT_DELAY_RMAP : 0; struct mmu_gather_batch *batch; VM_BUG_ON(!tlb->end); @@ -132,7 +133,7 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, * Add the page and check if we are full. If so * force a flush. */ - batch->encoded_pages[batch->nr++] = encode_page(page, delay_rmap); + batch->encoded_pages[batch->nr++] = encode_page(page, flags); if (batch->nr == batch->max) { if (!tlb_next_batch(tlb)) return true; -- cgit v1.2.3 From d7f861b9c43aadbe384ab1382d2e76750bedc91e Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 14 Feb 2024 21:44:33 +0100 Subject: mm/mmu_gather: add __tlb_remove_folio_pages() Add __tlb_remove_folio_pages(), which will remove multiple consecutive pages that belong to the same large folio, instead of only a single page. We'll be using this function when optimizing unmapping/zapping of large folios that are mapped by PTEs. We're using the remaining spare bit in an encoded_page to indicate that the next enoced page in an array contains actually shifted "nr_pages". Teach swap/freeing code about putting multiple folio references, and delayed rmap handling to remove page ranges of a folio. This extension allows for still gathering almost as many small folios as we used to (-1, because we have to prepare for a possibly bigger next entry), but still allows for gathering consecutive pages that belong to the same large folio. Note that we don't pass the folio pointer, because it is not required for now. Further, we don't support page_size != PAGE_SIZE, it won't be required for simple PTE batching. We have to provide a separate s390 implementation, but it's fairly straight forward. Another, more invasive and likely more expensive, approach would be to use folio+range or a PFN range instead of page+nr_pages. But, we should do that consistently for the whole mmu_gather. For now, let's keep it simple and add "nr_pages" only. Note that it is now possible to gather significantly more pages: In the past, we were able to gather ~10000 pages, now we can also gather ~5000 folio fragments that span multiple pages. A folio fragment on x86-64 can span up to 512 pages (2 MiB THP) and on arm64 with 64k in theory 8192 pages (512 MiB THP). Gathering more memory is not considered something we should worry about, especially because these are already corner cases. While we can gather more total memory, we won't free more folio fragments. As long as page freeing time primarily only depends on the number of involved folios, there is no effective change for !preempt configurations. However, we'll adjust tlb_batch_pages_flush() separately to handle corner cases where page freeing time grows proportionally with the actual memory size. Link: https://lkml.kernel.org/r/20240214204435.167852-9-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Ryan Roberts Cc: Alexander Gordeev Cc: Aneesh Kumar K.V Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Christophe Leroy Cc: Heiko Carstens Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: Michal Hocko Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Peter Zijlstra (Intel) Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Will Deacon Cc: Yin Fengwei Signed-off-by: Andrew Morton --- arch/s390/include/asm/tlb.h | 17 +++++++++++++ include/asm-generic/tlb.h | 8 ++++++ include/linux/mm_types.h | 20 +++++++++++++++ mm/mmu_gather.c | 61 ++++++++++++++++++++++++++++++++++++++------- mm/swap.c | 12 ++++++--- mm/swap_state.c | 15 +++++++++-- 6 files changed, 119 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 48df896d5b79..e95b2c8081eb 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -26,6 +26,8 @@ void __tlb_remove_table(void *_table); static inline void tlb_flush(struct mmu_gather *tlb); static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, bool delay_rmap, int page_size); +static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb, + struct page *page, unsigned int nr_pages, bool delay_rmap); #define tlb_flush tlb_flush #define pte_free_tlb pte_free_tlb @@ -52,6 +54,21 @@ static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, return false; } +static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb, + struct page *page, unsigned int nr_pages, bool delay_rmap) +{ + struct encoded_page *encoded_pages[] = { + encode_page(page, ENCODED_PAGE_BIT_NR_PAGES_NEXT), + encode_nr_pages(nr_pages), + }; + + VM_WARN_ON_ONCE(delay_rmap); + VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); + + free_pages_and_swap_cache(encoded_pages, ARRAY_SIZE(encoded_pages)); + return false; +} + static inline void tlb_flush(struct mmu_gather *tlb) { __tlb_flush_mm_lazy(tlb->mm); diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 127a8230a40a..709830274b75 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -69,6 +69,7 @@ * * - tlb_remove_page() / __tlb_remove_page() * - tlb_remove_page_size() / __tlb_remove_page_size() + * - __tlb_remove_folio_pages() * * __tlb_remove_page_size() is the basic primitive that queues a page for * freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a @@ -78,6 +79,11 @@ * tlb_remove_page() and tlb_remove_page_size() imply the call to * tlb_flush_mmu() when required and has no return value. * + * __tlb_remove_folio_pages() is similar to __tlb_remove_page(), however, + * instead of removing a single page, remove the given number of consecutive + * pages that are all part of the same (large) folio: just like calling + * __tlb_remove_page() on each page individually. + * * - tlb_change_page_size() * * call before __tlb_remove_page*() to set the current page-size; implies a @@ -262,6 +268,8 @@ struct mmu_gather_batch { extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, bool delay_rmap, int page_size); +bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, + unsigned int nr_pages, bool delay_rmap); #ifdef CONFIG_SMP /* diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 1b89eec0d6df..a7223ba3ea1e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -226,6 +226,15 @@ struct encoded_page; /* Perform rmap removal after we have flushed the TLB. */ #define ENCODED_PAGE_BIT_DELAY_RMAP 1ul +/* + * The next item in an encoded_page array is the "nr_pages" argument, specifying + * the number of consecutive pages starting from this page, that all belong to + * the same folio. For example, "nr_pages" corresponds to the number of folio + * references that must be dropped. If this bit is not set, "nr_pages" is + * implicitly 1. + */ +#define ENCODED_PAGE_BIT_NR_PAGES_NEXT 2ul + static __always_inline struct encoded_page *encode_page(struct page *page, unsigned long flags) { BUILD_BUG_ON(flags > ENCODED_PAGE_BITS); @@ -242,6 +251,17 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page) return (struct page *)(~ENCODED_PAGE_BITS & (unsigned long)page); } +static __always_inline struct encoded_page *encode_nr_pages(unsigned long nr) +{ + VM_WARN_ON_ONCE((nr << 2) >> 2 != nr); + return (struct encoded_page *)(nr << 2); +} + +static __always_inline unsigned long encoded_nr_pages(struct encoded_page *page) +{ + return ((unsigned long)page) >> 2; +} + /* * A swap entry has to fit into a "unsigned long", as the entry is hidden * in the "index" field of the swapper address space. diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 6540c99c6758..d175c0f1e2c8 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -50,12 +50,21 @@ static bool tlb_next_batch(struct mmu_gather *tlb) #ifdef CONFIG_SMP static void tlb_flush_rmap_batch(struct mmu_gather_batch *batch, struct vm_area_struct *vma) { + struct encoded_page **pages = batch->encoded_pages; + for (int i = 0; i < batch->nr; i++) { - struct encoded_page *enc = batch->encoded_pages[i]; + struct encoded_page *enc = pages[i]; if (encoded_page_flags(enc) & ENCODED_PAGE_BIT_DELAY_RMAP) { struct page *page = encoded_page_ptr(enc); - folio_remove_rmap_pte(page_folio(page), page, vma); + unsigned int nr_pages = 1; + + if (unlikely(encoded_page_flags(enc) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr_pages = encoded_nr_pages(pages[++i]); + + folio_remove_rmap_ptes(page_folio(page), page, nr_pages, + vma); } } } @@ -89,18 +98,26 @@ static void tlb_batch_pages_flush(struct mmu_gather *tlb) for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { struct encoded_page **pages = batch->encoded_pages; - do { + while (batch->nr) { /* * limit free batch count when PAGE_SIZE > 4K */ unsigned int nr = min(512U, batch->nr); + /* + * Make sure we cover page + nr_pages, and don't leave + * nr_pages behind when capping the number of entries. + */ + if (unlikely(encoded_page_flags(pages[nr - 1]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr++; + free_pages_and_swap_cache(pages, nr); pages += nr; batch->nr -= nr; cond_resched(); - } while (batch->nr); + } } tlb->active = &tlb->local; } @@ -116,8 +133,9 @@ static void tlb_batch_list_free(struct mmu_gather *tlb) tlb->local.next = NULL; } -bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, - bool delay_rmap, int page_size) +static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, + struct page *page, unsigned int nr_pages, bool delay_rmap, + int page_size) { int flags = delay_rmap ? ENCODED_PAGE_BIT_DELAY_RMAP : 0; struct mmu_gather_batch *batch; @@ -126,6 +144,8 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, #ifdef CONFIG_MMU_GATHER_PAGE_SIZE VM_WARN_ON(tlb->page_size != page_size); + VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); #endif batch = tlb->active; @@ -133,17 +153,40 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, * Add the page and check if we are full. If so * force a flush. */ - batch->encoded_pages[batch->nr++] = encode_page(page, flags); - if (batch->nr == batch->max) { + if (likely(nr_pages == 1)) { + batch->encoded_pages[batch->nr++] = encode_page(page, flags); + } else { + flags |= ENCODED_PAGE_BIT_NR_PAGES_NEXT; + batch->encoded_pages[batch->nr++] = encode_page(page, flags); + batch->encoded_pages[batch->nr++] = encode_nr_pages(nr_pages); + } + /* + * Make sure that we can always add another "page" + "nr_pages", + * requiring two entries instead of only a single one. + */ + if (batch->nr >= batch->max - 1) { if (!tlb_next_batch(tlb)) return true; batch = tlb->active; } - VM_BUG_ON_PAGE(batch->nr > batch->max, page); + VM_BUG_ON_PAGE(batch->nr > batch->max - 1, page); return false; } +bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, + unsigned int nr_pages, bool delay_rmap) +{ + return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap, + PAGE_SIZE); +} + +bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, + bool delay_rmap, int page_size) +{ + return __tlb_remove_folio_pages_size(tlb, page, 1, delay_rmap, page_size); +} + #endif /* MMU_GATHER_NO_GATHER */ #ifdef CONFIG_MMU_GATHER_TABLE_FREE diff --git a/mm/swap.c b/mm/swap.c index cd8f0150ba3a..e5380d732c0d 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -967,11 +967,17 @@ void release_pages(release_pages_arg arg, int nr) unsigned int lock_batch; for (i = 0; i < nr; i++) { + unsigned int nr_refs = 1; struct folio *folio; /* Turn any of the argument types into a folio */ folio = page_folio(encoded_page_ptr(encoded[i])); + /* Is our next entry actually "nr_pages" -> "nr_refs" ? */ + if (unlikely(encoded_page_flags(encoded[i]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + nr_refs = encoded_nr_pages(encoded[++i]); + /* * Make sure the IRQ-safe lock-holding time does not get * excessive with a continuous string of pages from the @@ -990,14 +996,14 @@ void release_pages(release_pages_arg arg, int nr) unlock_page_lruvec_irqrestore(lruvec, flags); lruvec = NULL; } - if (put_devmap_managed_page(&folio->page)) + if (put_devmap_managed_page_refs(&folio->page, nr_refs)) continue; - if (folio_put_testzero(folio)) + if (folio_ref_sub_and_test(folio, nr_refs)) free_zone_device_page(&folio->page); continue; } - if (!folio_put_testzero(folio)) + if (!folio_ref_sub_and_test(folio, nr_refs)) continue; if (folio_test_large(folio)) { diff --git a/mm/swap_state.c b/mm/swap_state.c index 7255c01a1e4e..2f540748f7c0 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -311,8 +311,19 @@ void free_page_and_swap_cache(struct page *page) void free_pages_and_swap_cache(struct encoded_page **pages, int nr) { lru_add_drain(); - for (int i = 0; i < nr; i++) - free_swap_cache(encoded_page_ptr(pages[i])); + for (int i = 0; i < nr; i++) { + struct page *page = encoded_page_ptr(pages[i]); + + /* + * Skip over the "nr_pages" entry. It's sufficient to call + * free_swap_cache() only once per folio. + */ + if (unlikely(encoded_page_flags(pages[i]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + i++; + + free_swap_cache(page); + } release_pages(pages, nr); } -- cgit v1.2.3 From 10ebac4f95e7a9951c453d6c66d9beb5a35db338 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 14 Feb 2024 21:44:35 +0100 Subject: mm/memory: optimize unmap/zap with PTE-mapped THP Similar to how we optimized fork(), let's implement PTE batching when consecutive (present) PTEs map consecutive pages of the same large folio. Most infrastructure we need for batching (mmu gather, rmap) is already there. We only have to add get_and_clear_full_ptes() and clear_full_ptes(). Similarly, extend zap_install_uffd_wp_if_needed() to process a PTE range. We won't bother sanity-checking the mapcount of all subpages, but only check the mapcount of the first subpage we process. If there is a real problem hiding somewhere, we can trigger it simply by using small folios, or when we zap single pages of a large folio. Ideally, we had that check in rmap code (including for delayed rmap), but then we cannot print the PTE. Let's keep it simple for now. If we ever have a cheap folio_mapcount(), we might just want to check for underflows there. To keep small folios as fast as possible force inlining of a specialized variant using __always_inline with nr=1. Link: https://lkml.kernel.org/r/20240214204435.167852-11-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Ryan Roberts Cc: Alexander Gordeev Cc: Aneesh Kumar K.V Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Christophe Leroy Cc: Heiko Carstens Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: Michal Hocko Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Peter Zijlstra (Intel) Cc: Sven Schnelle Cc: Vasily Gorbik Cc: Will Deacon Cc: Yin Fengwei Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 70 +++++++++++++++++++++++++++++++++++++ mm/memory.c | 92 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 136 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index aab227e12493..49ab1f73b5c2 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -580,6 +580,76 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, } #endif +#ifndef get_and_clear_full_ptes +/** + * get_and_clear_full_ptes - Clear present PTEs that map consecutive pages of + * the same folio, collecting dirty/accessed bits. + * @mm: Address space the pages are mapped into. + * @addr: Address the first page is mapped at. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to clear. + * @full: Whether we are clearing a full mm. + * + * May be overridden by the architecture; otherwise, implemented as a simple + * loop over ptep_get_and_clear_full(), merging dirty/accessed bits into the + * returned PTE. + * + * Note that PTE bits in the PTE range besides the PFN can differ. For example, + * some PTEs might be write-protected. + * + * Context: The caller holds the page table lock. The PTEs map consecutive + * pages that belong to the same folio. The PTEs are all in the same PMD. + */ +static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr, int full) +{ + pte_t pte, tmp_pte; + + pte = ptep_get_and_clear_full(mm, addr, ptep, full); + while (--nr) { + ptep++; + addr += PAGE_SIZE; + tmp_pte = ptep_get_and_clear_full(mm, addr, ptep, full); + if (pte_dirty(tmp_pte)) + pte = pte_mkdirty(pte); + if (pte_young(tmp_pte)) + pte = pte_mkyoung(pte); + } + return pte; +} +#endif + +#ifndef clear_full_ptes +/** + * clear_full_ptes - Clear present PTEs that map consecutive pages of the same + * folio. + * @mm: Address space the pages are mapped into. + * @addr: Address the first page is mapped at. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to clear. + * @full: Whether we are clearing a full mm. + * + * May be overridden by the architecture; otherwise, implemented as a simple + * loop over ptep_get_and_clear_full(). + * + * Note that PTE bits in the PTE range besides the PFN can differ. For example, + * some PTEs might be write-protected. + * + * Context: The caller holds the page table lock. The PTEs map consecutive + * pages that belong to the same folio. The PTEs are all in the same PMD. + */ +static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned int nr, int full) +{ + for (;;) { + ptep_get_and_clear_full(mm, addr, ptep, full); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} +#endif /* * If two threads concurrently fault at the same page, the thread that diff --git a/mm/memory.c b/mm/memory.c index 168096f9360e..465ada39c2b7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1517,7 +1517,7 @@ static inline bool zap_drop_file_uffd_wp(struct zap_details *details) */ static inline void zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, - unsigned long addr, pte_t *pte, + unsigned long addr, pte_t *pte, int nr, struct zap_details *details, pte_t pteval) { /* Zap on anonymous always means dropping everything */ @@ -1527,20 +1527,27 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (zap_drop_file_uffd_wp(details)) return; - pte_install_uffd_wp_if_needed(vma, addr, pte, pteval); + for (;;) { + /* the PFN in the PTE is irrelevant. */ + pte_install_uffd_wp_if_needed(vma, addr, pte, pteval); + if (--nr == 0) + break; + pte++; + addr += PAGE_SIZE; + } } -static inline void zap_present_folio_pte(struct mmu_gather *tlb, +static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, struct vm_area_struct *vma, struct folio *folio, - struct page *page, pte_t *pte, pte_t ptent, unsigned long addr, - struct zap_details *details, int *rss, bool *force_flush, - bool *force_break) + struct page *page, pte_t *pte, pte_t ptent, unsigned int nr, + unsigned long addr, struct zap_details *details, int *rss, + bool *force_flush, bool *force_break) { struct mm_struct *mm = tlb->mm; bool delay_rmap = false; if (!folio_test_anon(folio)) { - ptent = ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm); + ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { @@ -1550,36 +1557,49 @@ static inline void zap_present_folio_pte(struct mmu_gather *tlb, } if (pte_young(ptent) && likely(vma_has_recency(vma))) folio_mark_accessed(folio); - rss[mm_counter(folio)]--; + rss[mm_counter(folio)] -= nr; } else { /* We don't need up-to-date accessed/dirty bits. */ - ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm); - rss[MM_ANONPAGES]--; + clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); + rss[MM_ANONPAGES] -= nr; } + /* Checking a single PTE in a batch is sufficient. */ arch_check_zapped_pte(vma, ptent); - tlb_remove_tlb_entry(tlb, pte, addr); + tlb_remove_tlb_entries(tlb, pte, nr, addr); if (unlikely(userfaultfd_pte_wp(vma, ptent))) - zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent); + zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, + ptent); if (!delay_rmap) { - folio_remove_rmap_pte(folio, page, vma); + folio_remove_rmap_ptes(folio, page, nr, vma); + + /* Only sanity-check the first page in a batch. */ if (unlikely(page_mapcount(page) < 0)) print_bad_pte(vma, addr, ptent, page); } - if (unlikely(__tlb_remove_page(tlb, page, delay_rmap))) { + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { *force_flush = true; *force_break = true; } } -static inline void zap_present_pte(struct mmu_gather *tlb, +/* + * Zap or skip at least one present PTE, trying to batch-process subsequent + * PTEs that map consecutive pages of the same folio. + * + * Returns the number of processed (skipped or zapped) PTEs (at least 1). + */ +static inline int zap_present_ptes(struct mmu_gather *tlb, struct vm_area_struct *vma, pte_t *pte, pte_t ptent, - unsigned long addr, struct zap_details *details, - int *rss, bool *force_flush, bool *force_break) + unsigned int max_nr, unsigned long addr, + struct zap_details *details, int *rss, bool *force_flush, + bool *force_break) { + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; struct mm_struct *mm = tlb->mm; struct folio *folio; struct page *page; + int nr; page = vm_normal_page(vma, addr, ptent); if (!page) { @@ -1589,14 +1609,29 @@ static inline void zap_present_pte(struct mmu_gather *tlb, tlb_remove_tlb_entry(tlb, pte, addr); VM_WARN_ON_ONCE(userfaultfd_wp(vma)); ksm_might_unmap_zero_page(mm, ptent); - return; + return 1; } folio = page_folio(page); if (unlikely(!should_zap_folio(details, folio))) - return; - zap_present_folio_pte(tlb, vma, folio, page, pte, ptent, addr, details, - rss, force_flush, force_break); + return 1; + + /* + * Make sure that the common "small folio" case is as fast as possible + * by keeping the batching logic separate. + */ + if (unlikely(folio_test_large(folio) && max_nr != 1)) { + nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, + NULL); + + zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, + addr, details, rss, force_flush, + force_break); + return nr; + } + zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr, + details, rss, force_flush, force_break); + return 1; } static unsigned long zap_pte_range(struct mmu_gather *tlb, @@ -1611,6 +1646,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pte_t *start_pte; pte_t *pte; swp_entry_t entry; + int nr; tlb_change_page_size(tlb, PAGE_SIZE); init_rss_vec(rss); @@ -1624,7 +1660,9 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pte_t ptent = ptep_get(pte); struct folio *folio; struct page *page; + int max_nr; + nr = 1; if (pte_none(ptent)) continue; @@ -1632,10 +1670,12 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, break; if (pte_present(ptent)) { - zap_present_pte(tlb, vma, pte, ptent, addr, details, - rss, &force_flush, &force_break); + max_nr = (end - addr) / PAGE_SIZE; + nr = zap_present_ptes(tlb, vma, pte, ptent, max_nr, + addr, details, rss, &force_flush, + &force_break); if (unlikely(force_break)) { - addr += PAGE_SIZE; + addr += nr * PAGE_SIZE; break; } continue; @@ -1689,8 +1729,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, WARN_ON_ONCE(1); } pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); - zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent); - } while (pte++, addr += PAGE_SIZE, addr != end); + zap_install_uffd_wp_if_needed(vma, addr, pte, 1, details, ptent); + } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); add_mm_rss_vec(mm, rss); arch_leave_lazy_mmu_mode(); -- cgit v1.2.3 From 6280d7317ccae19c776a3b6cf9848c964f958091 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 15 Feb 2024 10:31:48 +0000 Subject: mm: clarify the spec for set_ptes() Patch series "Transparent Contiguous PTEs for User Mappings", v6. This is a series to opportunistically and transparently use contpte mappings (set the contiguous bit in ptes) for user memory when those mappings meet the requirements. The change benefits arm64, but there is some (very) minor refactoring for x86 to enable its integration with core-mm. It is part of a wider effort to improve performance by allocating and mapping variable-sized blocks of memory (folios). One aim is for the 4K kernel to approach the performance of the 16K kernel, but without breaking compatibility and without the associated increase in memory. Another aim is to benefit the 16K and 64K kernels by enabling 2M THP, since this is the contpte size for those kernels. We have good performance data that demonstrates both aims are being met (see below). Of course this is only one half of the change. We require the mapped physical memory to be the correct size and alignment for this to actually be useful (i.e. 64K for 4K pages, or 2M for 16K/64K pages). Fortunately folios are solving this problem for us. Filesystems that support it (XFS, AFS, EROFS, tmpfs, ...) will allocate large folios up to the PMD size today, and more filesystems are coming. And for anonymous memory, "multi-size THP" is now upstream. Patch Layout ============ In this version, I've split the patches to better show each optimization: - 1-2: mm prep: misc code and docs cleanups - 3-6: mm,arm64,x86 prep: Add pte_advance_pfn() and make pte_next_pfn() a generic wrapper around it - 7-11: arm64 prep: Refactor ptep helpers into new layer - 12: functional contpte implementation - 23-18: various optimizations on top of the contpte implementation Testing ======= I've tested this series on both Ampere Altra (bare metal) and Apple M2 (VM): - mm selftests (inc new tests written for multi-size THP); no regressions - Speedometer Java script benchmark in Chromium web browser; no issues - Kernel compilation; no issues - Various tests under high memory pressure with swap enabled; no issues Performance =========== High Level Use Cases ~~~~~~~~~~~~~~~~~~~~ First some high level use cases (kernel compilation and speedometer JavaScript benchmarks). These are running on Ampere Altra (I've seen similar improvements on Android/Pixel 6). baseline: mm-unstable (mTHP switched off) mTHP: + enable 16K, 32K, 64K mTHP sizes "always" mTHP + contpte: + this series mTHP + contpte + exefolio: + patch at [6], which series supports Kernel Compilation with -j8 (negative is faster): | kernel | real-time | kern-time | user-time | |---------------------------|-----------|-----------|-----------| | baseline | 0.0% | 0.0% | 0.0% | | mTHP | -5.0% | -39.1% | -0.7% | | mTHP + contpte | -6.0% | -41.4% | -1.5% | | mTHP + contpte + exefolio | -7.8% | -43.1% | -3.4% | Kernel Compilation with -j80 (negative is faster): | kernel | real-time | kern-time | user-time | |---------------------------|-----------|-----------|-----------| | baseline | 0.0% | 0.0% | 0.0% | | mTHP | -5.0% | -36.6% | -0.6% | | mTHP + contpte | -6.1% | -38.2% | -1.6% | | mTHP + contpte + exefolio | -7.4% | -39.2% | -3.2% | Speedometer (positive is faster): | kernel | runs_per_min | |:--------------------------|--------------| | baseline | 0.0% | | mTHP | 1.5% | | mTHP + contpte | 3.2% | | mTHP + contpte + exefolio | 4.5% | Micro Benchmarks ~~~~~~~~~~~~~~~~ The following microbenchmarks are intended to demonstrate the performance of fork() and munmap() do not regress. I'm showing results for order-0 (4K) mappings, and for order-9 (2M) PTE-mapped THP. Thanks to David for sharing his benchmarks. baseline: mm-unstable + batch zap [7] series contpte-basic: + patches 0-19; functional contpte implementation contpte-batch: + patches 20-23; implement new batched APIs contpte-inline: + patch 24; __always_inline to help compiler contpte-fold: + patch 25; fold contpte mapping when sensible Primary platform is Ampere Altra bare metal. I'm also showing results for M2 VM (on top of MacOS) for reference, although experience suggests this might not be the most reliable for performance numbers of this sort: | FORK | order-0 | order-9 | | Ampere Altra |------------------------|------------------------| | (pte-map) | mean | stdev | mean | stdev | |----------------|------------|-----------|------------|-----------| | baseline | 0.0% | 2.7% | 0.0% | 0.2% | | contpte-basic | 6.3% | 1.4% | 1948.7% | 0.2% | | contpte-batch | 7.6% | 2.0% | -1.9% | 0.4% | | contpte-inline | 3.6% | 1.5% | -1.0% | 0.2% | | contpte-fold | 4.6% | 2.1% | -1.8% | 0.2% | | MUNMAP | order-0 | order-9 | | Ampere Altra |------------------------|------------------------| | (pte-map) | mean | stdev | mean | stdev | |----------------|------------|-----------|------------|-----------| | baseline | 0.0% | 0.5% | 0.0% | 0.3% | | contpte-basic | 1.8% | 0.3% | 1104.8% | 0.1% | | contpte-batch | -0.3% | 0.4% | 2.7% | 0.1% | | contpte-inline | -0.1% | 0.6% | 0.9% | 0.1% | | contpte-fold | 0.1% | 0.6% | 0.8% | 0.1% | | FORK | order-0 | order-9 | | Apple M2 VM |------------------------|------------------------| | (pte-map) | mean | stdev | mean | stdev | |----------------|------------|-----------|------------|-----------| | baseline | 0.0% | 1.4% | 0.0% | 0.8% | | contpte-basic | 6.8% | 1.2% | 469.4% | 1.4% | | contpte-batch | -7.7% | 2.0% | -8.9% | 0.7% | | contpte-inline | -6.0% | 2.1% | -6.0% | 2.0% | | contpte-fold | 5.9% | 1.4% | -6.4% | 1.4% | | MUNMAP | order-0 | order-9 | | Apple M2 VM |------------------------|------------------------| | (pte-map) | mean | stdev | mean | stdev | |----------------|------------|-----------|------------|-----------| | baseline | 0.0% | 0.6% | 0.0% | 0.4% | | contpte-basic | 1.6% | 0.6% | 233.6% | 0.7% | | contpte-batch | 1.9% | 0.3% | -3.9% | 0.4% | | contpte-inline | 2.2% | 0.8% | -1.6% | 0.9% | | contpte-fold | 1.5% | 0.7% | -1.7% | 0.7% | Misc ~~~~ John Hubbard at Nvidia has indicated dramatic 10x performance improvements for some workloads at [8], when using 64K base page kernel. [1] https://lore.kernel.org/linux-arm-kernel/20230622144210.2623299-1-ryan.roberts@arm.com/ [2] https://lore.kernel.org/linux-arm-kernel/20231115163018.1303287-1-ryan.roberts@arm.com/ [3] https://lore.kernel.org/linux-arm-kernel/20231204105440.61448-1-ryan.roberts@arm.com/ [4] https://lore.kernel.org/lkml/20231218105100.172635-1-ryan.roberts@arm.com/ [5] https://lore.kernel.org/linux-mm/633af0a7-0823-424f-b6ef-374d99483f05@arm.com/ [6] https://lore.kernel.org/lkml/08c16f7d-f3b3-4f22-9acc-da943f647dc3@arm.com/ [7] https://lore.kernel.org/linux-mm/20240214204435.167852-1-david@redhat.com/ [8] https://lore.kernel.org/linux-mm/c507308d-bdd4-5f9e-d4ff-e96e4520be85@nvidia.com/ [9] https://gitlab.arm.com/linux-arm/linux-rr/-/tree/features/granule_perf/contpte-lkml_v6 This patch (of 18): set_ptes() spec implies that it can only be used to set a present pte because it interprets the PFN field to increment it. However, set_pte_at() has been implemented on top of set_ptes() since set_ptes() was introduced, and set_pte_at() allows setting a pte to a not-present state. So clarify the spec to state that when nr==1, new state of pte may be present or not present. When nr>1, new state of all ptes must be present. While we are at it, tighten the spec to set requirements around the initial state of ptes; when nr==1 it may be either present or not-present. But when nr>1 all ptes must initially be not-present. All set_ptes() callsites already conform to this requirement. Stating it explicitly is useful because it allows for a simplification to the upcoming arm64 contpte implementation. Link: https://lkml.kernel.org/r/20240215103205.2607016-1-ryan.roberts@arm.com Link: https://lkml.kernel.org/r/20240215103205.2607016-2-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Acked-by: David Hildenbrand Cc: Alistair Popple Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Barry Song <21cnbao@gmail.com> Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Morse Cc: John Hubbard Cc: Kefeng Wang Cc: Marc Zyngier Cc: Mark Rutland Cc: Matthew Wilcox (Oracle) Cc: Thomas Gleixner Cc: Will Deacon Cc: Yang Shi Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 49ab1f73b5c2..231370e1b80f 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -229,6 +229,10 @@ static inline pte_t pte_next_pfn(pte_t pte) * @pte: Page table entry for the first page. * @nr: Number of pages to map. * + * When nr==1, initial state of pte may be present or not present, and new state + * may be present or not present. When nr>1, initial state of all ptes must be + * not present, and new state must be present. + * * May be overridden by the architecture, or the architecture can define * set_pte() and PFN_PTE_SHIFT. * -- cgit v1.2.3 From 583ceaaa339960e673ac0029f323bb1c6ffc96d7 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 15 Feb 2024 10:31:50 +0000 Subject: mm: introduce pte_advance_pfn() and use for pte_next_pfn() The goal is to be able to advance a PTE by an arbitrary number of PFNs. So introduce a new API that takes a nr param. Define the default implementation here and allow for architectures to override. pte_next_pfn() becomes a wrapper around pte_advance_pfn(). Follow up commits will convert each overriding architecture's pte_next_pfn() to pte_advance_pfn(). Link: https://lkml.kernel.org/r/20240215103205.2607016-4-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Acked-by: David Hildenbrand Cc: Alistair Popple Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Barry Song <21cnbao@gmail.com> Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Morse Cc: John Hubbard Cc: Kefeng Wang Cc: Marc Zyngier Cc: Mark Rutland Cc: Matthew Wilcox (Oracle) Cc: Thomas Gleixner Cc: Will Deacon Cc: Yang Shi Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 231370e1b80f..b7ac8358f2aa 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -212,14 +212,17 @@ static inline int pmd_dirty(pmd_t pmd) #define arch_flush_lazy_mmu_mode() do {} while (0) #endif - #ifndef pte_next_pfn -static inline pte_t pte_next_pfn(pte_t pte) +#ifndef pte_advance_pfn +static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) { - return __pte(pte_val(pte) + (1UL << PFN_PTE_SHIFT)); + return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT)); } #endif +#define pte_next_pfn(pte) pte_advance_pfn(pte, 1) +#endif + #ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. -- cgit v1.2.3 From fb23bf6bd288db3187c27b971e558a3e9f70ae96 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 15 Feb 2024 10:31:53 +0000 Subject: mm: tidy up pte_next_pfn() definition Now that the all architecture overrides of pte_next_pfn() have been replaced with pte_advance_pfn(), we can simplify the definition of the generic pte_next_pfn() macro so that it is unconditionally defined. Link: https://lkml.kernel.org/r/20240215103205.2607016-7-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Acked-by: David Hildenbrand Cc: Alistair Popple Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Barry Song <21cnbao@gmail.com> Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Morse Cc: John Hubbard Cc: Kefeng Wang Cc: Marc Zyngier Cc: Mark Rutland Cc: Matthew Wilcox (Oracle) Cc: Thomas Gleixner Cc: Will Deacon Cc: Yang Shi Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index b7ac8358f2aa..bc005d84f764 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -212,7 +212,6 @@ static inline int pmd_dirty(pmd_t pmd) #define arch_flush_lazy_mmu_mode() do {} while (0) #endif -#ifndef pte_next_pfn #ifndef pte_advance_pfn static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) { @@ -221,7 +220,6 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) #endif #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) -#endif #ifndef set_ptes /** -- cgit v1.2.3 From 4602e5757bcceb231c3a13c36c373ad4a750eddb Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 15 Feb 2024 10:31:59 +0000 Subject: arm64/mm: wire up PTE_CONT for user mappings With the ptep API sufficiently refactored, we can now introduce a new "contpte" API layer, which transparently manages the PTE_CONT bit for user mappings. In this initial implementation, only suitable batches of PTEs, set via set_ptes(), are mapped with the PTE_CONT bit. Any subsequent modification of individual PTEs will cause an "unfold" operation to repaint the contpte block as individual PTEs before performing the requested operation. While, a modification of a single PTE could cause the block of PTEs to which it belongs to become eligible for "folding" into a contpte entry, "folding" is not performed in this initial implementation due to the costs of checking the requirements are met. Due to this, contpte mappings will degrade back to normal pte mappings over time if/when protections are changed. This will be solved in a future patch. Since a contpte block only has a single access and dirty bit, the semantic here changes slightly; when getting a pte (e.g. ptep_get()) that is part of a contpte mapping, the access and dirty information are pulled from the block (so all ptes in the block return the same access/dirty info). When changing the access/dirty info on a pte (e.g. ptep_set_access_flags()) that is part of a contpte mapping, this change will affect the whole contpte block. This is works fine in practice since we guarantee that only a single folio is mapped by a contpte block, and the core-mm tracks access/dirty information per folio. In order for the public functions, which used to be pure inline, to continue to be callable by modules, export all the contpte_* symbols that are now called by those public inline functions. The feature is enabled/disabled with the ARM64_CONTPTE Kconfig parameter at build time. It defaults to enabled as long as its dependency, TRANSPARENT_HUGEPAGE is also enabled. The core-mm depends upon TRANSPARENT_HUGEPAGE to be able to allocate large folios, so if its not enabled, then there is no chance of meeting the physical contiguity requirement for contpte mappings. Link: https://lkml.kernel.org/r/20240215103205.2607016-13-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Acked-by: Ard Biesheuvel Tested-by: John Hubbard Acked-by: Mark Rutland Reviewed-by: Catalin Marinas Cc: Alistair Popple Cc: Andrey Ryabinin Cc: Barry Song <21cnbao@gmail.com> Cc: Borislav Petkov (AMD) Cc: Dave Hansen Cc: David Hildenbrand Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Morse Cc: Kefeng Wang Cc: Marc Zyngier Cc: Matthew Wilcox (Oracle) Cc: Thomas Gleixner Cc: Will Deacon Cc: Yang Shi Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 9 ++ arch/arm64/include/asm/pgtable.h | 167 +++++++++++++++++++++++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/contpte.c | 285 +++++++++++++++++++++++++++++++++++++++ include/linux/efi.h | 5 + 5 files changed, 467 insertions(+) create mode 100644 arch/arm64/mm/contpte.c (limited to 'include/linux') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index aa7c1d435139..386566138620 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2229,6 +2229,15 @@ config UNWIND_PATCH_PAC_INTO_SCS select UNWIND_TABLES select DYNAMIC_SCS +config ARM64_CONTPTE + bool "Contiguous PTE mappings for user memory" if EXPERT + depends on TRANSPARENT_HUGEPAGE + default y + help + When enabled, user mappings are configured using the PTE contiguous + bit, for any mappings that meet the size and alignment requirements. + This reduces TLB pressure and improves performance. + endmenu # "Kernel Features" menu "Boot options" diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 7336d40a893a..831099cfc96b 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -133,6 +133,10 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) */ #define pte_valid_not_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN)) +/* + * Returns true if the pte is valid and has the contiguous bit set. + */ +#define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) /* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been @@ -1128,6 +1132,167 @@ extern void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t new_pte); +#ifdef CONFIG_ARM64_CONTPTE + +/* + * The contpte APIs are used to transparently manage the contiguous bit in ptes + * where it is possible and makes sense to do so. The PTE_CONT bit is considered + * a private implementation detail of the public ptep API (see below). + */ +extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); +extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte); +extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep); +extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr); +extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty); + +static inline void contpte_try_unfold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + if (unlikely(pte_valid_cont(pte))) + __contpte_try_unfold(mm, addr, ptep, pte); +} + +/* + * The below functions constitute the public API that arm64 presents to the + * core-mm to manipulate PTE entries within their page tables (or at least this + * is the subset of the API that arm64 needs to implement). These public + * versions will automatically and transparently apply the contiguous bit where + * it makes sense to do so. Therefore any users that are contig-aware (e.g. + * hugetlb, kernel mapper) should NOT use these APIs, but instead use the + * private versions, which are prefixed with double underscore. All of these + * APIs except for ptep_get_lockless() are expected to be called with the PTL + * held. Although the contiguous bit is considered private to the + * implementation, it is deliberately allowed to leak through the getters (e.g. + * ptep_get()), back to core code. This is required so that pte_leaf_size() can + * provide an accurate size for perf_get_pgtable_size(). But this leakage means + * its possible a pte will be passed to a setter with the contiguous bit set, so + * we explicitly clear the contiguous bit in those cases to prevent accidentally + * setting it in the pgtable. + */ + +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + pte_t pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(pte))) + return pte; + + return contpte_ptep_get(ptep, pte); +} + +#define ptep_get_lockless ptep_get_lockless +static inline pte_t ptep_get_lockless(pte_t *ptep) +{ + pte_t pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(pte))) + return pte; + + return contpte_ptep_get_lockless(ptep); +} + +static inline void set_pte(pte_t *ptep, pte_t pte) +{ + /* + * We don't have the mm or vaddr so cannot unfold contig entries (since + * it requires tlb maintenance). set_pte() is not used in core code, so + * this should never even be called. Regardless do our best to service + * any call and emit a warning if there is any attempt to set a pte on + * top of an existing contig range. + */ + pte_t orig_pte = __ptep_get(ptep); + + WARN_ON_ONCE(pte_valid_cont(orig_pte)); + __set_pte(ptep, pte_mknoncont(pte)); +} + +#define set_ptes set_ptes +static inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + pte = pte_mknoncont(pte); + + if (likely(nr == 1)) { + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __set_ptes(mm, addr, ptep, pte, 1); + } else { + contpte_set_ptes(mm, addr, ptep, pte, nr); + } +} + +static inline void pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __pte_clear(mm, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + return __ptep_get_and_clear(mm, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t orig_pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_test_and_clear_young(vma, addr, ptep); + + return contpte_ptep_test_and_clear_young(vma, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +static inline int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t orig_pte = __ptep_get(ptep); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_clear_flush_young(vma, addr, ptep); + + return contpte_ptep_clear_flush_young(vma, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); + __ptep_set_wrprotect(mm, addr, ptep); +} + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty) +{ + pte_t orig_pte = __ptep_get(ptep); + + entry = pte_mknoncont(entry); + + if (likely(!pte_valid_cont(orig_pte))) + return __ptep_set_access_flags(vma, addr, ptep, entry, dirty); + + return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty); +} + +#else /* CONFIG_ARM64_CONTPTE */ + #define ptep_get __ptep_get #define set_pte __set_pte #define set_ptes __set_ptes @@ -1143,6 +1308,8 @@ extern void ptep_modify_prot_commit(struct vm_area_struct *vma, #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define ptep_set_access_flags __ptep_set_access_flags +#endif /* CONFIG_ARM64_CONTPTE */ + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index dbd1bc95967d..60454256945b 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -3,6 +3,7 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ cache.o copypage.o flush.o \ ioremap.o mmap.o pgd.o mmu.o \ context.o proc.o pageattr.o fixmap.o +obj-$(CONFIG_ARM64_CONTPTE) += contpte.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PTDUMP_CORE) += ptdump.o obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c new file mode 100644 index 000000000000..6d7f40667fa2 --- /dev/null +++ b/arch/arm64/mm/contpte.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 ARM Ltd. + */ + +#include +#include +#include +#include + +static inline bool mm_is_user(struct mm_struct *mm) +{ + /* + * Don't attempt to apply the contig bit to kernel mappings, because + * dynamically adding/removing the contig bit can cause page faults. + * These racing faults are ok for user space, since they get serialized + * on the PTL. But kernel mappings can't tolerate faults. + */ + if (unlikely(mm_is_efi(mm))) + return false; + return mm != &init_mm; +} + +static inline pte_t *contpte_align_down(pte_t *ptep) +{ + return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); +} + +static void contpte_convert(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); + unsigned long start_addr; + pte_t *start_ptep; + int i; + + start_ptep = ptep = contpte_align_down(ptep); + start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + pte = pfn_pte(ALIGN_DOWN(pte_pfn(pte), CONT_PTES), pte_pgprot(pte)); + + for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { + pte_t ptent = __ptep_get_and_clear(mm, addr, ptep); + + if (pte_dirty(ptent)) + pte = pte_mkdirty(pte); + + if (pte_young(ptent)) + pte = pte_mkyoung(pte); + } + + __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3); + + __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); +} + +void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + /* + * We have already checked that the ptes are contiguous in + * contpte_try_unfold(), so just check that the mm is user space. + */ + if (!mm_is_user(mm)) + return; + + pte = pte_mknoncont(pte); + contpte_convert(mm, addr, ptep, pte); +} +EXPORT_SYMBOL(__contpte_try_unfold); + +pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) +{ + /* + * Gather access/dirty bits, which may be populated in any of the ptes + * of the contig range. We are guaranteed to be holding the PTL, so any + * contiguous range cannot be unfolded or otherwise modified under our + * feet. + */ + + pte_t pte; + int i; + + ptep = contpte_align_down(ptep); + + for (i = 0; i < CONT_PTES; i++, ptep++) { + pte = __ptep_get(ptep); + + if (pte_dirty(pte)) + orig_pte = pte_mkdirty(orig_pte); + + if (pte_young(pte)) + orig_pte = pte_mkyoung(orig_pte); + } + + return orig_pte; +} +EXPORT_SYMBOL(contpte_ptep_get); + +pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) +{ + /* + * Gather access/dirty bits, which may be populated in any of the ptes + * of the contig range. We may not be holding the PTL, so any contiguous + * range may be unfolded/modified/refolded under our feet. Therefore we + * ensure we read a _consistent_ contpte range by checking that all ptes + * in the range are valid and have CONT_PTE set, that all pfns are + * contiguous and that all pgprots are the same (ignoring access/dirty). + * If we find a pte that is not consistent, then we must be racing with + * an update so start again. If the target pte does not have CONT_PTE + * set then that is considered consistent on its own because it is not + * part of a contpte range. + */ + + pgprot_t orig_prot; + unsigned long pfn; + pte_t orig_pte; + pgprot_t prot; + pte_t *ptep; + pte_t pte; + int i; + +retry: + orig_pte = __ptep_get(orig_ptep); + + if (!pte_valid_cont(orig_pte)) + return orig_pte; + + orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte))); + ptep = contpte_align_down(orig_ptep); + pfn = pte_pfn(orig_pte) - (orig_ptep - ptep); + + for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { + pte = __ptep_get(ptep); + prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); + + if (!pte_valid_cont(pte) || + pte_pfn(pte) != pfn || + pgprot_val(prot) != pgprot_val(orig_prot)) + goto retry; + + if (pte_dirty(pte)) + orig_pte = pte_mkdirty(orig_pte); + + if (pte_young(pte)) + orig_pte = pte_mkyoung(orig_pte); + } + + return orig_pte; +} +EXPORT_SYMBOL(contpte_ptep_get_lockless); + +void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + unsigned long next; + unsigned long end; + unsigned long pfn; + pgprot_t prot; + + /* + * The set_ptes() spec guarantees that when nr > 1, the initial state of + * all ptes is not-present. Therefore we never need to unfold or + * otherwise invalidate a range before we set the new ptes. + * contpte_set_ptes() should never be called for nr < 2. + */ + VM_WARN_ON(nr == 1); + + if (!mm_is_user(mm)) + return __set_ptes(mm, addr, ptep, pte, nr); + + end = addr + (nr << PAGE_SHIFT); + pfn = pte_pfn(pte); + prot = pte_pgprot(pte); + + do { + next = pte_cont_addr_end(addr, end); + nr = (next - addr) >> PAGE_SHIFT; + pte = pfn_pte(pfn, prot); + + if (((addr | next | (pfn << PAGE_SHIFT)) & ~CONT_PTE_MASK) == 0) + pte = pte_mkcont(pte); + else + pte = pte_mknoncont(pte); + + __set_ptes(mm, addr, ptep, pte, nr); + + addr = next; + ptep += nr; + pfn += nr; + + } while (addr != end); +} +EXPORT_SYMBOL(contpte_set_ptes); + +int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + /* + * ptep_clear_flush_young() technically requires us to clear the access + * flag for a _single_ pte. However, the core-mm code actually tracks + * access/dirty per folio, not per page. And since we only create a + * contig range when the range is covered by a single folio, we can get + * away with clearing young for the whole contig range here, so we avoid + * having to unfold. + */ + + int young = 0; + int i; + + ptep = contpte_align_down(ptep); + addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + + for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + young |= __ptep_test_and_clear_young(vma, addr, ptep); + + return young; +} +EXPORT_SYMBOL(contpte_ptep_test_and_clear_young); + +int contpte_ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + int young; + + young = contpte_ptep_test_and_clear_young(vma, addr, ptep); + + if (young) { + /* + * See comment in __ptep_clear_flush_young(); same rationale for + * eliding the trailing DSB applies here. + */ + addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + __flush_tlb_range_nosync(vma, addr, addr + CONT_PTE_SIZE, + PAGE_SIZE, true, 3); + } + + return young; +} +EXPORT_SYMBOL(contpte_ptep_clear_flush_young); + +int contpte_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t entry, int dirty) +{ + unsigned long start_addr; + pte_t orig_pte; + int i; + + /* + * Gather the access/dirty bits for the contiguous range. If nothing has + * changed, its a noop. + */ + orig_pte = pte_mknoncont(ptep_get(ptep)); + if (pte_val(orig_pte) == pte_val(entry)) + return 0; + + /* + * We can fix up access/dirty bits without having to unfold the contig + * range. But if the write bit is changing, we must unfold. + */ + if (pte_write(orig_pte) == pte_write(entry)) { + /* + * For HW access management, we technically only need to update + * the flag on a single pte in the range. But for SW access + * management, we need to update all the ptes to prevent extra + * faults. Avoid per-page tlb flush in __ptep_set_access_flags() + * and instead flush the whole range at the end. + */ + ptep = contpte_align_down(ptep); + start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + + for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + __ptep_set_access_flags(vma, addr, ptep, entry, 0); + + if (dirty) + __flush_tlb_range(vma, start_addr, addr, + PAGE_SIZE, true, 3); + } else { + __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); + __ptep_set_access_flags(vma, addr, ptep, entry, dirty); + } + + return 1; +} +EXPORT_SYMBOL(contpte_ptep_set_access_flags); diff --git a/include/linux/efi.h b/include/linux/efi.h index c74f47711f0b..57da15e7429c 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -692,6 +692,11 @@ extern struct efi { extern struct mm_struct efi_mm; +static inline bool mm_is_efi(struct mm_struct *mm) +{ + return IS_ENABLED(CONFIG_EFI) && mm == &efi_mm; +} + static inline int efi_guidcmp (efi_guid_t left, efi_guid_t right) { -- cgit v1.2.3 From c6ec76a2ebc5829e5826b218d2e1475ec11b333e Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 15 Feb 2024 10:32:02 +0000 Subject: mm: add pte_batch_hint() to reduce scanning in folio_pte_batch() Some architectures (e.g. arm64) can tell from looking at a pte, if some follow-on ptes also map contiguous physical memory with the same pgprot. (for arm64, these are contpte mappings). Take advantage of this knowledge to optimize folio_pte_batch() so that it can skip these ptes when scanning to create a batch. By default, if an arch does not opt-in, folio_pte_batch() returns a compile-time 1, so the changes are optimized out and the behaviour is as before. arm64 will opt-in to providing this hint in the next patch, which will greatly reduce the cost of ptep_get() when scanning a range of contptes. Link: https://lkml.kernel.org/r/20240215103205.2607016-16-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Acked-by: David Hildenbrand Tested-by: John Hubbard Cc: Alistair Popple Cc: Andrey Ryabinin Cc: Ard Biesheuvel Cc: Barry Song <21cnbao@gmail.com> Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: James Morse Cc: Kefeng Wang Cc: Marc Zyngier Cc: Mark Rutland Cc: Matthew Wilcox (Oracle) Cc: Thomas Gleixner Cc: Will Deacon Cc: Yang Shi Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 21 +++++++++++++++++++++ mm/memory.c | 19 ++++++++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index bc005d84f764..a36cf4e124b0 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -212,6 +212,27 @@ static inline int pmd_dirty(pmd_t pmd) #define arch_flush_lazy_mmu_mode() do {} while (0) #endif +#ifndef pte_batch_hint +/** + * pte_batch_hint - Number of pages that can be added to batch without scanning. + * @ptep: Page table pointer for the entry. + * @pte: Page table entry. + * + * Some architectures know that a set of contiguous ptes all map the same + * contiguous memory with the same permissions. In this case, it can provide a + * hint to aid pte batching without the core code needing to scan every pte. + * + * An architecture implementation may ignore the PTE accessed state. Further, + * the dirty state must apply atomically to all the PTEs described by the hint. + * + * May be overridden by the architecture, else pte_batch_hint is always 1. + */ +static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) +{ + return 1; +} +#endif + #ifndef pte_advance_pfn static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) { diff --git a/mm/memory.c b/mm/memory.c index 465ada39c2b7..642b4f2be523 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -988,16 +988,20 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, { unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio); const pte_t *end_ptep = start_ptep + max_nr; - pte_t expected_pte = __pte_batch_clear_ignored(pte_next_pfn(pte), flags); - pte_t *ptep = start_ptep + 1; + pte_t expected_pte, *ptep; bool writable; + int nr; if (any_writable) *any_writable = false; VM_WARN_ON_FOLIO(!pte_present(pte), folio); - while (ptep != end_ptep) { + nr = pte_batch_hint(start_ptep, pte); + expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags); + ptep = start_ptep + nr; + + while (ptep < end_ptep) { pte = ptep_get(ptep); if (any_writable) writable = !!pte_write(pte); @@ -1011,17 +1015,18 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, * corner cases the next PFN might fall into a different * folio. */ - if (pte_pfn(pte) == folio_end_pfn) + if (pte_pfn(pte) >= folio_end_pfn) break; if (any_writable) *any_writable |= writable; - expected_pte = pte_next_pfn(expected_pte); - ptep++; + nr = pte_batch_hint(ptep, pte); + expected_pte = pte_advance_pfn(expected_pte, nr); + ptep += nr; } - return ptep - start_ptep; + return min(ptep - start_ptep, max_nr); } /* -- cgit v1.2.3 From 2807c54b3809ca5e51e5a9dfa5e9ddb8993a0e6f Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 15 Feb 2024 09:46:25 -0500 Subject: dax: add empty static inline for CONFIG_DAX=n Patch series "Introduce cpu_dcache_is_aliasing() to fix DAX regression", v6. This commit introduced in v4.0 prevents building FS_DAX on 32-bit ARM, even on ARMv7 which does not have virtually aliased data caches: commit d92576f1167c ("dax: does not work correctly with virtual aliasing caches") Even though it used to work fine before. The root of the issue here is the fact that DAX was never designed to handle virtually aliasing data caches (VIVT and VIPT with aliasing data cache). It touches the pages through their linear mapping, which is not consistent with the userspace mappings with virtually aliasing data caches. This patch series introduces cpu_dcache_is_aliasing() with the new Kconfig option ARCH_HAS_CPU_CACHE_ALIASING and implements it for all architectures. The implementation of cpu_dcache_is_aliasing() is either evaluated to a constant at compile-time or a runtime check, which is what is needed on ARM. With this we can basically narrow down the list of architectures which are unsupported by DAX to those which are really affected. This patch (of 9): When building a kernel with CONFIG_DAX=n, all uses of set_dax_nocache() and set_dax_nomc() need to be either within regions of code or compile units which are explicitly not compiled, or they need to rely on compiler optimizations to eliminate calls to those undefined symbols. It appears that at least the openrisc and loongarch architectures don't end up eliminating those undefined symbols even if they are provably within code which is eliminated due to conditional branches depending on constants. Implement empty static inline functions for set_dax_nocache() and set_dax_nomc() in CONFIG_DAX=n to ensure those undefined references are removed. Link: https://lkml.kernel.org/r/20240215144633.96437-1-mathieu.desnoyers@efficios.com Link: https://lkml.kernel.org/r/20240215144633.96437-2-mathieu.desnoyers@efficios.com Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402140037.wGfA1kqX-lkp@intel.com/ Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402131351.a0FZOgEG-lkp@intel.com/ Fixes: 7ac5360cd4d0 ("dax: remove the copy_from_iter and copy_to_iter methods") Signed-off-by: Mathieu Desnoyers Cc: Christoph Hellwig Cc: Dan Williams Cc: Vishal Verma Cc: Dave Jiang Cc: Matthew Wilcox Cc: Arnd Bergmann Cc: Russell King Cc: Dave Chinner Cc: Michael Sclafani Cc: Alasdair Kergon Cc: Heiko Carstens Cc: Mike Snitzer Cc: Mikulas Patocka Signed-off-by: Andrew Morton --- include/linux/dax.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dax.h b/include/linux/dax.h index b463502b16e1..e3ffe7c7f01d 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -63,6 +63,8 @@ void kill_dax(struct dax_device *dax_dev); void dax_write_cache(struct dax_device *dax_dev, bool wc); bool dax_write_cache_enabled(struct dax_device *dax_dev); bool dax_synchronous(struct dax_device *dax_dev); +void set_dax_nocache(struct dax_device *dax_dev); +void set_dax_nomc(struct dax_device *dax_dev); void set_dax_synchronous(struct dax_device *dax_dev); size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); @@ -109,6 +111,12 @@ static inline bool dax_synchronous(struct dax_device *dax_dev) { return true; } +static inline void set_dax_nocache(struct dax_device *dax_dev) +{ +} +static inline void set_dax_nomc(struct dax_device *dax_dev) +{ +} static inline void set_dax_synchronous(struct dax_device *dax_dev) { } @@ -124,9 +132,6 @@ static inline size_t dax_recovery_write(struct dax_device *dax_dev, } #endif -void set_dax_nocache(struct dax_device *dax_dev); -void set_dax_nomc(struct dax_device *dax_dev); - struct writeback_control; #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); -- cgit v1.2.3 From 6d439c18d9b190ab1e0f1196bc45590f95752bf1 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 15 Feb 2024 09:46:26 -0500 Subject: dax: alloc_dax() return ERR_PTR(-EOPNOTSUPP) for CONFIG_DAX=n Change the return value from NULL to PTR_ERR(-EOPNOTSUPP) for CONFIG_DAX=n to be consistent with the fact that CONFIG_DAX=y never returns NULL. This is done in preparation for using cpu_dcache_is_aliasing() in a following change which will properly support architectures which detect data cache aliasing at runtime. Link: https://lkml.kernel.org/r/20240215144633.96437-3-mathieu.desnoyers@efficios.com Fixes: 4e4ced93794a ("dax: Move mandatory ->zero_page_range() check in alloc_dax()") Signed-off-by: Mathieu Desnoyers Reviewed-by: Dan Williams Cc: Dan Williams Cc: Vishal Verma Cc: Dave Jiang Cc: Matthew Wilcox Cc: Arnd Bergmann Cc: Russell King Cc: Alasdair Kergon Cc: Christoph Hellwig Cc: Dave Chinner Cc: Heiko Carstens Cc: kernel test robot Cc: Michael Sclafani Cc: Mike Snitzer Cc: Mikulas Patocka Signed-off-by: Andrew Morton --- drivers/dax/super.c | 5 +++++ include/linux/dax.h | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index f4b635526345..97ef2a9d878d 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -319,6 +319,11 @@ EXPORT_SYMBOL_GPL(dax_alive); * that any fault handlers or operations that might have seen * dax_alive(), have completed. Any operations that start after * synchronize_srcu() has run will abort upon seeing !dax_alive(). + * + * Note, because alloc_dax() returns an ERR_PTR() on error, callers + * typically store its result into a local variable in order to check + * the result. Therefore, care must be taken to populate the struct + * device dax_dev field make sure the dax_dev is not leaked. */ void kill_dax(struct dax_device *dax_dev) { diff --git a/include/linux/dax.h b/include/linux/dax.h index e3ffe7c7f01d..9d3e3327af4c 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -88,11 +88,7 @@ static inline void *dax_holder(struct dax_device *dax_dev) static inline struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) { - /* - * Callers should check IS_ENABLED(CONFIG_DAX) to know if this - * NULL is an error or expected. - */ - return NULL; + return ERR_PTR(-EOPNOTSUPP); } static inline void put_dax(struct dax_device *dax_dev) { -- cgit v1.2.3 From 8690bbcf3b7010b31fdbf3851e1add6ae19b8624 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 15 Feb 2024 09:46:32 -0500 Subject: Introduce cpu_dcache_is_aliasing() across all architectures Introduce a generic way to query whether the data cache is virtually aliased on all architectures. Its purpose is to ensure that subsystems which are incompatible with virtually aliased data caches (e.g. FS_DAX) can reliably query this. For data cache aliasing, there are three scenarios dependending on the architecture. Here is a breakdown based on my understanding: A) The data cache is always aliasing: * arc * csky * m68k (note: shared memory mappings are incoherent ? SHMLBA is missing there.) * sh * parisc B) The data cache aliasing is statically known or depends on querying CPU state at runtime: * arm (cache_is_vivt() || cache_is_vipt_aliasing()) * mips (cpu_has_dc_aliases) * nios2 (NIOS2_DCACHE_SIZE > PAGE_SIZE) * sparc32 (vac_cache_size > PAGE_SIZE) * sparc64 (L1DCACHE_SIZE > PAGE_SIZE) * xtensa (DCACHE_WAY_SIZE > PAGE_SIZE) C) The data cache is never aliasing: * alpha * arm64 (aarch64) * hexagon * loongarch (but with incoherent write buffers, which are disabled since commit d23b7795 ("LoongArch: Change SHMLBA from SZ_64K to PAGE_SIZE")) * microblaze * openrisc * powerpc * riscv * s390 * um * x86 Require architectures in A) and B) to select ARCH_HAS_CPU_CACHE_ALIASING and implement "cpu_dcache_is_aliasing()". Architectures in C) don't select ARCH_HAS_CPU_CACHE_ALIASING, and thus cpu_dcache_is_aliasing() simply evaluates to "false". Note that this leaves "cpu_icache_is_aliasing()" to be implemented as future work. This would be useful to gate features like XIP on architectures which have aliasing CPU dcache-icache but not CPU dcache-dcache. Use "cpu_dcache" and "cpu_cache" rather than just "dcache" and "cache" to clarify that we really mean "CPU data cache" and "CPU cache" to eliminate any possible confusion with VFS "dentry cache" and "page cache". Link: https://lore.kernel.org/lkml/20030910210416.GA24258@mail.jlokier.co.uk/ Link: https://lkml.kernel.org/r/20240215144633.96437-9-mathieu.desnoyers@efficios.com Fixes: d92576f1167c ("dax: does not work correctly with virtual aliasing caches") Signed-off-by: Mathieu Desnoyers Cc: Dan Williams Cc: Vishal Verma Cc: Dave Jiang Cc: Matthew Wilcox Cc: Arnd Bergmann Cc: Russell King Cc: Alasdair Kergon Cc: Christoph Hellwig Cc: Dave Chinner Cc: Heiko Carstens Cc: kernel test robot Cc: Michael Sclafani Cc: Mike Snitzer Cc: Mikulas Patocka Signed-off-by: Andrew Morton --- arch/arc/Kconfig | 1 + arch/arc/include/asm/cachetype.h | 9 +++++++++ arch/arm/Kconfig | 1 + arch/arm/include/asm/cachetype.h | 2 ++ arch/csky/Kconfig | 1 + arch/csky/include/asm/cachetype.h | 9 +++++++++ arch/m68k/Kconfig | 1 + arch/m68k/include/asm/cachetype.h | 9 +++++++++ arch/mips/Kconfig | 1 + arch/mips/include/asm/cachetype.h | 9 +++++++++ arch/nios2/Kconfig | 1 + arch/nios2/include/asm/cachetype.h | 10 ++++++++++ arch/parisc/Kconfig | 1 + arch/parisc/include/asm/cachetype.h | 9 +++++++++ arch/sh/Kconfig | 1 + arch/sh/include/asm/cachetype.h | 9 +++++++++ arch/sparc/Kconfig | 1 + arch/sparc/include/asm/cachetype.h | 14 ++++++++++++++ arch/xtensa/Kconfig | 1 + arch/xtensa/include/asm/cachetype.h | 10 ++++++++++ include/linux/cacheinfo.h | 6 ++++++ mm/Kconfig | 6 ++++++ 22 files changed, 112 insertions(+) create mode 100644 arch/arc/include/asm/cachetype.h create mode 100644 arch/csky/include/asm/cachetype.h create mode 100644 arch/m68k/include/asm/cachetype.h create mode 100644 arch/mips/include/asm/cachetype.h create mode 100644 arch/nios2/include/asm/cachetype.h create mode 100644 arch/parisc/include/asm/cachetype.h create mode 100644 arch/sh/include/asm/cachetype.h create mode 100644 arch/sparc/include/asm/cachetype.h create mode 100644 arch/xtensa/include/asm/cachetype.h (limited to 'include/linux') diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 1b0483c51cc1..7d294a3242a4 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -6,6 +6,7 @@ config ARC def_bool y select ARC_TIMERS + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DMA_PREP_COHERENT diff --git a/arch/arc/include/asm/cachetype.h b/arch/arc/include/asm/cachetype.h new file mode 100644 index 000000000000..05fc7ed59712 --- /dev/null +++ b/arch/arc/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ARC_CACHETYPE_H +#define __ASM_ARC_CACHETYPE_H + +#include + +#define cpu_dcache_is_aliasing() true + +#endif diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 0af6709570d1..66a8e64b226e 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -5,6 +5,7 @@ config ARM select ARCH_32BIT_OFF_T select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE if HAVE_KRETPROBES && FRAME_POINTER && !ARM_UNWIND select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_CPU_FINALIZE_INIT if MMU select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL if MMU diff --git a/arch/arm/include/asm/cachetype.h b/arch/arm/include/asm/cachetype.h index e8c30430be33..b9dbe1d4c8fe 100644 --- a/arch/arm/include/asm/cachetype.h +++ b/arch/arm/include/asm/cachetype.h @@ -20,6 +20,8 @@ extern unsigned int cacheid; #define icache_is_vipt_aliasing() cacheid_is(CACHEID_VIPT_I_ALIASING) #define icache_is_pipt() cacheid_is(CACHEID_PIPT) +#define cpu_dcache_is_aliasing() (cache_is_vivt() || cache_is_vipt_aliasing()) + /* * __LINUX_ARM_ARCH__ is the minimum supported CPU architecture * Mask out support which will never be present on newer CPUs. diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index cf2a6fd7dff8..8a91eccf76dc 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -2,6 +2,7 @@ config CSKY def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SYNC_DMA_FOR_CPU diff --git a/arch/csky/include/asm/cachetype.h b/arch/csky/include/asm/cachetype.h new file mode 100644 index 000000000000..98cbe3af662f --- /dev/null +++ b/arch/csky/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_CSKY_CACHETYPE_H +#define __ASM_CSKY_CACHETYPE_H + +#include + +#define cpu_dcache_is_aliasing() true + +#endif diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 4b3e93cac723..a9c3e3de0c6d 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -3,6 +3,7 @@ config M68K bool default y select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CPU_FINALIZE_INIT if MMU select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/m68k/include/asm/cachetype.h b/arch/m68k/include/asm/cachetype.h new file mode 100644 index 000000000000..7fad5d9ab8fe --- /dev/null +++ b/arch/m68k/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_M68K_CACHETYPE_H +#define __ASM_M68K_CACHETYPE_H + +#include + +#define cpu_dcache_is_aliasing() true + +#endif diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 797ae590ebdb..ab1c8bd96666 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -4,6 +4,7 @@ config MIPS default y select ARCH_32BIT_OFF_T if !64BIT select ARCH_BINFMT_ELF_STATE if MIPS_FP_SUPPORT + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER if !CC_IS_CLANG || CLANG_VERSION >= 140000 select ARCH_HAS_DEBUG_VIRTUAL if !64BIT diff --git a/arch/mips/include/asm/cachetype.h b/arch/mips/include/asm/cachetype.h new file mode 100644 index 000000000000..9f4ba2fe1155 --- /dev/null +++ b/arch/mips/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MIPS_CACHETYPE_H +#define __ASM_MIPS_CACHETYPE_H + +#include + +#define cpu_dcache_is_aliasing() cpu_has_dc_aliases + +#endif diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 58d9565dc2c7..6b3a14633d2f 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -2,6 +2,7 @@ config NIOS2 def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE diff --git a/arch/nios2/include/asm/cachetype.h b/arch/nios2/include/asm/cachetype.h new file mode 100644 index 000000000000..eb9c416b8a1c --- /dev/null +++ b/arch/nios2/include/asm/cachetype.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NIOS2_CACHETYPE_H +#define __ASM_NIOS2_CACHETYPE_H + +#include +#include + +#define cpu_dcache_is_aliasing() (NIOS2_DCACHE_SIZE > PAGE_SIZE) + +#endif diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 5c845e8d59d9..da6e97ba46a6 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -8,6 +8,7 @@ config PARISC select HAVE_FUNCTION_GRAPH_TRACER select HAVE_SYSCALL_TRACEPOINTS select ARCH_WANT_FRAME_POINTERS + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_DMA_ALLOC if PA11 select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_STRICT_KERNEL_RWX diff --git a/arch/parisc/include/asm/cachetype.h b/arch/parisc/include/asm/cachetype.h new file mode 100644 index 000000000000..e0868a1d3c47 --- /dev/null +++ b/arch/parisc/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_PARISC_CACHETYPE_H +#define __ASM_PARISC_CACHETYPE_H + +#include + +#define cpu_dcache_is_aliasing() true + +#endif diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 7500521b2b98..2ad3e29f0ebe 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -2,6 +2,7 @@ config SUPERH def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM && MMU select ARCH_ENABLE_MEMORY_HOTREMOVE if SPARSEMEM && MMU select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A) diff --git a/arch/sh/include/asm/cachetype.h b/arch/sh/include/asm/cachetype.h new file mode 100644 index 000000000000..a5fffe536068 --- /dev/null +++ b/arch/sh/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SH_CACHETYPE_H +#define __ASM_SH_CACHETYPE_H + +#include + +#define cpu_dcache_is_aliasing() true + +#endif diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 204c43cb3d43..cbec48219d9e 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -13,6 +13,7 @@ config 64BIT config SPARC bool default y + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI select ARCH_MIGHT_HAVE_PC_SERIO select DMA_OPS diff --git a/arch/sparc/include/asm/cachetype.h b/arch/sparc/include/asm/cachetype.h new file mode 100644 index 000000000000..caf1c0045892 --- /dev/null +++ b/arch/sparc/include/asm/cachetype.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC_CACHETYPE_H +#define __ASM_SPARC_CACHETYPE_H + +#include + +#ifdef CONFIG_SPARC32 +extern int vac_cache_size; +#define cpu_dcache_is_aliasing() (vac_cache_size > PAGE_SIZE) +#else +#define cpu_dcache_is_aliasing() (L1DCACHE_SIZE > PAGE_SIZE) +#endif + +#endif diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 6f248d87e496..6689a8547346 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -2,6 +2,7 @@ config XTENSA def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_BINFMT_FLAT if !MMU select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VM_PGTABLE diff --git a/arch/xtensa/include/asm/cachetype.h b/arch/xtensa/include/asm/cachetype.h new file mode 100644 index 000000000000..51bd49e2a1c5 --- /dev/null +++ b/arch/xtensa/include/asm/cachetype.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_XTENSA_CACHETYPE_H +#define __ASM_XTENSA_CACHETYPE_H + +#include +#include + +#define cpu_dcache_is_aliasing() (DCACHE_WAY_SIZE > PAGE_SIZE) + +#endif diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index d504eb4b49ab..2cb15fe4fe12 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -138,4 +138,10 @@ static inline int get_cpu_cacheinfo_id(int cpu, int level) #define use_arch_cache_info() (false) #endif +#ifndef CONFIG_ARCH_HAS_CPU_CACHE_ALIASING +#define cpu_dcache_is_aliasing() false +#else +#include +#endif + #endif /* _LINUX_CACHEINFO_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 2b267553f793..b924f4a5a3ef 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -973,6 +973,12 @@ config IDLE_PAGE_TRACKING See Documentation/admin-guide/mm/idle_page_tracking.rst for more details. +# Architectures which implement cpu_dcache_is_aliasing() to query +# whether the data caches are aliased (VIVT or VIPT with dcache +# aliasing) need to select this. +config ARCH_HAS_CPU_CACHE_ALIASING + bool + config ARCH_HAS_CACHE_LINE_SIZE bool -- cgit v1.2.3 From f91e6b41dd11daffb138e3afdb4804aefc3d4e1b Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:53 -0800 Subject: userfaultfd: move userfaultfd_ctx struct to header file Patch series "per-vma locks in userfaultfd", v7. Performing userfaultfd operations (like copy/move etc.) in critical section of mmap_lock (read-mode) causes significant contention on the lock when operations requiring the lock in write-mode are taking place concurrently. We can use per-vma locks instead to significantly reduce the contention issue. Android runtime's Garbage Collector uses userfaultfd for concurrent compaction. mmap-lock contention during compaction potentially causes jittery experience for the user. During one such reproducible scenario, we observed the following improvements with this patch-set: - Wall clock time of compaction phase came down from ~3s to <500ms - Uninterruptible sleep time (across all threads in the process) was ~10ms (none in mmap_lock) during compaction, instead of >20s This patch (of 4): Move the struct to userfaultfd_k.h to be accessible from mm/userfaultfd.c. There are no other changes in the struct. This is required to prepare for using per-vma locks in userfaultfd operations. Link: https://lkml.kernel.org/r/20240215182756.3448972-1-lokeshgidra@google.com Link: https://lkml.kernel.org/r/20240215182756.3448972-2-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Mike Rapoport (IBM) Reviewed-by: Liam R. Howlett Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tim Murray Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 39 --------------------------------------- include/linux/userfaultfd_k.h | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 05c8e8a05427..58331b83d648 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -50,45 +50,6 @@ static struct ctl_table vm_userfaultfd_table[] = { static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init; -/* - * Start with fault_pending_wqh and fault_wqh so they're more likely - * to be in the same cacheline. - * - * Locking order: - * fd_wqh.lock - * fault_pending_wqh.lock - * fault_wqh.lock - * event_wqh.lock - * - * To avoid deadlocks, IRQs must be disabled when taking any of the above locks, - * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's - * also taken in IRQ context. - */ -struct userfaultfd_ctx { - /* waitqueue head for the pending (i.e. not read) userfaults */ - wait_queue_head_t fault_pending_wqh; - /* waitqueue head for the userfaults */ - wait_queue_head_t fault_wqh; - /* waitqueue head for the pseudo fd to wakeup poll/read */ - wait_queue_head_t fd_wqh; - /* waitqueue head for events */ - wait_queue_head_t event_wqh; - /* a refile sequence protected by fault_pending_wqh lock */ - seqcount_spinlock_t refile_seq; - /* pseudo fd refcounting */ - refcount_t refcount; - /* userfaultfd syscall flags */ - unsigned int flags; - /* features requested from the userspace */ - unsigned int features; - /* released */ - bool released; - /* memory mappings are changing because of non-cooperative event */ - atomic_t mmap_changing; - /* mm with one ore more vmas attached to this userfaultfd_ctx */ - struct mm_struct *mm; -}; - struct userfaultfd_fork_ctx { struct userfaultfd_ctx *orig; struct userfaultfd_ctx *new; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index e4056547fbe6..691d928ee864 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -36,6 +36,45 @@ #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) +/* + * Start with fault_pending_wqh and fault_wqh so they're more likely + * to be in the same cacheline. + * + * Locking order: + * fd_wqh.lock + * fault_pending_wqh.lock + * fault_wqh.lock + * event_wqh.lock + * + * To avoid deadlocks, IRQs must be disabled when taking any of the above locks, + * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's + * also taken in IRQ context. + */ +struct userfaultfd_ctx { + /* waitqueue head for the pending (i.e. not read) userfaults */ + wait_queue_head_t fault_pending_wqh; + /* waitqueue head for the userfaults */ + wait_queue_head_t fault_wqh; + /* waitqueue head for the pseudo fd to wakeup poll/read */ + wait_queue_head_t fd_wqh; + /* waitqueue head for events */ + wait_queue_head_t event_wqh; + /* a refile sequence protected by fault_pending_wqh lock */ + seqcount_spinlock_t refile_seq; + /* pseudo fd refcounting */ + refcount_t refcount; + /* userfaultfd syscall flags */ + unsigned int flags; + /* features requested from the userspace */ + unsigned int features; + /* released */ + bool released; + /* memory mappings are changing because of non-cooperative event */ + atomic_t mmap_changing; + /* mm with one ore more vmas attached to this userfaultfd_ctx */ + struct mm_struct *mm; +}; + extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); /* A combined operation mode + behavior flags. */ -- cgit v1.2.3 From 5e4c24a57b0c126686534b5b159a406c5dd02400 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:54 -0800 Subject: userfaultfd: protect mmap_changing with rw_sem in userfaulfd_ctx Increments and loads to mmap_changing are always in mmap_lock critical section. This ensures that if userspace requests event notification for non-cooperative operations (e.g. mremap), userfaultfd operations don't occur concurrently. This can be achieved by using a separate read-write semaphore in userfaultfd_ctx such that increments are done in write-mode and loads in read-mode, thereby eliminating the dependency on mmap_lock for this purpose. This is a preparatory step before we replace mmap_lock usage with per-vma locks in fill/move ioctls. Link: https://lkml.kernel.org/r/20240215182756.3448972-3-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Mike Rapoport (IBM) Reviewed-by: Liam R. Howlett Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tim Murray Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 40 ++++++++++++++++------------ include/linux/userfaultfd_k.h | 31 ++++++++++++---------- mm/userfaultfd.c | 62 ++++++++++++++++++++++++------------------- 3 files changed, 75 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 58331b83d648..c00a021bcce4 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -685,12 +685,15 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) ctx->flags = octx->flags; ctx->features = octx->features; ctx->released = false; + init_rwsem(&ctx->map_changing_lock); atomic_set(&ctx->mmap_changing, 0); ctx->mm = vma->vm_mm; mmgrab(ctx->mm); userfaultfd_ctx_get(octx); + down_write(&octx->map_changing_lock); atomic_inc(&octx->mmap_changing); + up_write(&octx->map_changing_lock); fctx->orig = octx; fctx->new = ctx; list_add_tail(&fctx->list, fcs); @@ -737,7 +740,9 @@ void mremap_userfaultfd_prep(struct vm_area_struct *vma, if (ctx->features & UFFD_FEATURE_EVENT_REMAP) { vm_ctx->ctx = ctx; userfaultfd_ctx_get(ctx); + down_write(&ctx->map_changing_lock); atomic_inc(&ctx->mmap_changing); + up_write(&ctx->map_changing_lock); } else { /* Drop uffd context if remap feature not enabled */ vma_start_write(vma); @@ -783,7 +788,9 @@ bool userfaultfd_remove(struct vm_area_struct *vma, return true; userfaultfd_ctx_get(ctx); + down_write(&ctx->map_changing_lock); atomic_inc(&ctx->mmap_changing); + up_write(&ctx->map_changing_lock); mmap_read_unlock(mm); msg_init(&ewq.msg); @@ -825,7 +832,9 @@ int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, return -ENOMEM; userfaultfd_ctx_get(ctx); + down_write(&ctx->map_changing_lock); atomic_inc(&ctx->mmap_changing); + up_write(&ctx->map_changing_lock); unmap_ctx->ctx = ctx; unmap_ctx->start = start; unmap_ctx->end = end; @@ -1709,9 +1718,8 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP) flags |= MFILL_ATOMIC_WP; if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_copy(ctx->mm, uffdio_copy.dst, uffdio_copy.src, - uffdio_copy.len, &ctx->mmap_changing, - flags); + ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src, + uffdio_copy.len, flags); mmput(ctx->mm); } else { return -ESRCH; @@ -1761,9 +1769,8 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, goto out; if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_zeropage(ctx->mm, uffdio_zeropage.range.start, - uffdio_zeropage.range.len, - &ctx->mmap_changing); + ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start, + uffdio_zeropage.range.len); mmput(ctx->mm); } else { return -ESRCH; @@ -1818,9 +1825,8 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, return -EINVAL; if (mmget_not_zero(ctx->mm)) { - ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start, - uffdio_wp.range.len, mode_wp, - &ctx->mmap_changing); + ret = mwriteprotect_range(ctx, uffdio_wp.range.start, + uffdio_wp.range.len, mode_wp); mmput(ctx->mm); } else { return -ESRCH; @@ -1870,9 +1876,8 @@ static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) flags |= MFILL_ATOMIC_WP; if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_continue(ctx->mm, uffdio_continue.range.start, - uffdio_continue.range.len, - &ctx->mmap_changing, flags); + ret = mfill_atomic_continue(ctx, uffdio_continue.range.start, + uffdio_continue.range.len, flags); mmput(ctx->mm); } else { return -ESRCH; @@ -1925,9 +1930,8 @@ static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long goto out; if (mmget_not_zero(ctx->mm)) { - ret = mfill_atomic_poison(ctx->mm, uffdio_poison.range.start, - uffdio_poison.range.len, - &ctx->mmap_changing, 0); + ret = mfill_atomic_poison(ctx, uffdio_poison.range.start, + uffdio_poison.range.len, 0); mmput(ctx->mm); } else { return -ESRCH; @@ -2003,13 +2007,14 @@ static int userfaultfd_move(struct userfaultfd_ctx *ctx, if (mmget_not_zero(mm)) { mmap_read_lock(mm); - /* Re-check after taking mmap_lock */ + /* Re-check after taking map_changing_lock */ + down_read(&ctx->map_changing_lock); if (likely(!atomic_read(&ctx->mmap_changing))) ret = move_pages(ctx, mm, uffdio_move.dst, uffdio_move.src, uffdio_move.len, uffdio_move.mode); else ret = -EAGAIN; - + up_read(&ctx->map_changing_lock); mmap_read_unlock(mm); mmput(mm); } else { @@ -2216,6 +2221,7 @@ static int new_userfaultfd(int flags) ctx->flags = flags; ctx->features = 0; ctx->released = false; + init_rwsem(&ctx->map_changing_lock); atomic_set(&ctx->mmap_changing, 0); ctx->mm = current->mm; /* prevent the mm struct to be freed */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 691d928ee864..3210c3552976 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -69,6 +69,13 @@ struct userfaultfd_ctx { unsigned int features; /* released */ bool released; + /* + * Prevents userfaultfd operations (fill/move/wp) from happening while + * some non-cooperative event(s) is taking place. Increments are done + * in write-mode. Whereas, userfaultfd operations, which includes + * reading mmap_changing, is done under read-mode. + */ + struct rw_semaphore map_changing_lock; /* memory mappings are changing because of non-cooperative event */ atomic_t mmap_changing; /* mm with one ore more vmas attached to this userfaultfd_ctx */ @@ -113,22 +120,18 @@ extern int mfill_atomic_install_pte(pmd_t *dst_pmd, unsigned long dst_addr, struct page *page, bool newly_allocated, uffd_flags_t flags); -extern ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, +extern ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, uffd_flags_t flags); -extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, + uffd_flags_t flags); +extern ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx, unsigned long dst_start, - unsigned long len, - atomic_t *mmap_changing); -extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long len, atomic_t *mmap_changing, - uffd_flags_t flags); -extern ssize_t mfill_atomic_poison(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing, - uffd_flags_t flags); -extern int mwriteprotect_range(struct mm_struct *dst_mm, - unsigned long start, unsigned long len, - bool enable_wp, atomic_t *mmap_changing); + unsigned long len); +extern ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long len, uffd_flags_t flags); +extern ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, uffd_flags_t flags); +extern int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_wp); extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 9cc93cc1330b..74aad0831e40 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -353,11 +353,11 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) * called with mmap_lock held, it will release mmap_lock before returning. */ static __always_inline ssize_t mfill_atomic_hugetlb( + struct userfaultfd_ctx *ctx, struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, uffd_flags_t flags) { struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -379,6 +379,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( * feature is not supported. */ if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE)) { + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); return -EINVAL; } @@ -463,6 +464,7 @@ retry: cond_resched(); if (unlikely(err == -ENOENT)) { + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); BUG_ON(!folio); @@ -473,12 +475,13 @@ retry: goto out; } mmap_read_lock(dst_mm); + down_read(&ctx->map_changing_lock); /* * If memory mappings are changing because of non-cooperative * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ - if (mmap_changing && atomic_read(mmap_changing)) { + if (atomic_read(&ctx->mmap_changing)) { err = -EAGAIN; break; } @@ -501,6 +504,7 @@ retry: } out_unlock: + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); out: if (folio) @@ -512,11 +516,11 @@ out: } #else /* !CONFIG_HUGETLB_PAGE */ /* fail at build time if gcc attempts to use this */ -extern ssize_t mfill_atomic_hugetlb(struct vm_area_struct *dst_vma, +extern ssize_t mfill_atomic_hugetlb(struct userfaultfd_ctx *ctx, + struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, uffd_flags_t flags); #endif /* CONFIG_HUGETLB_PAGE */ @@ -564,13 +568,13 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, return err; } -static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, +static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, uffd_flags_t flags) { + struct mm_struct *dst_mm = ctx->mm; struct vm_area_struct *dst_vma; ssize_t err; pmd_t *dst_pmd; @@ -600,8 +604,9 @@ retry: * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ + down_read(&ctx->map_changing_lock); err = -EAGAIN; - if (mmap_changing && atomic_read(mmap_changing)) + if (atomic_read(&ctx->mmap_changing)) goto out_unlock; /* @@ -633,8 +638,8 @@ retry: * If this is a HUGETLB vma, pass off to appropriate routine */ if (is_vm_hugetlb_page(dst_vma)) - return mfill_atomic_hugetlb(dst_vma, dst_start, src_start, - len, mmap_changing, flags); + return mfill_atomic_hugetlb(ctx, dst_vma, dst_start, + src_start, len, flags); if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) goto out_unlock; @@ -693,6 +698,7 @@ retry: if (unlikely(err == -ENOENT)) { void *kaddr; + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); BUG_ON(!folio); @@ -723,6 +729,7 @@ retry: } out_unlock: + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); out: if (folio) @@ -733,34 +740,33 @@ out: return copied ? copied : err; } -ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, +ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, uffd_flags_t flags) + uffd_flags_t flags) { - return mfill_atomic(dst_mm, dst_start, src_start, len, mmap_changing, + return mfill_atomic(ctx, dst_start, src_start, len, uffd_flags_set_mode(flags, MFILL_ATOMIC_COPY)); } -ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing) +ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx, + unsigned long start, + unsigned long len) { - return mfill_atomic(dst_mm, start, 0, len, mmap_changing, + return mfill_atomic(ctx, start, 0, len, uffd_flags_set_mode(0, MFILL_ATOMIC_ZEROPAGE)); } -ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing, - uffd_flags_t flags) +ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, uffd_flags_t flags) { - return mfill_atomic(dst_mm, start, 0, len, mmap_changing, + return mfill_atomic(ctx, start, 0, len, uffd_flags_set_mode(flags, MFILL_ATOMIC_CONTINUE)); } -ssize_t mfill_atomic_poison(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing, - uffd_flags_t flags) +ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, uffd_flags_t flags) { - return mfill_atomic(dst_mm, start, 0, len, mmap_changing, + return mfill_atomic(ctx, start, 0, len, uffd_flags_set_mode(flags, MFILL_ATOMIC_POISON)); } @@ -793,10 +799,10 @@ long uffd_wp_range(struct vm_area_struct *dst_vma, return ret; } -int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, bool enable_wp, - atomic_t *mmap_changing) +int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_wp) { + struct mm_struct *dst_mm = ctx->mm; unsigned long end = start + len; unsigned long _start, _end; struct vm_area_struct *dst_vma; @@ -820,8 +826,9 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ + down_read(&ctx->map_changing_lock); err = -EAGAIN; - if (mmap_changing && atomic_read(mmap_changing)) + if (atomic_read(&ctx->mmap_changing)) goto out_unlock; err = -ENOENT; @@ -850,6 +857,7 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, err = 0; } out_unlock: + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); return err; } -- cgit v1.2.3 From 32af81af2f6f4c23b1b4ff68410e91da660af102 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:55 -0800 Subject: mm: add vma_assert_locked() for !CONFIG_PER_VMA_LOCK vma_assert_locked() is needed to replace mmap_assert_locked() once we start using per-vma locks in userfaultfd operations. In !CONFIG_PER_VMA_LOCK case when mm is locked, it implies that the given VMA is locked. Link: https://lkml.kernel.org/r/20240215182756.3448972-4-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Suren Baghdasaryan Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Tim Murray Signed-off-by: Andrew Morton --- include/linux/mm.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index ac6b71cbdffb..6f4825d82965 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -781,6 +781,11 @@ static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, return NULL; } +static inline void vma_assert_locked(struct vm_area_struct *vma) +{ + mmap_assert_locked(vma->vm_mm); +} + static inline void release_fault_lock(struct vm_fault *vmf) { mmap_read_unlock(vmf->vma->vm_mm); -- cgit v1.2.3 From 867a43a34ff8a38772212045262b2c9b77807ea3 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:56 -0800 Subject: userfaultfd: use per-vma locks in userfaultfd operations All userfaultfd operations, except write-protect, opportunistically use per-vma locks to lock vmas. On failure, attempt again inside mmap_lock critical section. Write-protect operation requires mmap_lock as it iterates over multiple vmas. Link: https://lkml.kernel.org/r/20240215182756.3448972-5-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Liam R. Howlett Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tim Murray Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 13 +- include/linux/userfaultfd_k.h | 5 +- mm/huge_memory.c | 5 +- mm/userfaultfd.c | 380 ++++++++++++++++++++++++++++++++---------- 4 files changed, 299 insertions(+), 104 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index c00a021bcce4..60dcfafdc11a 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -2005,17 +2005,8 @@ static int userfaultfd_move(struct userfaultfd_ctx *ctx, return -EINVAL; if (mmget_not_zero(mm)) { - mmap_read_lock(mm); - - /* Re-check after taking map_changing_lock */ - down_read(&ctx->map_changing_lock); - if (likely(!atomic_read(&ctx->mmap_changing))) - ret = move_pages(ctx, mm, uffdio_move.dst, uffdio_move.src, - uffdio_move.len, uffdio_move.mode); - else - ret = -EAGAIN; - up_read(&ctx->map_changing_lock); - mmap_read_unlock(mm); + ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src, + uffdio_move.len, uffdio_move.mode); mmput(mm); } else { return -ESRCH; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 3210c3552976..05d59f74fc88 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -138,9 +138,8 @@ extern long uffd_wp_range(struct vm_area_struct *vma, /* move_pages */ void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2); -ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, - unsigned long dst_start, unsigned long src_start, - unsigned long len, __u64 flags); +ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long src_start, unsigned long len, __u64 flags); int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 14888b15121e..28341a5067fb 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2158,7 +2158,7 @@ unlock: #ifdef CONFIG_USERFAULTFD /* - * The PT lock for src_pmd and the mmap_lock for reading are held by + * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by * the caller, but it must return after releasing the page_table_lock. * Just move the page from src_pmd to dst_pmd if possible. * Return zero if succeeded in moving the page, -EAGAIN if it needs to be @@ -2181,7 +2181,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm src_ptl = pmd_lockptr(mm, src_pmd); lockdep_assert_held(src_ptl); - mmap_assert_locked(mm); + vma_assert_locked(src_vma); + vma_assert_locked(dst_vma); /* Sanity checks before the operation */ if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 74aad0831e40..4744d6a96f96 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -20,19 +20,11 @@ #include "internal.h" static __always_inline -struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long len) +bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) { - /* - * Make sure that the dst range is both valid and fully within a - * single existing vma. - */ - struct vm_area_struct *dst_vma; - - dst_vma = find_vma(dst_mm, dst_start); - if (!range_in_vma(dst_vma, dst_start, dst_start + len)) - return NULL; + /* Make sure that the dst range is fully within dst_vma. */ + if (dst_end > dst_vma->vm_end) + return false; /* * Check the vma is registered in uffd, this is required to @@ -40,11 +32,122 @@ struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, * time. */ if (!dst_vma->vm_userfaultfd_ctx.ctx) - return NULL; + return false; + + return true; +} + +static __always_inline +struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, + unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_assert_locked(mm); + vma = vma_lookup(mm, addr); + if (!vma) + vma = ERR_PTR(-ENOENT); + else if (!(vma->vm_flags & VM_SHARED) && + unlikely(anon_vma_prepare(vma))) + vma = ERR_PTR(-ENOMEM); + + return vma; +} + +#ifdef CONFIG_PER_VMA_LOCK +/* + * lock_vma() - Lookup and lock vma corresponding to @address. + * @mm: mm to search vma in. + * @address: address that the vma should contain. + * + * Should be called without holding mmap_lock. vma should be unlocked after use + * with unlock_vma(). + * + * Return: A locked vma containing @address, -ENOENT if no vma is found, or + * -ENOMEM if anon_vma couldn't be allocated. + */ +static struct vm_area_struct *lock_vma(struct mm_struct *mm, + unsigned long address) +{ + struct vm_area_struct *vma; + + vma = lock_vma_under_rcu(mm, address); + if (vma) { + /* + * lock_vma_under_rcu() only checks anon_vma for private + * anonymous mappings. But we need to ensure it is assigned in + * private file-backed vmas as well. + */ + if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma)) + vma_end_read(vma); + else + return vma; + } + + mmap_read_lock(mm); + vma = find_vma_and_prepare_anon(mm, address); + if (!IS_ERR(vma)) { + /* + * We cannot use vma_start_read() as it may fail due to + * false locked (see comment in vma_start_read()). We + * can avoid that by directly locking vm_lock under + * mmap_lock, which guarantees that nobody can lock the + * vma for write (vma_start_write()) under us. + */ + down_read(&vma->vm_lock->lock); + } + + mmap_read_unlock(mm); + return vma; +} + +static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, + unsigned long dst_start, + unsigned long len) +{ + struct vm_area_struct *dst_vma; + dst_vma = lock_vma(dst_mm, dst_start); + if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) + return dst_vma; + + vma_end_read(dst_vma); + return ERR_PTR(-ENOENT); +} + +static void uffd_mfill_unlock(struct vm_area_struct *vma) +{ + vma_end_read(vma); +} + +#else + +static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, + unsigned long dst_start, + unsigned long len) +{ + struct vm_area_struct *dst_vma; + + mmap_read_lock(dst_mm); + dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); + if (IS_ERR(dst_vma)) + goto out_unlock; + + if (validate_dst_vma(dst_vma, dst_start + len)) + return dst_vma; + + dst_vma = ERR_PTR(-ENOENT); +out_unlock: + mmap_read_unlock(dst_mm); return dst_vma; } +static void uffd_mfill_unlock(struct vm_area_struct *vma) +{ + mmap_read_unlock(vma->vm_mm); +} +#endif + /* Check if dst_addr is outside of file's size. Must be called with ptl held. */ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) @@ -350,7 +453,8 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) #ifdef CONFIG_HUGETLB_PAGE /* * mfill_atomic processing for HUGETLB vmas. Note that this routine is - * called with mmap_lock held, it will release mmap_lock before returning. + * called with either vma-lock or mmap_lock held, it will release the lock + * before returning. */ static __always_inline ssize_t mfill_atomic_hugetlb( struct userfaultfd_ctx *ctx, @@ -361,7 +465,6 @@ static __always_inline ssize_t mfill_atomic_hugetlb( uffd_flags_t flags) { struct mm_struct *dst_mm = dst_vma->vm_mm; - int vm_shared = dst_vma->vm_flags & VM_SHARED; ssize_t err; pte_t *dst_pte; unsigned long src_addr, dst_addr; @@ -380,7 +483,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( */ if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE)) { up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); return -EINVAL; } @@ -403,24 +506,28 @@ retry: * retry, dst_vma will be set to NULL and we must lookup again. */ if (!dst_vma) { + dst_vma = uffd_mfill_lock(dst_mm, dst_start, len); + if (IS_ERR(dst_vma)) { + err = PTR_ERR(dst_vma); + goto out; + } + err = -ENOENT; - dst_vma = find_dst_vma(dst_mm, dst_start, len); - if (!dst_vma || !is_vm_hugetlb_page(dst_vma)) - goto out_unlock; + if (!is_vm_hugetlb_page(dst_vma)) + goto out_unlock_vma; err = -EINVAL; if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) - goto out_unlock; - - vm_shared = dst_vma->vm_flags & VM_SHARED; - } + goto out_unlock_vma; - /* - * If not shared, ensure the dst_vma has a anon_vma. - */ - err = -ENOMEM; - if (!vm_shared) { - if (unlikely(anon_vma_prepare(dst_vma))) + /* + * If memory mappings are changing because of non-cooperative + * operation (e.g. mremap) running in parallel, bail out and + * request the user to retry later + */ + down_read(&ctx->map_changing_lock); + err = -EAGAIN; + if (atomic_read(&ctx->mmap_changing)) goto out_unlock; } @@ -465,7 +572,7 @@ retry: if (unlikely(err == -ENOENT)) { up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); BUG_ON(!folio); err = copy_folio_from_user(folio, @@ -474,17 +581,6 @@ retry: err = -EFAULT; goto out; } - mmap_read_lock(dst_mm); - down_read(&ctx->map_changing_lock); - /* - * If memory mappings are changing because of non-cooperative - * operation (e.g. mremap) running in parallel, bail out and - * request the user to retry later - */ - if (atomic_read(&ctx->mmap_changing)) { - err = -EAGAIN; - break; - } dst_vma = NULL; goto retry; @@ -505,7 +601,8 @@ retry: out_unlock: up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); +out_unlock_vma: + uffd_mfill_unlock(dst_vma); out: if (folio) folio_put(folio); @@ -597,7 +694,15 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, copied = 0; folio = NULL; retry: - mmap_read_lock(dst_mm); + /* + * Make sure the vma is not shared, that the dst range is + * both valid and fully within a single existing vma. + */ + dst_vma = uffd_mfill_lock(dst_mm, dst_start, len); + if (IS_ERR(dst_vma)) { + err = PTR_ERR(dst_vma); + goto out; + } /* * If memory mappings are changing because of non-cooperative @@ -609,15 +714,6 @@ retry: if (atomic_read(&ctx->mmap_changing)) goto out_unlock; - /* - * Make sure the vma is not shared, that the dst range is - * both valid and fully within a single existing vma. - */ - err = -ENOENT; - dst_vma = find_dst_vma(dst_mm, dst_start, len); - if (!dst_vma) - goto out_unlock; - err = -EINVAL; /* * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but @@ -647,16 +743,6 @@ retry: uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE)) goto out_unlock; - /* - * Ensure the dst_vma has a anon_vma or this page - * would get a NULL anon_vma when moved in the - * dst_vma. - */ - err = -ENOMEM; - if (!(dst_vma->vm_flags & VM_SHARED) && - unlikely(anon_vma_prepare(dst_vma))) - goto out_unlock; - while (src_addr < src_start + len) { pmd_t dst_pmdval; @@ -699,7 +785,7 @@ retry: void *kaddr; up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); BUG_ON(!folio); kaddr = kmap_local_folio(folio, 0); @@ -730,7 +816,7 @@ retry: out_unlock: up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); out: if (folio) folio_put(folio); @@ -1267,27 +1353,136 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, if (!vma_is_anonymous(src_vma) || !vma_is_anonymous(dst_vma)) return -EINVAL; + return 0; +} + +static __always_inline +int find_vmas_mm_locked(struct mm_struct *mm, + unsigned long dst_start, + unsigned long src_start, + struct vm_area_struct **dst_vmap, + struct vm_area_struct **src_vmap) +{ + struct vm_area_struct *vma; + + mmap_assert_locked(mm); + vma = find_vma_and_prepare_anon(mm, dst_start); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + *dst_vmap = vma; + /* Skip finding src_vma if src_start is in dst_vma */ + if (src_start >= vma->vm_start && src_start < vma->vm_end) + goto out_success; + + vma = vma_lookup(mm, src_start); + if (!vma) + return -ENOENT; +out_success: + *src_vmap = vma; + return 0; +} + +#ifdef CONFIG_PER_VMA_LOCK +static int uffd_move_lock(struct mm_struct *mm, + unsigned long dst_start, + unsigned long src_start, + struct vm_area_struct **dst_vmap, + struct vm_area_struct **src_vmap) +{ + struct vm_area_struct *vma; + int err; + + vma = lock_vma(mm, dst_start); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + *dst_vmap = vma; /* - * Ensure the dst_vma has a anon_vma or this page - * would get a NULL anon_vma when moved in the - * dst_vma. + * Skip finding src_vma if src_start is in dst_vma. This also ensures + * that we don't lock the same vma twice. */ - if (unlikely(anon_vma_prepare(dst_vma))) - return -ENOMEM; + if (src_start >= vma->vm_start && src_start < vma->vm_end) { + *src_vmap = vma; + return 0; + } - return 0; + /* + * Using lock_vma() to get src_vma can lead to following deadlock: + * + * Thread1 Thread2 + * ------- ------- + * vma_start_read(dst_vma) + * mmap_write_lock(mm) + * vma_start_write(src_vma) + * vma_start_read(src_vma) + * mmap_read_lock(mm) + * vma_start_write(dst_vma) + */ + *src_vmap = lock_vma_under_rcu(mm, src_start); + if (likely(*src_vmap)) + return 0; + + /* Undo any locking and retry in mmap_lock critical section */ + vma_end_read(*dst_vmap); + + mmap_read_lock(mm); + err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); + if (!err) { + /* + * See comment in lock_vma() as to why not using + * vma_start_read() here. + */ + down_read(&(*dst_vmap)->vm_lock->lock); + if (*dst_vmap != *src_vmap) + down_read(&(*src_vmap)->vm_lock->lock); + } + mmap_read_unlock(mm); + return err; +} + +static void uffd_move_unlock(struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma) +{ + vma_end_read(src_vma); + if (src_vma != dst_vma) + vma_end_read(dst_vma); } +#else + +static int uffd_move_lock(struct mm_struct *mm, + unsigned long dst_start, + unsigned long src_start, + struct vm_area_struct **dst_vmap, + struct vm_area_struct **src_vmap) +{ + int err; + + mmap_read_lock(mm); + err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); + if (err) + mmap_read_unlock(mm); + return err; +} + +static void uffd_move_unlock(struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma) +{ + mmap_assert_locked(src_vma->vm_mm); + mmap_read_unlock(dst_vma->vm_mm); +} +#endif + /** * move_pages - move arbitrary anonymous pages of an existing vma * @ctx: pointer to the userfaultfd context - * @mm: the address space to move pages * @dst_start: start of the destination virtual memory range * @src_start: start of the source virtual memory range * @len: length of the virtual memory range * @mode: flags from uffdio_move.mode * - * Must be called with mmap_lock held for read. + * It will either use the mmap_lock in read mode or per-vma locks * * move_pages() remaps arbitrary anonymous pages atomically in zero * copy. It only works on non shared anonymous pages because those can @@ -1355,10 +1550,10 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, * could be obtained. This is the only additional complexity added to * the rmap code to provide this anonymous page remapping functionality. */ -ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, - unsigned long dst_start, unsigned long src_start, - unsigned long len, __u64 mode) +ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long src_start, unsigned long len, __u64 mode) { + struct mm_struct *mm = ctx->mm; struct vm_area_struct *src_vma, *dst_vma; unsigned long src_addr, dst_addr; pmd_t *src_pmd, *dst_pmd; @@ -1376,28 +1571,34 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, WARN_ON_ONCE(dst_start + len <= dst_start)) goto out; + err = uffd_move_lock(mm, dst_start, src_start, &dst_vma, &src_vma); + if (err) + goto out; + + /* Re-check after taking map_changing_lock */ + err = -EAGAIN; + down_read(&ctx->map_changing_lock); + if (likely(atomic_read(&ctx->mmap_changing))) + goto out_unlock; /* * Make sure the vma is not shared, that the src and dst remap * ranges are both valid and fully within a single existing * vma. */ - src_vma = find_vma(mm, src_start); - if (!src_vma || (src_vma->vm_flags & VM_SHARED)) - goto out; - if (src_start < src_vma->vm_start || - src_start + len > src_vma->vm_end) - goto out; + err = -EINVAL; + if (src_vma->vm_flags & VM_SHARED) + goto out_unlock; + if (src_start + len > src_vma->vm_end) + goto out_unlock; - dst_vma = find_vma(mm, dst_start); - if (!dst_vma || (dst_vma->vm_flags & VM_SHARED)) - goto out; - if (dst_start < dst_vma->vm_start || - dst_start + len > dst_vma->vm_end) - goto out; + if (dst_vma->vm_flags & VM_SHARED) + goto out_unlock; + if (dst_start + len > dst_vma->vm_end) + goto out_unlock; err = validate_move_areas(ctx, src_vma, dst_vma); if (err) - goto out; + goto out_unlock; for (src_addr = src_start, dst_addr = dst_start; src_addr < src_start + len;) { @@ -1514,6 +1715,9 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, moved += step_size; } +out_unlock: + up_read(&ctx->map_changing_lock); + uffd_move_unlock(dst_vma, src_vma); out: VM_WARN_ON(moved < 0); VM_WARN_ON(err > 0); -- cgit v1.2.3 From fafdea34194a10deefc0a0f1dace4280079ce0e7 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 9 Jan 2024 15:16:30 -0700 Subject: arch and include: update LLVM Phabricator links reviews.llvm.org was LLVM's Phabricator instances for code review. It has been abandoned in favor of GitHub pull requests. While the majority of links in the kernel sources still work because of the work Fangrui has done turning the dynamic Phabricator instance into a static archive, there are some issues with that work, so preemptively convert all the links in the kernel sources to point to the commit on GitHub. Most of the commits have the corresponding differential review link in the commit message itself so there should not be any loss of fidelity in the relevant information. Link: https://discourse.llvm.org/t/update-on-github-pull-requests/71540/172 Link: https://lkml.kernel.org/r/20240109-update-llvm-links-v1-2-eb09b59db071@kernel.org Signed-off-by: Nathan Chancellor Reviewed-by: Conor Dooley Reviewed-by: Kees Cook Acked-by: Fangrui Song Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Daniel Borkmann Cc: Mykola Lysenko Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 4 ++-- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/ftrace.h | 2 +- include/linux/compiler-clang.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index aa7c1d435139..5a8acca4dbf4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -382,7 +382,7 @@ config BROKEN_GAS_INST config BUILTIN_RETURN_ADDRESS_STRIPS_PAC bool # Clang's __builtin_return_adddress() strips the PAC since 12.0.0 - # https://reviews.llvm.org/D75044 + # https://github.com/llvm/llvm-project/commit/2a96f47c5ffca84cd774ad402cacd137f4bf45e2 default y if CC_IS_CLANG && (CLANG_VERSION >= 120000) # GCC's __builtin_return_address() strips the PAC since 11.1.0, # and this was backported to 10.2.0, 9.4.0, 8.5.0, but not earlier @@ -2222,7 +2222,7 @@ config STACKPROTECTOR_PER_TASK config UNWIND_PATCH_PAC_INTO_SCS bool "Enable shadow call stack dynamically using code patching" - # needs Clang with https://reviews.llvm.org/D111780 incorporated + # needs Clang with https://github.com/llvm/llvm-project/commit/de07cde67b5d205d58690be012106022aea6d2b3 incorporated depends on CC_IS_CLANG && CLANG_VERSION >= 150000 depends on ARM64_PTR_AUTH_KERNEL && CC_HAS_BRANCH_PROT_PAC_RET depends on SHADOW_CALL_STACK diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index bffbd869a068..69d24f513922 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -312,7 +312,7 @@ config AS_HAS_INSN def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero) config AS_HAS_OPTION_ARCH - # https://reviews.llvm.org/D123515 + # https://github.com/llvm/llvm-project/commit/9e8ed3403c191ab9c4903e8eeb8f732ff8a43cb4 def_bool y depends on $(as-instr, .option arch$(comma) +m) depends on !$(as-instr, .option arch$(comma) -i) diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index 329172122952..06874fb1311e 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -15,7 +15,7 @@ /* * Clang prior to 13 had "mcount" instead of "_mcount": - * https://reviews.llvm.org/D98881 + * https://github.com/llvm/llvm-project/commit/ef58ae86ba778ed7d01cd3f6bd6d08f943abab44 */ #if defined(CONFIG_CC_IS_GCC) || CONFIG_CLANG_VERSION >= 130000 #define MCOUNT_NAME _mcount diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index ddab1ef22bee..f0a47afef125 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -9,7 +9,7 @@ * Clang prior to 17 is being silly and considers many __cleanup() variables * as unused (because they are, their sole purpose is to go out of scope). * - * https://reviews.llvm.org/D152180 + * https://github.com/llvm/llvm-project/commit/877210faa447f4cc7db87812f8ed80e398fedd61 */ #undef __cleanup #define __cleanup(func) __maybe_unused __attribute__((__cleanup__(func))) -- cgit v1.2.3 From a43c47561e46cbefc6a2fe2c3f8f6bda0e553a83 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 4 Jan 2024 17:49:33 +0100 Subject: list: add hlist_count_nodes() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a generic hlist_count_nodes() function and use it in two drivers. This patch (of 3): Add a function to count nodes in a hlist. hlist_count_nodes() is similar to list_count_nodes(). Link: https://lkml.kernel.org/r/20240104164937.424320-1-pierre.gondois@arm.com Link: https://lkml.kernel.org/r/20240104164937.424320-2-pierre.gondois@arm.com Signed-off-by: Pierre Gondois Reviewed-by: Carlos Llamas Acked-by: Coly Li Acked-by: Marco Elver Reviewed-by: Andy Shevchenko Cc: Arve Hjønnevåg Cc: Christian Brauner Cc: Greg Kroah-Hartman Cc: Ingo Molnar Cc: Jani Nikula Cc: Joel Fernandes (Google) Cc: Kees Cook Cc: Kent Overstreet Cc: Martijn Coenen Cc: Suren Baghdasaryan Cc: Todd Kjos Signed-off-by: Andrew Morton --- include/linux/list.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/list.h b/include/linux/list.h index 059aa1fff41e..523b7c4d000a 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -1195,4 +1195,19 @@ static inline void hlist_splice_init(struct hlist_head *from, pos && ({ n = pos->member.next; 1; }); \ pos = hlist_entry_safe(n, typeof(*pos), member)) +/** + * hlist_count_nodes - count nodes in the hlist + * @head: the head for your hlist. + */ +static inline size_t hlist_count_nodes(struct hlist_head *head) +{ + struct hlist_node *pos; + size_t count = 0; + + hlist_for_each(pos, head) + count++; + + return count; +} + #endif -- cgit v1.2.3 From 3911fb647b65f11d42bc97e0890bba8ef7e1e0e6 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sun, 7 Jan 2024 14:01:55 -0800 Subject: lib/win_minmax: fix header comments Don't use "/**" kernel-doc comment marker for non-kernel-doc comment. Correct the filename but omit the path since we know where it is and it could change (but not likely). Link: https://lkml.kernel.org/r/20240107220155.29013-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Signed-off-by: Andrew Morton --- include/linux/win_minmax.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/win_minmax.h b/include/linux/win_minmax.h index 4ca2842d2842..6a5bb052fcc2 100644 --- a/include/linux/win_minmax.h +++ b/include/linux/win_minmax.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/** - * lib/minmax.c: windowed min/max tracker by Kathleen Nichols. +/* + * win_minmax.h: windowed min/max tracker by Kathleen Nichols. * */ #ifndef MINMAX_H -- cgit v1.2.3 From c499c717ee7cc07f47d7ee38a1791a58dcf1d4eb Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Wed, 10 Jan 2024 16:12:12 +0800 Subject: lib min_heap: optimize number of calls to min_heapify() Patch series "lib min_heap: Min heap optimizations". The purpose of this patch series is to enhance the existing min heap implementation. The optimization focuses on both the heap construction process and the number of comparisons made during the heapify operation. This patch (of 2): Improve the heap construction process by reducing unnecessary heapify operations. Specifically, adjust the starting condition from n / 2 to n / 2 - 1 in the loop that iterates over all non-leaf elements. Link: https://lkml.kernel.org/r/20240110081213.2289636-1-visitorckw@gmail.com Link: https://lkml.kernel.org/r/20240110081213.2289636-2-visitorckw@gmail.com Signed-off-by: Kuan-Wei Chiu Acked-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Arnaldo Carvalho de Melo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- include/linux/min_heap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/min_heap.h b/include/linux/min_heap.h index 44077837385f..18a581310eb3 100644 --- a/include/linux/min_heap.h +++ b/include/linux/min_heap.h @@ -70,7 +70,7 @@ void min_heapify_all(struct min_heap *heap, { int i; - for (i = heap->nr / 2; i >= 0; i--) + for (i = heap->nr / 2 - 1; i >= 0; i--) min_heapify(heap, i, func); } -- cgit v1.2.3 From c641722e0c944e423572dd6222c677678d793ed5 Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Wed, 10 Jan 2024 16:12:13 +0800 Subject: lib min_heap: optimize number of comparisons in min_heapify() Optimize the min_heapify() function, resulting in a significant reduction of approximately 50% in the number of comparisons for large random inputs, while maintaining identical results. The current implementation performs two comparisons per level to identify the minimum among three elements. In contrast, the proposed bottom-up variation uses only one comparison per level to assess two children until reaching the leaves. Then, it sifts up until the correct position is determined. Typically, the process of sifting down proceeds to the leaf level, resulting in O(1) secondary comparisons instead of log2(n). This optimization significantly reduces the number of costly indirect function calls and improves overall performance. Link: https://lkml.kernel.org/r/20240110081213.2289636-3-visitorckw@gmail.com Signed-off-by: Kuan-Wei Chiu Acked-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Arnaldo Carvalho de Melo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- include/linux/min_heap.h | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/min_heap.h b/include/linux/min_heap.h index 18a581310eb3..d52daf45861b 100644 --- a/include/linux/min_heap.h +++ b/include/linux/min_heap.h @@ -35,31 +35,33 @@ static __always_inline void min_heapify(struct min_heap *heap, int pos, const struct min_heap_callbacks *func) { - void *left, *right, *parent, *smallest; + void *left, *right; void *data = heap->data; + void *root = data + pos * func->elem_size; + int i = pos, j; + /* Find the sift-down path all the way to the leaves. */ for (;;) { - if (pos * 2 + 1 >= heap->nr) + if (i * 2 + 2 >= heap->nr) break; + left = data + (i * 2 + 1) * func->elem_size; + right = data + (i * 2 + 2) * func->elem_size; + i = func->less(left, right) ? i * 2 + 1 : i * 2 + 2; + } - left = data + ((pos * 2 + 1) * func->elem_size); - parent = data + (pos * func->elem_size); - smallest = parent; - if (func->less(left, smallest)) - smallest = left; - - if (pos * 2 + 2 < heap->nr) { - right = data + ((pos * 2 + 2) * func->elem_size); - if (func->less(right, smallest)) - smallest = right; - } - if (smallest == parent) - break; - func->swp(smallest, parent); - if (smallest == left) - pos = (pos * 2) + 1; - else - pos = (pos * 2) + 2; + /* Special case for the last leaf with no sibling. */ + if (i * 2 + 2 == heap->nr) + i = i * 2 + 1; + + /* Backtrack to the correct location. */ + while (i != pos && func->less(root, data + i * func->elem_size)) + i = (i - 1) / 2; + + /* Shift the element into its correct place. */ + j = i; + while (i != pos) { + i = (i - 1) / 2; + func->swp(data + i * func->elem_size, data + j * func->elem_size); } } -- cgit v1.2.3 From d6bbab8f352efb0533d3fa4af09bb60da770ecc5 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Fri, 19 Jan 2024 04:13:21 +0800 Subject: flex_proportions: remove unused fprop_local_single The single variant of flex_proportions is not used. Simply remove it. Link: https://lkml.kernel.org/r/20240118201321.759174-1-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Signed-off-by: Andrew Morton --- include/linux/flex_proportions.h | 32 ----------------- lib/flex_proportions.c | 77 ---------------------------------------- 2 files changed, 109 deletions(-) (limited to 'include/linux') diff --git a/include/linux/flex_proportions.h b/include/linux/flex_proportions.h index 3e378b1fb0bc..e9a72fd0bfe7 100644 --- a/include/linux/flex_proportions.h +++ b/include/linux/flex_proportions.h @@ -38,38 +38,6 @@ int fprop_global_init(struct fprop_global *p, gfp_t gfp); void fprop_global_destroy(struct fprop_global *p); bool fprop_new_period(struct fprop_global *p, int periods); -/* - * ---- SINGLE ---- - */ -struct fprop_local_single { - /* the local events counter */ - unsigned long events; - /* Period in which we last updated events */ - unsigned int period; - raw_spinlock_t lock; /* Protect period and numerator */ -}; - -#define INIT_FPROP_LOCAL_SINGLE(name) \ -{ .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ -} - -int fprop_local_init_single(struct fprop_local_single *pl); -void fprop_local_destroy_single(struct fprop_local_single *pl); -void __fprop_inc_single(struct fprop_global *p, struct fprop_local_single *pl); -void fprop_fraction_single(struct fprop_global *p, - struct fprop_local_single *pl, unsigned long *numerator, - unsigned long *denominator); - -static inline -void fprop_inc_single(struct fprop_global *p, struct fprop_local_single *pl) -{ - unsigned long flags; - - local_irq_save(flags); - __fprop_inc_single(p, pl); - local_irq_restore(flags); -} - /* * ---- PERCPU ---- */ diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c index 83332fefa6f4..84ecccddc771 100644 --- a/lib/flex_proportions.c +++ b/lib/flex_proportions.c @@ -83,83 +83,6 @@ bool fprop_new_period(struct fprop_global *p, int periods) return true; } -/* - * ---- SINGLE ---- - */ - -int fprop_local_init_single(struct fprop_local_single *pl) -{ - pl->events = 0; - pl->period = 0; - raw_spin_lock_init(&pl->lock); - return 0; -} - -void fprop_local_destroy_single(struct fprop_local_single *pl) -{ -} - -static void fprop_reflect_period_single(struct fprop_global *p, - struct fprop_local_single *pl) -{ - unsigned int period = p->period; - unsigned long flags; - - /* Fast path - period didn't change */ - if (pl->period == period) - return; - raw_spin_lock_irqsave(&pl->lock, flags); - /* Someone updated pl->period while we were spinning? */ - if (pl->period >= period) { - raw_spin_unlock_irqrestore(&pl->lock, flags); - return; - } - /* Aging zeroed our fraction? */ - if (period - pl->period < BITS_PER_LONG) - pl->events >>= period - pl->period; - else - pl->events = 0; - pl->period = period; - raw_spin_unlock_irqrestore(&pl->lock, flags); -} - -/* Event of type pl happened */ -void __fprop_inc_single(struct fprop_global *p, struct fprop_local_single *pl) -{ - fprop_reflect_period_single(p, pl); - pl->events++; - percpu_counter_add(&p->events, 1); -} - -/* Return fraction of events of type pl */ -void fprop_fraction_single(struct fprop_global *p, - struct fprop_local_single *pl, - unsigned long *numerator, unsigned long *denominator) -{ - unsigned int seq; - s64 num, den; - - do { - seq = read_seqcount_begin(&p->sequence); - fprop_reflect_period_single(p, pl); - num = pl->events; - den = percpu_counter_read_positive(&p->events); - } while (read_seqcount_retry(&p->sequence, seq)); - - /* - * Make fraction <= 1 and denominator > 0 even in presence of percpu - * counter errors - */ - if (den <= num) { - if (num) - den = num; - else - den = 1; - } - *denominator = den; - *numerator = num; -} - /* * ---- PERCPU ---- */ -- cgit v1.2.3 From e5efd80a9a7688a26ef34b9c1d3801c25abdf350 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 25 Jan 2024 15:55:17 -0700 Subject: compiler-clang.h: update __diag_clang() macros for minimum version bump The minimum supported version of LLVM for building the kernel has been bumped to 13.0.1. Update the __diag_clang() macros for this bump. Link: https://lkml.kernel.org/r/20240125-bump-min-llvm-ver-to-13-0-1-v1-11-f5ff9bda41c5@kernel.org Signed-off-by: Nathan Chancellor Reviewed-by: Kees Cook Cc: Albert Ou Cc: "Aneesh Kumar K.V (IBM)" Cc: Ard Biesheuvel Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Conor Dooley Cc: Dave Hansen Cc: Ingo Molnar Cc: Mark Rutland Cc: Masahiro Yamada Cc: Michael Ellerman Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Nicolas Schier Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Russell King Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/compiler-clang.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index f0a47afef125..49feac0162a5 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -114,11 +114,7 @@ #define __diag_str(s) __diag_str1(s) #define __diag(s) _Pragma(__diag_str(clang diagnostic s)) -#if CONFIG_CLANG_VERSION >= 110000 -#define __diag_clang_11(s) __diag(s) -#else -#define __diag_clang_11(s) -#endif +#define __diag_clang_13(s) __diag(s) #define __diag_ignore_all(option, comment) \ - __diag_clang(11, ignore, option) + __diag_clang(13, ignore, option) -- cgit v1.2.3 From ac4db926e17a669c788efc89b81a4a0f40648445 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 24 Jan 2024 15:27:35 +0100 Subject: init: remove obsolete arch_call_rest_init() wrapper Since commit 3570ee046c46b5dc ("s390/smp: keep the original lowcore for CPU 0"), there is no longer any architecture that needs to override arch_call_rest_init(). Remove the weak wrapper around rest_init(), call rest_init() directly, and make rest_init() static. Link: https://lkml.kernel.org/r/aa10868bfb176eef4abb8bb4a710b85330792694.1706106183.git.geert@linux-m68k.org Signed-off-by: Geert Uytterhoeven Cc: Arnd Bergmann Cc: Ilya Leoshkevich Cc: Josh Poimboeuf Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- include/linux/start_kernel.h | 2 -- init/main.c | 9 ++------- tools/objtool/noreturns.h | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/start_kernel.h b/include/linux/start_kernel.h index a9806a44a605..09f994ac87df 100644 --- a/include/linux/start_kernel.h +++ b/include/linux/start_kernel.h @@ -9,7 +9,5 @@ up something else. */ extern asmlinkage void __init __noreturn start_kernel(void); -extern void __init __noreturn arch_call_rest_init(void); -extern void __ref __noreturn rest_init(void); #endif /* _LINUX_START_KERNEL_H */ diff --git a/init/main.c b/init/main.c index e24b0780fdff..521f40770e67 100644 --- a/init/main.c +++ b/init/main.c @@ -681,7 +681,7 @@ static void __init setup_command_line(char *command_line) static __initdata DECLARE_COMPLETION(kthreadd_done); -noinline void __ref __noreturn rest_init(void) +static noinline void __ref __noreturn rest_init(void) { struct task_struct *tsk; int pid; @@ -822,11 +822,6 @@ static int __init early_randomize_kstack_offset(char *buf) early_param("randomize_kstack_offset", early_randomize_kstack_offset); #endif -void __init __weak __noreturn arch_call_rest_init(void) -{ - rest_init(); -} - static void __init print_unknown_bootoptions(void) { char *unknown_options; @@ -1069,7 +1064,7 @@ void start_kernel(void) kcsan_init(); /* Do the rest non-__init'ed, we're now alive */ - arch_call_rest_init(); + rest_init(); /* * Avoid stack canaries in callers of boot_init_stack_canary for gcc-10 diff --git a/tools/objtool/noreturns.h b/tools/objtool/noreturns.h index 1685d7ea6a9f..7cda577da897 100644 --- a/tools/objtool/noreturns.h +++ b/tools/objtool/noreturns.h @@ -12,7 +12,6 @@ NORETURN(__reiserfs_panic) NORETURN(__stack_chk_fail) NORETURN(__tdx_hypercall_failed) NORETURN(__ubsan_handle_builtin_unreachable) -NORETURN(arch_call_rest_init) NORETURN(arch_cpu_idle_dead) NORETURN(bch2_trans_in_restart_error) NORETURN(bch2_trans_restart_error) -- cgit v1.2.3 From 022b973a3de9b45148e4ae2da1480ec9e61e74a2 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 5 Feb 2024 12:39:30 +0300 Subject: smp: make __smp_processor_id() 0-argument macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smp_processor_id family of macros never accepted any arguments. #define __smp_processor_id(x) works by accident (see C99 6.10.3 §4). __smp_processor_id() gets 1 (empty) argument and passes it down to raw_smp_processor_id() which doesn't accept arguments. Link: https://lkml.kernel.org/r/0037d1f2-8153-4b33-b43e-f4b6ecd710ac@p183 Signed-off-by: Alexey Dobriyan Cc: Peter Zijlstra (Intel) Signed-off-by: Andrew Morton --- include/linux/smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/smp.h b/include/linux/smp.h index e87520dc2959..cc517002c599 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -261,7 +261,7 @@ static inline int get_boot_cpu_id(void) * regular asm read for the stable. */ #ifndef __smp_processor_id -#define __smp_processor_id(x) raw_smp_processor_id(x) +#define __smp_processor_id() raw_smp_processor_id() #endif #ifdef CONFIG_DEBUG_PREEMPT -- cgit v1.2.3 From 2932fb0a927d30690b8cb70c71d511fd9054bb61 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Thu, 8 Feb 2024 02:14:23 +0000 Subject: list: leverage list_is_head() for list_entry_is_head() This is what list_is_head() exactly do. Link: https://lkml.kernel.org/r/20240208021423.15704-1-richard.weiyang@gmail.com Signed-off-by: Wei Yang Cc: Andy Shevchenko Signed-off-by: Andrew Morton --- include/linux/list.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/list.h b/include/linux/list.h index 523b7c4d000a..5f4b0a39cf46 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -766,7 +766,7 @@ static inline size_t list_count_nodes(struct list_head *head) * @member: the name of the list_head within the struct. */ #define list_entry_is_head(pos, head, member) \ - (&pos->member == (head)) + list_is_head(&pos->member, (head)) /** * list_for_each_entry - iterate over list of given type -- cgit v1.2.3 From 77bcd9e6231a5297ef417a7d7f734d61c2bcceb6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 9 Jan 2024 16:39:35 -0800 Subject: KVM: Add dedicated arch hook for querying if vCPU was preempted in-kernel Plumb in a dedicated hook for querying whether or not a vCPU was preempted in-kernel. Unlike literally every other architecture, x86's VMX can check if a vCPU is in kernel context if and only if the vCPU is loaded on the current pCPU. x86's kvm_arch_vcpu_in_kernel() works around the limitation by querying kvm_get_running_vcpu() and redirecting to vcpu->arch.preempted_in_kernel as needed. But that's unnecessary, confusing, and fragile, e.g. x86 has had at least one bug where KVM incorrectly used a stale preempted_in_kernel. No functional change intended. Reviewed-by: Yuan Yao Link: https://lore.kernel.org/r/20240110003938.490206-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 5 +++++ include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 14 +++++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 019320580a84..94346490f407 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -13054,6 +13054,11 @@ bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) return false; } +bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) +{ + return kvm_arch_vcpu_in_kernel(vcpu); +} + bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) { if (READ_ONCE(vcpu->arch.pv.pv_unhalted)) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 7e7fd25b09b3..28b020404a41 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1505,6 +1505,7 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu); bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu); +bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu); int kvm_arch_post_init_vm(struct kvm *kvm); void kvm_arch_pre_destroy_vm(struct kvm *kvm); int kvm_arch_create_vm_debugfs(struct kvm *kvm); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 10bfc88a69f7..9b92858c8b72 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4042,6 +4042,18 @@ static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu) return false; } +/* + * By default, simply query the target vCPU's current mode when checking if a + * vCPU was preempted in kernel mode. All architectures except x86 (or more + * specifical, except VMX) allow querying whether or not a vCPU is in kernel + * mode even if the vCPU is NOT loaded, i.e. using kvm_arch_vcpu_in_kernel() + * directly for cross-vCPU checks is functionally correct and accurate. + */ +bool __weak kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) +{ + return kvm_arch_vcpu_in_kernel(vcpu); +} + bool __weak kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) { return false; @@ -4080,7 +4092,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) continue; if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode && !kvm_arch_dy_has_pending_interrupt(vcpu) && - !kvm_arch_vcpu_in_kernel(vcpu)) + !kvm_arch_vcpu_preempted_in_kernel(vcpu)) continue; if (!kvm_vcpu_eligible_for_directed_yield(vcpu)) continue; -- cgit v1.2.3 From aa3c88990f77bb9acb3d445337bc088031ac63f9 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 22 Feb 2024 12:41:00 -0800 Subject: sysfs: Document new "group visible" helpers Add documentation and examples for how to use DEFINE_SYSFS_GROUP_VISIBLE() and SYSFS_GROUP_VISIBLE(). Recall that the motivation for this work is that it is easier to reason about the lifetime of statically defined sysfs attributes that become visible at device_add() time rather than dynamically adding them later. DEFINE_SYSFS_GROUP_VISIBLE() tackles one of the reasons to opt for dynamically created attributes which did not have a facility for hiding empty directories. Signed-off-by: Dan Williams Link: https://lore.kernel.org/r/170863446065.1479840.10697164014098377292.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/sysfs.h | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index a42642b277dd..dabf7f4f3581 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -105,8 +105,42 @@ struct attribute_group { #define SYSFS_GROUP_INVISIBLE 020000 /* - * The first call to is_visible() in the create / update path may - * indicate visibility for the entire group + * DEFINE_SYSFS_GROUP_VISIBLE(name): + * A helper macro to pair with the assignment of ".is_visible = + * SYSFS_GROUP_VISIBLE(name)", that arranges for the directory + * associated with a named attribute_group to optionally be hidden. + * This allows for static declaration of attribute_groups, and the + * simplification of attribute visibility lifetime that implies, + * without polluting sysfs with empty attribute directories. + * Ex. + * + * static umode_t example_attr_visible(struct kobject *kobj, + * struct attribute *attr, int n) + * { + * if (example_attr_condition) + * return 0; + * else if (ro_attr_condition) + * return 0444; + * return a->mode; + * } + * + * static bool example_group_visible(struct kobject *kobj) + * { + * if (example_group_condition) + * return false; + * return true; + * } + * + * DEFINE_SYSFS_GROUP_VISIBLE(example); + * + * static struct attribute_group example_group = { + * .name = "example", + * .is_visible = SYSFS_GROUP_VISIBLE(example), + * .attrs = &example_attrs, + * }; + * + * Note that it expects _attr_visible and _group_visible to + * be defined. */ #define DEFINE_SYSFS_GROUP_VISIBLE(name) \ static inline umode_t sysfs_group_visible_##name( \ @@ -119,7 +153,9 @@ struct attribute_group { /* * Same as DEFINE_SYSFS_GROUP_VISIBLE, but for groups with only binary - * attributes + * attributes. If an attribute_group defines both text and binary + * attributes, the group visibility is determined by the function + * specified to is_visible() not is_bin_visible() */ #define DEFINE_SYSFS_BIN_GROUP_VISIBLE(name) \ static inline umode_t sysfs_group_visible_##name( \ -- cgit v1.2.3 From 04edfa7fa059ba50d3236b55ba0ae23b1721e868 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 22 Feb 2024 12:41:06 -0800 Subject: sysfs: Introduce DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE() One of the first users of DEFINE_SYSFS_GROUP_VISIBLE() did this: static umode_t dp0_attr_visible(struct kobject *kobj, struct attribute *attr, int n) { struct sdw_slave *slave = dev_to_sdw_dev(kobj_to_dev(kobj)); if (slave->prop.dp0_prop) return attr->mode; return 0; } static bool dp0_group_visible(struct kobject *kobj) { struct sdw_slave *slave = dev_to_sdw_dev(kobj_to_dev(kobj)); if (slave->prop.dp0_prop) return true; return false; } DEFINE_SYSFS_GROUP_VISIBLE(dp0); ...i.e. the _group_visible() helper is identical to the _attr_visible() helper. Use the "simple" helper to reduce that to: static bool dp0_group_visible(struct kobject *kobj) { struct sdw_slave *slave = dev_to_sdw_dev(kobj_to_dev(kobj)); if (slave->prop.dp0_prop) return true; return false; } DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(dp0); Remove the need to specify per attribute visibility if the goal is to hide the entire group. Signed-off-by: Dan Williams Link: https://lore.kernel.org/r/170863446625.1479840.10593839479268727913.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/sysfs.h | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index dabf7f4f3581..326341c62385 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -140,7 +140,9 @@ struct attribute_group { * }; * * Note that it expects _attr_visible and _group_visible to - * be defined. + * be defined. For cases where individual attributes do not need + * separate visibility consideration, only entire group visibility at + * once, see DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(). */ #define DEFINE_SYSFS_GROUP_VISIBLE(name) \ static inline umode_t sysfs_group_visible_##name( \ @@ -151,6 +153,38 @@ struct attribute_group { return name##_attr_visible(kobj, attr, n); \ } +/* + * DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(name): + * A helper macro to pair with SYSFS_GROUP_VISIBLE() that like + * DEFINE_SYSFS_GROUP_VISIBLE() controls group visibility, but does + * not require the implementation of a per-attribute visibility + * callback. + * Ex. + * + * static bool example_group_visible(struct kobject *kobj) + * { + * if (example_group_condition) + * return false; + * return true; + * } + * + * DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(example); + * + * static struct attribute_group example_group = { + * .name = "example", + * .is_visible = SYSFS_GROUP_VISIBLE(example), + * .attrs = &example_attrs, + * }; + */ +#define DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(name) \ + static inline umode_t sysfs_group_visible_##name( \ + struct kobject *kobj, struct attribute *a, int n) \ + { \ + if (n == 0 && !name##_group_visible(kobj)) \ + return SYSFS_GROUP_INVISIBLE; \ + return a->mode; \ + } + /* * Same as DEFINE_SYSFS_GROUP_VISIBLE, but for groups with only binary * attributes. If an attribute_group defines both text and binary @@ -166,6 +200,15 @@ struct attribute_group { return name##_attr_visible(kobj, attr, n); \ } +#define DEFINE_SIMPLE_SYSFS_BIN_GROUP_VISIBLE(name) \ + static inline umode_t sysfs_group_visible_##name( \ + struct kobject *kobj, struct bin_attribute *a, int n) \ + { \ + if (n == 0 && !name##_group_visible(kobj)) \ + return SYSFS_GROUP_INVISIBLE; \ + return a->mode; \ + } + #define SYSFS_GROUP_VISIBLE(fn) sysfs_group_visible_##fn /* -- cgit v1.2.3 From 00ca8a15dafa990d391abc37f2b8256ddf909b35 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 17 Feb 2024 10:39:37 +0100 Subject: phy: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Acked-by: Thierry Reding Acked-by: Linus Walleij Acked-by: Florian Fainelli #Broadcom Link: https://lore.kernel.org/r/20240217093937.58234-1-krzysztof.kozlowski@linaro.org Signed-off-by: Vinod Koul --- drivers/phy/allwinner/phy-sun4i-usb.c | 2 +- drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c | 2 +- drivers/phy/broadcom/phy-bcm-sr-pcie.c | 2 +- drivers/phy/broadcom/phy-bcm-sr-usb.c | 2 +- drivers/phy/broadcom/phy-bcm63xx-usbh.c | 2 +- drivers/phy/broadcom/phy-brcm-usb.c | 2 +- drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c | 2 +- drivers/phy/freescale/phy-fsl-lynx-28g.c | 2 +- drivers/phy/hisilicon/phy-histb-combphy.c | 2 +- drivers/phy/intel/phy-intel-lgm-combo.c | 2 +- drivers/phy/lantiq/phy-lantiq-vrx200-pcie.c | 2 +- drivers/phy/marvell/phy-armada375-usb2.c | 2 +- drivers/phy/marvell/phy-armada38x-comphy.c | 2 +- drivers/phy/marvell/phy-berlin-sata.c | 2 +- drivers/phy/marvell/phy-mvebu-a3700-comphy.c | 2 +- drivers/phy/marvell/phy-mvebu-cp110-comphy.c | 2 +- drivers/phy/mediatek/phy-mtk-mipi-csi-0-5.c | 2 +- drivers/phy/mediatek/phy-mtk-tphy.c | 2 +- drivers/phy/mediatek/phy-mtk-xsphy.c | 2 +- drivers/phy/microchip/lan966x_serdes.c | 2 +- drivers/phy/microchip/sparx5_serdes.c | 2 +- drivers/phy/mscc/phy-ocelot-serdes.c | 2 +- drivers/phy/phy-core.c | 8 ++++---- drivers/phy/phy-xgene.c | 2 +- drivers/phy/qualcomm/phy-qcom-qmp-combo.c | 2 +- drivers/phy/ralink/phy-mt7621-pci.c | 2 +- drivers/phy/renesas/phy-rcar-gen2.c | 2 +- drivers/phy/renesas/phy-rcar-gen3-usb2.c | 2 +- drivers/phy/renesas/r8a779f0-ether-serdes.c | 2 +- drivers/phy/rockchip/phy-rockchip-naneng-combphy.c | 2 +- drivers/phy/rockchip/phy-rockchip-pcie.c | 2 +- drivers/phy/samsung/phy-exynos-mipi-video.c | 2 +- drivers/phy/samsung/phy-exynos5-usbdrd.c | 2 +- drivers/phy/samsung/phy-samsung-usb2.c | 2 +- drivers/phy/socionext/phy-uniphier-usb2.c | 2 +- drivers/phy/st/phy-miphy28lp.c | 2 +- drivers/phy/st/phy-spear1310-miphy.c | 2 +- drivers/phy/st/phy-spear1340-miphy.c | 2 +- drivers/phy/st/phy-stm32-usbphyc.c | 2 +- drivers/phy/tegra/xusb.c | 2 +- drivers/phy/ti/phy-am654-serdes.c | 2 +- drivers/phy/ti/phy-da8xx-usb.c | 2 +- drivers/phy/ti/phy-gmii-sel.c | 2 +- drivers/phy/xilinx/phy-zynqmp.c | 2 +- drivers/pinctrl/tegra/pinctrl-tegra-xusb.c | 2 +- include/linux/phy/phy.h | 14 +++++++------- 46 files changed, 55 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/drivers/phy/allwinner/phy-sun4i-usb.c b/drivers/phy/allwinner/phy-sun4i-usb.c index e53a9a9317bc..b0f19e950601 100644 --- a/drivers/phy/allwinner/phy-sun4i-usb.c +++ b/drivers/phy/allwinner/phy-sun4i-usb.c @@ -683,7 +683,7 @@ static int sun4i_usb_phy0_vbus_notify(struct notifier_block *nb, } static struct phy *sun4i_usb_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct sun4i_usb_phy_data *data = dev_get_drvdata(dev); diff --git a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c index 2712c4bd549d..5468831d6ab9 100644 --- a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c +++ b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c @@ -350,7 +350,7 @@ static int phy_g12a_usb3_pcie_exit(struct phy *phy) } static struct phy *phy_g12a_usb3_pcie_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct phy_g12a_usb3_pcie_priv *priv = dev_get_drvdata(dev); unsigned int mode; diff --git a/drivers/phy/broadcom/phy-bcm-sr-pcie.c b/drivers/phy/broadcom/phy-bcm-sr-pcie.c index 8a4aadf166cf..ff9b3862bf7a 100644 --- a/drivers/phy/broadcom/phy-bcm-sr-pcie.c +++ b/drivers/phy/broadcom/phy-bcm-sr-pcie.c @@ -195,7 +195,7 @@ static const struct phy_ops sr_paxc_phy_ops = { }; static struct phy *sr_pcie_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct sr_pcie_phy_core *core; int phy_idx; diff --git a/drivers/phy/broadcom/phy-bcm-sr-usb.c b/drivers/phy/broadcom/phy-bcm-sr-usb.c index b0bd18a5df87..6bcfe83609c8 100644 --- a/drivers/phy/broadcom/phy-bcm-sr-usb.c +++ b/drivers/phy/broadcom/phy-bcm-sr-usb.c @@ -209,7 +209,7 @@ static const struct phy_ops sr_phy_ops = { }; static struct phy *bcm_usb_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct bcm_usb_phy_cfg *phy_cfg; int phy_idx; diff --git a/drivers/phy/broadcom/phy-bcm63xx-usbh.c b/drivers/phy/broadcom/phy-bcm63xx-usbh.c index f8183dea774b..647644de041b 100644 --- a/drivers/phy/broadcom/phy-bcm63xx-usbh.c +++ b/drivers/phy/broadcom/phy-bcm63xx-usbh.c @@ -366,7 +366,7 @@ static const struct phy_ops bcm63xx_usbh_phy_ops = { }; static struct phy *bcm63xx_usbh_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct bcm63xx_usbh_phy *usbh = dev_get_drvdata(dev); diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c index a16f0b58eb74..ad2eec095601 100644 --- a/drivers/phy/broadcom/phy-brcm-usb.c +++ b/drivers/phy/broadcom/phy-brcm-usb.c @@ -175,7 +175,7 @@ static const struct phy_ops brcm_usb_phy_ops = { }; static struct phy *brcm_usb_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct brcm_usb_phy_data *data = dev_get_drvdata(dev); diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c b/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c index 0ae052df3765..38388dd04bdc 100644 --- a/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c +++ b/drivers/phy/freescale/phy-fsl-imx8qm-lvds-phy.c @@ -294,7 +294,7 @@ static int mixel_lvds_phy_reset(struct device *dev) } static struct phy *mixel_lvds_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mixel_lvds_phy_priv *priv = dev_get_drvdata(dev); unsigned int phy_id; diff --git a/drivers/phy/freescale/phy-fsl-lynx-28g.c b/drivers/phy/freescale/phy-fsl-lynx-28g.c index e2187767ce00..b86da8e9daa4 100644 --- a/drivers/phy/freescale/phy-fsl-lynx-28g.c +++ b/drivers/phy/freescale/phy-fsl-lynx-28g.c @@ -556,7 +556,7 @@ static void lynx_28g_lane_read_configuration(struct lynx_28g_lane *lane) } static struct phy *lynx_28g_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct lynx_28g_priv *priv = dev_get_drvdata(dev); int idx = args->args[0]; diff --git a/drivers/phy/hisilicon/phy-histb-combphy.c b/drivers/phy/hisilicon/phy-histb-combphy.c index c44588fd5a53..7436dcae3981 100644 --- a/drivers/phy/hisilicon/phy-histb-combphy.c +++ b/drivers/phy/hisilicon/phy-histb-combphy.c @@ -163,7 +163,7 @@ static const struct phy_ops histb_combphy_ops = { }; static struct phy *histb_combphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct histb_combphy_priv *priv = dev_get_drvdata(dev); struct histb_combphy_mode *mode = &priv->mode; diff --git a/drivers/phy/intel/phy-intel-lgm-combo.c b/drivers/phy/intel/phy-intel-lgm-combo.c index d32e267c0001..f8e3054a9e59 100644 --- a/drivers/phy/intel/phy-intel-lgm-combo.c +++ b/drivers/phy/intel/phy-intel-lgm-combo.c @@ -508,7 +508,7 @@ static const struct phy_ops intel_cbphy_ops = { }; static struct phy *intel_cbphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct intel_combo_phy *cbphy = dev_get_drvdata(dev); u32 iphy_id; diff --git a/drivers/phy/lantiq/phy-lantiq-vrx200-pcie.c b/drivers/phy/lantiq/phy-lantiq-vrx200-pcie.c index ef93bf2cba10..406a87c8b759 100644 --- a/drivers/phy/lantiq/phy-lantiq-vrx200-pcie.c +++ b/drivers/phy/lantiq/phy-lantiq-vrx200-pcie.c @@ -358,7 +358,7 @@ static const struct phy_ops ltq_vrx200_pcie_phy_ops = { }; static struct phy *ltq_vrx200_pcie_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct ltq_vrx200_pcie_phy_priv *priv = dev_get_drvdata(dev); unsigned int mode; diff --git a/drivers/phy/marvell/phy-armada375-usb2.c b/drivers/phy/marvell/phy-armada375-usb2.c index b141e3cd8a94..3731f9b25655 100644 --- a/drivers/phy/marvell/phy-armada375-usb2.c +++ b/drivers/phy/marvell/phy-armada375-usb2.c @@ -61,7 +61,7 @@ static const struct phy_ops armada375_usb_phy_ops = { * USB3 case it still optional and we use ENODEV. */ static struct phy *armada375_usb_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct armada375_cluster_phy *cluster_phy = dev_get_drvdata(dev); diff --git a/drivers/phy/marvell/phy-armada38x-comphy.c b/drivers/phy/marvell/phy-armada38x-comphy.c index d3259984ee8e..5063361b0120 100644 --- a/drivers/phy/marvell/phy-armada38x-comphy.c +++ b/drivers/phy/marvell/phy-armada38x-comphy.c @@ -160,7 +160,7 @@ static const struct phy_ops a38x_comphy_ops = { }; static struct phy *a38x_comphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct a38x_comphy_lane *lane; struct phy *phy; diff --git a/drivers/phy/marvell/phy-berlin-sata.c b/drivers/phy/marvell/phy-berlin-sata.c index f972d78372ea..c90e2867900c 100644 --- a/drivers/phy/marvell/phy-berlin-sata.c +++ b/drivers/phy/marvell/phy-berlin-sata.c @@ -155,7 +155,7 @@ static int phy_berlin_sata_power_off(struct phy *phy) } static struct phy *phy_berlin_sata_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct phy_berlin_priv *priv = dev_get_drvdata(dev); int i; diff --git a/drivers/phy/marvell/phy-mvebu-a3700-comphy.c b/drivers/phy/marvell/phy-mvebu-a3700-comphy.c index 24c3371e2bb2..41162d7228c9 100644 --- a/drivers/phy/marvell/phy-mvebu-a3700-comphy.c +++ b/drivers/phy/marvell/phy-mvebu-a3700-comphy.c @@ -1213,7 +1213,7 @@ static const struct phy_ops mvebu_a3700_comphy_ops = { }; static struct phy *mvebu_a3700_comphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mvebu_a3700_comphy_lane *lane; unsigned int port; diff --git a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c index b0dd13366598..da5e8f405749 100644 --- a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c +++ b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c @@ -917,7 +917,7 @@ static const struct phy_ops mvebu_comphy_ops = { }; static struct phy *mvebu_comphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mvebu_comphy_lane *lane; struct phy *phy; diff --git a/drivers/phy/mediatek/phy-mtk-mipi-csi-0-5.c b/drivers/phy/mediatek/phy-mtk-mipi-csi-0-5.c index 972c129185f7..058e1d926630 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-csi-0-5.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-csi-0-5.c @@ -165,7 +165,7 @@ static int mtk_mipi_phy_power_off(struct phy *phy) } static struct phy *mtk_mipi_cdphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mtk_mipi_cdphy_port *priv = dev_get_drvdata(dev); diff --git a/drivers/phy/mediatek/phy-mtk-tphy.c b/drivers/phy/mediatek/phy-mtk-tphy.c index a4746f6cb8a1..25b86bbb9cec 100644 --- a/drivers/phy/mediatek/phy-mtk-tphy.c +++ b/drivers/phy/mediatek/phy-mtk-tphy.c @@ -1467,7 +1467,7 @@ static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) } static struct phy *mtk_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mtk_tphy *tphy = dev_get_drvdata(dev); struct mtk_phy_instance *instance = NULL; diff --git a/drivers/phy/mediatek/phy-mtk-xsphy.c b/drivers/phy/mediatek/phy-mtk-xsphy.c index b222fbbd71d1..064fd0941727 100644 --- a/drivers/phy/mediatek/phy-mtk-xsphy.c +++ b/drivers/phy/mediatek/phy-mtk-xsphy.c @@ -378,7 +378,7 @@ static int mtk_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) } static struct phy *mtk_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mtk_xsphy *xsphy = dev_get_drvdata(dev); struct xsphy_instance *inst = NULL; diff --git a/drivers/phy/microchip/lan966x_serdes.c b/drivers/phy/microchip/lan966x_serdes.c index c1a41b6cd29b..0188b0f01a8d 100644 --- a/drivers/phy/microchip/lan966x_serdes.c +++ b/drivers/phy/microchip/lan966x_serdes.c @@ -516,7 +516,7 @@ static const struct phy_ops serdes_ops = { }; static struct phy *serdes_simple_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct serdes_ctrl *ctrl = dev_get_drvdata(dev); unsigned int port, idx, i; diff --git a/drivers/phy/microchip/sparx5_serdes.c b/drivers/phy/microchip/sparx5_serdes.c index 01bd5ea620c5..7cb85029fab3 100644 --- a/drivers/phy/microchip/sparx5_serdes.c +++ b/drivers/phy/microchip/sparx5_serdes.c @@ -2509,7 +2509,7 @@ static struct sparx5_serdes_io_resource sparx5_serdes_iomap[] = { /* Client lookup function, uses serdes index */ static struct phy *sparx5_serdes_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct sparx5_serdes_private *priv = dev_get_drvdata(dev); int idx; diff --git a/drivers/phy/mscc/phy-ocelot-serdes.c b/drivers/phy/mscc/phy-ocelot-serdes.c index d9443e865a78..1cd1b5db2ad7 100644 --- a/drivers/phy/mscc/phy-ocelot-serdes.c +++ b/drivers/phy/mscc/phy-ocelot-serdes.c @@ -441,7 +441,7 @@ static const struct phy_ops serdes_ops = { }; static struct phy *serdes_simple_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct serdes_ctrl *ctrl = dev_get_drvdata(dev); unsigned int port, idx, i; diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c index d9be6a4d5383..7f9b4de772ee 100644 --- a/drivers/phy/phy-core.c +++ b/drivers/phy/phy-core.c @@ -700,8 +700,8 @@ EXPORT_SYMBOL_GPL(devm_phy_put); * should provide a custom of_xlate function that reads the *args* and returns * the appropriate phy. */ -struct phy *of_phy_simple_xlate(struct device *dev, struct of_phandle_args - *args) +struct phy *of_phy_simple_xlate(struct device *dev, + const struct of_phandle_args *args) { struct phy *phy; struct class_dev_iter iter; @@ -1095,7 +1095,7 @@ EXPORT_SYMBOL_GPL(devm_phy_destroy); struct phy_provider *__of_phy_provider_register(struct device *dev, struct device_node *children, struct module *owner, struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { struct phy_provider *phy_provider; @@ -1158,7 +1158,7 @@ EXPORT_SYMBOL_GPL(__of_phy_provider_register); struct phy_provider *__devm_of_phy_provider_register(struct device *dev, struct device_node *children, struct module *owner, struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { struct phy_provider **ptr, *phy_provider; diff --git a/drivers/phy/phy-xgene.c b/drivers/phy/phy-xgene.c index 1f0f908323f0..5007dc7a357c 100644 --- a/drivers/phy/phy-xgene.c +++ b/drivers/phy/phy-xgene.c @@ -1611,7 +1611,7 @@ static const struct phy_ops xgene_phy_ops = { }; static struct phy *xgene_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct xgene_phy_ctx *ctx = dev_get_drvdata(dev); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index b6908a03da58..546d3c6bee32 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -3454,7 +3454,7 @@ static int qmp_combo_parse_dt(struct qmp_combo *qmp) return 0; } -static struct phy *qmp_combo_phy_xlate(struct device *dev, struct of_phandle_args *args) +static struct phy *qmp_combo_phy_xlate(struct device *dev, const struct of_phandle_args *args) { struct qmp_combo *qmp = dev_get_drvdata(dev); diff --git a/drivers/phy/ralink/phy-mt7621-pci.c b/drivers/phy/ralink/phy-mt7621-pci.c index 2f876f158e1d..a591ad95347c 100644 --- a/drivers/phy/ralink/phy-mt7621-pci.c +++ b/drivers/phy/ralink/phy-mt7621-pci.c @@ -263,7 +263,7 @@ static const struct phy_ops mt7621_pci_phy_ops = { }; static struct phy *mt7621_pcie_phy_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct mt7621_pci_phy *mt7621_phy = dev_get_drvdata(dev); diff --git a/drivers/phy/renesas/phy-rcar-gen2.c b/drivers/phy/renesas/phy-rcar-gen2.c index 507435af2656..c0221e7258c0 100644 --- a/drivers/phy/renesas/phy-rcar-gen2.c +++ b/drivers/phy/renesas/phy-rcar-gen2.c @@ -306,7 +306,7 @@ static const struct of_device_id rcar_gen2_phy_match_table[] = { MODULE_DEVICE_TABLE(of, rcar_gen2_phy_match_table); static struct phy *rcar_gen2_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct rcar_gen2_phy_driver *drv; struct device_node *np = args->np; diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c index e53eace7c91e..c8b58607b037 100644 --- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c +++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c @@ -608,7 +608,7 @@ static const unsigned int rcar_gen3_phy_cable[] = { }; static struct phy *rcar_gen3_phy_usb2_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct rcar_gen3_chan *ch = dev_get_drvdata(dev); diff --git a/drivers/phy/renesas/r8a779f0-ether-serdes.c b/drivers/phy/renesas/r8a779f0-ether-serdes.c index fc6e398fa3bf..f1f1da4a0b1f 100644 --- a/drivers/phy/renesas/r8a779f0-ether-serdes.c +++ b/drivers/phy/renesas/r8a779f0-ether-serdes.c @@ -334,7 +334,7 @@ static const struct phy_ops r8a779f0_eth_serdes_ops = { }; static struct phy *r8a779f0_eth_serdes_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct r8a779f0_eth_serdes_drv_data *dd = dev_get_drvdata(dev); diff --git a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c index 5de5e2e97ffa..76b9cf417591 100644 --- a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c +++ b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c @@ -251,7 +251,7 @@ static const struct phy_ops rochchip_combphy_ops = { .owner = THIS_MODULE, }; -static struct phy *rockchip_combphy_xlate(struct device *dev, struct of_phandle_args *args) +static struct phy *rockchip_combphy_xlate(struct device *dev, const struct of_phandle_args *args) { struct rockchip_combphy_priv *priv = dev_get_drvdata(dev); diff --git a/drivers/phy/rockchip/phy-rockchip-pcie.c b/drivers/phy/rockchip/phy-rockchip-pcie.c index 1bbd6be2a584..51cc5ece0e63 100644 --- a/drivers/phy/rockchip/phy-rockchip-pcie.c +++ b/drivers/phy/rockchip/phy-rockchip-pcie.c @@ -82,7 +82,7 @@ static struct rockchip_pcie_phy *to_pcie_phy(struct phy_pcie_instance *inst) } static struct phy *rockchip_pcie_phy_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct rockchip_pcie_phy *rk_phy = dev_get_drvdata(dev); diff --git a/drivers/phy/samsung/phy-exynos-mipi-video.c b/drivers/phy/samsung/phy-exynos-mipi-video.c index 592d8067e848..f6756a609a9a 100644 --- a/drivers/phy/samsung/phy-exynos-mipi-video.c +++ b/drivers/phy/samsung/phy-exynos-mipi-video.c @@ -274,7 +274,7 @@ static int exynos_mipi_video_phy_power_off(struct phy *phy) } static struct phy *exynos_mipi_video_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct exynos_mipi_video_phy *state = dev_get_drvdata(dev); diff --git a/drivers/phy/samsung/phy-exynos5-usbdrd.c b/drivers/phy/samsung/phy-exynos5-usbdrd.c index 3f310b28bfff..04171eed5b16 100644 --- a/drivers/phy/samsung/phy-exynos5-usbdrd.c +++ b/drivers/phy/samsung/phy-exynos5-usbdrd.c @@ -715,7 +715,7 @@ static int exynos5420_usbdrd_phy_calibrate(struct exynos5_usbdrd_phy *phy_drd) } static struct phy *exynos5_usbdrd_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct exynos5_usbdrd_phy *phy_drd = dev_get_drvdata(dev); diff --git a/drivers/phy/samsung/phy-samsung-usb2.c b/drivers/phy/samsung/phy-samsung-usb2.c index 68a174eca0ba..9de744cd6f39 100644 --- a/drivers/phy/samsung/phy-samsung-usb2.c +++ b/drivers/phy/samsung/phy-samsung-usb2.c @@ -87,7 +87,7 @@ static const struct phy_ops samsung_usb2_phy_ops = { }; static struct phy *samsung_usb2_phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct samsung_usb2_phy_driver *drv; diff --git a/drivers/phy/socionext/phy-uniphier-usb2.c b/drivers/phy/socionext/phy-uniphier-usb2.c index 3f2086ed4fe4..21c201717d95 100644 --- a/drivers/phy/socionext/phy-uniphier-usb2.c +++ b/drivers/phy/socionext/phy-uniphier-usb2.c @@ -81,7 +81,7 @@ static int uniphier_u2phy_init(struct phy *phy) } static struct phy *uniphier_u2phy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct uniphier_u2phy_priv *priv = dev_get_drvdata(dev); diff --git a/drivers/phy/st/phy-miphy28lp.c b/drivers/phy/st/phy-miphy28lp.c index e30305b77f0d..063fc38788ed 100644 --- a/drivers/phy/st/phy-miphy28lp.c +++ b/drivers/phy/st/phy-miphy28lp.c @@ -1074,7 +1074,7 @@ static int miphy28lp_get_addr(struct miphy28lp_phy *miphy_phy) } static struct phy *miphy28lp_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct miphy28lp_dev *miphy_dev = dev_get_drvdata(dev); struct miphy28lp_phy *miphy_phy = NULL; diff --git a/drivers/phy/st/phy-spear1310-miphy.c b/drivers/phy/st/phy-spear1310-miphy.c index 35a9831b5161..c661ab63505f 100644 --- a/drivers/phy/st/phy-spear1310-miphy.c +++ b/drivers/phy/st/phy-spear1310-miphy.c @@ -183,7 +183,7 @@ static const struct phy_ops spear1310_miphy_ops = { }; static struct phy *spear1310_miphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct spear1310_miphy_priv *priv = dev_get_drvdata(dev); diff --git a/drivers/phy/st/phy-spear1340-miphy.c b/drivers/phy/st/phy-spear1340-miphy.c index 34a1cf21015f..85a60d64ebb7 100644 --- a/drivers/phy/st/phy-spear1340-miphy.c +++ b/drivers/phy/st/phy-spear1340-miphy.c @@ -220,7 +220,7 @@ static SIMPLE_DEV_PM_OPS(spear1340_miphy_pm_ops, spear1340_miphy_suspend, spear1340_miphy_resume); static struct phy *spear1340_miphy_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct spear1340_miphy_priv *priv = dev_get_drvdata(dev); diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index d5e7e44000b5..9dbe60dcf319 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -574,7 +574,7 @@ static void stm32_usbphyc_switch_setup(struct stm32_usbphyc *usbphyc, } static struct phy *stm32_usbphyc_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct stm32_usbphyc *usbphyc = dev_get_drvdata(dev); struct stm32_usbphyc_phy *usbphyc_phy = NULL; diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c index 142ebe0247cc..0dc86a7740e3 100644 --- a/drivers/phy/tegra/xusb.c +++ b/drivers/phy/tegra/xusb.c @@ -22,7 +22,7 @@ #include "xusb.h" static struct phy *tegra_xusb_pad_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct tegra_xusb_pad *pad = dev_get_drvdata(dev); struct phy *phy = NULL; diff --git a/drivers/phy/ti/phy-am654-serdes.c b/drivers/phy/ti/phy-am654-serdes.c index 3f1d43e8b7ad..8b3b937de624 100644 --- a/drivers/phy/ti/phy-am654-serdes.c +++ b/drivers/phy/ti/phy-am654-serdes.c @@ -495,7 +495,7 @@ static void serdes_am654_release(struct phy *x) } static struct phy *serdes_am654_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct serdes_am654 *am654_phy; struct phy *phy; diff --git a/drivers/phy/ti/phy-da8xx-usb.c b/drivers/phy/ti/phy-da8xx-usb.c index b7a9ef3f4654..0fe577f0d6c1 100644 --- a/drivers/phy/ti/phy-da8xx-usb.c +++ b/drivers/phy/ti/phy-da8xx-usb.c @@ -119,7 +119,7 @@ static const struct phy_ops da8xx_usb20_phy_ops = { }; static struct phy *da8xx_usb_phy_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct da8xx_usb_phy *d_phy = dev_get_drvdata(dev); diff --git a/drivers/phy/ti/phy-gmii-sel.c b/drivers/phy/ti/phy-gmii-sel.c index 81dd1c3449d9..b30bf740e2e0 100644 --- a/drivers/phy/ti/phy-gmii-sel.c +++ b/drivers/phy/ti/phy-gmii-sel.c @@ -297,7 +297,7 @@ static const struct phy_ops phy_gmii_sel_ops = { }; static struct phy *phy_gmii_sel_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct phy_gmii_sel_priv *priv = dev_get_drvdata(dev); int phy_id = args->args[0]; diff --git a/drivers/phy/xilinx/phy-zynqmp.c b/drivers/phy/xilinx/phy-zynqmp.c index 2559c6594cea..f72c5257d712 100644 --- a/drivers/phy/xilinx/phy-zynqmp.c +++ b/drivers/phy/xilinx/phy-zynqmp.c @@ -768,7 +768,7 @@ static const unsigned int icm_matrix[NUM_LANES][CONTROLLERS_PER_LANE] = { /* Translate OF phandle and args to PHY instance. */ static struct phy *xpsgtr_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct xpsgtr_dev *gtr_dev = dev_get_drvdata(dev); struct xpsgtr_phy *gtr_phy; diff --git a/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c b/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c index 7641848be4de..96ef57a7d385 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c @@ -685,7 +685,7 @@ static const struct phy_ops sata_phy_ops = { }; static struct phy *tegra_xusb_padctl_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct tegra_xusb_padctl *padctl = dev_get_drvdata(dev); unsigned int index = args->args[0]; diff --git a/include/linux/phy/phy.h b/include/linux/phy/phy.h index f6d607ef0e80..58be86e6fe83 100644 --- a/include/linux/phy/phy.h +++ b/include/linux/phy/phy.h @@ -176,7 +176,7 @@ struct phy_provider { struct module *owner; struct list_head list; struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args); + const struct of_phandle_args *args); }; /** @@ -265,7 +265,7 @@ void phy_put(struct device *dev, struct phy *phy); void devm_phy_put(struct device *dev, struct phy *phy); struct phy *of_phy_get(struct device_node *np, const char *con_id); struct phy *of_phy_simple_xlate(struct device *dev, - struct of_phandle_args *args); + const struct of_phandle_args *args); struct phy *phy_create(struct device *dev, struct device_node *node, const struct phy_ops *ops); struct phy *devm_phy_create(struct device *dev, struct device_node *node, @@ -275,11 +275,11 @@ void devm_phy_destroy(struct device *dev, struct phy *phy); struct phy_provider *__of_phy_provider_register(struct device *dev, struct device_node *children, struct module *owner, struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args)); + const struct of_phandle_args *args)); struct phy_provider *__devm_of_phy_provider_register(struct device *dev, struct device_node *children, struct module *owner, struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args)); + const struct of_phandle_args *args)); void of_phy_provider_unregister(struct phy_provider *phy_provider); void devm_of_phy_provider_unregister(struct device *dev, struct phy_provider *phy_provider); @@ -479,7 +479,7 @@ static inline struct phy *of_phy_get(struct device_node *np, const char *con_id) } static inline struct phy *of_phy_simple_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { return ERR_PTR(-ENOSYS); } @@ -509,7 +509,7 @@ static inline void devm_phy_destroy(struct device *dev, struct phy *phy) static inline struct phy_provider *__of_phy_provider_register( struct device *dev, struct device_node *children, struct module *owner, struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { return ERR_PTR(-ENOSYS); } @@ -517,7 +517,7 @@ static inline struct phy_provider *__of_phy_provider_register( static inline struct phy_provider *__devm_of_phy_provider_register(struct device *dev, struct device_node *children, struct module *owner, struct phy * (*of_xlate)(struct device *dev, - struct of_phandle_args *args)) + const struct of_phandle_args *args)) { return ERR_PTR(-ENOSYS); } -- cgit v1.2.3 From dfebe38e46c2e866c6f3ff54d8ed8dabbed193de Mon Sep 17 00:00:00 2001 From: Fuyao Kashizuku Date: Wed, 27 Dec 2023 10:01:17 +0800 Subject: mfd: sun4i-gpadc: Correct specified GPADC interrupt numbers The identifiers are used as IRQ resource numbers, where 0 is treated specially. This fixes sun4i-gpadc-iio probe failed when request irq. The backstack: WARNING: CPU: 3 PID: 1 at drivers/base/platform.c:451 __platform_get_irq_byname+0xb8/0xc4 0 is an invalid IRQ number Modules linked in: CPU: 3 PID: 1 Comm: swapper/0 Not tainted 6.7.0-rc6 #9 Hardware name: Allwinner sun8i Family unwind_backtrace show_stack dump_stack_lvl __warn warn_slowpath_fmt __platform_get_irq_byname platform_get_irq_byname sun4i_irq_init sun4i_gpadc_probe platform_probe really_probe __driver_probe_device driver_probe_device __driver_attach bus_for_each_dev bus_add_driver driver_register do_one_initcall do_initcall_level do_initcalls kernel_init_freeable kernel_init Log reports: sun4i-gpadc-iio sun6i-a31-gpadc-iio.0: error -EINVAL: IRQ FIFO_DATA_PENDING not found sun4i-gpadc-iio: probe of sun6i-a31-gpadc-iio.0 failed with error -22 Signed-off-by: Fuyao Kashizuku Acked-by: Jernej Skrabec Link: https://lore.kernel.org/r/ZYuFbUUus9apiCpq@debian.cyg Signed-off-by: Lee Jones --- include/linux/mfd/sun4i-gpadc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/sun4i-gpadc.h b/include/linux/mfd/sun4i-gpadc.h index ea0ccf33a459..021f820f9d52 100644 --- a/include/linux/mfd/sun4i-gpadc.h +++ b/include/linux/mfd/sun4i-gpadc.h @@ -81,8 +81,8 @@ #define SUN4I_GPADC_TEMP_DATA 0x20 #define SUN4I_GPADC_DATA 0x24 -#define SUN4I_GPADC_IRQ_FIFO_DATA 0 -#define SUN4I_GPADC_IRQ_TEMP_DATA 1 +#define SUN4I_GPADC_IRQ_FIFO_DATA 1 +#define SUN4I_GPADC_IRQ_TEMP_DATA 2 /* 10s delay before suspending the IP */ #define SUN4I_GPADC_AUTOSUSPEND_DELAY 10000 -- cgit v1.2.3 From 8b9a1f5ef43b8d26c4df3b4e3cbebec04d7be1c5 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Mon, 29 Jan 2024 15:25:53 +0000 Subject: mfd: cs42l43: Tidy up header includes Use more forward declarations, move header guards to cover other includes, and rely less on including headers through other headers. Suggested-by: Andy Shevchenko Signed-off-by: Charles Keepax Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240129152557.3221212-2-ckeepax@opensource.cirrus.com Signed-off-by: Lee Jones --- drivers/mfd/cs42l43-i2c.c | 6 +++++- drivers/mfd/cs42l43-sdw.c | 6 +++++- drivers/mfd/cs42l43.c | 8 +++++++- drivers/mfd/cs42l43.h | 10 ++++++---- include/linux/mfd/cs42l43.h | 13 +++++++------ 5 files changed, 30 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/cs42l43-i2c.c b/drivers/mfd/cs42l43-i2c.c index 4922211680c9..7162274a0b55 100644 --- a/drivers/mfd/cs42l43-i2c.c +++ b/drivers/mfd/cs42l43-i2c.c @@ -6,11 +6,15 @@ * Cirrus Logic International Semiconductor Ltd. */ +#include #include -#include #include +#include #include +#include #include +#include +#include #include "cs42l43.h" diff --git a/drivers/mfd/cs42l43-sdw.c b/drivers/mfd/cs42l43-sdw.c index 1d85bbf8cdd5..d6962a5a35f6 100644 --- a/drivers/mfd/cs42l43-sdw.c +++ b/drivers/mfd/cs42l43-sdw.c @@ -6,11 +6,15 @@ * Cirrus Logic International Semiconductor Ltd. */ +#include #include #include -#include +#include #include +#include #include +#include +#include #include #include #include diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c index 7b6d07cbe6fc..4e2bc5ad244a 100644 --- a/drivers/mfd/cs42l43.c +++ b/drivers/mfd/cs42l43.c @@ -6,18 +6,24 @@ * Cirrus Logic International Semiconductor Ltd. */ +#include #include #include #include +#include #include -#include #include +#include #include #include +#include #include #include +#include #include +#include #include +#include #include "cs42l43.h" diff --git a/drivers/mfd/cs42l43.h b/drivers/mfd/cs42l43.h index eb4caf393833..8d1b1b0f5a47 100644 --- a/drivers/mfd/cs42l43.h +++ b/drivers/mfd/cs42l43.h @@ -6,15 +6,17 @@ * Cirrus Logic International Semiconductor Ltd. */ -#include -#include -#include - #ifndef CS42L43_CORE_INT_H #define CS42L43_CORE_INT_H #define CS42L43_N_DEFAULTS 176 +struct dev_pm_ops; +struct device; +struct reg_default; + +struct cs42l43; + extern const struct dev_pm_ops cs42l43_pm_ops; extern const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS]; diff --git a/include/linux/mfd/cs42l43.h b/include/linux/mfd/cs42l43.h index cf8263aab41b..2239d8585e78 100644 --- a/include/linux/mfd/cs42l43.h +++ b/include/linux/mfd/cs42l43.h @@ -6,20 +6,21 @@ * Cirrus Logic International Semiconductor Ltd. */ +#ifndef CS42L43_CORE_EXT_H +#define CS42L43_CORE_EXT_H + #include -#include -#include #include #include #include -#include #include -#ifndef CS42L43_CORE_EXT_H -#define CS42L43_CORE_EXT_H - #define CS42L43_N_SUPPLIES 3 +struct device; +struct gpio_desc; +struct sdw_slave; + enum cs42l43_irq_numbers { CS42L43_PLL_LOST_LOCK, CS42L43_PLL_READY, -- cgit v1.2.3 From 0db017f8edd9b9af818bc1d68ba578df1b4c4628 Mon Sep 17 00:00:00 2001 From: Peter Griffin Date: Tue, 20 Feb 2024 11:50:11 +0000 Subject: mfd: syscon: Remove extern from function prototypes The kernel coding style does not require 'extern' in function prototypes in .h files, so remove them as they are not needed. To avoid checkpatch warnings such as CHECK: Lines should not end with a '(' +struct regmap *syscon_regmap_lookup_by_phandle( The indentation is also updated. No functional changes in this patch. Signed-off-by: Peter Griffin Link: https://lore.kernel.org/r/20240220115012.471689-3-peter.griffin@linaro.org Signed-off-by: Lee Jones --- include/linux/mfd/syscon.h | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/syscon.h b/include/linux/mfd/syscon.h index fecc2fa2a364..c315903f6dab 100644 --- a/include/linux/mfd/syscon.h +++ b/include/linux/mfd/syscon.h @@ -17,20 +17,17 @@ struct device_node; #ifdef CONFIG_MFD_SYSCON -extern struct regmap *device_node_to_regmap(struct device_node *np); -extern struct regmap *syscon_node_to_regmap(struct device_node *np); -extern struct regmap *syscon_regmap_lookup_by_compatible(const char *s); -extern struct regmap *syscon_regmap_lookup_by_phandle( - struct device_node *np, - const char *property); -extern struct regmap *syscon_regmap_lookup_by_phandle_args( - struct device_node *np, - const char *property, - int arg_count, - unsigned int *out_args); -extern struct regmap *syscon_regmap_lookup_by_phandle_optional( - struct device_node *np, - const char *property); +struct regmap *device_node_to_regmap(struct device_node *np); +struct regmap *syscon_node_to_regmap(struct device_node *np); +struct regmap *syscon_regmap_lookup_by_compatible(const char *s); +struct regmap *syscon_regmap_lookup_by_phandle(struct device_node *np, + const char *property); +struct regmap *syscon_regmap_lookup_by_phandle_args(struct device_node *np, + const char *property, + int arg_count, + unsigned int *out_args); +struct regmap *syscon_regmap_lookup_by_phandle_optional(struct device_node *np, + const char *property); #else static inline struct regmap *device_node_to_regmap(struct device_node *np) { -- cgit v1.2.3 From ca9414a1d08756c8392f9219caee607e1b7bade1 Mon Sep 17 00:00:00 2001 From: Andreas Kemnade Date: Sat, 17 Feb 2024 09:20:04 +0100 Subject: mfd: twl-core: Add power off implementation for twl603x If the system-power-controller property is there, enable power off. Implementation is based on a Linux v3.0 vendor kernel. Signed-off-by: Andreas Kemnade Link: https://lore.kernel.org/r/20240217082007.3238948-3-andreas@kemnade.info Signed-off-by: Lee Jones --- drivers/mfd/twl-core.c | 28 ++++++++++++++++++++++++++++ include/linux/mfd/twl.h | 1 + 2 files changed, 29 insertions(+) (limited to 'include/linux') diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c index 6e384a79e341..c130ffef182f 100644 --- a/drivers/mfd/twl-core.c +++ b/drivers/mfd/twl-core.c @@ -124,6 +124,11 @@ #define TWL6030_BASEADD_RSV 0x0000 #define TWL6030_BASEADD_ZERO 0x0000 +/* Some fields in TWL6030_PHOENIX_DEV_ON */ +#define TWL6030_APP_DEVOFF BIT(0) +#define TWL6030_CON_DEVOFF BIT(1) +#define TWL6030_MOD_DEVOFF BIT(2) + /* Few power values */ #define R_CFG_BOOT 0x05 @@ -687,6 +692,20 @@ static void twl_remove(struct i2c_client *client) twl_priv->ready = false; } +static void twl6030_power_off(void) +{ + int err; + u8 val; + + err = twl_i2c_read_u8(TWL_MODULE_PM_MASTER, &val, TWL6030_PHOENIX_DEV_ON); + if (err) + return; + + val |= TWL6030_APP_DEVOFF | TWL6030_CON_DEVOFF | TWL6030_MOD_DEVOFF; + twl_i2c_write_u8(TWL_MODULE_PM_MASTER, val, TWL6030_PHOENIX_DEV_ON); +} + + static struct of_dev_auxdata twl_auxdata_lookup[] = { OF_DEV_AUXDATA("ti,twl4030-gpio", 0, "twl4030-gpio", NULL), { /* sentinel */ }, @@ -852,6 +871,15 @@ twl_probe(struct i2c_client *client) goto free; } + if (twl_class_is_6030()) { + if (of_device_is_system_power_controller(node)) { + if (!pm_power_off) + pm_power_off = twl6030_power_off; + else + dev_warn(&client->dev, "Poweroff callback already assigned\n"); + } + } + status = of_platform_populate(node, NULL, twl_auxdata_lookup, &client->dev); diff --git a/include/linux/mfd/twl.h b/include/linux/mfd/twl.h index c062d91a67d9..85dc406173db 100644 --- a/include/linux/mfd/twl.h +++ b/include/linux/mfd/twl.h @@ -461,6 +461,7 @@ static inline int twl6030_mmc_card_detect(struct device *dev, int slot) #define TWL4030_PM_MASTER_GLOBAL_TST 0xb6 +#define TWL6030_PHOENIX_DEV_ON 0x06 /*----------------------------------------------------------------------*/ /* Power bus message definitions */ -- cgit v1.2.3 From 9b0a62758665e4d76269bba61eb63e5b8d18e499 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 14:52:01 +0100 Subject: thermal: core: Store zone trips table in struct thermal_zone_device The current code expects thermal zone creators to pass a pointer to a writable trips table to thermal_zone_device_register_with_trips() and that trips table is then used by the thermal core going forward. Consequently, the callers of thermal_zone_device_register_with_trips() are required to hold on to the trips table passed to it until the given thermal zone is unregistered, at which point the trips table can be freed, but at the same time they are not expected to access that table directly. This is both error prone and confusing. To address it, turn the trips table pointer in struct thermal_zone_device into a flex array (counted by its num_trips field), allocate it during thermal zone device allocation and copy the contents of the trips table supplied by the zone creator (which can be const now) into it, which will allow the callers of thermal_zone_device_register_with_trips() to drop their trip tables right after the zone registration. This requires the imx thermal driver to be adjusted to store the new temperature in its internal trips table in imx_set_trip_temp(), because it will be separate from the core's trips table now and it has to be explicitly kept in sync with the latter. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Daniel Lezcano --- drivers/thermal/imx_thermal.c | 1 + drivers/thermal/thermal_core.c | 18 +++++++++--------- drivers/thermal/thermal_of.c | 4 ++-- drivers/thermal/thermal_trip.c | 2 +- include/linux/thermal.h | 10 +++++----- 5 files changed, 18 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 7019c4fdd549..499eff858f3f 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -354,6 +354,7 @@ static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip_id, return -EINVAL; imx_set_alarm_temp(data, temp); + trips[IMX_TRIP_PASSIVE].temperature = temp; pm_runtime_put(data->dev); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index dfaa6341694a..8bdece8b1536 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1227,9 +1227,6 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp) if (tz->ops->get_crit_temp) return tz->ops->get_crit_temp(tz, temp); - if (!tz->trips) - return -EINVAL; - mutex_lock(&tz->lock); for (i = 0; i < tz->num_trips; i++) { @@ -1272,10 +1269,13 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); * IS_ERR*() helpers. */ struct thermal_zone_device * -thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *trips, int num_trips, int mask, - void *devdata, struct thermal_zone_device_ops *ops, - const struct thermal_zone_params *tzp, int passive_delay, - int polling_delay) +thermal_zone_device_register_with_trips(const char *type, + const struct thermal_trip *trips, + int num_trips, int mask, + void *devdata, + struct thermal_zone_device_ops *ops, + const struct thermal_zone_params *tzp, + int passive_delay, int polling_delay) { struct thermal_zone_device *tz; int id; @@ -1322,7 +1322,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t if (!thermal_class) return ERR_PTR(-ENODEV); - tz = kzalloc(sizeof(*tz), GFP_KERNEL); + tz = kzalloc(struct_size(tz, trips, num_trips), GFP_KERNEL); if (!tz) return ERR_PTR(-ENOMEM); @@ -1354,7 +1354,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t tz->ops = ops; tz->device.class = thermal_class; tz->devdata = devdata; - tz->trips = trips; + memcpy(tz->trips, trips, num_trips * sizeof(*trips)); tz->num_trips = num_trips; thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 4d6c22e0ed85..20fa7d5ac536 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -438,12 +438,10 @@ static int thermal_of_unbind(struct thermal_zone_device *tz, */ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - struct thermal_trip *trips = tz->trips; struct thermal_zone_device_ops *ops = tz->ops; thermal_zone_device_disable(tz); thermal_zone_device_unregister(tz); - kfree(trips); kfree(ops); } @@ -526,6 +524,8 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * goto out_kfree_trips; } + kfree(trips); + ret = thermal_zone_device_enable(tz); if (ret) { pr_err("Failed to enabled thermal zone '%s', id=%d: %d\n", diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index c875a26d5adf..a80ca78f8473 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -122,7 +122,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) int __thermal_zone_get_trip(struct thermal_zone_device *tz, int trip_id, struct thermal_trip *trip) { - if (!tz || !tz->trips || trip_id < 0 || trip_id >= tz->num_trips || !trip) + if (!tz || trip_id < 0 || trip_id >= tz->num_trips || !trip) return -EINVAL; *trip = tz->trips[trip_id]; diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 65d8f92a9a0d..572a24f29a10 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -130,7 +130,6 @@ struct thermal_cooling_device { * @trip_hyst_attrs: attributes for trip points for sysfs: trip hysteresis * @mode: current mode of this thermal zone * @devdata: private pointer for device private data - * @trips: an array of struct thermal_trip * @num_trips: number of trip points the thermal zone supports * @passive_delay_jiffies: number of jiffies to wait between polls when * performing passive cooling. @@ -160,6 +159,7 @@ struct thermal_cooling_device { * @poll_queue: delayed work for polling * @notify_event: Last notification event * @suspended: thermal zone suspend indicator + * @trips: array of struct thermal_trip objects */ struct thermal_zone_device { int id; @@ -172,7 +172,6 @@ struct thermal_zone_device { struct thermal_attr *trip_hyst_attrs; enum thermal_device_mode mode; void *devdata; - struct thermal_trip *trips; int num_trips; unsigned long passive_delay_jiffies; unsigned long polling_delay_jiffies; @@ -193,10 +192,11 @@ struct thermal_zone_device { struct list_head node; struct delayed_work poll_queue; enum thermal_notify_event notify_event; + bool suspended; #ifdef CONFIG_THERMAL_DEBUGFS struct thermal_debugfs *debugfs; #endif - bool suspended; + struct thermal_trip trips[] __counted_by(num_trips); }; /** @@ -315,7 +315,7 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp); #ifdef CONFIG_THERMAL struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, - struct thermal_trip *trips, + const struct thermal_trip *trips, int num_trips, int mask, void *devdata, struct thermal_zone_device_ops *ops, @@ -375,7 +375,7 @@ void thermal_zone_device_critical(struct thermal_zone_device *tz); #else static inline struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, - struct thermal_trip *trips, + const struct thermal_trip *trips, int num_trips, int mask, void *devdata, struct thermal_zone_device_ops *ops, -- cgit v1.2.3 From 698a1eb1f75eb6ac957d2af7721a3b1a94281e5d Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:18:01 +0100 Subject: thermal: core: Store zone ops in struct thermal_zone_device The current code requires thermal zone creators to pass pointers to writable ops structures to thermal_zone_device_register_with_trips() which needs to modify the target struct thermal_zone_device_ops object if the "critical" operation in it is NULL. Moreover, the callers of thermal_zone_device_register_with_trips() are required to hold on to the struct thermal_zone_device_ops object passed to it until the given thermal zone is unregistered. Both of these requirements are quite inconvenient, so modify struct thermal_zone_device to contain struct thermal_zone_device_ops as field and make thermal_zone_device_register_with_trips() copy the contents of the struct thermal_zone_device_ops passed to it via a pointer (which can be const now) to that field. Also adjust the code using thermal zone ops accordingly and modify thermal_of_zone_register() to use a local ops variable during thermal zone registration so ops do not need to be freed in thermal_of_zone_unregister() any more. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_core.c | 40 +++++++++++++++++++-------------------- drivers/thermal/thermal_helpers.c | 10 +++++----- drivers/thermal/thermal_hwmon.c | 4 ++-- drivers/thermal/thermal_of.c | 27 ++++++++------------------ drivers/thermal/thermal_sysfs.c | 14 +++++++------- drivers/thermal/thermal_trip.c | 4 ++-- include/linux/thermal.h | 8 ++++---- 7 files changed, 48 insertions(+), 59 deletions(-) (limited to 'include/linux') diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 8bdece8b1536..bb21f78b4bfa 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -356,9 +356,9 @@ static void handle_critical_trips(struct thermal_zone_device *tz, trace_thermal_zone_trip(tz, thermal_zone_trip_id(tz, trip), trip->type); if (trip->type == THERMAL_TRIP_CRITICAL) - tz->ops->critical(tz); - else if (tz->ops->hot) - tz->ops->hot(tz); + tz->ops.critical(tz); + else if (tz->ops.hot) + tz->ops.hot(tz); } static void handle_thermal_trip(struct thermal_zone_device *tz, @@ -493,8 +493,8 @@ static int thermal_zone_device_set_mode(struct thermal_zone_device *tz, return ret; } - if (tz->ops->change_mode) - ret = tz->ops->change_mode(tz, mode); + if (tz->ops.change_mode) + ret = tz->ops.change_mode(tz, mode); if (!ret) tz->mode = mode; @@ -867,8 +867,8 @@ static void bind_cdev(struct thermal_cooling_device *cdev) struct thermal_zone_device *pos = NULL; list_for_each_entry(pos, &thermal_tz_list, node) { - if (pos->ops->bind) { - ret = pos->ops->bind(pos, cdev); + if (pos->ops.bind) { + ret = pos->ops.bind(pos, cdev); if (ret) print_bind_err_msg(pos, cdev, ret); } @@ -1184,8 +1184,8 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev) /* Unbind all thermal zones associated with 'this' cdev */ list_for_each_entry(tz, &thermal_tz_list, node) { - if (tz->ops->unbind) - tz->ops->unbind(tz, cdev); + if (tz->ops.unbind) + tz->ops.unbind(tz, cdev); } mutex_unlock(&thermal_list_lock); @@ -1199,13 +1199,13 @@ static void bind_tz(struct thermal_zone_device *tz) int ret; struct thermal_cooling_device *pos = NULL; - if (!tz->ops->bind) + if (!tz->ops.bind) return; mutex_lock(&thermal_list_lock); list_for_each_entry(pos, &thermal_cdev_list, node) { - ret = tz->ops->bind(tz, pos); + ret = tz->ops.bind(tz, pos); if (ret) print_bind_err_msg(tz, pos, ret); } @@ -1224,8 +1224,8 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp) { int i, ret = -EINVAL; - if (tz->ops->get_crit_temp) - return tz->ops->get_crit_temp(tz, temp); + if (tz->ops.get_crit_temp) + return tz->ops.get_crit_temp(tz, temp); mutex_lock(&tz->lock); @@ -1273,7 +1273,7 @@ thermal_zone_device_register_with_trips(const char *type, const struct thermal_trip *trips, int num_trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { @@ -1348,10 +1348,10 @@ thermal_zone_device_register_with_trips(const char *type, tz->id = id; strscpy(tz->type, type, sizeof(tz->type)); - if (!ops->critical) - ops->critical = thermal_zone_device_critical; + tz->ops = *ops; + if (!tz->ops.critical) + tz->ops.critical = thermal_zone_device_critical; - tz->ops = ops; tz->device.class = thermal_class; tz->devdata = devdata; memcpy(tz->trips, trips, num_trips * sizeof(*trips)); @@ -1437,7 +1437,7 @@ EXPORT_SYMBOL_GPL(thermal_zone_device_register_with_trips); struct thermal_zone_device *thermal_tripless_zone_device_register( const char *type, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp) { return thermal_zone_device_register_with_trips(type, NULL, 0, 0, devdata, @@ -1499,8 +1499,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) /* Unbind all cdevs associated with 'this' thermal zone */ list_for_each_entry(cdev, &thermal_cdev_list, node) - if (tz->ops->unbind) - tz->ops->unbind(tz, cdev); + if (tz->ops.unbind) + tz->ops.unbind(tz, cdev); mutex_unlock(&thermal_list_lock); diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index 0329f4a71b02..c5a057b59c42 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -26,8 +26,8 @@ int get_tz_trend(struct thermal_zone_device *tz, const struct thermal_trip *trip { enum thermal_trend trend; - if (tz->emul_temperature || !tz->ops->get_trend || - tz->ops->get_trend(tz, trip, &trend)) { + if (tz->emul_temperature || !tz->ops.get_trend || + tz->ops.get_trend(tz, trip, &trend)) { if (tz->temperature > tz->last_temperature) trend = THERMAL_TREND_RAISING; else if (tz->temperature < tz->last_temperature) @@ -75,7 +75,7 @@ EXPORT_SYMBOL(get_thermal_instance); * temperature and fill @temp. * * Both tz and tz->ops must be valid pointers when calling this function, - * and the tz->ops->get_temp callback must be provided. + * and the tz->ops.get_temp callback must be provided. * The function must be called under tz->lock. * * Return: On success returns 0, an error code otherwise @@ -88,7 +88,7 @@ int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) lockdep_assert_held(&tz->lock); - ret = tz->ops->get_temp(tz, temp); + ret = tz->ops.get_temp(tz, temp); if (IS_ENABLED(CONFIG_THERMAL_EMULATION) && tz->emul_temperature) { for_each_trip(tz, trip) { @@ -132,7 +132,7 @@ int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) mutex_lock(&tz->lock); - if (!tz->ops->get_temp) { + if (!tz->ops.get_temp) { ret = -EINVAL; goto unlock; } diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index 252116f1e535..f0e504fd866a 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -80,7 +80,7 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf) mutex_lock(&tz->lock); - ret = tz->ops->get_crit_temp(tz, &temperature); + ret = tz->ops.get_crit_temp(tz, &temperature); mutex_unlock(&tz->lock); @@ -132,7 +132,7 @@ thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon, static bool thermal_zone_crit_temp_valid(struct thermal_zone_device *tz) { int temp; - return tz->ops->get_crit_temp && !tz->ops->get_crit_temp(tz, &temp); + return tz->ops.get_crit_temp && !tz->ops.get_crit_temp(tz, &temp); } int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 20fa7d5ac536..405743e90531 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -438,11 +438,8 @@ static int thermal_of_unbind(struct thermal_zone_device *tz, */ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - struct thermal_zone_device_ops *ops = tz->ops; - thermal_zone_device_disable(tz); thermal_zone_device_unregister(tz); - kfree(ops); } /** @@ -468,33 +465,27 @@ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) static struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data, const struct thermal_zone_device_ops *ops) { + struct thermal_zone_device_ops of_ops = *ops; struct thermal_zone_device *tz; struct thermal_trip *trips; struct thermal_zone_params tzp = {}; - struct thermal_zone_device_ops *of_ops; struct device_node *np; const char *action; int delay, pdelay; int ntrips, mask; int ret; - of_ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL); - if (!of_ops) - return ERR_PTR(-ENOMEM); - np = of_thermal_zone_find(sensor, id); if (IS_ERR(np)) { if (PTR_ERR(np) != -ENODEV) pr_err("Failed to find thermal zone for %pOFn id=%d\n", sensor, id); - ret = PTR_ERR(np); - goto out_kfree_of_ops; + return ERR_CAST(np); } trips = thermal_of_trips_init(np, &ntrips); if (IS_ERR(trips)) { pr_err("Failed to find trip points for %pOFn id=%d\n", sensor, id); - ret = PTR_ERR(trips); - goto out_kfree_of_ops; + return ERR_CAST(trips); } ret = thermal_of_monitor_init(np, &delay, &pdelay); @@ -505,18 +496,18 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * thermal_of_parameters_init(np, &tzp); - of_ops->bind = thermal_of_bind; - of_ops->unbind = thermal_of_unbind; + of_ops.bind = thermal_of_bind; + of_ops.unbind = thermal_of_unbind; mask = GENMASK_ULL((ntrips) - 1, 0); ret = of_property_read_string(np, "critical-action", &action); if (!ret) - if (!of_ops->critical && !strcasecmp(action, "reboot")) - of_ops->critical = thermal_zone_device_critical_reboot; + if (!of_ops.critical && !strcasecmp(action, "reboot")) + of_ops.critical = thermal_zone_device_critical_reboot; tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, - mask, data, of_ops, &tzp, + mask, data, &of_ops, &tzp, pdelay, delay); if (IS_ERR(tz)) { ret = PTR_ERR(tz); @@ -538,8 +529,6 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * out_kfree_trips: kfree(trips); -out_kfree_of_ops: - kfree(of_ops); return ERR_PTR(ret); } diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index d55f9303afb5..2eace6b33b8c 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -123,8 +123,8 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (temp != trip->temperature) { - if (tz->ops->set_trip_temp) { - ret = tz->ops->set_trip_temp(tz, trip_id, temp); + if (tz->ops.set_trip_temp) { + ret = tz->ops.set_trip_temp(tz, trip_id, temp); if (ret) goto unlock; } @@ -174,8 +174,8 @@ trip_point_hyst_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (hyst != trip->hysteresis) { - if (tz->ops->set_trip_hyst) { - ret = tz->ops->set_trip_hyst(tz, trip_id, hyst); + if (tz->ops.set_trip_hyst) { + ret = tz->ops.set_trip_hyst(tz, trip_id, hyst); if (ret) goto unlock; } @@ -250,10 +250,10 @@ emul_temp_store(struct device *dev, struct device_attribute *attr, mutex_lock(&tz->lock); - if (!tz->ops->set_emul_temp) + if (!tz->ops.set_emul_temp) tz->emul_temperature = temperature; else - ret = tz->ops->set_emul_temp(tz, temperature); + ret = tz->ops.set_emul_temp(tz, temperature); if (!ret) __thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED); @@ -474,7 +474,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_hyst_attrs[indx].name; tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show; - if (tz->ops->set_trip_hyst) { + if (tz->ops.set_trip_hyst) { tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_hyst_attrs[indx].attr.store = trip_point_hyst_store; diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index a80ca78f8473..09f6050dd041 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -70,7 +70,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) lockdep_assert_held(&tz->lock); - if (!tz->ops->set_trips) + if (!tz->ops.set_trips) return; for_each_trip(tz, trip) { @@ -114,7 +114,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) * Set a temperature window. When this window is left the driver * must inform the thermal core via thermal_zone_device_update. */ - ret = tz->ops->set_trips(tz, low, high); + ret = tz->ops.set_trips(tz, low, high); if (ret) dev_err(&tz->device, "Failed to set trips: %d\n", ret); } diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 572a24f29a10..ec0559e98d6f 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -182,7 +182,7 @@ struct thermal_zone_device { int prev_low_trip; int prev_high_trip; atomic_t need_update; - struct thermal_zone_device_ops *ops; + struct thermal_zone_device_ops ops; struct thermal_zone_params *tzp; struct thermal_governor *governor; void *governor_data; @@ -318,14 +318,14 @@ struct thermal_zone_device *thermal_zone_device_register_with_trips( const struct thermal_trip *trips, int num_trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay); struct thermal_zone_device *thermal_tripless_zone_device_register( const char *type, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp); void thermal_zone_device_unregister(struct thermal_zone_device *tz); @@ -378,7 +378,7 @@ static inline struct thermal_zone_device *thermal_zone_device_register_with_trip const struct thermal_trip *trips, int num_trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { return ERR_PTR(-ENODEV); } -- cgit v1.2.3 From 284851ee5caef1b42b513752bf1642ce4570bdc1 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Fri, 16 Feb 2024 15:59:41 +0000 Subject: KVM: Get rid of return value from kvm_arch_create_vm_debugfs() The general expectation with debugfs is that any initialization failure is nonfatal. Nevertheless, kvm_arch_create_vm_debugfs() allows implementations to return an error and kvm_create_vm_debugfs() allows that to fail VM creation. Change to a void return to discourage architectures from making debugfs failures fatal for the VM. Seems like everyone already had the right idea, as all implementations already return 0 unconditionally. Acked-by: Marc Zyngier Acked-by: Paolo Bonzini Link: https://lore.kernel.org/r/20240216155941.2029458-1-oliver.upton@linux.dev Signed-off-by: Oliver Upton --- arch/powerpc/kvm/powerpc.c | 3 +-- arch/x86/kvm/debugfs.c | 3 +-- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 8 ++------ 4 files changed, 5 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 23407fbd73c9..d32abe7fe6ab 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -2538,9 +2538,8 @@ void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_ vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs(vcpu, debugfs_dentry); } -int kvm_arch_create_vm_debugfs(struct kvm *kvm) +void kvm_arch_create_vm_debugfs(struct kvm *kvm) { if (kvm->arch.kvm_ops->create_vm_debugfs) kvm->arch.kvm_ops->create_vm_debugfs(kvm); - return 0; } diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index 95ea1a1f7403..999227fc7c66 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -189,9 +189,8 @@ static const struct file_operations mmu_rmaps_stat_fops = { .release = kvm_mmu_rmaps_stat_release, }; -int kvm_arch_create_vm_debugfs(struct kvm *kvm) +void kvm_arch_create_vm_debugfs(struct kvm *kvm) { debugfs_create_file("mmu_rmaps_stat", 0644, kvm->debugfs_dentry, kvm, &mmu_rmaps_stat_fops); - return 0; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 7e7fd25b09b3..9a45f673f687 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1507,7 +1507,7 @@ bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu); bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu); int kvm_arch_post_init_vm(struct kvm *kvm); void kvm_arch_pre_destroy_vm(struct kvm *kvm); -int kvm_arch_create_vm_debugfs(struct kvm *kvm); +void kvm_arch_create_vm_debugfs(struct kvm *kvm); #ifndef __KVM_HAVE_ARCH_VM_ALLOC /* diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 10bfc88a69f7..c681149c382a 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1150,10 +1150,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) &stat_fops_per_vm); } - ret = kvm_arch_create_vm_debugfs(kvm); - if (ret) - goto out_err; - + kvm_arch_create_vm_debugfs(kvm); return 0; out_err: kvm_destroy_vm_debugfs(kvm); @@ -1183,9 +1180,8 @@ void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm) * Cleanup should be automatic done in kvm_destroy_vm_debugfs() recursively, so * a per-arch destroy interface is not needed. */ -int __weak kvm_arch_create_vm_debugfs(struct kvm *kvm) +void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm) { - return 0; } static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) -- cgit v1.2.3 From 025f8ad20f2e3264d11683aa9cbbf0083eefbdcd Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Thu, 22 Feb 2024 15:03:10 +0100 Subject: net: mpls: error out if inner headers are not set mpls_gso_segment() assumes skb_inner_network_header() returns a valid result: mpls_hlen = skb_inner_network_header(skb) - skb_network_header(skb); if (unlikely(!mpls_hlen || mpls_hlen % MPLS_HLEN)) goto out; if (unlikely(!pskb_may_pull(skb, mpls_hlen))) With syzbot reproducer, skb_inner_network_header() yields 0, skb_network_header() returns 108, so this will "pskb_may_pull(skb, -108)))" which triggers a newly added DEBUG_NET_WARN_ON_ONCE() check: ------------[ cut here ]------------ WARNING: CPU: 0 PID: 5068 at include/linux/skbuff.h:2723 pskb_may_pull_reason include/linux/skbuff.h:2723 [inline] WARNING: CPU: 0 PID: 5068 at include/linux/skbuff.h:2723 pskb_may_pull include/linux/skbuff.h:2739 [inline] WARNING: CPU: 0 PID: 5068 at include/linux/skbuff.h:2723 mpls_gso_segment+0x773/0xaa0 net/mpls/mpls_gso.c:34 [..] skb_mac_gso_segment+0x383/0x740 net/core/gso.c:53 nsh_gso_segment+0x40a/0xad0 net/nsh/nsh.c:108 skb_mac_gso_segment+0x383/0x740 net/core/gso.c:53 __skb_gso_segment+0x324/0x4c0 net/core/gso.c:124 skb_gso_segment include/net/gso.h:83 [inline] [..] sch_direct_xmit+0x11a/0x5f0 net/sched/sch_generic.c:327 [..] packet_sendmsg+0x46a9/0x6130 net/packet/af_packet.c:3113 [..] First iteration of this patch made mpls_hlen signed and changed test to error out to "mpls_hlen <= 0 || ..". Eric Dumazet said: > I was thinking about adding a debug check in skb_inner_network_header() > if inner_network_header is zero (that would mean it is not 'set' yet), > but this would trigger even after your patch. So add new skb_inner_network_header_was_set() helper and use that. The syzbot reproducer injects data via packet socket. The skb that gets allocated and passed down the stack has ->protocol set to NSH (0x894f) and gso_type set to SKB_GSO_UDP | SKB_GSO_DODGY. This gets passed to skb_mac_gso_segment(), which sees NSH as ptype to find a callback for. nsh_gso_segment() retrieves next type: proto = tun_p_to_eth_p(nsh_hdr(skb)->np); ... which is MPLS (TUN_P_MPLS_UC). It updates skb->protocol and then calls mpls_gso_segment(). Inner offsets are all 0, so mpls_gso_segment() ends up with a negative header size. In case more callers rely on silent handling of such large may_pull values we could also 'legalize' this behaviour, either replacing the debug check with (len > INT_MAX) test or removing it and instead adding a comment before existing if (unlikely(len > skb->len)) return SKB_DROP_REASON_PKT_TOO_SMALL; test in pskb_may_pull_reason(), saying that this check also implicitly takes care of callers that miscompute header sizes. Cc: Simon Horman Fixes: 219eee9c0d16 ("net: skbuff: add overflow debug check to pull/push helpers") Reported-by: syzbot+99d15fcdb0132a1e1a82@syzkaller.appspotmail.com Closes: https://lore.kernel.org/netdev/00000000000043b1310611e388aa@google.com/raw Signed-off-by: Florian Westphal Link: https://lore.kernel.org/r/20240222140321.14080-1-fw@strlen.de Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 5 +++++ net/mpls/mpls_gso.c | 3 +++ 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 28c7cb7ce251..1470b74fb6d2 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2894,6 +2894,11 @@ static inline void skb_set_inner_network_header(struct sk_buff *skb, skb->inner_network_header += offset; } +static inline bool skb_inner_network_header_was_set(const struct sk_buff *skb) +{ + return skb->inner_network_header > 0; +} + static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb) { return skb->head + skb->inner_mac_header; diff --git a/net/mpls/mpls_gso.c b/net/mpls/mpls_gso.c index 533d082f0701..45d1e6a157fc 100644 --- a/net/mpls/mpls_gso.c +++ b/net/mpls/mpls_gso.c @@ -27,6 +27,9 @@ static struct sk_buff *mpls_gso_segment(struct sk_buff *skb, __be16 mpls_protocol; unsigned int mpls_hlen; + if (!skb_inner_network_header_was_set(skb)) + goto out; + skb_reset_network_header(skb); mpls_hlen = skb_inner_network_header(skb) - skb_network_header(skb); if (unlikely(!mpls_hlen || mpls_hlen % MPLS_HLEN)) -- cgit v1.2.3 From 8151c7a35d8bd8a12e93538ef7963ea209b6ab41 Mon Sep 17 00:00:00 2001 From: Oscar Salvador Date: Thu, 15 Feb 2024 22:59:02 +0100 Subject: lib/stackdepot: move stack_record struct definition into the header In order to move the heavy lifting into page_owner code, this one needs to have access to the stack_record structure, which right now sits in lib/stackdepot.c. Move it to the stackdepot.h header so page_owner can access stack_record's struct fields. Link: https://lkml.kernel.org/r/20240215215907.20121-3-osalvador@suse.de Signed-off-by: Oscar Salvador Reviewed-by: Marco Elver Reviewed-by: Vlastimil Babka Acked-by: Andrey Konovalov Cc: Alexander Potapenko Cc: Michal Hocko Signed-off-by: Andrew Morton --- include/linux/stackdepot.h | 47 ++++++++++++++++++++++++++++++++++++++++++++++ lib/stackdepot.c | 43 ------------------------------------------ 2 files changed, 47 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index adcbb8f23600..c4b5ad57c066 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -30,6 +30,53 @@ typedef u32 depot_stack_handle_t; */ #define STACK_DEPOT_EXTRA_BITS 5 +#define DEPOT_HANDLE_BITS (sizeof(depot_stack_handle_t) * 8) + +#define DEPOT_POOL_ORDER 2 /* Pool size order, 4 pages */ +#define DEPOT_POOL_SIZE (1LL << (PAGE_SHIFT + DEPOT_POOL_ORDER)) +#define DEPOT_STACK_ALIGN 4 +#define DEPOT_OFFSET_BITS (DEPOT_POOL_ORDER + PAGE_SHIFT - DEPOT_STACK_ALIGN) +#define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_OFFSET_BITS - \ + STACK_DEPOT_EXTRA_BITS) + +#ifdef CONFIG_STACKDEPOT +/* Compact structure that stores a reference to a stack. */ +union handle_parts { + depot_stack_handle_t handle; + struct { + /* pool_index is offset by 1 */ + u32 pool_index : DEPOT_POOL_INDEX_BITS; + u32 offset : DEPOT_OFFSET_BITS; + u32 extra : STACK_DEPOT_EXTRA_BITS; + }; +}; + +struct stack_record { + struct list_head hash_list; /* Links in the hash table */ + u32 hash; /* Hash in hash table */ + u32 size; /* Number of stored frames */ + union handle_parts handle; /* Constant after initialization */ + refcount_t count; + union { + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ + struct { + /* + * An important invariant of the implementation is to + * only place a stack record onto the freelist iff its + * refcount is zero. Because stack records with a zero + * refcount are never considered as valid, it is safe to + * union @entries and freelist management state below. + * Conversely, as soon as an entry is off the freelist + * and its refcount becomes non-zero, the below must not + * be accessed until being placed back on the freelist. + */ + struct list_head free_list; /* Links in the freelist */ + unsigned long rcu_state; /* RCU cookie */ + }; + }; +}; +#endif + typedef u32 depot_flags_t; /* diff --git a/lib/stackdepot.c b/lib/stackdepot.c index c043a4186bc5..514b8d40ff57 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -36,55 +36,12 @@ #include #include -#define DEPOT_HANDLE_BITS (sizeof(depot_stack_handle_t) * 8) - -#define DEPOT_POOL_ORDER 2 /* Pool size order, 4 pages */ -#define DEPOT_POOL_SIZE (1LL << (PAGE_SHIFT + DEPOT_POOL_ORDER)) -#define DEPOT_STACK_ALIGN 4 -#define DEPOT_OFFSET_BITS (DEPOT_POOL_ORDER + PAGE_SHIFT - DEPOT_STACK_ALIGN) -#define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_OFFSET_BITS - \ - STACK_DEPOT_EXTRA_BITS) #define DEPOT_POOLS_CAP 8192 /* The pool_index is offset by 1 so the first record does not have a 0 handle. */ #define DEPOT_MAX_POOLS \ (((1LL << (DEPOT_POOL_INDEX_BITS)) - 1 < DEPOT_POOLS_CAP) ? \ (1LL << (DEPOT_POOL_INDEX_BITS)) - 1 : DEPOT_POOLS_CAP) -/* Compact structure that stores a reference to a stack. */ -union handle_parts { - depot_stack_handle_t handle; - struct { - u32 pool_index : DEPOT_POOL_INDEX_BITS; /* pool_index is offset by 1 */ - u32 offset : DEPOT_OFFSET_BITS; - u32 extra : STACK_DEPOT_EXTRA_BITS; - }; -}; - -struct stack_record { - struct list_head hash_list; /* Links in the hash table */ - u32 hash; /* Hash in hash table */ - u32 size; /* Number of stored frames */ - union handle_parts handle; /* Constant after initialization */ - refcount_t count; - union { - unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ - struct { - /* - * An important invariant of the implementation is to - * only place a stack record onto the freelist iff its - * refcount is zero. Because stack records with a zero - * refcount are never considered as valid, it is safe to - * union @entries and freelist management state below. - * Conversely, as soon as an entry is off the freelist - * and its refcount becomes non-zero, the below must not - * be accessed until being placed back on the freelist. - */ - struct list_head free_list; /* Links in the freelist */ - unsigned long rcu_state; /* RCU cookie */ - }; - }; -}; - static bool stack_depot_disabled; static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; -- cgit v1.2.3 From 4bedfb314bdd85c1662ecc46fa25b33b998f994d Mon Sep 17 00:00:00 2001 From: Oscar Salvador Date: Thu, 15 Feb 2024 22:59:03 +0100 Subject: mm,page_owner: maintain own list of stack_records structs page_owner needs to increment a stack_record refcount when a new allocation occurs, and decrement it on a free operation. In order to do that, we need to have a way to get a stack_record from a handle. Implement __stack_depot_get_stack_record() which just does that, and make it public so page_owner can use it. Also, traversing all stackdepot buckets comes with its own complexity, plus we would have to implement a way to mark only those stack_records that were originated from page_owner, as those are the ones we are interested in. For that reason, page_owner maintains its own list of stack_records, because traversing that list is faster than traversing all buckets while keeping at the same time a low complexity. For now, add to stack_list only the stack_records of dummy_handle and failure_handle, and set their refcount of 1. Further patches will add code to increment or decrement stack_records count on allocation and free operation. Link: https://lkml.kernel.org/r/20240215215907.20121-4-osalvador@suse.de Signed-off-by: Oscar Salvador Reviewed-by: Vlastimil Babka Reviewed-by: Marco Elver Acked-by: Andrey Konovalov Cc: Alexander Potapenko Cc: Michal Hocko Signed-off-by: Andrew Morton --- include/linux/stackdepot.h | 11 +++++++++++ lib/stackdepot.c | 8 ++++++++ mm/page_owner.c | 15 +++++++++++++++ 3 files changed, 34 insertions(+) (limited to 'include/linux') diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index c4b5ad57c066..3c6caa5abc7c 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -178,6 +178,17 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t gfp_flags); +/** + * __stack_depot_get_stack_record - Get a pointer to a stack_record struct + * + * @handle: Stack depot handle + * + * This function is only for internal purposes. + * + * Return: Returns a pointer to a stack_record struct + */ +struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); + /** * stack_depot_fetch - Fetch a stack trace from stack depot * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 514b8d40ff57..8c795bb20afb 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -687,6 +687,14 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, } EXPORT_SYMBOL_GPL(stack_depot_save); +struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) +{ + if (!handle) + return NULL; + + return depot_fetch_stack(handle); +} + unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { diff --git a/mm/page_owner.c b/mm/page_owner.c index 5634e5d890f8..33e342b15d9b 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -36,6 +36,14 @@ struct page_owner { pid_t free_tgid; }; +struct stack { + struct stack_record *stack_record; + struct stack *next; +}; +static struct stack dummy_stack; +static struct stack failure_stack; +static struct stack *stack_list; + static bool page_owner_enabled __initdata; DEFINE_STATIC_KEY_FALSE(page_owner_inited); @@ -95,6 +103,13 @@ static __init void init_page_owner(void) register_early_stack(); static_branch_enable(&page_owner_inited); init_early_allocated_pages(); + /* Initialize dummy and failure stacks and link them to stack_list */ + dummy_stack.stack_record = __stack_depot_get_stack_record(dummy_handle); + failure_stack.stack_record = __stack_depot_get_stack_record(failure_handle); + refcount_set(&dummy_stack.stack_record->count, 1); + refcount_set(&failure_stack.stack_record->count, 1); + dummy_stack.next = &failure_stack; + stack_list = &dummy_stack; } struct page_ext_operations page_owner_ops = { -- cgit v1.2.3 From 55c49fee57af99f3c663e69dedc5b85e691bbe50 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 2 Jan 2024 19:46:27 +0100 Subject: mm/vmalloc: remove vmap_area_list Earlier, vmap_area_list is exported to vmcoreinfo so that makedumpfile get the base address of vmalloc area. Now, vmap_area_list is empty, so export VMALLOC_START to vmcoreinfo instead, and remove vmap_area_list. [urezki@gmail.com: fix a warning in the crash_save_vmcoreinfo_init()] Link: https://lkml.kernel.org/r/20240111192329.449189-1-urezki@gmail.com Link: https://lkml.kernel.org/r/20240102184633.748113-6-urezki@gmail.com Signed-off-by: Baoquan He Signed-off-by: Uladzislau Rezki (Sony) Acked-by: Lorenzo Stoakes Cc: Christoph Hellwig Cc: Dave Chinner Cc: Joel Fernandes (Google) Cc: Kazuhito Hagio Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Oleksiy Avramchenko Cc: Paul E. McKenney Signed-off-by: Andrew Morton --- Documentation/admin-guide/kdump/vmcoreinfo.rst | 8 ++++---- arch/arm64/kernel/crash_core.c | 1 - arch/riscv/kernel/crash_core.c | 1 - include/linux/vmalloc.h | 1 - kernel/crash_core.c | 4 +--- kernel/kallsyms_selftest.c | 1 - mm/nommu.c | 2 -- mm/vmalloc.c | 2 -- 8 files changed, 5 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst index bced9e4b6e08..0f714fc945ac 100644 --- a/Documentation/admin-guide/kdump/vmcoreinfo.rst +++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst @@ -65,11 +65,11 @@ Defines the beginning of the text section. In general, _stext indicates the kernel start address. Used to convert a virtual address from the direct kernel map to a physical address. -vmap_area_list --------------- +VMALLOC_START +------------- -Stores the virtual area list. makedumpfile gets the vmalloc start value -from this variable and its value is necessary for vmalloc translation. +Stores the base address of vmalloc area. makedumpfile gets this value +since is necessary for vmalloc translation. mem_map ------- diff --git a/arch/arm64/kernel/crash_core.c b/arch/arm64/kernel/crash_core.c index 66cde752cd74..2a24199a9b81 100644 --- a/arch/arm64/kernel/crash_core.c +++ b/arch/arm64/kernel/crash_core.c @@ -23,7 +23,6 @@ void arch_crash_save_vmcoreinfo(void) /* Please note VMCOREINFO_NUMBER() uses "%d", not "%x" */ vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR); vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END); - vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START); vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START); vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END); diff --git a/arch/riscv/kernel/crash_core.c b/arch/riscv/kernel/crash_core.c index 8706736fd4e2..d18d529fd9b9 100644 --- a/arch/riscv/kernel/crash_core.c +++ b/arch/riscv/kernel/crash_core.c @@ -8,7 +8,6 @@ void arch_crash_save_vmcoreinfo(void) VMCOREINFO_NUMBER(phys_ram_base); vmcoreinfo_append_str("NUMBER(PAGE_OFFSET)=0x%lx\n", PAGE_OFFSET); - vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START); vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); #ifdef CONFIG_MMU VMCOREINFO_NUMBER(VA_BITS); diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index c720be70c8dd..91810b4e9510 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -253,7 +253,6 @@ extern long vread_iter(struct iov_iter *iter, const char *addr, size_t count); /* * Internals. Don't use.. */ -extern struct list_head vmap_area_list; extern __init void vm_area_add_early(struct vm_struct *vm); extern __init void vm_area_register_early(struct vm_struct *vm, size_t align); diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 75cd6a736d03..49b31e59d3cc 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -748,7 +748,7 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_SYMBOL_ARRAY(swapper_pg_dir); #endif VMCOREINFO_SYMBOL(_stext); - VMCOREINFO_SYMBOL(vmap_area_list); + vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", (unsigned long) VMALLOC_START); #ifndef CONFIG_NUMA VMCOREINFO_SYMBOL(mem_map); @@ -789,8 +789,6 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_OFFSET(free_area, free_list); VMCOREINFO_OFFSET(list_head, next); VMCOREINFO_OFFSET(list_head, prev); - VMCOREINFO_OFFSET(vmap_area, va_start); - VMCOREINFO_OFFSET(vmap_area, list); VMCOREINFO_LENGTH(zone.free_area, NR_PAGE_ORDERS); log_buf_vmcoreinfo_setup(); VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c index b4cac76ea5e9..8a689b4ff4f9 100644 --- a/kernel/kallsyms_selftest.c +++ b/kernel/kallsyms_selftest.c @@ -89,7 +89,6 @@ static struct test_item test_items[] = { ITEM_DATA(kallsyms_test_var_data_static), ITEM_DATA(kallsyms_test_var_bss), ITEM_DATA(kallsyms_test_var_data), - ITEM_DATA(vmap_area_list), #endif }; diff --git a/mm/nommu.c b/mm/nommu.c index b6dc558d3144..5ec8f44e7ce9 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -131,8 +131,6 @@ int follow_pfn(struct vm_area_struct *vma, unsigned long address, } EXPORT_SYMBOL(follow_pfn); -LIST_HEAD(vmap_area_list); - void vfree(const void *addr) { kfree(addr); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 86efebf0e0c8..b5882790da00 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -729,8 +729,6 @@ EXPORT_SYMBOL(vmalloc_to_pfn); static DEFINE_SPINLOCK(free_vmap_area_lock); -/* Export for kexec only */ -LIST_HEAD(vmap_area_list); static bool vmap_initialized __read_mostly; static struct rb_root purge_vmap_area_root = RB_ROOT; -- cgit v1.2.3 From 85fcde402db191b5f222ebfecda653777d7d084e Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Jan 2024 13:12:41 +0800 Subject: kexec: split crashkernel reservation code out from crash_core.c Patch series "Split crash out from kexec and clean up related config items", v3. Motivation: ============= Previously, LKP reported a building error. When investigating, it can't be resolved reasonablly with the present messy kdump config items. https://lore.kernel.org/oe-kbuild-all/202312182200.Ka7MzifQ-lkp@intel.com/ The kdump (crash dumping) related config items could causes confusions: Firstly, CRASH_CORE enables codes including - crashkernel reservation; - elfcorehdr updating; - vmcoreinfo exporting; - crash hotplug handling; Now fadump of powerpc, kcore dynamic debugging and kdump all selects CRASH_CORE, while fadump - fadump needs crashkernel parsing, vmcoreinfo exporting, and accessing global variable 'elfcorehdr_addr'; - kcore only needs vmcoreinfo exporting; - kdump needs all of the current kernel/crash_core.c. So only enabling PROC_CORE or FA_DUMP will enable CRASH_CORE, this mislead people that we enable crash dumping, actual it's not. Secondly, It's not reasonable to allow KEXEC_CORE select CRASH_CORE. Because KEXEC_CORE enables codes which allocate control pages, copy kexec/kdump segments, and prepare for switching. These codes are shared by both kexec reboot and kdump. We could want kexec reboot, but disable kdump. In that case, CRASH_CORE should not be selected. -------------------- CONFIG_CRASH_CORE=y CONFIG_KEXEC_CORE=y CONFIG_KEXEC=y CONFIG_KEXEC_FILE=y --------------------- Thirdly, It's not reasonable to allow CRASH_DUMP select KEXEC_CORE. That could make KEXEC_CORE, CRASH_DUMP are enabled independently from KEXEC or KEXEC_FILE. However, w/o KEXEC or KEXEC_FILE, the KEXEC_CORE code built in doesn't make any sense because no kernel loading or switching will happen to utilize the KEXEC_CORE code. --------------------- CONFIG_CRASH_CORE=y CONFIG_KEXEC_CORE=y CONFIG_CRASH_DUMP=y --------------------- In this case, what is worse, on arch sh and arm, KEXEC relies on MMU, while CRASH_DUMP can still be enabled when !MMU, then compiling error is seen as the lkp test robot reported in above link. ------arch/sh/Kconfig------ config ARCH_SUPPORTS_KEXEC def_bool MMU config ARCH_SUPPORTS_CRASH_DUMP def_bool BROKEN_ON_SMP --------------------------- Changes: =========== 1, split out crash_reserve.c from crash_core.c; 2, split out vmcore_infoc. from crash_core.c; 3, move crash related codes in kexec_core.c into crash_core.c; 4, remove dependency of FA_DUMP on CRASH_DUMP; 5, clean up kdump related config items; 6, wrap up crash codes in crash related ifdefs on all 8 arch-es which support crash dumping, except of ppc; Achievement: =========== With above changes, I can rearrange the config item logic as below (the right item depends on or is selected by the left item): PROC_KCORE -----------> VMCORE_INFO |----------> VMCORE_INFO FA_DUMP----| |----------> CRASH_RESERVE ---->VMCORE_INFO / |---->CRASH_RESERVE KEXEC --| /| |--> KEXEC_CORE--> CRASH_DUMP-->/-|---->PROC_VMCORE KEXEC_FILE --| \ | \---->CRASH_HOTPLUG KEXEC --| |--> KEXEC_CORE (for kexec reboot only) KEXEC_FILE --| Test ======== On all 8 architectures, including x86_64, arm64, s390x, sh, arm, mips, riscv, loongarch, I did below three cases of config item setting and building all passed. Take configs on x86_64 as exampmle here: (1) Both CONFIG_KEXEC and KEXEC_FILE is unset, then all kexec/kdump items are unset automatically: # Kexec and crash features # CONFIG_KEXEC is not set # CONFIG_KEXEC_FILE is not set # end of Kexec and crash features (2) set CONFIG_KEXEC_FILE and 'make olddefconfig': --------------- # Kexec and crash features CONFIG_CRASH_RESERVE=y CONFIG_VMCORE_INFO=y CONFIG_KEXEC_CORE=y CONFIG_KEXEC_FILE=y CONFIG_CRASH_DUMP=y CONFIG_CRASH_HOTPLUG=y CONFIG_CRASH_MAX_MEMORY_RANGES=8192 # end of Kexec and crash features --------------- (3) unset CONFIG_CRASH_DUMP in case 2 and execute 'make olddefconfig': ------------------------ # Kexec and crash features CONFIG_KEXEC_CORE=y CONFIG_KEXEC_FILE=y # end of Kexec and crash features ------------------------ Note: For ppc, it needs investigation to make clear how to split out crash code in arch folder. Hope Hari and Pingfan can help have a look, see if it's doable. Now, I make it either have both kexec and crash enabled, or disable both of them altogether. This patch (of 14): Both kdump and fa_dump of ppc rely on crashkernel reservation. Move the relevant codes into separate files: crash_reserve.c, include/linux/crash_reserve.h. And also add config item CRASH_RESERVE to control its enabling of the codes. And update config items which has relationship with crashkernel reservation. And also change ifdeffery from CONFIG_CRASH_CORE to CONFIG_CRASH_RESERVE when those scopes are only crashkernel reservation related. And also rename arch/XXX/include/asm/{crash_core.h => crash_reserve.h} on arm64, x86 and risc-v because those architectures' crash_core.h is only related to crashkernel reservation. [akpm@linux-foundation.org: s/CRASH_RESEERVE/CRASH_RESERVE/, per Klara Modin] Link: https://lkml.kernel.org/r/20240124051254.67105-1-bhe@redhat.com Link: https://lkml.kernel.org/r/20240124051254.67105-2-bhe@redhat.com Signed-off-by: Baoquan He Acked-by: Hari Bathini Cc: Al Viro Cc: Eric W. Biederman Cc: Pingfan Liu Cc: Klara Modin Cc: Michael Kelley Cc: Nathan Chancellor Cc: Stephen Rothwell Cc: Yang Li Signed-off-by: Andrew Morton --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/crash_core.h | 10 - arch/arm64/include/asm/crash_reserve.h | 10 + arch/powerpc/Kconfig | 1 + arch/powerpc/mm/nohash/kaslr_booke.c | 4 +- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/crash_core.h | 11 - arch/riscv/include/asm/crash_reserve.h | 11 + arch/x86/Kconfig | 2 +- arch/x86/include/asm/crash_core.h | 42 --- arch/x86/include/asm/crash_reserve.h | 42 +++ include/linux/crash_core.h | 40 --- include/linux/crash_reserve.h | 48 ++++ include/linux/kexec.h | 1 + kernel/Kconfig.kexec | 5 +- kernel/Makefile | 1 + kernel/crash_core.c | 438 ------------------------------- kernel/crash_reserve.c | 464 +++++++++++++++++++++++++++++++++ 18 files changed, 587 insertions(+), 547 deletions(-) delete mode 100644 arch/arm64/include/asm/crash_core.h create mode 100644 arch/arm64/include/asm/crash_reserve.h delete mode 100644 arch/riscv/include/asm/crash_core.h create mode 100644 arch/riscv/include/asm/crash_reserve.h delete mode 100644 arch/x86/include/asm/crash_core.h create mode 100644 arch/x86/include/asm/crash_reserve.h create mode 100644 include/linux/crash_reserve.h create mode 100644 kernel/crash_reserve.c (limited to 'include/linux') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 386566138620..5a7ac1f37bdc 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1519,7 +1519,7 @@ config ARCH_SUPPORTS_CRASH_DUMP def_bool y config ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION - def_bool CRASH_CORE + def_bool CRASH_RESERVE config TRANS_TABLE def_bool y diff --git a/arch/arm64/include/asm/crash_core.h b/arch/arm64/include/asm/crash_core.h deleted file mode 100644 index 9f5c8d339f44..000000000000 --- a/arch/arm64/include/asm/crash_core.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _ARM64_CRASH_CORE_H -#define _ARM64_CRASH_CORE_H - -/* Current arm64 boot protocol requires 2MB alignment */ -#define CRASH_ALIGN SZ_2M - -#define CRASH_ADDR_LOW_MAX arm64_dma_phys_limit -#define CRASH_ADDR_HIGH_MAX (PHYS_MASK + 1) -#endif diff --git a/arch/arm64/include/asm/crash_reserve.h b/arch/arm64/include/asm/crash_reserve.h new file mode 100644 index 000000000000..4afe027a4e7b --- /dev/null +++ b/arch/arm64/include/asm/crash_reserve.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ARM64_CRASH_RESERVE_H +#define _ARM64_CRASH_RESERVE_H + +/* Current arm64 boot protocol requires 2MB alignment */ +#define CRASH_ALIGN SZ_2M + +#define CRASH_ADDR_LOW_MAX arm64_dma_phys_limit +#define CRASH_ADDR_HIGH_MAX (PHYS_MASK + 1) +#endif diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index b9fc064d38d2..7f704ae5c5ef 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -691,6 +691,7 @@ config FA_DUMP bool "Firmware-assisted dump" depends on PPC64 && (PPC_RTAS || PPC_POWERNV) select CRASH_CORE + select CRASH_RESERVE select CRASH_DUMP help A robust mechanism to get reliable kernel crash dump with diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c index b4f2786a7d2b..cdff129abb14 100644 --- a/arch/powerpc/mm/nohash/kaslr_booke.c +++ b/arch/powerpc/mm/nohash/kaslr_booke.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -173,7 +173,7 @@ static __init bool overlaps_region(const void *fdt, u32 start, static void __init get_crash_kernel(void *fdt, unsigned long size) { -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_CRASH_RESERVE unsigned long long crash_size, crash_base; int ret; diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index bffbd869a068..bd06ad1bb97c 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -767,7 +767,7 @@ config ARCH_SUPPORTS_CRASH_DUMP def_bool y config ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION - def_bool CRASH_CORE + def_bool CRASH_RESERVE config COMPAT bool "Kernel support for 32-bit U-mode" diff --git a/arch/riscv/include/asm/crash_core.h b/arch/riscv/include/asm/crash_core.h deleted file mode 100644 index e1874b23feaf..000000000000 --- a/arch/riscv/include/asm/crash_core.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _RISCV_CRASH_CORE_H -#define _RISCV_CRASH_CORE_H - -#define CRASH_ALIGN PMD_SIZE - -#define CRASH_ADDR_LOW_MAX dma32_phys_limit -#define CRASH_ADDR_HIGH_MAX memblock_end_of_DRAM() - -extern phys_addr_t memblock_end_of_DRAM(void); -#endif diff --git a/arch/riscv/include/asm/crash_reserve.h b/arch/riscv/include/asm/crash_reserve.h new file mode 100644 index 000000000000..013962e63587 --- /dev/null +++ b/arch/riscv/include/asm/crash_reserve.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _RISCV_CRASH_RESERVE_H +#define _RISCV_CRASH_RESERVE_H + +#define CRASH_ALIGN PMD_SIZE + +#define CRASH_ADDR_LOW_MAX dma32_phys_limit +#define CRASH_ADDR_HIGH_MAX memblock_end_of_DRAM() + +extern phys_addr_t memblock_end_of_DRAM(void); +#endif diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5edec175b9bf..5bd925815154 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2106,7 +2106,7 @@ config ARCH_SUPPORTS_CRASH_HOTPLUG def_bool y config ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION - def_bool CRASH_CORE + def_bool CRASH_RESERVE config PHYSICAL_START hex "Physical address where the kernel is loaded" if (EXPERT || CRASH_DUMP) diff --git a/arch/x86/include/asm/crash_core.h b/arch/x86/include/asm/crash_core.h deleted file mode 100644 index 76af98f4e801..000000000000 --- a/arch/x86/include/asm/crash_core.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _X86_CRASH_CORE_H -#define _X86_CRASH_CORE_H - -/* 16M alignment for crash kernel regions */ -#define CRASH_ALIGN SZ_16M - -/* - * Keep the crash kernel below this limit. - * - * Earlier 32-bits kernels would limit the kernel to the low 512 MB range - * due to mapping restrictions. - * - * 64-bit kdump kernels need to be restricted to be under 64 TB, which is - * the upper limit of system RAM in 4-level paging mode. Since the kdump - * jump could be from 5-level paging to 4-level paging, the jump will fail if - * the kernel is put above 64 TB, and during the 1st kernel bootup there's - * no good way to detect the paging mode of the target kernel which will be - * loaded for dumping. - */ -extern unsigned long swiotlb_size_or_default(void); - -#ifdef CONFIG_X86_32 -# define CRASH_ADDR_LOW_MAX SZ_512M -# define CRASH_ADDR_HIGH_MAX SZ_512M -#else -# define CRASH_ADDR_LOW_MAX SZ_4G -# define CRASH_ADDR_HIGH_MAX SZ_64T -#endif - -# define DEFAULT_CRASH_KERNEL_LOW_SIZE crash_low_size_default() - -static inline unsigned long crash_low_size_default(void) -{ -#ifdef CONFIG_X86_64 - return max(swiotlb_size_or_default() + (8UL << 20), 256UL << 20); -#else - return 0; -#endif -} - -#endif /* _X86_CRASH_CORE_H */ diff --git a/arch/x86/include/asm/crash_reserve.h b/arch/x86/include/asm/crash_reserve.h new file mode 100644 index 000000000000..152239f95541 --- /dev/null +++ b/arch/x86/include/asm/crash_reserve.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_CRASH_RESERVE_H +#define _X86_CRASH_RESERVE_H + +/* 16M alignment for crash kernel regions */ +#define CRASH_ALIGN SZ_16M + +/* + * Keep the crash kernel below this limit. + * + * Earlier 32-bits kernels would limit the kernel to the low 512 MB range + * due to mapping restrictions. + * + * 64-bit kdump kernels need to be restricted to be under 64 TB, which is + * the upper limit of system RAM in 4-level paging mode. Since the kdump + * jump could be from 5-level paging to 4-level paging, the jump will fail if + * the kernel is put above 64 TB, and during the 1st kernel bootup there's + * no good way to detect the paging mode of the target kernel which will be + * loaded for dumping. + */ +extern unsigned long swiotlb_size_or_default(void); + +#ifdef CONFIG_X86_32 +# define CRASH_ADDR_LOW_MAX SZ_512M +# define CRASH_ADDR_HIGH_MAX SZ_512M +#else +# define CRASH_ADDR_LOW_MAX SZ_4G +# define CRASH_ADDR_HIGH_MAX SZ_64T +#endif + +# define DEFAULT_CRASH_KERNEL_LOW_SIZE crash_low_size_default() + +static inline unsigned long crash_low_size_default(void) +{ +#ifdef CONFIG_X86_64 + return max(swiotlb_size_or_default() + (8UL << 20), 256UL << 20); +#else + return 0; +#endif +} + +#endif /* _X86_CRASH_RESERVE_H */ diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index 9eaeaafe0cad..1fde49246fa6 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -5,14 +5,6 @@ #include #include #include -#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION -#include -#endif - -/* Location of a reserved region to hold the crash kernel. - */ -extern struct resource crashk_res; -extern struct resource crashk_low_res; #define CRASH_CORE_NOTE_NAME "CORE" #define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4) @@ -87,38 +79,6 @@ Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, void *data, size_t data_len); void final_note(Elf_Word *buf); -int __init parse_crashkernel(char *cmdline, unsigned long long system_ram, - unsigned long long *crash_size, unsigned long long *crash_base, - unsigned long long *low_size, bool *high); - -#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION -#ifndef DEFAULT_CRASH_KERNEL_LOW_SIZE -#define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20) -#endif -#ifndef CRASH_ALIGN -#define CRASH_ALIGN SZ_2M -#endif -#ifndef CRASH_ADDR_LOW_MAX -#define CRASH_ADDR_LOW_MAX SZ_4G -#endif -#ifndef CRASH_ADDR_HIGH_MAX -#define CRASH_ADDR_HIGH_MAX memblock_end_of_DRAM() -#endif - -void __init reserve_crashkernel_generic(char *cmdline, - unsigned long long crash_size, - unsigned long long crash_base, - unsigned long long crash_low_size, - bool high); -#else -static inline void __init reserve_crashkernel_generic(char *cmdline, - unsigned long long crash_size, - unsigned long long crash_base, - unsigned long long crash_low_size, - bool high) -{} -#endif - /* Alignment required for elf header segment */ #define ELF_CORE_HEADER_ALIGN 4096 diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h new file mode 100644 index 000000000000..5a9df944fb80 --- /dev/null +++ b/include/linux/crash_reserve.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_CRASH_RESERVE_H +#define LINUX_CRASH_RESERVE_H + +#include +#include +#include +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +#include +#endif + +/* Location of a reserved region to hold the crash kernel. + */ +extern struct resource crashk_res; +extern struct resource crashk_low_res; + +int __init parse_crashkernel(char *cmdline, unsigned long long system_ram, + unsigned long long *crash_size, unsigned long long *crash_base, + unsigned long long *low_size, bool *high); + +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +#ifndef DEFAULT_CRASH_KERNEL_LOW_SIZE +#define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20) +#endif +#ifndef CRASH_ALIGN +#define CRASH_ALIGN SZ_2M +#endif +#ifndef CRASH_ADDR_LOW_MAX +#define CRASH_ADDR_LOW_MAX SZ_4G +#endif +#ifndef CRASH_ADDR_HIGH_MAX +#define CRASH_ADDR_HIGH_MAX memblock_end_of_DRAM() +#endif + +void __init reserve_crashkernel_generic(char *cmdline, + unsigned long long crash_size, + unsigned long long crash_base, + unsigned long long crash_low_size, + bool high); +#else +static inline void __init reserve_crashkernel_generic(char *cmdline, + unsigned long long crash_size, + unsigned long long crash_base, + unsigned long long crash_low_size, + bool high) +{} +#endif +#endif /* LINUX_CRASH_RESERVE_H */ diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 400cb6c02176..6d79bfb52e5b 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -16,6 +16,7 @@ #if !defined(__ASSEMBLY__) #include +#include #include #include diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec index 946dffa048b7..8b7be71edd85 100644 --- a/kernel/Kconfig.kexec +++ b/kernel/Kconfig.kexec @@ -2,11 +2,15 @@ menu "Kexec and crash features" +config CRASH_RESERVE + bool + config CRASH_CORE bool config KEXEC_CORE select CRASH_CORE + select CRASH_RESERVE bool config KEXEC_ELF @@ -96,7 +100,6 @@ config KEXEC_JUMP config CRASH_DUMP bool "kernel crash dumps" depends on ARCH_SUPPORTS_CRASH_DUMP - select CRASH_CORE select KEXEC_CORE help Generate crash dump after being started by kexec. diff --git a/kernel/Makefile b/kernel/Makefile index ce105a5558fc..05fa88b3ab74 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -69,6 +69,7 @@ obj-$(CONFIG_KALLSYMS) += kallsyms.o obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o obj-$(CONFIG_CRASH_CORE) += crash_core.o +obj-$(CONFIG_CRASH_RESERVE) += crash_reserve.o obj-$(CONFIG_KEXEC_CORE) += kexec_core.o obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_KEXEC_FILE) += kexec_file.o diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 49b31e59d3cc..ae0d1ce89b46 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -34,444 +34,6 @@ u32 *vmcoreinfo_note; /* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */ static unsigned char *vmcoreinfo_data_safecopy; -/* Location of the reserved area for the crash kernel */ -struct resource crashk_res = { - .name = "Crash kernel", - .start = 0, - .end = 0, - .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM, - .desc = IORES_DESC_CRASH_KERNEL -}; -struct resource crashk_low_res = { - .name = "Crash kernel", - .start = 0, - .end = 0, - .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM, - .desc = IORES_DESC_CRASH_KERNEL -}; - -/* - * parsing the "crashkernel" commandline - * - * this code is intended to be called from architecture specific code - */ - - -/* - * This function parses command lines in the format - * - * crashkernel=ramsize-range:size[,...][@offset] - * - * The function returns 0 on success and -EINVAL on failure. - */ -static int __init parse_crashkernel_mem(char *cmdline, - unsigned long long system_ram, - unsigned long long *crash_size, - unsigned long long *crash_base) -{ - char *cur = cmdline, *tmp; - unsigned long long total_mem = system_ram; - - /* - * Firmware sometimes reserves some memory regions for its own use, - * so the system memory size is less than the actual physical memory - * size. Work around this by rounding up the total size to 128M, - * which is enough for most test cases. - */ - total_mem = roundup(total_mem, SZ_128M); - - /* for each entry of the comma-separated list */ - do { - unsigned long long start, end = ULLONG_MAX, size; - - /* get the start of the range */ - start = memparse(cur, &tmp); - if (cur == tmp) { - pr_warn("crashkernel: Memory value expected\n"); - return -EINVAL; - } - cur = tmp; - if (*cur != '-') { - pr_warn("crashkernel: '-' expected\n"); - return -EINVAL; - } - cur++; - - /* if no ':' is here, than we read the end */ - if (*cur != ':') { - end = memparse(cur, &tmp); - if (cur == tmp) { - pr_warn("crashkernel: Memory value expected\n"); - return -EINVAL; - } - cur = tmp; - if (end <= start) { - pr_warn("crashkernel: end <= start\n"); - return -EINVAL; - } - } - - if (*cur != ':') { - pr_warn("crashkernel: ':' expected\n"); - return -EINVAL; - } - cur++; - - size = memparse(cur, &tmp); - if (cur == tmp) { - pr_warn("Memory value expected\n"); - return -EINVAL; - } - cur = tmp; - if (size >= total_mem) { - pr_warn("crashkernel: invalid size\n"); - return -EINVAL; - } - - /* match ? */ - if (total_mem >= start && total_mem < end) { - *crash_size = size; - break; - } - } while (*cur++ == ','); - - if (*crash_size > 0) { - while (*cur && *cur != ' ' && *cur != '@') - cur++; - if (*cur == '@') { - cur++; - *crash_base = memparse(cur, &tmp); - if (cur == tmp) { - pr_warn("Memory value expected after '@'\n"); - return -EINVAL; - } - } - } else - pr_info("crashkernel size resulted in zero bytes\n"); - - return 0; -} - -/* - * That function parses "simple" (old) crashkernel command lines like - * - * crashkernel=size[@offset] - * - * It returns 0 on success and -EINVAL on failure. - */ -static int __init parse_crashkernel_simple(char *cmdline, - unsigned long long *crash_size, - unsigned long long *crash_base) -{ - char *cur = cmdline; - - *crash_size = memparse(cmdline, &cur); - if (cmdline == cur) { - pr_warn("crashkernel: memory value expected\n"); - return -EINVAL; - } - - if (*cur == '@') - *crash_base = memparse(cur+1, &cur); - else if (*cur != ' ' && *cur != '\0') { - pr_warn("crashkernel: unrecognized char: %c\n", *cur); - return -EINVAL; - } - - return 0; -} - -#define SUFFIX_HIGH 0 -#define SUFFIX_LOW 1 -#define SUFFIX_NULL 2 -static __initdata char *suffix_tbl[] = { - [SUFFIX_HIGH] = ",high", - [SUFFIX_LOW] = ",low", - [SUFFIX_NULL] = NULL, -}; - -/* - * That function parses "suffix" crashkernel command lines like - * - * crashkernel=size,[high|low] - * - * It returns 0 on success and -EINVAL on failure. - */ -static int __init parse_crashkernel_suffix(char *cmdline, - unsigned long long *crash_size, - const char *suffix) -{ - char *cur = cmdline; - - *crash_size = memparse(cmdline, &cur); - if (cmdline == cur) { - pr_warn("crashkernel: memory value expected\n"); - return -EINVAL; - } - - /* check with suffix */ - if (strncmp(cur, suffix, strlen(suffix))) { - pr_warn("crashkernel: unrecognized char: %c\n", *cur); - return -EINVAL; - } - cur += strlen(suffix); - if (*cur != ' ' && *cur != '\0') { - pr_warn("crashkernel: unrecognized char: %c\n", *cur); - return -EINVAL; - } - - return 0; -} - -static __init char *get_last_crashkernel(char *cmdline, - const char *name, - const char *suffix) -{ - char *p = cmdline, *ck_cmdline = NULL; - - /* find crashkernel and use the last one if there are more */ - p = strstr(p, name); - while (p) { - char *end_p = strchr(p, ' '); - char *q; - - if (!end_p) - end_p = p + strlen(p); - - if (!suffix) { - int i; - - /* skip the one with any known suffix */ - for (i = 0; suffix_tbl[i]; i++) { - q = end_p - strlen(suffix_tbl[i]); - if (!strncmp(q, suffix_tbl[i], - strlen(suffix_tbl[i]))) - goto next; - } - ck_cmdline = p; - } else { - q = end_p - strlen(suffix); - if (!strncmp(q, suffix, strlen(suffix))) - ck_cmdline = p; - } -next: - p = strstr(p+1, name); - } - - return ck_cmdline; -} - -static int __init __parse_crashkernel(char *cmdline, - unsigned long long system_ram, - unsigned long long *crash_size, - unsigned long long *crash_base, - const char *suffix) -{ - char *first_colon, *first_space; - char *ck_cmdline; - char *name = "crashkernel="; - - BUG_ON(!crash_size || !crash_base); - *crash_size = 0; - *crash_base = 0; - - ck_cmdline = get_last_crashkernel(cmdline, name, suffix); - if (!ck_cmdline) - return -ENOENT; - - ck_cmdline += strlen(name); - - if (suffix) - return parse_crashkernel_suffix(ck_cmdline, crash_size, - suffix); - /* - * if the commandline contains a ':', then that's the extended - * syntax -- if not, it must be the classic syntax - */ - first_colon = strchr(ck_cmdline, ':'); - first_space = strchr(ck_cmdline, ' '); - if (first_colon && (!first_space || first_colon < first_space)) - return parse_crashkernel_mem(ck_cmdline, system_ram, - crash_size, crash_base); - - return parse_crashkernel_simple(ck_cmdline, crash_size, crash_base); -} - -/* - * That function is the entry point for command line parsing and should be - * called from the arch-specific code. - * - * If crashkernel=,high|low is supported on architecture, non-NULL values - * should be passed to parameters 'low_size' and 'high'. - */ -int __init parse_crashkernel(char *cmdline, - unsigned long long system_ram, - unsigned long long *crash_size, - unsigned long long *crash_base, - unsigned long long *low_size, - bool *high) -{ - int ret; - - /* crashkernel=X[@offset] */ - ret = __parse_crashkernel(cmdline, system_ram, crash_size, - crash_base, NULL); -#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION - /* - * If non-NULL 'high' passed in and no normal crashkernel - * setting detected, try parsing crashkernel=,high|low. - */ - if (high && ret == -ENOENT) { - ret = __parse_crashkernel(cmdline, 0, crash_size, - crash_base, suffix_tbl[SUFFIX_HIGH]); - if (ret || !*crash_size) - return -EINVAL; - - /* - * crashkernel=Y,low can be specified or not, but invalid value - * is not allowed. - */ - ret = __parse_crashkernel(cmdline, 0, low_size, - crash_base, suffix_tbl[SUFFIX_LOW]); - if (ret == -ENOENT) { - *low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; - ret = 0; - } else if (ret) { - return ret; - } - - *high = true; - } -#endif - if (!*crash_size) - ret = -EINVAL; - - return ret; -} - -/* - * Add a dummy early_param handler to mark crashkernel= as a known command line - * parameter and suppress incorrect warnings in init/main.c. - */ -static int __init parse_crashkernel_dummy(char *arg) -{ - return 0; -} -early_param("crashkernel", parse_crashkernel_dummy); - -#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION -static int __init reserve_crashkernel_low(unsigned long long low_size) -{ -#ifdef CONFIG_64BIT - unsigned long long low_base; - - low_base = memblock_phys_alloc_range(low_size, CRASH_ALIGN, 0, CRASH_ADDR_LOW_MAX); - if (!low_base) { - pr_err("cannot allocate crashkernel low memory (size:0x%llx).\n", low_size); - return -ENOMEM; - } - - pr_info("crashkernel low memory reserved: 0x%08llx - 0x%08llx (%lld MB)\n", - low_base, low_base + low_size, low_size >> 20); - - crashk_low_res.start = low_base; - crashk_low_res.end = low_base + low_size - 1; -#endif - return 0; -} - -void __init reserve_crashkernel_generic(char *cmdline, - unsigned long long crash_size, - unsigned long long crash_base, - unsigned long long crash_low_size, - bool high) -{ - unsigned long long search_end = CRASH_ADDR_LOW_MAX, search_base = 0; - bool fixed_base = false; - - /* User specifies base address explicitly. */ - if (crash_base) { - fixed_base = true; - search_base = crash_base; - search_end = crash_base + crash_size; - } else if (high) { - search_base = CRASH_ADDR_LOW_MAX; - search_end = CRASH_ADDR_HIGH_MAX; - } - -retry: - crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, - search_base, search_end); - if (!crash_base) { - /* - * For crashkernel=size[KMG]@offset[KMG], print out failure - * message if can't reserve the specified region. - */ - if (fixed_base) { - pr_warn("crashkernel reservation failed - memory is in use.\n"); - return; - } - - /* - * For crashkernel=size[KMG], if the first attempt was for - * low memory, fall back to high memory, the minimum required - * low memory will be reserved later. - */ - if (!high && search_end == CRASH_ADDR_LOW_MAX) { - search_end = CRASH_ADDR_HIGH_MAX; - search_base = CRASH_ADDR_LOW_MAX; - crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; - goto retry; - } - - /* - * For crashkernel=size[KMG],high, if the first attempt was - * for high memory, fall back to low memory. - */ - if (high && search_end == CRASH_ADDR_HIGH_MAX) { - search_end = CRASH_ADDR_LOW_MAX; - search_base = 0; - goto retry; - } - pr_warn("cannot allocate crashkernel (size:0x%llx)\n", - crash_size); - return; - } - - if ((crash_base >= CRASH_ADDR_LOW_MAX) && - crash_low_size && reserve_crashkernel_low(crash_low_size)) { - memblock_phys_free(crash_base, crash_size); - return; - } - - pr_info("crashkernel reserved: 0x%016llx - 0x%016llx (%lld MB)\n", - crash_base, crash_base + crash_size, crash_size >> 20); - - /* - * The crashkernel memory will be removed from the kernel linear - * map. Inform kmemleak so that it won't try to access it. - */ - kmemleak_ignore_phys(crash_base); - if (crashk_low_res.end) - kmemleak_ignore_phys(crashk_low_res.start); - - crashk_res.start = crash_base; - crashk_res.end = crash_base + crash_size - 1; -} - -static __init int insert_crashkernel_resources(void) -{ - if (crashk_res.start < crashk_res.end) - insert_resource(&iomem_resource, &crashk_res); - - if (crashk_low_res.start < crashk_low_res.end) - insert_resource(&iomem_resource, &crashk_low_res); - - return 0; -} -early_initcall(insert_crashkernel_resources); -#endif - int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz) { diff --git a/kernel/crash_reserve.c b/kernel/crash_reserve.c new file mode 100644 index 000000000000..bbb6c3cb00e4 --- /dev/null +++ b/kernel/crash_reserve.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * crash.c - kernel crash support code. + * Copyright (C) 2002-2004 Eric Biederman + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "kallsyms_internal.h" +#include "kexec_internal.h" + +/* Location of the reserved area for the crash kernel */ +struct resource crashk_res = { + .name = "Crash kernel", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM, + .desc = IORES_DESC_CRASH_KERNEL +}; +struct resource crashk_low_res = { + .name = "Crash kernel", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM, + .desc = IORES_DESC_CRASH_KERNEL +}; + +/* + * parsing the "crashkernel" commandline + * + * this code is intended to be called from architecture specific code + */ + + +/* + * This function parses command lines in the format + * + * crashkernel=ramsize-range:size[,...][@offset] + * + * The function returns 0 on success and -EINVAL on failure. + */ +static int __init parse_crashkernel_mem(char *cmdline, + unsigned long long system_ram, + unsigned long long *crash_size, + unsigned long long *crash_base) +{ + char *cur = cmdline, *tmp; + unsigned long long total_mem = system_ram; + + /* + * Firmware sometimes reserves some memory regions for its own use, + * so the system memory size is less than the actual physical memory + * size. Work around this by rounding up the total size to 128M, + * which is enough for most test cases. + */ + total_mem = roundup(total_mem, SZ_128M); + + /* for each entry of the comma-separated list */ + do { + unsigned long long start, end = ULLONG_MAX, size; + + /* get the start of the range */ + start = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("crashkernel: Memory value expected\n"); + return -EINVAL; + } + cur = tmp; + if (*cur != '-') { + pr_warn("crashkernel: '-' expected\n"); + return -EINVAL; + } + cur++; + + /* if no ':' is here, than we read the end */ + if (*cur != ':') { + end = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("crashkernel: Memory value expected\n"); + return -EINVAL; + } + cur = tmp; + if (end <= start) { + pr_warn("crashkernel: end <= start\n"); + return -EINVAL; + } + } + + if (*cur != ':') { + pr_warn("crashkernel: ':' expected\n"); + return -EINVAL; + } + cur++; + + size = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("Memory value expected\n"); + return -EINVAL; + } + cur = tmp; + if (size >= total_mem) { + pr_warn("crashkernel: invalid size\n"); + return -EINVAL; + } + + /* match ? */ + if (total_mem >= start && total_mem < end) { + *crash_size = size; + break; + } + } while (*cur++ == ','); + + if (*crash_size > 0) { + while (*cur && *cur != ' ' && *cur != '@') + cur++; + if (*cur == '@') { + cur++; + *crash_base = memparse(cur, &tmp); + if (cur == tmp) { + pr_warn("Memory value expected after '@'\n"); + return -EINVAL; + } + } + } else + pr_info("crashkernel size resulted in zero bytes\n"); + + return 0; +} + +/* + * That function parses "simple" (old) crashkernel command lines like + * + * crashkernel=size[@offset] + * + * It returns 0 on success and -EINVAL on failure. + */ +static int __init parse_crashkernel_simple(char *cmdline, + unsigned long long *crash_size, + unsigned long long *crash_base) +{ + char *cur = cmdline; + + *crash_size = memparse(cmdline, &cur); + if (cmdline == cur) { + pr_warn("crashkernel: memory value expected\n"); + return -EINVAL; + } + + if (*cur == '@') + *crash_base = memparse(cur+1, &cur); + else if (*cur != ' ' && *cur != '\0') { + pr_warn("crashkernel: unrecognized char: %c\n", *cur); + return -EINVAL; + } + + return 0; +} + +#define SUFFIX_HIGH 0 +#define SUFFIX_LOW 1 +#define SUFFIX_NULL 2 +static __initdata char *suffix_tbl[] = { + [SUFFIX_HIGH] = ",high", + [SUFFIX_LOW] = ",low", + [SUFFIX_NULL] = NULL, +}; + +/* + * That function parses "suffix" crashkernel command lines like + * + * crashkernel=size,[high|low] + * + * It returns 0 on success and -EINVAL on failure. + */ +static int __init parse_crashkernel_suffix(char *cmdline, + unsigned long long *crash_size, + const char *suffix) +{ + char *cur = cmdline; + + *crash_size = memparse(cmdline, &cur); + if (cmdline == cur) { + pr_warn("crashkernel: memory value expected\n"); + return -EINVAL; + } + + /* check with suffix */ + if (strncmp(cur, suffix, strlen(suffix))) { + pr_warn("crashkernel: unrecognized char: %c\n", *cur); + return -EINVAL; + } + cur += strlen(suffix); + if (*cur != ' ' && *cur != '\0') { + pr_warn("crashkernel: unrecognized char: %c\n", *cur); + return -EINVAL; + } + + return 0; +} + +static __init char *get_last_crashkernel(char *cmdline, + const char *name, + const char *suffix) +{ + char *p = cmdline, *ck_cmdline = NULL; + + /* find crashkernel and use the last one if there are more */ + p = strstr(p, name); + while (p) { + char *end_p = strchr(p, ' '); + char *q; + + if (!end_p) + end_p = p + strlen(p); + + if (!suffix) { + int i; + + /* skip the one with any known suffix */ + for (i = 0; suffix_tbl[i]; i++) { + q = end_p - strlen(suffix_tbl[i]); + if (!strncmp(q, suffix_tbl[i], + strlen(suffix_tbl[i]))) + goto next; + } + ck_cmdline = p; + } else { + q = end_p - strlen(suffix); + if (!strncmp(q, suffix, strlen(suffix))) + ck_cmdline = p; + } +next: + p = strstr(p+1, name); + } + + return ck_cmdline; +} + +static int __init __parse_crashkernel(char *cmdline, + unsigned long long system_ram, + unsigned long long *crash_size, + unsigned long long *crash_base, + const char *suffix) +{ + char *first_colon, *first_space; + char *ck_cmdline; + char *name = "crashkernel="; + + BUG_ON(!crash_size || !crash_base); + *crash_size = 0; + *crash_base = 0; + + ck_cmdline = get_last_crashkernel(cmdline, name, suffix); + if (!ck_cmdline) + return -ENOENT; + + ck_cmdline += strlen(name); + + if (suffix) + return parse_crashkernel_suffix(ck_cmdline, crash_size, + suffix); + /* + * if the commandline contains a ':', then that's the extended + * syntax -- if not, it must be the classic syntax + */ + first_colon = strchr(ck_cmdline, ':'); + first_space = strchr(ck_cmdline, ' '); + if (first_colon && (!first_space || first_colon < first_space)) + return parse_crashkernel_mem(ck_cmdline, system_ram, + crash_size, crash_base); + + return parse_crashkernel_simple(ck_cmdline, crash_size, crash_base); +} + +/* + * That function is the entry point for command line parsing and should be + * called from the arch-specific code. + * + * If crashkernel=,high|low is supported on architecture, non-NULL values + * should be passed to parameters 'low_size' and 'high'. + */ +int __init parse_crashkernel(char *cmdline, + unsigned long long system_ram, + unsigned long long *crash_size, + unsigned long long *crash_base, + unsigned long long *low_size, + bool *high) +{ + int ret; + + /* crashkernel=X[@offset] */ + ret = __parse_crashkernel(cmdline, system_ram, crash_size, + crash_base, NULL); +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION + /* + * If non-NULL 'high' passed in and no normal crashkernel + * setting detected, try parsing crashkernel=,high|low. + */ + if (high && ret == -ENOENT) { + ret = __parse_crashkernel(cmdline, 0, crash_size, + crash_base, suffix_tbl[SUFFIX_HIGH]); + if (ret || !*crash_size) + return -EINVAL; + + /* + * crashkernel=Y,low can be specified or not, but invalid value + * is not allowed. + */ + ret = __parse_crashkernel(cmdline, 0, low_size, + crash_base, suffix_tbl[SUFFIX_LOW]); + if (ret == -ENOENT) { + *low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; + ret = 0; + } else if (ret) { + return ret; + } + + *high = true; + } +#endif + if (!*crash_size) + ret = -EINVAL; + + return ret; +} + +/* + * Add a dummy early_param handler to mark crashkernel= as a known command line + * parameter and suppress incorrect warnings in init/main.c. + */ +static int __init parse_crashkernel_dummy(char *arg) +{ + return 0; +} +early_param("crashkernel", parse_crashkernel_dummy); + +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +static int __init reserve_crashkernel_low(unsigned long long low_size) +{ +#ifdef CONFIG_64BIT + unsigned long long low_base; + + low_base = memblock_phys_alloc_range(low_size, CRASH_ALIGN, 0, CRASH_ADDR_LOW_MAX); + if (!low_base) { + pr_err("cannot allocate crashkernel low memory (size:0x%llx).\n", low_size); + return -ENOMEM; + } + + pr_info("crashkernel low memory reserved: 0x%08llx - 0x%08llx (%lld MB)\n", + low_base, low_base + low_size, low_size >> 20); + + crashk_low_res.start = low_base; + crashk_low_res.end = low_base + low_size - 1; + insert_resource(&iomem_resource, &crashk_low_res); +#endif + return 0; +} + +void __init reserve_crashkernel_generic(char *cmdline, + unsigned long long crash_size, + unsigned long long crash_base, + unsigned long long crash_low_size, + bool high) +{ + unsigned long long search_end = CRASH_ADDR_LOW_MAX, search_base = 0; + bool fixed_base = false; + + /* User specifies base address explicitly. */ + if (crash_base) { + fixed_base = true; + search_base = crash_base; + search_end = crash_base + crash_size; + } else if (high) { + search_base = CRASH_ADDR_LOW_MAX; + search_end = CRASH_ADDR_HIGH_MAX; + } + +retry: + crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, + search_base, search_end); + if (!crash_base) { + /* + * For crashkernel=size[KMG]@offset[KMG], print out failure + * message if can't reserve the specified region. + */ + if (fixed_base) { + pr_warn("crashkernel reservation failed - memory is in use.\n"); + return; + } + + /* + * For crashkernel=size[KMG], if the first attempt was for + * low memory, fall back to high memory, the minimum required + * low memory will be reserved later. + */ + if (!high && search_end == CRASH_ADDR_LOW_MAX) { + search_end = CRASH_ADDR_HIGH_MAX; + search_base = CRASH_ADDR_LOW_MAX; + crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; + goto retry; + } + + /* + * For crashkernel=size[KMG],high, if the first attempt was + * for high memory, fall back to low memory. + */ + if (high && search_end == CRASH_ADDR_HIGH_MAX) { + search_end = CRASH_ADDR_LOW_MAX; + search_base = 0; + goto retry; + } + pr_warn("cannot allocate crashkernel (size:0x%llx)\n", + crash_size); + return; + } + + if ((crash_base >= CRASH_ADDR_LOW_MAX) && + crash_low_size && reserve_crashkernel_low(crash_low_size)) { + memblock_phys_free(crash_base, crash_size); + return; + } + + pr_info("crashkernel reserved: 0x%016llx - 0x%016llx (%lld MB)\n", + crash_base, crash_base + crash_size, crash_size >> 20); + + /* + * The crashkernel memory will be removed from the kernel linear + * map. Inform kmemleak so that it won't try to access it. + */ + kmemleak_ignore_phys(crash_base); + if (crashk_low_res.end) + kmemleak_ignore_phys(crashk_low_res.start); + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; +} + +static __init int insert_crashkernel_resources(void) +{ + if (crashk_res.start < crashk_res.end) + insert_resource(&iomem_resource, &crashk_res); + + if (crashk_low_res.start < crashk_low_res.end) + insert_resource(&iomem_resource, &crashk_low_res); + + return 0; +} +early_initcall(insert_crashkernel_resources); +#endif -- cgit v1.2.3 From 443cbaf9e2fdbef7d7cae457434a6cb8a679441b Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Jan 2024 13:12:42 +0800 Subject: crash: split vmcoreinfo exporting code out from crash_core.c Now move the relevant codes into separate files: kernel/crash_reserve.c, include/linux/crash_reserve.h. And add config item CRASH_RESERVE to control its enabling. And also update the old ifdeffery of CONFIG_CRASH_CORE, including of and config item dependency on CRASH_CORE accordingly. And also do renaming as follows: - arch/xxx/kernel/{crash_core.c => vmcore_info.c} because they are only related to vmcoreinfo exporting on x86, arm64, riscv. And also Remove config item CRASH_CORE, and rely on CONFIG_KEXEC_CORE to decide if build in crash_core.c. [yang.lee@linux.alibaba.com: remove duplicated include in vmcore_info.c] Link: https://lkml.kernel.org/r/20240126005744.16561-1-yang.lee@linux.alibaba.com Link: https://lkml.kernel.org/r/20240124051254.67105-3-bhe@redhat.com Signed-off-by: Baoquan He Signed-off-by: Yang Li Acked-by: Hari Bathini Cc: Al Viro Cc: Eric W. Biederman Cc: Pingfan Liu Cc: Klara Modin Cc: Michael Kelley Cc: Nathan Chancellor Cc: Stephen Rothwell Cc: Yang Li Signed-off-by: Andrew Morton --- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/crash_core.c | 39 ----- arch/arm64/kernel/vmcore_info.c | 39 +++++ arch/powerpc/Kconfig | 2 +- arch/powerpc/kernel/setup-common.c | 2 +- arch/powerpc/platforms/powernv/opal-core.c | 2 +- arch/riscv/kernel/Makefile | 2 +- arch/riscv/kernel/crash_core.c | 24 --- arch/riscv/kernel/vmcore_info.c | 24 +++ arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/crash_core_32.c | 17 --- arch/x86/kernel/crash_core_64.c | 24 --- arch/x86/kernel/vmcore_info_32.c | 17 +++ arch/x86/kernel/vmcore_info_64.c | 24 +++ drivers/firmware/qemu_fw_cfg.c | 14 +- fs/proc/Kconfig | 2 +- fs/proc/kcore.c | 2 +- include/linux/buildid.h | 2 +- include/linux/crash_core.h | 73 --------- include/linux/kexec.h | 1 + include/linux/vmcore_info.h | 81 ++++++++++ kernel/Kconfig.kexec | 4 +- kernel/Makefile | 4 +- kernel/crash_core.c | 206 -------------------------- kernel/ksysfs.c | 6 +- kernel/printk/printk.c | 4 +- kernel/vmcore_info.c | 230 +++++++++++++++++++++++++++++ lib/buildid.c | 2 +- 28 files changed, 442 insertions(+), 409 deletions(-) delete mode 100644 arch/arm64/kernel/crash_core.c create mode 100644 arch/arm64/kernel/vmcore_info.c delete mode 100644 arch/riscv/kernel/crash_core.c create mode 100644 arch/riscv/kernel/vmcore_info.c delete mode 100644 arch/x86/kernel/crash_core_32.c delete mode 100644 arch/x86/kernel/crash_core_64.c create mode 100644 arch/x86/kernel/vmcore_info_32.c create mode 100644 arch/x86/kernel/vmcore_info_64.c create mode 100644 include/linux/vmcore_info.h create mode 100644 kernel/vmcore_info.c (limited to 'include/linux') diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 467cb7117273..a3882cccf049 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -66,7 +66,7 @@ obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o -obj-$(CONFIG_CRASH_CORE) += crash_core.o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o obj-$(CONFIG_ARM64_MTE) += mte.o diff --git a/arch/arm64/kernel/crash_core.c b/arch/arm64/kernel/crash_core.c deleted file mode 100644 index 2a24199a9b81..000000000000 --- a/arch/arm64/kernel/crash_core.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) Linaro. - * Copyright (C) Huawei Futurewei Technologies. - */ - -#include -#include -#include -#include -#include - -static inline u64 get_tcr_el1_t1sz(void); - -static inline u64 get_tcr_el1_t1sz(void) -{ - return (read_sysreg(tcr_el1) & TCR_T1SZ_MASK) >> TCR_T1SZ_OFFSET; -} - -void arch_crash_save_vmcoreinfo(void) -{ - VMCOREINFO_NUMBER(VA_BITS); - /* Please note VMCOREINFO_NUMBER() uses "%d", not "%x" */ - vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR); - vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END); - vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); - vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START); - vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END); - vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n", - kimage_voffset); - vmcoreinfo_append_str("NUMBER(PHYS_OFFSET)=0x%llx\n", - PHYS_OFFSET); - vmcoreinfo_append_str("NUMBER(TCR_EL1_T1SZ)=0x%llx\n", - get_tcr_el1_t1sz()); - vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - vmcoreinfo_append_str("NUMBER(KERNELPACMASK)=0x%llx\n", - system_supports_address_auth() ? - ptrauth_kernel_pac_mask() : 0); -} diff --git a/arch/arm64/kernel/vmcore_info.c b/arch/arm64/kernel/vmcore_info.c new file mode 100644 index 000000000000..b19d5d6cb8b3 --- /dev/null +++ b/arch/arm64/kernel/vmcore_info.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Linaro. + * Copyright (C) Huawei Futurewei Technologies. + */ + +#include +#include +#include +#include +#include + +static inline u64 get_tcr_el1_t1sz(void); + +static inline u64 get_tcr_el1_t1sz(void) +{ + return (read_sysreg(tcr_el1) & TCR_T1SZ_MASK) >> TCR_T1SZ_OFFSET; +} + +void arch_crash_save_vmcoreinfo(void) +{ + VMCOREINFO_NUMBER(VA_BITS); + /* Please note VMCOREINFO_NUMBER() uses "%d", not "%x" */ + vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR); + vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END); + vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); + vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START); + vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END); + vmcoreinfo_append_str("NUMBER(kimage_voffset)=0x%llx\n", + kimage_voffset); + vmcoreinfo_append_str("NUMBER(PHYS_OFFSET)=0x%llx\n", + PHYS_OFFSET); + vmcoreinfo_append_str("NUMBER(TCR_EL1_T1SZ)=0x%llx\n", + get_tcr_el1_t1sz()); + vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); + vmcoreinfo_append_str("NUMBER(KERNELPACMASK)=0x%llx\n", + system_supports_address_auth() ? + ptrauth_kernel_pac_mask() : 0); +} diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 7f704ae5c5ef..495d197c9b27 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -690,7 +690,7 @@ config ARCH_SELECTS_CRASH_DUMP config FA_DUMP bool "Firmware-assisted dump" depends on PPC64 && (PPC_RTAS || PPC_POWERNV) - select CRASH_CORE + select VMCORE_INFO select CRASH_RESERVE select CRASH_DUMP help diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 9b142b9d5187..733f210ffda1 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -109,7 +109,7 @@ int ppc_do_canonicalize_irqs; EXPORT_SYMBOL(ppc_do_canonicalize_irqs); #endif -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO /* This keeps a track of which one is the crashing cpu. */ int crashing_cpu = -1; #endif diff --git a/arch/powerpc/platforms/powernv/opal-core.c b/arch/powerpc/platforms/powernv/opal-core.c index bb7657115f1d..c9a9b759cc92 100644 --- a/arch/powerpc/platforms/powernv/opal-core.c +++ b/arch/powerpc/platforms/powernv/opal-core.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index f71910718053..d6fd8dcfceb5 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -92,7 +92,7 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_KEXEC_CORE) += kexec_relocate.o crash_save_regs.o machine_kexec.o obj-$(CONFIG_KEXEC_FILE) += elf_kexec.o machine_kexec_file.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o -obj-$(CONFIG_CRASH_CORE) += crash_core.o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/riscv/kernel/crash_core.c b/arch/riscv/kernel/crash_core.c deleted file mode 100644 index d18d529fd9b9..000000000000 --- a/arch/riscv/kernel/crash_core.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -#include -#include - -void arch_crash_save_vmcoreinfo(void) -{ - VMCOREINFO_NUMBER(phys_ram_base); - - vmcoreinfo_append_str("NUMBER(PAGE_OFFSET)=0x%lx\n", PAGE_OFFSET); - vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); -#ifdef CONFIG_MMU - VMCOREINFO_NUMBER(VA_BITS); - vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START); - vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END); -#ifdef CONFIG_64BIT - vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR); - vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END); -#endif -#endif - vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR); - vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n", - kernel_map.va_kernel_pa_offset); -} diff --git a/arch/riscv/kernel/vmcore_info.c b/arch/riscv/kernel/vmcore_info.c new file mode 100644 index 000000000000..6d7a22522d63 --- /dev/null +++ b/arch/riscv/kernel/vmcore_info.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +void arch_crash_save_vmcoreinfo(void) +{ + VMCOREINFO_NUMBER(phys_ram_base); + + vmcoreinfo_append_str("NUMBER(PAGE_OFFSET)=0x%lx\n", PAGE_OFFSET); + vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); +#ifdef CONFIG_MMU + VMCOREINFO_NUMBER(VA_BITS); + vmcoreinfo_append_str("NUMBER(VMEMMAP_START)=0x%lx\n", VMEMMAP_START); + vmcoreinfo_append_str("NUMBER(VMEMMAP_END)=0x%lx\n", VMEMMAP_END); +#ifdef CONFIG_64BIT + vmcoreinfo_append_str("NUMBER(MODULES_VADDR)=0x%lx\n", MODULES_VADDR); + vmcoreinfo_append_str("NUMBER(MODULES_END)=0x%lx\n", MODULES_END); +#endif +#endif + vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR); + vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n", + kernel_map.va_kernel_pa_offset); +} diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 0000325ab98f..913d4022131e 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -98,7 +98,7 @@ obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o obj-$(CONFIG_X86_TSC) += trace_clock.o obj-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_RETHOOK) += rethook.o -obj-$(CONFIG_CRASH_CORE) += crash_core_$(BITS).o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info_$(BITS).o obj-$(CONFIG_KEXEC_CORE) += machine_kexec_$(BITS).o obj-$(CONFIG_KEXEC_CORE) += relocate_kernel_$(BITS).o crash.o obj-$(CONFIG_KEXEC_FILE) += kexec-bzimage64.o diff --git a/arch/x86/kernel/crash_core_32.c b/arch/x86/kernel/crash_core_32.c deleted file mode 100644 index 8a89c109e20a..000000000000 --- a/arch/x86/kernel/crash_core_32.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -#include -#include - -#include - -void arch_crash_save_vmcoreinfo(void) -{ -#ifdef CONFIG_NUMA - VMCOREINFO_SYMBOL(node_data); - VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); -#endif -#ifdef CONFIG_X86_PAE - VMCOREINFO_CONFIG(X86_PAE); -#endif -} diff --git a/arch/x86/kernel/crash_core_64.c b/arch/x86/kernel/crash_core_64.c deleted file mode 100644 index 7d255f882afe..000000000000 --- a/arch/x86/kernel/crash_core_64.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -#include -#include - -#include - -void arch_crash_save_vmcoreinfo(void) -{ - u64 sme_mask = sme_me_mask; - - VMCOREINFO_NUMBER(phys_base); - VMCOREINFO_SYMBOL(init_top_pgt); - vmcoreinfo_append_str("NUMBER(pgtable_l5_enabled)=%d\n", - pgtable_l5_enabled()); - -#ifdef CONFIG_NUMA - VMCOREINFO_SYMBOL(node_data); - VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); -#endif - vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE); - VMCOREINFO_NUMBER(sme_mask); -} diff --git a/arch/x86/kernel/vmcore_info_32.c b/arch/x86/kernel/vmcore_info_32.c new file mode 100644 index 000000000000..5995a749288a --- /dev/null +++ b/arch/x86/kernel/vmcore_info_32.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +#include + +void arch_crash_save_vmcoreinfo(void) +{ +#ifdef CONFIG_NUMA + VMCOREINFO_SYMBOL(node_data); + VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); +#endif +#ifdef CONFIG_X86_PAE + VMCOREINFO_CONFIG(X86_PAE); +#endif +} diff --git a/arch/x86/kernel/vmcore_info_64.c b/arch/x86/kernel/vmcore_info_64.c new file mode 100644 index 000000000000..0dec7d868754 --- /dev/null +++ b/arch/x86/kernel/vmcore_info_64.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +#include + +void arch_crash_save_vmcoreinfo(void) +{ + u64 sme_mask = sme_me_mask; + + VMCOREINFO_NUMBER(phys_base); + VMCOREINFO_SYMBOL(init_top_pgt); + vmcoreinfo_append_str("NUMBER(pgtable_l5_enabled)=%d\n", + pgtable_l5_enabled()); + +#ifdef CONFIG_NUMA + VMCOREINFO_SYMBOL(node_data); + VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); +#endif + vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); + VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE); + VMCOREINFO_NUMBER(sme_mask); +} diff --git a/drivers/firmware/qemu_fw_cfg.c b/drivers/firmware/qemu_fw_cfg.c index 03da9a4354f8..5f43dfa22f79 100644 --- a/drivers/firmware/qemu_fw_cfg.c +++ b/drivers/firmware/qemu_fw_cfg.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include MODULE_AUTHOR("Gabriel L. Somlo "); MODULE_DESCRIPTION("QEMU fw_cfg sysfs support"); @@ -67,7 +67,7 @@ static void fw_cfg_sel_endianness(u16 key) iowrite16(key, fw_cfg_reg_ctrl); } -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO static inline bool fw_cfg_dma_enabled(void) { return (fw_cfg_rev & FW_CFG_VERSION_DMA) && fw_cfg_reg_dma; @@ -156,7 +156,7 @@ static ssize_t fw_cfg_read_blob(u16 key, return count; } -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO /* write chunk of given fw_cfg blob (caller responsible for sanity-check) */ static ssize_t fw_cfg_write_blob(u16 key, void *buf, loff_t pos, size_t count) @@ -195,7 +195,7 @@ end: return ret; } -#endif /* CONFIG_CRASH_CORE */ +#endif /* CONFIG_VMCORE_INFO */ /* clean up fw_cfg device i/o */ static void fw_cfg_io_cleanup(void) @@ -319,7 +319,7 @@ struct fw_cfg_sysfs_entry { struct list_head list; }; -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO static ssize_t fw_cfg_write_vmcoreinfo(const struct fw_cfg_file *f) { static struct fw_cfg_vmcoreinfo *data; @@ -343,7 +343,7 @@ static ssize_t fw_cfg_write_vmcoreinfo(const struct fw_cfg_file *f) kfree(data); return ret; } -#endif /* CONFIG_CRASH_CORE */ +#endif /* CONFIG_VMCORE_INFO */ /* get fw_cfg_sysfs_entry from kobject member */ static inline struct fw_cfg_sysfs_entry *to_entry(struct kobject *kobj) @@ -583,7 +583,7 @@ static int fw_cfg_register_file(const struct fw_cfg_file *f) int err; struct fw_cfg_sysfs_entry *entry; -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO if (fw_cfg_dma_enabled() && strcmp(f->name, FW_CFG_VMCOREINFO_FILENAME) == 0 && !is_kdump_kernel()) { diff --git a/fs/proc/Kconfig b/fs/proc/Kconfig index 32b1116ae137..d80a1431ef7b 100644 --- a/fs/proc/Kconfig +++ b/fs/proc/Kconfig @@ -32,7 +32,7 @@ config PROC_FS config PROC_KCORE bool "/proc/kcore support" if !ARM depends on PROC_FS && MMU - select CRASH_CORE + select VMCORE_INFO help Provides a virtual ELF core file of the live kernel. This can be read with gdb and other ELF tools. No modifications can be diff --git a/fs/proc/kcore.c b/fs/proc/kcore.c index 6422e569b080..8e08a9a1b7ed 100644 --- a/fs/proc/kcore.c +++ b/fs/proc/kcore.c @@ -10,7 +10,7 @@ * Safe accesses to vmalloc/direct-mapped discontiguous areas, Kanoj Sarcar */ -#include +#include #include #include #include diff --git a/include/linux/buildid.h b/include/linux/buildid.h index 8a582d242f06..20aa3c2d89f7 100644 --- a/include/linux/buildid.h +++ b/include/linux/buildid.h @@ -11,7 +11,7 @@ int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size); int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size); -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_CRASH_CORE) +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_VMCORE_INFO) extern unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX]; void init_vmlinux_build_id(void); #else diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index 1fde49246fa6..7f19f62018ef 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -6,79 +6,6 @@ #include #include -#define CRASH_CORE_NOTE_NAME "CORE" -#define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4) -#define CRASH_CORE_NOTE_NAME_BYTES ALIGN(sizeof(CRASH_CORE_NOTE_NAME), 4) -#define CRASH_CORE_NOTE_DESC_BYTES ALIGN(sizeof(struct elf_prstatus), 4) - -/* - * The per-cpu notes area is a list of notes terminated by a "NULL" - * note header. For kdump, the code in vmcore.c runs in the context - * of the second kernel to combine them into one note. - */ -#define CRASH_CORE_NOTE_BYTES ((CRASH_CORE_NOTE_HEAD_BYTES * 2) + \ - CRASH_CORE_NOTE_NAME_BYTES + \ - CRASH_CORE_NOTE_DESC_BYTES) - -#define VMCOREINFO_BYTES PAGE_SIZE -#define VMCOREINFO_NOTE_NAME "VMCOREINFO" -#define VMCOREINFO_NOTE_NAME_BYTES ALIGN(sizeof(VMCOREINFO_NOTE_NAME), 4) -#define VMCOREINFO_NOTE_SIZE ((CRASH_CORE_NOTE_HEAD_BYTES * 2) + \ - VMCOREINFO_NOTE_NAME_BYTES + \ - VMCOREINFO_BYTES) - -typedef u32 note_buf_t[CRASH_CORE_NOTE_BYTES/4]; -/* Per cpu memory for storing cpu states in case of system crash. */ -extern note_buf_t __percpu *crash_notes; - -void crash_update_vmcoreinfo_safecopy(void *ptr); -void crash_save_vmcoreinfo(void); -void arch_crash_save_vmcoreinfo(void); -__printf(1, 2) -void vmcoreinfo_append_str(const char *fmt, ...); -phys_addr_t paddr_vmcoreinfo_note(void); - -#define VMCOREINFO_OSRELEASE(value) \ - vmcoreinfo_append_str("OSRELEASE=%s\n", value) -#define VMCOREINFO_BUILD_ID() \ - ({ \ - static_assert(sizeof(vmlinux_build_id) == 20); \ - vmcoreinfo_append_str("BUILD-ID=%20phN\n", vmlinux_build_id); \ - }) - -#define VMCOREINFO_PAGESIZE(value) \ - vmcoreinfo_append_str("PAGESIZE=%ld\n", value) -#define VMCOREINFO_SYMBOL(name) \ - vmcoreinfo_append_str("SYMBOL(%s)=%lx\n", #name, (unsigned long)&name) -#define VMCOREINFO_SYMBOL_ARRAY(name) \ - vmcoreinfo_append_str("SYMBOL(%s)=%lx\n", #name, (unsigned long)name) -#define VMCOREINFO_SIZE(name) \ - vmcoreinfo_append_str("SIZE(%s)=%lu\n", #name, \ - (unsigned long)sizeof(name)) -#define VMCOREINFO_STRUCT_SIZE(name) \ - vmcoreinfo_append_str("SIZE(%s)=%lu\n", #name, \ - (unsigned long)sizeof(struct name)) -#define VMCOREINFO_OFFSET(name, field) \ - vmcoreinfo_append_str("OFFSET(%s.%s)=%lu\n", #name, #field, \ - (unsigned long)offsetof(struct name, field)) -#define VMCOREINFO_TYPE_OFFSET(name, field) \ - vmcoreinfo_append_str("OFFSET(%s.%s)=%lu\n", #name, #field, \ - (unsigned long)offsetof(name, field)) -#define VMCOREINFO_LENGTH(name, value) \ - vmcoreinfo_append_str("LENGTH(%s)=%lu\n", #name, (unsigned long)value) -#define VMCOREINFO_NUMBER(name) \ - vmcoreinfo_append_str("NUMBER(%s)=%ld\n", #name, (long)name) -#define VMCOREINFO_CONFIG(name) \ - vmcoreinfo_append_str("CONFIG_%s=y\n", #name) - -extern unsigned char *vmcoreinfo_data; -extern size_t vmcoreinfo_size; -extern u32 *vmcoreinfo_note; - -Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, - void *data, size_t data_len); -void final_note(Elf_Word *buf); - /* Alignment required for elf header segment */ #define ELF_CORE_HEADER_ALIGN 4096 diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 6d79bfb52e5b..9c7bb8b56ed6 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -16,6 +16,7 @@ #if !defined(__ASSEMBLY__) #include +#include #include #include #include diff --git a/include/linux/vmcore_info.h b/include/linux/vmcore_info.h new file mode 100644 index 000000000000..e1dec1a6a749 --- /dev/null +++ b/include/linux/vmcore_info.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_VMCORE_INFO_H +#define LINUX_VMCORE_INFO_H + +#include +#include +#include + +#define CRASH_CORE_NOTE_NAME "CORE" +#define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4) +#define CRASH_CORE_NOTE_NAME_BYTES ALIGN(sizeof(CRASH_CORE_NOTE_NAME), 4) +#define CRASH_CORE_NOTE_DESC_BYTES ALIGN(sizeof(struct elf_prstatus), 4) + +/* + * The per-cpu notes area is a list of notes terminated by a "NULL" + * note header. For kdump, the code in vmcore.c runs in the context + * of the second kernel to combine them into one note. + */ +#define CRASH_CORE_NOTE_BYTES ((CRASH_CORE_NOTE_HEAD_BYTES * 2) + \ + CRASH_CORE_NOTE_NAME_BYTES + \ + CRASH_CORE_NOTE_DESC_BYTES) + +#define VMCOREINFO_BYTES PAGE_SIZE +#define VMCOREINFO_NOTE_NAME "VMCOREINFO" +#define VMCOREINFO_NOTE_NAME_BYTES ALIGN(sizeof(VMCOREINFO_NOTE_NAME), 4) +#define VMCOREINFO_NOTE_SIZE ((CRASH_CORE_NOTE_HEAD_BYTES * 2) + \ + VMCOREINFO_NOTE_NAME_BYTES + \ + VMCOREINFO_BYTES) + +typedef u32 note_buf_t[CRASH_CORE_NOTE_BYTES/4]; +/* Per cpu memory for storing cpu states in case of system crash. */ +extern note_buf_t __percpu *crash_notes; + +void crash_update_vmcoreinfo_safecopy(void *ptr); +void crash_save_vmcoreinfo(void); +void arch_crash_save_vmcoreinfo(void); +__printf(1, 2) +void vmcoreinfo_append_str(const char *fmt, ...); +phys_addr_t paddr_vmcoreinfo_note(void); + +#define VMCOREINFO_OSRELEASE(value) \ + vmcoreinfo_append_str("OSRELEASE=%s\n", value) +#define VMCOREINFO_BUILD_ID() \ + ({ \ + static_assert(sizeof(vmlinux_build_id) == 20); \ + vmcoreinfo_append_str("BUILD-ID=%20phN\n", vmlinux_build_id); \ + }) + +#define VMCOREINFO_PAGESIZE(value) \ + vmcoreinfo_append_str("PAGESIZE=%ld\n", value) +#define VMCOREINFO_SYMBOL(name) \ + vmcoreinfo_append_str("SYMBOL(%s)=%lx\n", #name, (unsigned long)&name) +#define VMCOREINFO_SYMBOL_ARRAY(name) \ + vmcoreinfo_append_str("SYMBOL(%s)=%lx\n", #name, (unsigned long)name) +#define VMCOREINFO_SIZE(name) \ + vmcoreinfo_append_str("SIZE(%s)=%lu\n", #name, \ + (unsigned long)sizeof(name)) +#define VMCOREINFO_STRUCT_SIZE(name) \ + vmcoreinfo_append_str("SIZE(%s)=%lu\n", #name, \ + (unsigned long)sizeof(struct name)) +#define VMCOREINFO_OFFSET(name, field) \ + vmcoreinfo_append_str("OFFSET(%s.%s)=%lu\n", #name, #field, \ + (unsigned long)offsetof(struct name, field)) +#define VMCOREINFO_TYPE_OFFSET(name, field) \ + vmcoreinfo_append_str("OFFSET(%s.%s)=%lu\n", #name, #field, \ + (unsigned long)offsetof(name, field)) +#define VMCOREINFO_LENGTH(name, value) \ + vmcoreinfo_append_str("LENGTH(%s)=%lu\n", #name, (unsigned long)value) +#define VMCOREINFO_NUMBER(name) \ + vmcoreinfo_append_str("NUMBER(%s)=%ld\n", #name, (long)name) +#define VMCOREINFO_CONFIG(name) \ + vmcoreinfo_append_str("CONFIG_%s=y\n", #name) + +extern unsigned char *vmcoreinfo_data; +extern size_t vmcoreinfo_size; +extern u32 *vmcoreinfo_note; + +Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, + void *data, size_t data_len); +void final_note(Elf_Word *buf); +#endif /* LINUX_VMCORE_INFO_H */ diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec index 8b7be71edd85..8faf27043432 100644 --- a/kernel/Kconfig.kexec +++ b/kernel/Kconfig.kexec @@ -5,11 +5,11 @@ menu "Kexec and crash features" config CRASH_RESERVE bool -config CRASH_CORE +config VMCORE_INFO bool config KEXEC_CORE - select CRASH_CORE + select VMCORE_INFO select CRASH_RESERVE bool diff --git a/kernel/Makefile b/kernel/Makefile index 05fa88b3ab74..649272a1d6b9 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -68,9 +68,9 @@ obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o obj-$(CONFIG_KALLSYMS) += kallsyms.o obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o -obj-$(CONFIG_CRASH_CORE) += crash_core.o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_CRASH_RESERVE) += crash_reserve.o -obj-$(CONFIG_KEXEC_CORE) += kexec_core.o +obj-$(CONFIG_KEXEC_CORE) += kexec_core.o crash_core.o obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_KEXEC_FILE) += kexec_file.o obj-$(CONFIG_KEXEC_ELF) += kexec_elf.o diff --git a/kernel/crash_core.c b/kernel/crash_core.c index ae0d1ce89b46..2f4df1fe6f7a 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -26,14 +26,6 @@ /* Per cpu memory for storing cpu states in case of system crash. */ note_buf_t __percpu *crash_notes; -/* vmcoreinfo stuff */ -unsigned char *vmcoreinfo_data; -size_t vmcoreinfo_size; -u32 *vmcoreinfo_note; - -/* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */ -static unsigned char *vmcoreinfo_data_safecopy; - int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz) { @@ -195,204 +187,6 @@ int crash_exclude_mem_range(struct crash_mem *mem, return 0; } -Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, - void *data, size_t data_len) -{ - struct elf_note *note = (struct elf_note *)buf; - - note->n_namesz = strlen(name) + 1; - note->n_descsz = data_len; - note->n_type = type; - buf += DIV_ROUND_UP(sizeof(*note), sizeof(Elf_Word)); - memcpy(buf, name, note->n_namesz); - buf += DIV_ROUND_UP(note->n_namesz, sizeof(Elf_Word)); - memcpy(buf, data, data_len); - buf += DIV_ROUND_UP(data_len, sizeof(Elf_Word)); - - return buf; -} - -void final_note(Elf_Word *buf) -{ - memset(buf, 0, sizeof(struct elf_note)); -} - -static void update_vmcoreinfo_note(void) -{ - u32 *buf = vmcoreinfo_note; - - if (!vmcoreinfo_size) - return; - buf = append_elf_note(buf, VMCOREINFO_NOTE_NAME, 0, vmcoreinfo_data, - vmcoreinfo_size); - final_note(buf); -} - -void crash_update_vmcoreinfo_safecopy(void *ptr) -{ - if (ptr) - memcpy(ptr, vmcoreinfo_data, vmcoreinfo_size); - - vmcoreinfo_data_safecopy = ptr; -} - -void crash_save_vmcoreinfo(void) -{ - if (!vmcoreinfo_note) - return; - - /* Use the safe copy to generate vmcoreinfo note if have */ - if (vmcoreinfo_data_safecopy) - vmcoreinfo_data = vmcoreinfo_data_safecopy; - - vmcoreinfo_append_str("CRASHTIME=%lld\n", ktime_get_real_seconds()); - update_vmcoreinfo_note(); -} - -void vmcoreinfo_append_str(const char *fmt, ...) -{ - va_list args; - char buf[0x50]; - size_t r; - - va_start(args, fmt); - r = vscnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - - r = min(r, (size_t)VMCOREINFO_BYTES - vmcoreinfo_size); - - memcpy(&vmcoreinfo_data[vmcoreinfo_size], buf, r); - - vmcoreinfo_size += r; - - WARN_ONCE(vmcoreinfo_size == VMCOREINFO_BYTES, - "vmcoreinfo data exceeds allocated size, truncating"); -} - -/* - * provide an empty default implementation here -- architecture - * code may override this - */ -void __weak arch_crash_save_vmcoreinfo(void) -{} - -phys_addr_t __weak paddr_vmcoreinfo_note(void) -{ - return __pa(vmcoreinfo_note); -} -EXPORT_SYMBOL(paddr_vmcoreinfo_note); - -static int __init crash_save_vmcoreinfo_init(void) -{ - vmcoreinfo_data = (unsigned char *)get_zeroed_page(GFP_KERNEL); - if (!vmcoreinfo_data) { - pr_warn("Memory allocation for vmcoreinfo_data failed\n"); - return -ENOMEM; - } - - vmcoreinfo_note = alloc_pages_exact(VMCOREINFO_NOTE_SIZE, - GFP_KERNEL | __GFP_ZERO); - if (!vmcoreinfo_note) { - free_page((unsigned long)vmcoreinfo_data); - vmcoreinfo_data = NULL; - pr_warn("Memory allocation for vmcoreinfo_note failed\n"); - return -ENOMEM; - } - - VMCOREINFO_OSRELEASE(init_uts_ns.name.release); - VMCOREINFO_BUILD_ID(); - VMCOREINFO_PAGESIZE(PAGE_SIZE); - - VMCOREINFO_SYMBOL(init_uts_ns); - VMCOREINFO_OFFSET(uts_namespace, name); - VMCOREINFO_SYMBOL(node_online_map); -#ifdef CONFIG_MMU - VMCOREINFO_SYMBOL_ARRAY(swapper_pg_dir); -#endif - VMCOREINFO_SYMBOL(_stext); - vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", (unsigned long) VMALLOC_START); - -#ifndef CONFIG_NUMA - VMCOREINFO_SYMBOL(mem_map); - VMCOREINFO_SYMBOL(contig_page_data); -#endif -#ifdef CONFIG_SPARSEMEM - VMCOREINFO_SYMBOL_ARRAY(mem_section); - VMCOREINFO_LENGTH(mem_section, NR_SECTION_ROOTS); - VMCOREINFO_STRUCT_SIZE(mem_section); - VMCOREINFO_OFFSET(mem_section, section_mem_map); - VMCOREINFO_NUMBER(SECTION_SIZE_BITS); - VMCOREINFO_NUMBER(MAX_PHYSMEM_BITS); -#endif - VMCOREINFO_STRUCT_SIZE(page); - VMCOREINFO_STRUCT_SIZE(pglist_data); - VMCOREINFO_STRUCT_SIZE(zone); - VMCOREINFO_STRUCT_SIZE(free_area); - VMCOREINFO_STRUCT_SIZE(list_head); - VMCOREINFO_SIZE(nodemask_t); - VMCOREINFO_OFFSET(page, flags); - VMCOREINFO_OFFSET(page, _refcount); - VMCOREINFO_OFFSET(page, mapping); - VMCOREINFO_OFFSET(page, lru); - VMCOREINFO_OFFSET(page, _mapcount); - VMCOREINFO_OFFSET(page, private); - VMCOREINFO_OFFSET(page, compound_head); - VMCOREINFO_OFFSET(pglist_data, node_zones); - VMCOREINFO_OFFSET(pglist_data, nr_zones); -#ifdef CONFIG_FLATMEM - VMCOREINFO_OFFSET(pglist_data, node_mem_map); -#endif - VMCOREINFO_OFFSET(pglist_data, node_start_pfn); - VMCOREINFO_OFFSET(pglist_data, node_spanned_pages); - VMCOREINFO_OFFSET(pglist_data, node_id); - VMCOREINFO_OFFSET(zone, free_area); - VMCOREINFO_OFFSET(zone, vm_stat); - VMCOREINFO_OFFSET(zone, spanned_pages); - VMCOREINFO_OFFSET(free_area, free_list); - VMCOREINFO_OFFSET(list_head, next); - VMCOREINFO_OFFSET(list_head, prev); - VMCOREINFO_LENGTH(zone.free_area, NR_PAGE_ORDERS); - log_buf_vmcoreinfo_setup(); - VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); - VMCOREINFO_NUMBER(NR_FREE_PAGES); - VMCOREINFO_NUMBER(PG_lru); - VMCOREINFO_NUMBER(PG_private); - VMCOREINFO_NUMBER(PG_swapcache); - VMCOREINFO_NUMBER(PG_swapbacked); - VMCOREINFO_NUMBER(PG_slab); -#ifdef CONFIG_MEMORY_FAILURE - VMCOREINFO_NUMBER(PG_hwpoison); -#endif - VMCOREINFO_NUMBER(PG_head_mask); -#define PAGE_BUDDY_MAPCOUNT_VALUE (~PG_buddy) - VMCOREINFO_NUMBER(PAGE_BUDDY_MAPCOUNT_VALUE); -#ifdef CONFIG_HUGETLB_PAGE - VMCOREINFO_NUMBER(PG_hugetlb); -#define PAGE_OFFLINE_MAPCOUNT_VALUE (~PG_offline) - VMCOREINFO_NUMBER(PAGE_OFFLINE_MAPCOUNT_VALUE); -#endif - -#ifdef CONFIG_KALLSYMS - VMCOREINFO_SYMBOL(kallsyms_names); - VMCOREINFO_SYMBOL(kallsyms_num_syms); - VMCOREINFO_SYMBOL(kallsyms_token_table); - VMCOREINFO_SYMBOL(kallsyms_token_index); -#ifdef CONFIG_KALLSYMS_BASE_RELATIVE - VMCOREINFO_SYMBOL(kallsyms_offsets); - VMCOREINFO_SYMBOL(kallsyms_relative_base); -#else - VMCOREINFO_SYMBOL(kallsyms_addresses); -#endif /* CONFIG_KALLSYMS_BASE_RELATIVE */ -#endif /* CONFIG_KALLSYMS */ - - arch_crash_save_vmcoreinfo(); - update_vmcoreinfo_note(); - - return 0; -} - -subsys_initcall(crash_save_vmcoreinfo_init); - static int __init crash_notes_memory_init(void) { /* Allocate memory for saving cpu registers. */ diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 1d4bc493b2f4..11526fc42bc2 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -154,7 +154,7 @@ KERNEL_ATTR_RW(kexec_crash_size); #endif /* CONFIG_KEXEC_CORE */ -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO static ssize_t vmcoreinfo_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -177,7 +177,7 @@ KERNEL_ATTR_RO(crash_elfcorehdr_size); #endif -#endif /* CONFIG_CRASH_CORE */ +#endif /* CONFIG_VMCORE_INFO */ /* whether file capabilities are enabled */ static ssize_t fscaps_show(struct kobject *kobj, @@ -265,7 +265,7 @@ static struct attribute * kernel_attrs[] = { &kexec_crash_loaded_attr.attr, &kexec_crash_size_attr.attr, #endif -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO &vmcoreinfo_attr.attr, #ifdef CONFIG_CRASH_HOTPLUG &crash_elfcorehdr_size_attr.attr, diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index f2444b581e16..7d74b000b43a 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include #include @@ -951,7 +951,7 @@ const struct file_operations kmsg_fops = { .release = devkmsg_release, }; -#ifdef CONFIG_CRASH_CORE +#ifdef CONFIG_VMCORE_INFO /* * This appends the listed symbols to /proc/vmcore * diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c new file mode 100644 index 000000000000..8f77e238a54f --- /dev/null +++ b/kernel/vmcore_info.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * crash.c - kernel crash support code. + * Copyright (C) 2002-2004 Eric Biederman + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "kallsyms_internal.h" +#include "kexec_internal.h" + +/* vmcoreinfo stuff */ +unsigned char *vmcoreinfo_data; +size_t vmcoreinfo_size; +u32 *vmcoreinfo_note; + +/* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */ +static unsigned char *vmcoreinfo_data_safecopy; + +Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, + void *data, size_t data_len) +{ + struct elf_note *note = (struct elf_note *)buf; + + note->n_namesz = strlen(name) + 1; + note->n_descsz = data_len; + note->n_type = type; + buf += DIV_ROUND_UP(sizeof(*note), sizeof(Elf_Word)); + memcpy(buf, name, note->n_namesz); + buf += DIV_ROUND_UP(note->n_namesz, sizeof(Elf_Word)); + memcpy(buf, data, data_len); + buf += DIV_ROUND_UP(data_len, sizeof(Elf_Word)); + + return buf; +} + +void final_note(Elf_Word *buf) +{ + memset(buf, 0, sizeof(struct elf_note)); +} + +static void update_vmcoreinfo_note(void) +{ + u32 *buf = vmcoreinfo_note; + + if (!vmcoreinfo_size) + return; + buf = append_elf_note(buf, VMCOREINFO_NOTE_NAME, 0, vmcoreinfo_data, + vmcoreinfo_size); + final_note(buf); +} + +void crash_update_vmcoreinfo_safecopy(void *ptr) +{ + if (ptr) + memcpy(ptr, vmcoreinfo_data, vmcoreinfo_size); + + vmcoreinfo_data_safecopy = ptr; +} + +void crash_save_vmcoreinfo(void) +{ + if (!vmcoreinfo_note) + return; + + /* Use the safe copy to generate vmcoreinfo note if have */ + if (vmcoreinfo_data_safecopy) + vmcoreinfo_data = vmcoreinfo_data_safecopy; + + vmcoreinfo_append_str("CRASHTIME=%lld\n", ktime_get_real_seconds()); + update_vmcoreinfo_note(); +} + +void vmcoreinfo_append_str(const char *fmt, ...) +{ + va_list args; + char buf[0x50]; + size_t r; + + va_start(args, fmt); + r = vscnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + + r = min(r, (size_t)VMCOREINFO_BYTES - vmcoreinfo_size); + + memcpy(&vmcoreinfo_data[vmcoreinfo_size], buf, r); + + vmcoreinfo_size += r; + + WARN_ONCE(vmcoreinfo_size == VMCOREINFO_BYTES, + "vmcoreinfo data exceeds allocated size, truncating"); +} + +/* + * provide an empty default implementation here -- architecture + * code may override this + */ +void __weak arch_crash_save_vmcoreinfo(void) +{} + +phys_addr_t __weak paddr_vmcoreinfo_note(void) +{ + return __pa(vmcoreinfo_note); +} +EXPORT_SYMBOL(paddr_vmcoreinfo_note); + +static int __init crash_save_vmcoreinfo_init(void) +{ + vmcoreinfo_data = (unsigned char *)get_zeroed_page(GFP_KERNEL); + if (!vmcoreinfo_data) { + pr_warn("Memory allocation for vmcoreinfo_data failed\n"); + return -ENOMEM; + } + + vmcoreinfo_note = alloc_pages_exact(VMCOREINFO_NOTE_SIZE, + GFP_KERNEL | __GFP_ZERO); + if (!vmcoreinfo_note) { + free_page((unsigned long)vmcoreinfo_data); + vmcoreinfo_data = NULL; + pr_warn("Memory allocation for vmcoreinfo_note failed\n"); + return -ENOMEM; + } + + VMCOREINFO_OSRELEASE(init_uts_ns.name.release); + VMCOREINFO_BUILD_ID(); + VMCOREINFO_PAGESIZE(PAGE_SIZE); + + VMCOREINFO_SYMBOL(init_uts_ns); + VMCOREINFO_OFFSET(uts_namespace, name); + VMCOREINFO_SYMBOL(node_online_map); +#ifdef CONFIG_MMU + VMCOREINFO_SYMBOL_ARRAY(swapper_pg_dir); +#endif + VMCOREINFO_SYMBOL(_stext); + vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", (unsigned long) VMALLOC_START); + +#ifndef CONFIG_NUMA + VMCOREINFO_SYMBOL(mem_map); + VMCOREINFO_SYMBOL(contig_page_data); +#endif +#ifdef CONFIG_SPARSEMEM + VMCOREINFO_SYMBOL_ARRAY(mem_section); + VMCOREINFO_LENGTH(mem_section, NR_SECTION_ROOTS); + VMCOREINFO_STRUCT_SIZE(mem_section); + VMCOREINFO_OFFSET(mem_section, section_mem_map); + VMCOREINFO_NUMBER(SECTION_SIZE_BITS); + VMCOREINFO_NUMBER(MAX_PHYSMEM_BITS); +#endif + VMCOREINFO_STRUCT_SIZE(page); + VMCOREINFO_STRUCT_SIZE(pglist_data); + VMCOREINFO_STRUCT_SIZE(zone); + VMCOREINFO_STRUCT_SIZE(free_area); + VMCOREINFO_STRUCT_SIZE(list_head); + VMCOREINFO_SIZE(nodemask_t); + VMCOREINFO_OFFSET(page, flags); + VMCOREINFO_OFFSET(page, _refcount); + VMCOREINFO_OFFSET(page, mapping); + VMCOREINFO_OFFSET(page, lru); + VMCOREINFO_OFFSET(page, _mapcount); + VMCOREINFO_OFFSET(page, private); + VMCOREINFO_OFFSET(page, compound_head); + VMCOREINFO_OFFSET(pglist_data, node_zones); + VMCOREINFO_OFFSET(pglist_data, nr_zones); +#ifdef CONFIG_FLATMEM + VMCOREINFO_OFFSET(pglist_data, node_mem_map); +#endif + VMCOREINFO_OFFSET(pglist_data, node_start_pfn); + VMCOREINFO_OFFSET(pglist_data, node_spanned_pages); + VMCOREINFO_OFFSET(pglist_data, node_id); + VMCOREINFO_OFFSET(zone, free_area); + VMCOREINFO_OFFSET(zone, vm_stat); + VMCOREINFO_OFFSET(zone, spanned_pages); + VMCOREINFO_OFFSET(free_area, free_list); + VMCOREINFO_OFFSET(list_head, next); + VMCOREINFO_OFFSET(list_head, prev); + VMCOREINFO_LENGTH(zone.free_area, NR_PAGE_ORDERS); + log_buf_vmcoreinfo_setup(); + VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); + VMCOREINFO_NUMBER(NR_FREE_PAGES); + VMCOREINFO_NUMBER(PG_lru); + VMCOREINFO_NUMBER(PG_private); + VMCOREINFO_NUMBER(PG_swapcache); + VMCOREINFO_NUMBER(PG_swapbacked); + VMCOREINFO_NUMBER(PG_slab); +#ifdef CONFIG_MEMORY_FAILURE + VMCOREINFO_NUMBER(PG_hwpoison); +#endif + VMCOREINFO_NUMBER(PG_head_mask); +#define PAGE_BUDDY_MAPCOUNT_VALUE (~PG_buddy) + VMCOREINFO_NUMBER(PAGE_BUDDY_MAPCOUNT_VALUE); +#ifdef CONFIG_HUGETLB_PAGE + VMCOREINFO_NUMBER(PG_hugetlb); +#define PAGE_OFFLINE_MAPCOUNT_VALUE (~PG_offline) + VMCOREINFO_NUMBER(PAGE_OFFLINE_MAPCOUNT_VALUE); +#endif + +#ifdef CONFIG_KALLSYMS + VMCOREINFO_SYMBOL(kallsyms_names); + VMCOREINFO_SYMBOL(kallsyms_num_syms); + VMCOREINFO_SYMBOL(kallsyms_token_table); + VMCOREINFO_SYMBOL(kallsyms_token_index); +#ifdef CONFIG_KALLSYMS_BASE_RELATIVE + VMCOREINFO_SYMBOL(kallsyms_offsets); + VMCOREINFO_SYMBOL(kallsyms_relative_base); +#else + VMCOREINFO_SYMBOL(kallsyms_addresses); +#endif /* CONFIG_KALLSYMS_BASE_RELATIVE */ +#endif /* CONFIG_KALLSYMS */ + + arch_crash_save_vmcoreinfo(); + update_vmcoreinfo_note(); + + return 0; +} + +subsys_initcall(crash_save_vmcoreinfo_init); diff --git a/lib/buildid.c b/lib/buildid.c index e3a7acdeef0e..3e6868c86b45 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -174,7 +174,7 @@ int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) return parse_build_id_buf(build_id, NULL, buf, buf_size); } -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_CRASH_CORE) +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_VMCORE_INFO) unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; /** -- cgit v1.2.3 From 02aff8480533817a29e820729360866441d7403d Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Jan 2024 13:12:44 +0800 Subject: crash: split crash dumping code out from kexec_core.c Currently, KEXEC_CORE select CRASH_CORE automatically because crash codes need be built in to avoid compiling error when building kexec code even though the crash dumping functionality is not enabled. E.g -------------------- CONFIG_CRASH_CORE=y CONFIG_KEXEC_CORE=y CONFIG_KEXEC=y CONFIG_KEXEC_FILE=y --------------------- After splitting out crashkernel reservation code and vmcoreinfo exporting code, there's only crash related code left in kernel/crash_core.c. Now move crash related codes from kexec_core.c to crash_core.c and only build it in when CONFIG_CRASH_DUMP=y. And also wrap up crash codes inside CONFIG_CRASH_DUMP ifdeffery scope, or replace inappropriate CONFIG_KEXEC_CORE ifdef with CONFIG_CRASH_DUMP ifdef in generic kernel files. With these changes, crash_core codes are abstracted from kexec codes and can be disabled at all if only kexec reboot feature is wanted. Link: https://lkml.kernel.org/r/20240124051254.67105-5-bhe@redhat.com Signed-off-by: Baoquan He Cc: Al Viro Cc: Eric W. Biederman Cc: Hari Bathini Cc: Pingfan Liu Cc: Klara Modin Cc: Michael Kelley Cc: Nathan Chancellor Cc: Stephen Rothwell Cc: Yang Li Signed-off-by: Andrew Morton --- drivers/base/cpu.c | 6 +- include/linux/crash_core.h | 61 +++++++++++ include/linux/kexec.h | 45 +------- init/initramfs.c | 2 +- kernel/Makefile | 3 +- kernel/crash_core.c | 256 +++++++++++++++++++++++++++++++++++++++++++++ kernel/kexec.c | 11 +- kernel/kexec_core.c | 250 +++---------------------------------------- kernel/kexec_file.c | 13 ++- kernel/ksysfs.c | 4 + 10 files changed, 359 insertions(+), 292 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 47de0f140ba6..b621a0fc75e1 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -144,7 +144,7 @@ static DEVICE_ATTR(release, S_IWUSR, NULL, cpu_release_store); #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */ #endif /* CONFIG_HOTPLUG_CPU */ -#ifdef CONFIG_KEXEC_CORE +#ifdef CONFIG_CRASH_DUMP #include static ssize_t crash_notes_show(struct device *dev, @@ -189,14 +189,14 @@ static const struct attribute_group crash_note_cpu_attr_group = { #endif static const struct attribute_group *common_cpu_attr_groups[] = { -#ifdef CONFIG_KEXEC_CORE +#ifdef CONFIG_CRASH_DUMP &crash_note_cpu_attr_group, #endif NULL }; static const struct attribute_group *hotplugable_cpu_attr_groups[] = { -#ifdef CONFIG_KEXEC_CORE +#ifdef CONFIG_CRASH_DUMP &crash_note_cpu_attr_group, #endif NULL diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index 7f19f62018ef..23270b16e1db 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -6,6 +6,48 @@ #include #include +struct kimage; + +#ifdef CONFIG_CRASH_DUMP + +int crash_shrink_memory(unsigned long new_size); +ssize_t crash_get_memory_size(void); + +#ifndef arch_kexec_protect_crashkres +/* + * Protection mechanism for crashkernel reserved memory after + * the kdump kernel is loaded. + * + * Provide an empty default implementation here -- architecture + * code may override this + */ +static inline void arch_kexec_protect_crashkres(void) { } +#endif + +#ifndef arch_kexec_unprotect_crashkres +static inline void arch_kexec_unprotect_crashkres(void) { } +#endif + + + +#ifndef arch_crash_handle_hotplug_event +static inline void arch_crash_handle_hotplug_event(struct kimage *image) { } +#endif + +int crash_check_update_elfcorehdr(void); + +#ifndef crash_hotplug_cpu_support +static inline int crash_hotplug_cpu_support(void) { return 0; } +#endif + +#ifndef crash_hotplug_memory_support +static inline int crash_hotplug_memory_support(void) { return 0; } +#endif + +#ifndef crash_get_elfcorehdr_size +static inline unsigned int crash_get_elfcorehdr_size(void) { return 0; } +#endif + /* Alignment required for elf header segment */ #define ELF_CORE_HEADER_ALIGN 4096 @@ -31,4 +73,23 @@ struct kexec_segment; #define KEXEC_CRASH_HP_REMOVE_MEMORY 4 #define KEXEC_CRASH_HP_INVALID_CPU -1U +extern void __crash_kexec(struct pt_regs *regs); +extern void crash_kexec(struct pt_regs *regs); +int kexec_should_crash(struct task_struct *p); +int kexec_crash_loaded(void); +void crash_save_cpu(struct pt_regs *regs, int cpu); +extern int kimage_crash_copy_vmcoreinfo(struct kimage *image); + +#else /* !CONFIG_CRASH_DUMP*/ +struct pt_regs; +struct task_struct; +struct kimage; +static inline void __crash_kexec(struct pt_regs *regs) { } +static inline void crash_kexec(struct pt_regs *regs) { } +static inline int kexec_should_crash(struct task_struct *p) { return 0; } +static inline int kexec_crash_loaded(void) { return 0; } +static inline void crash_save_cpu(struct pt_regs *regs, int cpu) {}; +static inline int kimage_crash_copy_vmcoreinfo(struct kimage *image) { return 0; }; +#endif /* CONFIG_CRASH_DUMP*/ + #endif /* LINUX_CRASH_CORE_H */ diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 9c7bb8b56ed6..060835bb82d5 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -15,7 +15,6 @@ #if !defined(__ASSEMBLY__) -#include #include #include #include @@ -33,6 +32,7 @@ extern note_buf_t __percpu *crash_notes; #include #include #include +#include /* Verify architecture specific macros are defined */ @@ -380,13 +380,6 @@ extern struct page *kimage_alloc_control_pages(struct kimage *image, static inline int machine_kexec_post_load(struct kimage *image) { return 0; } #endif -extern void __crash_kexec(struct pt_regs *); -extern void crash_kexec(struct pt_regs *); -int kexec_should_crash(struct task_struct *); -int kexec_crash_loaded(void); -void crash_save_cpu(struct pt_regs *regs, int cpu); -extern int kimage_crash_copy_vmcoreinfo(struct kimage *image); - extern struct kimage *kexec_image; extern struct kimage *kexec_crash_image; @@ -410,24 +403,6 @@ bool kexec_load_permitted(int kexec_image_type); /* flag to track if kexec reboot is in progress */ extern bool kexec_in_progress; -int crash_shrink_memory(unsigned long new_size); -ssize_t crash_get_memory_size(void); - -#ifndef arch_kexec_protect_crashkres -/* - * Protection mechanism for crashkernel reserved memory after - * the kdump kernel is loaded. - * - * Provide an empty default implementation here -- architecture - * code may override this - */ -static inline void arch_kexec_protect_crashkres(void) { } -#endif - -#ifndef arch_kexec_unprotect_crashkres -static inline void arch_kexec_unprotect_crashkres(void) { } -#endif - #ifndef page_to_boot_pfn static inline unsigned long page_to_boot_pfn(struct page *page) { @@ -484,24 +459,6 @@ static inline int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, g static inline void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages) { } #endif -#ifndef arch_crash_handle_hotplug_event -static inline void arch_crash_handle_hotplug_event(struct kimage *image) { } -#endif - -int crash_check_update_elfcorehdr(void); - -#ifndef crash_hotplug_cpu_support -static inline int crash_hotplug_cpu_support(void) { return 0; } -#endif - -#ifndef crash_hotplug_memory_support -static inline int crash_hotplug_memory_support(void) { return 0; } -#endif - -#ifndef crash_get_elfcorehdr_size -static inline unsigned int crash_get_elfcorehdr_size(void) { return 0; } -#endif - extern bool kexec_file_dbg_print; #define kexec_dprintk(fmt, ...) \ diff --git a/init/initramfs.c b/init/initramfs.c index 76deb48c38cb..6f095f54eec9 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -642,7 +642,7 @@ void __weak __init free_initrd_mem(unsigned long start, unsigned long end) "initrd"); } -#ifdef CONFIG_KEXEC_CORE +#ifdef CONFIG_CRASH_RESERVE static bool __init kexec_free_initrd(void) { unsigned long crashk_start = (unsigned long)__va(crashk_res.start); diff --git a/kernel/Makefile b/kernel/Makefile index 35abc65e1f1a..3c13240dfc9f 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -70,7 +70,8 @@ obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o elfcorehdr.o obj-$(CONFIG_CRASH_RESERVE) += crash_reserve.o -obj-$(CONFIG_KEXEC_CORE) += kexec_core.o crash_core.o +obj-$(CONFIG_KEXEC_CORE) += kexec_core.o +obj-$(CONFIG_CRASH_DUMP) += crash_core.o obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_KEXEC_FILE) += kexec_file.o obj-$(CONFIG_KEXEC_ELF) += kexec_elf.o diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 2f4df1fe6f7a..78b5dc7cee3a 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -11,9 +11,14 @@ #include #include #include +#include #include #include #include +#include +#include +#include +#include #include #include @@ -26,6 +31,131 @@ /* Per cpu memory for storing cpu states in case of system crash. */ note_buf_t __percpu *crash_notes; +#ifdef CONFIG_CRASH_DUMP + +int kimage_crash_copy_vmcoreinfo(struct kimage *image) +{ + struct page *vmcoreinfo_page; + void *safecopy; + + if (!IS_ENABLED(CONFIG_CRASH_DUMP)) + return 0; + if (image->type != KEXEC_TYPE_CRASH) + return 0; + + /* + * For kdump, allocate one vmcoreinfo safe copy from the + * crash memory. as we have arch_kexec_protect_crashkres() + * after kexec syscall, we naturally protect it from write + * (even read) access under kernel direct mapping. But on + * the other hand, we still need to operate it when crash + * happens to generate vmcoreinfo note, hereby we rely on + * vmap for this purpose. + */ + vmcoreinfo_page = kimage_alloc_control_pages(image, 0); + if (!vmcoreinfo_page) { + pr_warn("Could not allocate vmcoreinfo buffer\n"); + return -ENOMEM; + } + safecopy = vmap(&vmcoreinfo_page, 1, VM_MAP, PAGE_KERNEL); + if (!safecopy) { + pr_warn("Could not vmap vmcoreinfo buffer\n"); + return -ENOMEM; + } + + image->vmcoreinfo_data_copy = safecopy; + crash_update_vmcoreinfo_safecopy(safecopy); + + return 0; +} + + + +int kexec_should_crash(struct task_struct *p) +{ + /* + * If crash_kexec_post_notifiers is enabled, don't run + * crash_kexec() here yet, which must be run after panic + * notifiers in panic(). + */ + if (crash_kexec_post_notifiers) + return 0; + /* + * There are 4 panic() calls in make_task_dead() path, each of which + * corresponds to each of these 4 conditions. + */ + if (in_interrupt() || !p->pid || is_global_init(p) || panic_on_oops) + return 1; + return 0; +} + +int kexec_crash_loaded(void) +{ + return !!kexec_crash_image; +} +EXPORT_SYMBOL_GPL(kexec_crash_loaded); + +/* + * No panic_cpu check version of crash_kexec(). This function is called + * only when panic_cpu holds the current CPU number; this is the only CPU + * which processes crash_kexec routines. + */ +void __noclone __crash_kexec(struct pt_regs *regs) +{ + /* Take the kexec_lock here to prevent sys_kexec_load + * running on one cpu from replacing the crash kernel + * we are using after a panic on a different cpu. + * + * If the crash kernel was not located in a fixed area + * of memory the xchg(&kexec_crash_image) would be + * sufficient. But since I reuse the memory... + */ + if (kexec_trylock()) { + if (kexec_crash_image) { + struct pt_regs fixed_regs; + + crash_setup_regs(&fixed_regs, regs); + crash_save_vmcoreinfo(); + machine_crash_shutdown(&fixed_regs); + machine_kexec(kexec_crash_image); + } + kexec_unlock(); + } +} +STACK_FRAME_NON_STANDARD(__crash_kexec); + +__bpf_kfunc void crash_kexec(struct pt_regs *regs) +{ + int old_cpu, this_cpu; + + /* + * Only one CPU is allowed to execute the crash_kexec() code as with + * panic(). Otherwise parallel calls of panic() and crash_kexec() + * may stop each other. To exclude them, we use panic_cpu here too. + */ + old_cpu = PANIC_CPU_INVALID; + this_cpu = raw_smp_processor_id(); + + if (atomic_try_cmpxchg(&panic_cpu, &old_cpu, this_cpu)) { + /* This is the 1st CPU which comes here, so go ahead. */ + __crash_kexec(regs); + + /* + * Reset panic_cpu to allow another panic()/crash_kexec() + * call. + */ + atomic_set(&panic_cpu, PANIC_CPU_INVALID); + } +} + +static inline resource_size_t crash_resource_size(const struct resource *res) +{ + return !res->end ? 0 : resource_size(res); +} + + + + int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz) { @@ -187,6 +317,130 @@ int crash_exclude_mem_range(struct crash_mem *mem, return 0; } +ssize_t crash_get_memory_size(void) +{ + ssize_t size = 0; + + if (!kexec_trylock()) + return -EBUSY; + + size += crash_resource_size(&crashk_res); + size += crash_resource_size(&crashk_low_res); + + kexec_unlock(); + return size; +} + +static int __crash_shrink_memory(struct resource *old_res, + unsigned long new_size) +{ + struct resource *ram_res; + + ram_res = kzalloc(sizeof(*ram_res), GFP_KERNEL); + if (!ram_res) + return -ENOMEM; + + ram_res->start = old_res->start + new_size; + ram_res->end = old_res->end; + ram_res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; + ram_res->name = "System RAM"; + + if (!new_size) { + release_resource(old_res); + old_res->start = 0; + old_res->end = 0; + } else { + crashk_res.end = ram_res->start - 1; + } + + crash_free_reserved_phys_range(ram_res->start, ram_res->end); + insert_resource(&iomem_resource, ram_res); + + return 0; +} + +int crash_shrink_memory(unsigned long new_size) +{ + int ret = 0; + unsigned long old_size, low_size; + + if (!kexec_trylock()) + return -EBUSY; + + if (kexec_crash_image) { + ret = -ENOENT; + goto unlock; + } + + low_size = crash_resource_size(&crashk_low_res); + old_size = crash_resource_size(&crashk_res) + low_size; + new_size = roundup(new_size, KEXEC_CRASH_MEM_ALIGN); + if (new_size >= old_size) { + ret = (new_size == old_size) ? 0 : -EINVAL; + goto unlock; + } + + /* + * (low_size > new_size) implies that low_size is greater than zero. + * This also means that if low_size is zero, the else branch is taken. + * + * If low_size is greater than 0, (low_size > new_size) indicates that + * crashk_low_res also needs to be shrunken. Otherwise, only crashk_res + * needs to be shrunken. + */ + if (low_size > new_size) { + ret = __crash_shrink_memory(&crashk_res, 0); + if (ret) + goto unlock; + + ret = __crash_shrink_memory(&crashk_low_res, new_size); + } else { + ret = __crash_shrink_memory(&crashk_res, new_size - low_size); + } + + /* Swap crashk_res and crashk_low_res if needed */ + if (!crashk_res.end && crashk_low_res.end) { + crashk_res.start = crashk_low_res.start; + crashk_res.end = crashk_low_res.end; + release_resource(&crashk_low_res); + crashk_low_res.start = 0; + crashk_low_res.end = 0; + insert_resource(&iomem_resource, &crashk_res); + } + +unlock: + kexec_unlock(); + return ret; +} + +void crash_save_cpu(struct pt_regs *regs, int cpu) +{ + struct elf_prstatus prstatus; + u32 *buf; + + if ((cpu < 0) || (cpu >= nr_cpu_ids)) + return; + + /* Using ELF notes here is opportunistic. + * I need a well defined structure format + * for the data I pass, and I need tags + * on the data to indicate what information I have + * squirrelled away. ELF notes happen to provide + * all of that, so there is no need to invent something new. + */ + buf = (u32 *)per_cpu_ptr(crash_notes, cpu); + if (!buf) + return; + memset(&prstatus, 0, sizeof(prstatus)); + prstatus.common.pr_pid = current->pid; + elf_core_copy_regs(&prstatus.pr_reg, regs); + buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, + &prstatus, sizeof(prstatus)); + final_note(buf); +} + + + static int __init crash_notes_memory_init(void) { /* Allocate memory for saving cpu registers. */ @@ -220,6 +474,8 @@ static int __init crash_notes_memory_init(void) } subsys_initcall(crash_notes_memory_init); +#endif /*CONFIG_CRASH_DUMP*/ + #ifdef CONFIG_CRASH_HOTPLUG #undef pr_fmt #define pr_fmt(fmt) "crash hp: " fmt diff --git a/kernel/kexec.c b/kernel/kexec.c index 8f35a5a42af8..bab542fc1463 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -28,12 +28,14 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, struct kimage *image; bool kexec_on_panic = flags & KEXEC_ON_CRASH; +#ifdef CONFIG_CRASH_DUMP if (kexec_on_panic) { /* Verify we have a valid entry point */ if ((entry < phys_to_boot_phys(crashk_res.start)) || (entry > phys_to_boot_phys(crashk_res.end))) return -EADDRNOTAVAIL; } +#endif /* Allocate and initialize a controlling structure */ image = do_kimage_alloc_init(); @@ -44,11 +46,13 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, image->nr_segments = nr_segments; memcpy(image->segment, segments, nr_segments * sizeof(*segments)); +#ifdef CONFIG_CRASH_DUMP if (kexec_on_panic) { /* Enable special crash kernel control page alloc policy. */ image->control_page = crashk_res.start; image->type = KEXEC_TYPE_CRASH; } +#endif ret = sanity_check_segment_list(image); if (ret) @@ -99,13 +103,14 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, if (!kexec_trylock()) return -EBUSY; +#ifdef CONFIG_CRASH_DUMP if (flags & KEXEC_ON_CRASH) { dest_image = &kexec_crash_image; if (kexec_crash_image) arch_kexec_unprotect_crashkres(); - } else { + } else +#endif dest_image = &kexec_image; - } if (nr_segments == 0) { /* Uninstall image */ @@ -162,8 +167,10 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, image = xchg(dest_image, image); out: +#ifdef CONFIG_CRASH_DUMP if ((flags & KEXEC_ON_CRASH) && kexec_crash_image) arch_kexec_protect_crashkres(); +#endif kimage_free(image); out_unlock: diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index d08fc7b5db97..ce3429e7972c 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -54,30 +54,6 @@ bool kexec_in_progress = false; bool kexec_file_dbg_print; -int kexec_should_crash(struct task_struct *p) -{ - /* - * If crash_kexec_post_notifiers is enabled, don't run - * crash_kexec() here yet, which must be run after panic - * notifiers in panic(). - */ - if (crash_kexec_post_notifiers) - return 0; - /* - * There are 4 panic() calls in make_task_dead() path, each of which - * corresponds to each of these 4 conditions. - */ - if (in_interrupt() || !p->pid || is_global_init(p) || panic_on_oops) - return 1; - return 0; -} - -int kexec_crash_loaded(void) -{ - return !!kexec_crash_image; -} -EXPORT_SYMBOL_GPL(kexec_crash_loaded); - /* * When kexec transitions to the new kernel there is a one-to-one * mapping between physical and virtual addresses. On processors @@ -209,6 +185,7 @@ int sanity_check_segment_list(struct kimage *image) if (total_pages > nr_pages / 2) return -EINVAL; +#ifdef CONFIG_CRASH_DUMP /* * Verify we have good destination addresses. Normally * the caller is responsible for making certain we don't @@ -231,6 +208,7 @@ int sanity_check_segment_list(struct kimage *image) return -EADDRNOTAVAIL; } } +#endif return 0; } @@ -403,6 +381,7 @@ static struct page *kimage_alloc_normal_control_pages(struct kimage *image, return pages; } +#ifdef CONFIG_CRASH_DUMP static struct page *kimage_alloc_crash_control_pages(struct kimage *image, unsigned int order) { @@ -468,6 +447,7 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image, return pages; } +#endif struct page *kimage_alloc_control_pages(struct kimage *image, @@ -479,48 +459,16 @@ struct page *kimage_alloc_control_pages(struct kimage *image, case KEXEC_TYPE_DEFAULT: pages = kimage_alloc_normal_control_pages(image, order); break; +#ifdef CONFIG_CRASH_DUMP case KEXEC_TYPE_CRASH: pages = kimage_alloc_crash_control_pages(image, order); break; +#endif } return pages; } -int kimage_crash_copy_vmcoreinfo(struct kimage *image) -{ - struct page *vmcoreinfo_page; - void *safecopy; - - if (image->type != KEXEC_TYPE_CRASH) - return 0; - - /* - * For kdump, allocate one vmcoreinfo safe copy from the - * crash memory. as we have arch_kexec_protect_crashkres() - * after kexec syscall, we naturally protect it from write - * (even read) access under kernel direct mapping. But on - * the other hand, we still need to operate it when crash - * happens to generate vmcoreinfo note, hereby we rely on - * vmap for this purpose. - */ - vmcoreinfo_page = kimage_alloc_control_pages(image, 0); - if (!vmcoreinfo_page) { - pr_warn("Could not allocate vmcoreinfo buffer\n"); - return -ENOMEM; - } - safecopy = vmap(&vmcoreinfo_page, 1, VM_MAP, PAGE_KERNEL); - if (!safecopy) { - pr_warn("Could not vmap vmcoreinfo buffer\n"); - return -ENOMEM; - } - - image->vmcoreinfo_data_copy = safecopy; - crash_update_vmcoreinfo_safecopy(safecopy); - - return 0; -} - static int kimage_add_entry(struct kimage *image, kimage_entry_t entry) { if (*image->entry != 0) @@ -603,10 +551,12 @@ void kimage_free(struct kimage *image) if (!image) return; +#ifdef CONFIG_CRASH_DUMP if (image->vmcoreinfo_data_copy) { crash_update_vmcoreinfo_safecopy(NULL); vunmap(image->vmcoreinfo_data_copy); } +#endif kimage_free_extra_pages(image); for_each_kimage_entry(image, ptr, entry) { @@ -824,6 +774,7 @@ out: return result; } +#ifdef CONFIG_CRASH_DUMP static int kimage_load_crash_segment(struct kimage *image, struct kexec_segment *segment) { @@ -891,6 +842,7 @@ static int kimage_load_crash_segment(struct kimage *image, out: return result; } +#endif int kimage_load_segment(struct kimage *image, struct kexec_segment *segment) @@ -901,9 +853,11 @@ int kimage_load_segment(struct kimage *image, case KEXEC_TYPE_DEFAULT: result = kimage_load_normal_segment(image, segment); break; +#ifdef CONFIG_CRASH_DUMP case KEXEC_TYPE_CRASH: result = kimage_load_crash_segment(image, segment); break; +#endif } return result; @@ -1027,186 +981,6 @@ bool kexec_load_permitted(int kexec_image_type) return true; } -/* - * No panic_cpu check version of crash_kexec(). This function is called - * only when panic_cpu holds the current CPU number; this is the only CPU - * which processes crash_kexec routines. - */ -void __noclone __crash_kexec(struct pt_regs *regs) -{ - /* Take the kexec_lock here to prevent sys_kexec_load - * running on one cpu from replacing the crash kernel - * we are using after a panic on a different cpu. - * - * If the crash kernel was not located in a fixed area - * of memory the xchg(&kexec_crash_image) would be - * sufficient. But since I reuse the memory... - */ - if (kexec_trylock()) { - if (kexec_crash_image) { - struct pt_regs fixed_regs; - - crash_setup_regs(&fixed_regs, regs); - crash_save_vmcoreinfo(); - machine_crash_shutdown(&fixed_regs); - machine_kexec(kexec_crash_image); - } - kexec_unlock(); - } -} -STACK_FRAME_NON_STANDARD(__crash_kexec); - -__bpf_kfunc void crash_kexec(struct pt_regs *regs) -{ - int old_cpu, this_cpu; - - /* - * Only one CPU is allowed to execute the crash_kexec() code as with - * panic(). Otherwise parallel calls of panic() and crash_kexec() - * may stop each other. To exclude them, we use panic_cpu here too. - */ - old_cpu = PANIC_CPU_INVALID; - this_cpu = raw_smp_processor_id(); - - if (atomic_try_cmpxchg(&panic_cpu, &old_cpu, this_cpu)) { - /* This is the 1st CPU which comes here, so go ahead. */ - __crash_kexec(regs); - - /* - * Reset panic_cpu to allow another panic()/crash_kexec() - * call. - */ - atomic_set(&panic_cpu, PANIC_CPU_INVALID); - } -} - -static inline resource_size_t crash_resource_size(const struct resource *res) -{ - return !res->end ? 0 : resource_size(res); -} - -ssize_t crash_get_memory_size(void) -{ - ssize_t size = 0; - - if (!kexec_trylock()) - return -EBUSY; - - size += crash_resource_size(&crashk_res); - size += crash_resource_size(&crashk_low_res); - - kexec_unlock(); - return size; -} - -static int __crash_shrink_memory(struct resource *old_res, - unsigned long new_size) -{ - struct resource *ram_res; - - ram_res = kzalloc(sizeof(*ram_res), GFP_KERNEL); - if (!ram_res) - return -ENOMEM; - - ram_res->start = old_res->start + new_size; - ram_res->end = old_res->end; - ram_res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; - ram_res->name = "System RAM"; - - if (!new_size) { - release_resource(old_res); - old_res->start = 0; - old_res->end = 0; - } else { - crashk_res.end = ram_res->start - 1; - } - - crash_free_reserved_phys_range(ram_res->start, ram_res->end); - insert_resource(&iomem_resource, ram_res); - - return 0; -} - -int crash_shrink_memory(unsigned long new_size) -{ - int ret = 0; - unsigned long old_size, low_size; - - if (!kexec_trylock()) - return -EBUSY; - - if (kexec_crash_image) { - ret = -ENOENT; - goto unlock; - } - - low_size = crash_resource_size(&crashk_low_res); - old_size = crash_resource_size(&crashk_res) + low_size; - new_size = roundup(new_size, KEXEC_CRASH_MEM_ALIGN); - if (new_size >= old_size) { - ret = (new_size == old_size) ? 0 : -EINVAL; - goto unlock; - } - - /* - * (low_size > new_size) implies that low_size is greater than zero. - * This also means that if low_size is zero, the else branch is taken. - * - * If low_size is greater than 0, (low_size > new_size) indicates that - * crashk_low_res also needs to be shrunken. Otherwise, only crashk_res - * needs to be shrunken. - */ - if (low_size > new_size) { - ret = __crash_shrink_memory(&crashk_res, 0); - if (ret) - goto unlock; - - ret = __crash_shrink_memory(&crashk_low_res, new_size); - } else { - ret = __crash_shrink_memory(&crashk_res, new_size - low_size); - } - - /* Swap crashk_res and crashk_low_res if needed */ - if (!crashk_res.end && crashk_low_res.end) { - crashk_res.start = crashk_low_res.start; - crashk_res.end = crashk_low_res.end; - release_resource(&crashk_low_res); - crashk_low_res.start = 0; - crashk_low_res.end = 0; - insert_resource(&iomem_resource, &crashk_res); - } - -unlock: - kexec_unlock(); - return ret; -} - -void crash_save_cpu(struct pt_regs *regs, int cpu) -{ - struct elf_prstatus prstatus; - u32 *buf; - - if ((cpu < 0) || (cpu >= nr_cpu_ids)) - return; - - /* Using ELF notes here is opportunistic. - * I need a well defined structure format - * for the data I pass, and I need tags - * on the data to indicate what information I have - * squirrelled away. ELF notes happen to provide - * all of that, so there is no need to invent something new. - */ - buf = (u32 *)per_cpu_ptr(crash_notes, cpu); - if (!buf) - return; - memset(&prstatus, 0, sizeof(prstatus)); - prstatus.common.pr_pid = current->pid; - elf_core_copy_regs(&prstatus.pr_reg, regs); - buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, - &prstatus, sizeof(prstatus)); - final_note(buf); -} - /* * Move into place and start executing a preloaded standalone * executable. If nothing was preloaded return an error. diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index bef2f6f2571b..ce7ce2ae27cd 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -285,11 +285,13 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, kexec_file_dbg_print = !!(flags & KEXEC_FILE_DEBUG); image->file_mode = 1; +#ifdef CONFIG_CRASH_DUMP if (kexec_on_panic) { /* Enable special crash kernel control page alloc policy. */ image->control_page = crashk_res.start; image->type = KEXEC_TYPE_CRASH; } +#endif ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd, cmdline_ptr, cmdline_len, flags); @@ -349,13 +351,14 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, if (!kexec_trylock()) return -EBUSY; +#ifdef CONFIG_CRASH_DUMP if (image_type == KEXEC_TYPE_CRASH) { dest_image = &kexec_crash_image; if (kexec_crash_image) arch_kexec_unprotect_crashkres(); - } else { + } else +#endif dest_image = &kexec_image; - } if (flags & KEXEC_FILE_UNLOAD) goto exchange; @@ -419,8 +422,10 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, exchange: image = xchg(dest_image, image); out: +#ifdef CONFIG_CRASH_DUMP if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image) arch_kexec_protect_crashkres(); +#endif kexec_unlock(); kimage_free(image); @@ -595,12 +600,14 @@ static int kexec_walk_memblock(struct kexec_buf *kbuf, static int kexec_walk_resources(struct kexec_buf *kbuf, int (*func)(struct resource *, void *)) { +#ifdef CONFIG_CRASH_DUMP if (kbuf->image->type == KEXEC_TYPE_CRASH) return walk_iomem_res_desc(crashk_res.desc, IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, crashk_res.start, crashk_res.end, kbuf, func); - else if (kbuf->top_down) +#endif + if (kbuf->top_down) return walk_system_ram_res_rev(0, ULONG_MAX, kbuf, func); else return walk_system_ram_res(0, ULONG_MAX, kbuf, func); diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 11526fc42bc2..fe7a517fc4ab 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -120,6 +120,7 @@ static ssize_t kexec_loaded_show(struct kobject *kobj, } KERNEL_ATTR_RO(kexec_loaded); +#ifdef CONFIG_CRASH_DUMP static ssize_t kexec_crash_loaded_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -152,6 +153,7 @@ static ssize_t kexec_crash_size_store(struct kobject *kobj, } KERNEL_ATTR_RW(kexec_crash_size); +#endif /* CONFIG_CRASH_DUMP*/ #endif /* CONFIG_KEXEC_CORE */ #ifdef CONFIG_VMCORE_INFO @@ -262,9 +264,11 @@ static struct attribute * kernel_attrs[] = { #endif #ifdef CONFIG_KEXEC_CORE &kexec_loaded_attr.attr, +#ifdef CONFIG_CRASH_DUMP &kexec_crash_loaded_attr.attr, &kexec_crash_size_attr.attr, #endif +#endif #ifdef CONFIG_VMCORE_INFO &vmcoreinfo_attr.attr, #ifdef CONFIG_CRASH_HOTPLUG -- cgit v1.2.3 From 78f2f60377ee43b7f27b4584dae754b8aa3f80e1 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:12 -0800 Subject: mm/damon/core: set damos_quota->esz as public field and document Patch series "mm/damon: let DAMOS feeds and tame/auto-tune itself". The Aim-oriented Feedback-driven DAMOS Aggressiveness Auto-tuning patchset[1] which has merged since commit 9294a037c015 ("mm/damon/core: implement goal-oriented feedback-driven quota auto-tuning") made the mechanism and the policy separated. That is, users can set a part of DAMOS control policies without a deep understanding of the mechanism but just their demands such as SLA. However, users are still required to do some additional work of manually collecting their target metric and feeding it to DAMOS. In the case of end-users who use DAMON sysfs interface, the context switches between user-space and kernel-space could also make it inefficient. The overhead is supposed to be only trivial in common cases, though. Meanwhile, in simple use cases, the target metric could be common system metrics that the kernel can efficiently self-retrieve, such as memory pressure stall time (PSI). Extend DAMOS quota auto-tuning to support multiple types of metrics including the DAMOS self-retrievable ones, and add support for memory pressure stall time metric. Different types of metrics can be supported in future. The auto-tuning capability is currently supported for only users of DAMOS kernel API and DAMON sysfs interface. Extend the support to DAMON_RECLAIM. Patches Sequence ================ First five patches are for helping debugging and fine-tuning existing quota control features. The first one (patch 1) exposes the effective quota that is made with given user inputs to DAMOS kernel API users and kernel-doc documents. Following four patches implement (patches 1, 2 and 3) and document (patches 4 and 5) a new DAMON sysfs file that exposes the value. Following six patches cleanup and simplify the existing DAMOS quota auto-tuning code by improving layout of comments and data structures (patches 6 and 7), supporting common use cases, namely multiple goals (patches 8, 9 and 10), and simplifying the interface (patch 11). Then six patches for the main purpose of this patchset follow. The first three changes extend the core logic for various target metrics (patch 12), implement memory pressure stall time-based target metric support (patch 13), and update DAMON sysfs interface to support the new target metric (patch 14). Then, documentation updates for the features on design (patch 15), ABI (patch 16), and usage (patch 17) follow. Last three patches add auto-tuning support on DAMON_RECLAIM. The patches implement DAMON_RECLAIM parameters for user-feedback driven quota auto-tuning (patch 18), memory pressure stall time-driven quota self-tuning (patch 19), and finally update the DAMON_RECLAIM usage document for the new parameters (patch 20). [1] https://lore.kernel.org/all/20231130023652.50284-1-sj@kernel.org/ This patch (of 20): DAMOS allow users to specify the quota as they want in multiple ways including time quota, size quota, and feedback-based auto-tuning. DAMOS makes one effective quota out of the inputs and use it at the end. Knowing the current effective quota helps understanding DAMOS' internal mechanism and fine-tuning quotas. DAMON kernel API users can get the information from ->esz field of damos_quota struct, but the field is marked as private purpose, and not kernel-doc documented. Make it public and document. Link: https://lkml.kernel.org/r/20240219194431.159606-1-sj@kernel.org Link: https://lkml.kernel.org/r/20240219194431.159606-2-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 6 ++++-- mm/damon/core.c | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 5881e4ac30be..93ef45b87b9c 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -138,6 +138,7 @@ enum damos_action { * * @get_score: Feedback function for self-tuning quota. * @get_score_arg: Parameter for @get_score + * @esz: Effective size quota in bytes. * * To avoid consuming too much CPU time or IO resources for applying the * &struct damos->action to large memory, DAMON allows users to set time and/or @@ -167,6 +168,8 @@ enum damos_action { * tuning is getting the feedback screo value of 10,000. If @ms and/or @sz are * set together, those work as a hard limit quota. If neither @ms nor @sz are * set, the mechanism starts from the quota of one byte. + * + * The resulting effective size quota in bytes is set to @esz. */ struct damos_quota { unsigned long ms; @@ -179,14 +182,13 @@ struct damos_quota { unsigned long (*get_score)(void *arg); void *get_score_arg; + unsigned long esz; /* private: */ /* For throughput estimation */ unsigned long total_charged_sz; unsigned long total_charged_ns; - unsigned long esz; /* Effective size quota in bytes */ - /* For charging the quota */ unsigned long charged_sz; unsigned long charged_from; diff --git a/mm/damon/core.c b/mm/damon/core.c index 5b325749fc12..0656966a6fc4 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -299,12 +299,12 @@ void damos_destroy_filter(struct damos_filter *f) damos_free_filter(f); } -/* initialize private fields of damos_quota and return the pointer */ -static struct damos_quota *damos_quota_init_priv(struct damos_quota *quota) +/* initialize fields of @quota that normally API users wouldn't set */ +static struct damos_quota *damos_quota_init(struct damos_quota *quota) { + quota->esz = 0; quota->total_charged_sz = 0; quota->total_charged_ns = 0; - quota->esz = 0; quota->charged_sz = 0; quota->charged_from = 0; quota->charge_target_from = NULL; @@ -336,7 +336,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, scheme->stat = (struct damos_stat){}; INIT_LIST_HEAD(&scheme->list); - scheme->quota = *(damos_quota_init_priv(quota)); + scheme->quota = *(damos_quota_init(quota)); scheme->wmarks = *wmarks; scheme->wmarks.activated = true; -- cgit v1.2.3 From 4d791a0a2ab47d70131909942009f12f61db20ab Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:17 -0800 Subject: mm/damon: move comments and fields for damos-quota-prioritization to the end The comments and definition of 'struct damos_quota' lists a few fields for effective quota generation first, fields for regions prioritization under the quota, and then remaining fields for effective quota generation. Readers' should unnecesssarily switch their context in the middle. List all the fields for the effective quota first, and then fields for the prioritization for making it easier to read. Link: https://lkml.kernel.org/r/20240219194431.159606-7-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 93ef45b87b9c..bd17b14828bc 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -128,18 +128,17 @@ enum damos_action { /** * struct damos_quota - Controls the aggressiveness of the given scheme. + * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @reset_interval: Charge reset interval in milliseconds. + * @get_score: Feedback function for self-tuning quota. + * @get_score_arg: Parameter for @get_score + * @esz: Effective size quota in bytes. * * @weight_sz: Weight of the region's size for prioritization. * @weight_nr_accesses: Weight of the region's nr_accesses for prioritization. * @weight_age: Weight of the region's age for prioritization. * - * @get_score: Feedback function for self-tuning quota. - * @get_score_arg: Parameter for @get_score - * @esz: Effective size quota in bytes. - * * To avoid consuming too much CPU time or IO resources for applying the * &struct damos->action to large memory, DAMON allows users to set time and/or * size quotas. The quotas can be set by writing non-zero values to &ms and @@ -152,12 +151,6 @@ enum damos_action { * throughput of the scheme's action. DAMON then compares it against &sz and * uses smaller one as the effective quota. * - * For selecting regions within the quota, DAMON prioritizes current scheme's - * target memory regions using the &struct damon_operations->get_scheme_score. - * You could customize the prioritization logic by setting &weight_sz, - * &weight_nr_accesses, and &weight_age, because monitoring operations are - * encouraged to respect those. - * * If @get_score function pointer is set, DAMON calls it back with * @get_score_arg and get the return value of it for every @reset_interval. * Then, DAMON adjusts the effective quota using the return value as a feedback @@ -170,20 +163,25 @@ enum damos_action { * set, the mechanism starts from the quota of one byte. * * The resulting effective size quota in bytes is set to @esz. + * + * For selecting regions within the quota, DAMON prioritizes current scheme's + * target memory regions using the &struct damon_operations->get_scheme_score. + * You could customize the prioritization logic by setting &weight_sz, + * &weight_nr_accesses, and &weight_age, because monitoring operations are + * encouraged to respect those. */ struct damos_quota { + unsigned long reset_interval; unsigned long ms; unsigned long sz; - unsigned long reset_interval; + unsigned long (*get_score)(void *arg); + void *get_score_arg; + unsigned long esz; unsigned int weight_sz; unsigned int weight_nr_accesses; unsigned int weight_age; - unsigned long (*get_score)(void *arg); - void *get_score_arg; - unsigned long esz; - /* private: */ /* For throughput estimation */ unsigned long total_charged_sz; -- cgit v1.2.3 From 106e26fc1c4c1a0e3747246e15df2bc3aa9d46b2 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:18 -0800 Subject: mm/damon/core: split out quota goal related fields to a struct 'struct damos_quota' is not small now. Split out fields for quota goal to a separate struct for easier reading. Link: https://lkml.kernel.org/r/20240219194431.159606-8-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 36 ++++++++++++++++++++++-------------- mm/damon/core.c | 13 +++++++------ mm/damon/sysfs-schemes.c | 10 +++++----- 3 files changed, 34 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index bd17b14828bc..2fe345adf6b2 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -126,13 +126,28 @@ enum damos_action { NR_DAMOS_ACTIONS, }; +/** + * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. + * @get_score: Function for getting current score of the goal. + * @get_score_arg: Parameter for @get_score + * + * Data structure for getting the current score of the quota tuning goal. + * Calling @get_score with @get_score_arg as the parameter should return the + * current score. Then the score is entered to DAMON's internal feedback loop + * mechanism to get the auto-tuned quota. The goal of the tuning is getting + * the feedback score value of 10,000. + */ +struct damos_quota_goal { + unsigned long (*get_score)(void *arg); + void *get_score_arg; +}; + /** * struct damos_quota - Controls the aggressiveness of the given scheme. * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @get_score: Feedback function for self-tuning quota. - * @get_score_arg: Parameter for @get_score + * @goal: Quota auto-tuning goal. * @esz: Effective size quota in bytes. * * @weight_sz: Weight of the region's size for prioritization. @@ -151,16 +166,10 @@ enum damos_action { * throughput of the scheme's action. DAMON then compares it against &sz and * uses smaller one as the effective quota. * - * If @get_score function pointer is set, DAMON calls it back with - * @get_score_arg and get the return value of it for every @reset_interval. - * Then, DAMON adjusts the effective quota using the return value as a feedback - * score to the current quota, using its internal feedback loop algorithm. - * - * The feedback loop algorithem assumes the quota input and the feedback score - * output are in a positive proportional relationship, and the goal of the - * tuning is getting the feedback screo value of 10,000. If @ms and/or @sz are - * set together, those work as a hard limit quota. If neither @ms nor @sz are - * set, the mechanism starts from the quota of one byte. + * If ->get_score field of @goal is set, DAMON calculates yet another size + * quota based on the goal using its internal feedback loop algorithm, for + * every @reset_interval. Then, if the new size quota is smaller than the + * effective quota, it uses the new size quota as the effective quota. * * The resulting effective size quota in bytes is set to @esz. * @@ -174,8 +183,7 @@ struct damos_quota { unsigned long reset_interval; unsigned long ms; unsigned long sz; - unsigned long (*get_score)(void *arg); - void *get_score_arg; + struct damos_quota_goal goal; unsigned long esz; unsigned int weight_sz; diff --git a/mm/damon/core.c b/mm/damon/core.c index 0656966a6fc4..fe4209672121 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1083,21 +1083,22 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, return min_input; } -/* Shouldn't be called if quota->ms, quota->sz, and quota->get_score unset */ +/* Called only if quota->ms, quota->sz, or quota->goal.get_score are set */ static void damos_set_effective_quota(struct damos_quota *quota) { unsigned long throughput; unsigned long esz; - if (!quota->ms && !quota->get_score) { + if (!quota->ms && !quota->goal.get_score) { quota->esz = quota->sz; return; } - if (quota->get_score) { + if (quota->goal.get_score) { quota->esz_bp = damon_feed_loop_next_input( max(quota->esz_bp, 10000UL), - quota->get_score(quota->get_score_arg)); + quota->goal.get_score( + quota->goal.get_score_arg)); esz = quota->esz_bp / 10000; } @@ -1107,7 +1108,7 @@ static void damos_set_effective_quota(struct damos_quota *quota) quota->total_charged_ns; else throughput = PAGE_SIZE * 1024; - if (quota->get_score) + if (quota->goal.get_score) esz = min(throughput * quota->ms, esz); else esz = throughput * quota->ms; @@ -1127,7 +1128,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) unsigned long cumulated_sz; unsigned int score, max_score = 0; - if (!quota->ms && !quota->sz && !quota->get_score) + if (!quota->ms && !quota->sz && !quota->goal.get_score) return; /* New charge window starts */ diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 9d90e7b757b7..85ef58f98a87 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1894,19 +1894,19 @@ static void damos_sysfs_set_quota_score( struct damos_sysfs_quota_goal *sysfs_goal; int i; - quota->get_score = NULL; - quota->get_score_arg = (void *)0; + quota->goal.get_score = NULL; + quota->goal.get_score_arg = (void *)0; for (i = 0; i < sysfs_goals->nr; i++) { sysfs_goal = sysfs_goals->goals_arr[i]; if (!sysfs_goal->target_value) continue; /* Higher score makes scheme less aggressive */ - quota->get_score_arg = (void *)max( - (unsigned long)quota->get_score_arg, + quota->goal.get_score_arg = (void *)max( + (unsigned long)quota->goal.get_score_arg, sysfs_goal->current_value * 10000 / sysfs_goal->target_value); - quota->get_score = damos_sysfs_get_quota_score; + quota->goal.get_score = damos_sysfs_get_quota_score; } } -- cgit v1.2.3 From 91f21216a79d00f7da380ed4ce100e8a7a0d0cff Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:19 -0800 Subject: mm/damon/core: add multiple goals per damos_quota and helpers for those The feedback-driven DAMOS quota auto-tuning feature allows only single goal to the DAMON kernel API users. The API users could implement multiple goals for the end-users on their level, and that's what DAMON sysfs interface is doing. More DAMON kernel API users such as DAMON_RECLAIM would need to do similar work. To reduce unnecessary future duplciated efforts, support multiple goals from DAMOS core layer. To make the support in minimum non-destructive change, keep the old single goal setup interface, and add multiple goals setup. The single goal will treated as one of the multiple goals, so old API users are not required to make any change. Link: https://lkml.kernel.org/r/20240219194431.159606-9-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 17 +++++++++++ mm/damon/core.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 88 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 2fe345adf6b2..4bd898eaf80e 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -130,6 +130,7 @@ enum damos_action { * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. * @get_score: Function for getting current score of the goal. * @get_score_arg: Parameter for @get_score + * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. * Calling @get_score with @get_score_arg as the parameter should return the @@ -140,6 +141,7 @@ enum damos_action { struct damos_quota_goal { unsigned long (*get_score)(void *arg); void *get_score_arg; + struct list_head list; }; /** @@ -148,6 +150,7 @@ struct damos_quota_goal { * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. * @goal: Quota auto-tuning goal. + * @goals: Head of quota tuning goals (&damos_quota_goal) list. * @esz: Effective size quota in bytes. * * @weight_sz: Weight of the region's size for prioritization. @@ -171,6 +174,8 @@ struct damos_quota_goal { * every @reset_interval. Then, if the new size quota is smaller than the * effective quota, it uses the new size quota as the effective quota. * + * If @goals is not empty, same action is taken for each goal of the list. + * * The resulting effective size quota in bytes is set to @esz. * * For selecting regions within the quota, DAMON prioritizes current scheme's @@ -184,6 +189,7 @@ struct damos_quota { unsigned long ms; unsigned long sz; struct damos_quota_goal goal; + struct list_head goals; unsigned long esz; unsigned int weight_sz; @@ -648,6 +654,12 @@ static inline unsigned long damon_sz_region(struct damon_region *r) #define damon_for_each_scheme_safe(s, next, ctx) \ list_for_each_entry_safe(s, next, &(ctx)->schemes, list) +#define damos_for_each_quota_goal(goal, quota) \ + list_for_each_entry(goal, "a->goals, list) + +#define damos_for_each_quota_goal_safe(goal, next, quota) \ + list_for_each_entry_safe(goal, next, &(quota)->goals, list) + #define damos_for_each_filter(f, scheme) \ list_for_each_entry(f, &(scheme)->filters, list) @@ -681,6 +693,11 @@ struct damos_filter *damos_new_filter(enum damos_filter_type type, void damos_add_filter(struct damos *s, struct damos_filter *f); void damos_destroy_filter(struct damos_filter *f); +struct damos_quota_goal *damos_new_quota_goal( + unsigned long (*get_score)(void *), void *get_score_arg); +void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g); +void damos_destroy_quota_goal(struct damos_quota_goal *goal); + struct damos *damon_new_scheme(struct damos_access_pattern *pattern, enum damos_action action, unsigned long apply_interval_us, diff --git a/mm/damon/core.c b/mm/damon/core.c index fe4209672121..b6cd99b64e85 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -299,6 +299,41 @@ void damos_destroy_filter(struct damos_filter *f) damos_free_filter(f); } +struct damos_quota_goal *damos_new_quota_goal( + unsigned long (*get_score)(void *), void *get_score_arg) +{ + struct damos_quota_goal *goal; + + goal = kmalloc(sizeof(*goal), GFP_KERNEL); + if (!goal) + return NULL; + goal->get_score = get_score; + goal->get_score_arg = get_score_arg; + INIT_LIST_HEAD(&goal->list); + return goal; +} + +void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g) +{ + list_add_tail(&g->list, &q->goals); +} + +static void damos_del_quota_goal(struct damos_quota_goal *g) +{ + list_del(&g->list); +} + +static void damos_free_quota_goal(struct damos_quota_goal *g) +{ + kfree(g); +} + +void damos_destroy_quota_goal(struct damos_quota_goal *g) +{ + damos_del_quota_goal(g); + damos_free_quota_goal(g); +} + /* initialize fields of @quota that normally API users wouldn't set */ static struct damos_quota *damos_quota_init(struct damos_quota *quota) { @@ -337,6 +372,8 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); + /* quota.goals should be separately set by caller */ + INIT_LIST_HEAD(&scheme->quota.goals); scheme->wmarks = *wmarks; scheme->wmarks.activated = true; @@ -373,8 +410,12 @@ static void damon_free_scheme(struct damos *s) void damon_destroy_scheme(struct damos *s) { + struct damos_quota_goal *g, *g_next; struct damos_filter *f, *next; + damos_for_each_quota_goal_safe(g, g_next, &s->quota) + damos_destroy_quota_goal(g); + damos_for_each_filter_safe(f, next, s) damos_destroy_filter(f); damon_del_scheme(s); @@ -1083,22 +1124,44 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, return min_input; } -/* Called only if quota->ms, quota->sz, or quota->goal.get_score are set */ +/* Return the highest score since it makes schemes least aggressive */ +static unsigned long damos_quota_score(struct damos_quota *quota) +{ + struct damos_quota_goal *goal; + unsigned long highest_score = 0; + + if (quota->goal.get_score) + highest_score = quota->goal.get_score( + quota->goal.get_score_arg); + + damos_for_each_quota_goal(goal, quota) + highest_score = max(highest_score, + goal->get_score(goal->get_score_arg)); + + return highest_score; +} + +/* + * Called only if quota->ms, quota->sz, or quota->goal.get_score are set, or + * quota->goals is not empty + */ static void damos_set_effective_quota(struct damos_quota *quota) { unsigned long throughput; unsigned long esz; - if (!quota->ms && !quota->goal.get_score) { + if (!quota->ms && !quota->goal.get_score && + list_empty("a->goals)) { quota->esz = quota->sz; return; } - if (quota->goal.get_score) { + if (quota->goal.get_score || !list_empty("a->goals)) { + unsigned long score = damos_quota_score(quota); + quota->esz_bp = damon_feed_loop_next_input( max(quota->esz_bp, 10000UL), - quota->goal.get_score( - quota->goal.get_score_arg)); + score); esz = quota->esz_bp / 10000; } @@ -1108,7 +1171,7 @@ static void damos_set_effective_quota(struct damos_quota *quota) quota->total_charged_ns; else throughput = PAGE_SIZE * 1024; - if (quota->goal.get_score) + if (quota->goal.get_score || !list_empty("a->goals)) esz = min(throughput * quota->ms, esz); else esz = throughput * quota->ms; @@ -1128,7 +1191,8 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) unsigned long cumulated_sz; unsigned int score, max_score = 0; - if (!quota->ms && !quota->sz && !quota->goal.get_score) + if (!quota->ms && !quota->sz && !quota->goal.get_score && + list_empty("a->goals)) return; /* New charge window starts */ -- cgit v1.2.3 From 89d347a545a704e0bd4fc61f9aea956d71bc72d2 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:21 -0800 Subject: mm/damon/core: remove ->goal field of damos_quota DAMOS quota auto-tuning feature supports static signle goal and dynamic multiple goals via DAMON kernel API, specifically via ->goal and ->goals fields of damos_quota struct, respectively. All in-tree DAMOS kernel API users are using only the dynamic multiple goals now. Remove the unsued static single goal interface. Link: https://lkml.kernel.org/r/20240219194431.159606-11-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 12 ++++-------- mm/damon/core.c | 17 +++++------------ 2 files changed, 9 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 4bd898eaf80e..76c965c1eea3 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -149,7 +149,6 @@ struct damos_quota_goal { * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @goal: Quota auto-tuning goal. * @goals: Head of quota tuning goals (&damos_quota_goal) list. * @esz: Effective size quota in bytes. * @@ -169,12 +168,10 @@ struct damos_quota_goal { * throughput of the scheme's action. DAMON then compares it against &sz and * uses smaller one as the effective quota. * - * If ->get_score field of @goal is set, DAMON calculates yet another size - * quota based on the goal using its internal feedback loop algorithm, for - * every @reset_interval. Then, if the new size quota is smaller than the - * effective quota, it uses the new size quota as the effective quota. - * - * If @goals is not empty, same action is taken for each goal of the list. + * If @goals is not empt, DAMON calculates yet another size quota based on the + * goals using its internal feedback loop algorithm, for every @reset_interval. + * Then, if the new size quota is smaller than the effective quota, it uses the + * new size quota as the effective quota. * * The resulting effective size quota in bytes is set to @esz. * @@ -188,7 +185,6 @@ struct damos_quota { unsigned long reset_interval; unsigned long ms; unsigned long sz; - struct damos_quota_goal goal; struct list_head goals; unsigned long esz; diff --git a/mm/damon/core.c b/mm/damon/core.c index b6cd99b64e85..7b06d926c552 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1130,10 +1130,6 @@ static unsigned long damos_quota_score(struct damos_quota *quota) struct damos_quota_goal *goal; unsigned long highest_score = 0; - if (quota->goal.get_score) - highest_score = quota->goal.get_score( - quota->goal.get_score_arg); - damos_for_each_quota_goal(goal, quota) highest_score = max(highest_score, goal->get_score(goal->get_score_arg)); @@ -1142,21 +1138,19 @@ static unsigned long damos_quota_score(struct damos_quota *quota) } /* - * Called only if quota->ms, quota->sz, or quota->goal.get_score are set, or - * quota->goals is not empty + * Called only if quota->ms, or quota->sz are set, or quota->goals is not empty */ static void damos_set_effective_quota(struct damos_quota *quota) { unsigned long throughput; unsigned long esz; - if (!quota->ms && !quota->goal.get_score && - list_empty("a->goals)) { + if (!quota->ms && list_empty("a->goals)) { quota->esz = quota->sz; return; } - if (quota->goal.get_score || !list_empty("a->goals)) { + if (!list_empty("a->goals)) { unsigned long score = damos_quota_score(quota); quota->esz_bp = damon_feed_loop_next_input( @@ -1171,7 +1165,7 @@ static void damos_set_effective_quota(struct damos_quota *quota) quota->total_charged_ns; else throughput = PAGE_SIZE * 1024; - if (quota->goal.get_score || !list_empty("a->goals)) + if (!list_empty("a->goals)) esz = min(throughput * quota->ms, esz); else esz = throughput * quota->ms; @@ -1191,8 +1185,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) unsigned long cumulated_sz; unsigned int score, max_score = 0; - if (!quota->ms && !quota->sz && !quota->goal.get_score && - list_empty("a->goals)) + if (!quota->ms && !quota->sz && list_empty("a->goals)) return; /* New charge window starts */ -- cgit v1.2.3 From 06ba5b309ed870cf1a0fedc611d0e7fbb6425a2d Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:22 -0800 Subject: mm/damon/core: let goal specified with only target and current values DAMOS quota auto-tuning feature let users to set the goal by providing a function for getting the current score of the tuned quota. It allows flexible goal setup, but only simple user-set quota is currently being used. As a result, the only user of the DAMOS quota auto-tuning is using a silly void pointer casting based score value passing function. Simplify the interface and the user code by letting user directly set the target and the current value. Link: https://lkml.kernel.org/r/20240219194431.159606-12-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 19 +++++++++---------- mm/damon/core.c | 9 +++++---- mm/damon/sysfs-schemes.c | 10 ++-------- 3 files changed, 16 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 76c965c1eea3..de0cdc7f96d2 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -128,19 +128,18 @@ enum damos_action { /** * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. - * @get_score: Function for getting current score of the goal. - * @get_score_arg: Parameter for @get_score + * @target_value: Target value to achieve with the tuning. + * @current_value: Current value that achieving with the tuning. * @list: List head for siblings. * - * Data structure for getting the current score of the quota tuning goal. - * Calling @get_score with @get_score_arg as the parameter should return the - * current score. Then the score is entered to DAMON's internal feedback loop - * mechanism to get the auto-tuned quota. The goal of the tuning is getting - * the feedback score value of 10,000. + * Data structure for getting the current score of the quota tuning goal. The + * score is calculated by how close @current_value and @target_value are. Then + * the score is entered to DAMON's internal feedback loop mechanism to get the + * auto-tuned quota. */ struct damos_quota_goal { - unsigned long (*get_score)(void *arg); - void *get_score_arg; + unsigned long target_value; + unsigned long current_value; struct list_head list; }; @@ -690,7 +689,7 @@ void damos_add_filter(struct damos *s, struct damos_filter *f); void damos_destroy_filter(struct damos_filter *f); struct damos_quota_goal *damos_new_quota_goal( - unsigned long (*get_score)(void *), void *get_score_arg); + unsigned long target_value, unsigned long current_value); void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g); void damos_destroy_quota_goal(struct damos_quota_goal *goal); diff --git a/mm/damon/core.c b/mm/damon/core.c index 7b06d926c552..907f467fc8c0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -300,15 +300,15 @@ void damos_destroy_filter(struct damos_filter *f) } struct damos_quota_goal *damos_new_quota_goal( - unsigned long (*get_score)(void *), void *get_score_arg) + unsigned long target_value, unsigned long current_value) { struct damos_quota_goal *goal; goal = kmalloc(sizeof(*goal), GFP_KERNEL); if (!goal) return NULL; - goal->get_score = get_score; - goal->get_score_arg = get_score_arg; + goal->target_value = target_value; + goal->current_value = current_value; INIT_LIST_HEAD(&goal->list); return goal; } @@ -1132,7 +1132,8 @@ static unsigned long damos_quota_score(struct damos_quota *quota) damos_for_each_quota_goal(goal, quota) highest_score = max(highest_score, - goal->get_score(goal->get_score_arg)); + goal->current_value * 10000 / + goal->target_value); return highest_score; } diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 7bf94b1ed6f7..50218a7bfa0a 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1882,11 +1882,6 @@ static int damon_sysfs_set_scheme_filters(struct damos *scheme, return 0; } -static unsigned long damos_sysfs_get_quota_score(void *arg) -{ - return (unsigned long)arg; -} - static int damos_sysfs_set_quota_score( struct damos_sysfs_quota_goals *sysfs_goals, struct damos_quota *quota) @@ -1904,9 +1899,8 @@ static int damos_sysfs_set_quota_score( if (!sysfs_goal->target_value) continue; - goal = damos_new_quota_goal(damos_sysfs_get_quota_score, - (void *)(sysfs_goal->current_value * 10000 / - sysfs_goal->target_value)); + goal = damos_new_quota_goal(sysfs_goal->target_value, + sysfs_goal->current_value); if (!goal) return -ENOMEM; damos_add_quota_goal(quota, goal); -- cgit v1.2.3 From bcce9bc16f56fbc254857fcb31674ab868b824d7 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:23 -0800 Subject: mm/damon/core: support multiple metrics for quota goal DAMOS quota auto-tuning asks users to assess the current tuned quota and provide the feedback in a manual and repeated way. It allows users generate the feedback from a source that the kernel cannot access, and writing a script or a function for doing the manual and repeated feeding is not a big deal. However, additional works are additional works, and it could be more efficient if DAMOS could do the fetch itself, especially in case of DAMON sysfs interface use case, since it can avoid the context switches between the user-space and the kernel-space, though the overhead would be only trivial in most cases. Also in many cases, feedbacks could be made from kernel-accessible sources, such as PSI, CPU usage, etc. Make the quota goal to support multiple types of metrics including such ones. Link: https://lkml.kernel.org/r/20240219194431.159606-13-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 26 +++++++++++++++++++++++--- mm/damon/core.c | 22 +++++++++++++++++++--- mm/damon/sysfs-schemes.c | 5 +++-- 3 files changed, 45 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index de0cdc7f96d2..5a06993d8479 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -126,18 +126,37 @@ enum damos_action { NR_DAMOS_ACTIONS, }; +/** + * enum damos_quota_goal_metric - Represents the metric to be used as the goal + * + * @DAMOS_QUOTA_USER_INPUT: User-input value. + * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. + * + * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. + */ +enum damos_quota_goal_metric { + DAMOS_QUOTA_USER_INPUT, + NR_DAMOS_QUOTA_GOAL_METRICS, +}; + /** * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. - * @target_value: Target value to achieve with the tuning. - * @current_value: Current value that achieving with the tuning. + * @metric: Metric to be used for representing the goal. + * @target_value: Target value of @metric to achieve with the tuning. + * @current_value: Current value of @metric. * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. The * score is calculated by how close @current_value and @target_value are. Then * the score is entered to DAMON's internal feedback loop mechanism to get the * auto-tuned quota. + * + * If @metric is DAMOS_QUOTA_USER_INPUT, @current_value should be manually + * entered by the user, probably inside the kdamond callbacks. Otherwise, + * DAMON sets @current_value with self-measured value of @metric. */ struct damos_quota_goal { + enum damos_quota_goal_metric metric; unsigned long target_value; unsigned long current_value; struct list_head list; @@ -689,7 +708,8 @@ void damos_add_filter(struct damos *s, struct damos_filter *f); void damos_destroy_filter(struct damos_filter *f); struct damos_quota_goal *damos_new_quota_goal( - unsigned long target_value, unsigned long current_value); + enum damos_quota_goal_metric metric, + unsigned long target_value); void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g); void damos_destroy_quota_goal(struct damos_quota_goal *goal); diff --git a/mm/damon/core.c b/mm/damon/core.c index 907f467fc8c0..973423166ee2 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -300,15 +300,16 @@ void damos_destroy_filter(struct damos_filter *f) } struct damos_quota_goal *damos_new_quota_goal( - unsigned long target_value, unsigned long current_value) + enum damos_quota_goal_metric metric, + unsigned long target_value) { struct damos_quota_goal *goal; goal = kmalloc(sizeof(*goal), GFP_KERNEL); if (!goal) return NULL; + goal->metric = metric; goal->target_value = target_value; - goal->current_value = current_value; INIT_LIST_HEAD(&goal->list); return goal; } @@ -1124,16 +1125,31 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, return min_input; } +static void damos_set_quota_goal_current_value(struct damos_quota_goal *goal) +{ + u64 now_psi_total; + + switch (goal->metric) { + case DAMOS_QUOTA_USER_INPUT: + /* User should already set goal->current_value */ + break; + default: + break; + } +} + /* Return the highest score since it makes schemes least aggressive */ static unsigned long damos_quota_score(struct damos_quota *quota) { struct damos_quota_goal *goal; unsigned long highest_score = 0; - damos_for_each_quota_goal(goal, quota) + damos_for_each_quota_goal(goal, quota) { + damos_set_quota_goal_current_value(goal); highest_score = max(highest_score, goal->current_value * 10000 / goal->target_value); + } return highest_score; } diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 50218a7bfa0a..7a8a39f2679b 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1899,10 +1899,11 @@ static int damos_sysfs_set_quota_score( if (!sysfs_goal->target_value) continue; - goal = damos_new_quota_goal(sysfs_goal->target_value, - sysfs_goal->current_value); + goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, + sysfs_goal->target_value); if (!goal) return -ENOMEM; + goal->current_value = sysfs_goal->current_value; damos_add_quota_goal(quota, goal); } return 0; -- cgit v1.2.3 From 2dbb60f789cbb5c0000a4664f40f9358b3a62ba2 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 19 Feb 2024 11:44:24 -0800 Subject: mm/damon/core: implement PSI metric DAMOS quota goal Extend DAMOS quota goal metric with system wide memory pressure stall time. Specifically, the system level 'some' PSI for memory is used. The target value can be set in microseconds. DAMOS measures the increased amount of the PSI metric in last quota_reset_interval and use the ratio of it versus the user-specified target PSI value as the score for the auto-tuning feedback loop. Link: https://lkml.kernel.org/r/20240219194431.159606-14-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- include/linux/damon.h | 7 +++++++ mm/damon/core.c | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 5a06993d8479..886d07294f4e 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -130,12 +130,14 @@ enum damos_action { * enum damos_quota_goal_metric - Represents the metric to be used as the goal * * @DAMOS_QUOTA_USER_INPUT: User-input value. + * @DAMOS_QUOTA_SOME_MEM_PSI_US: System level some memory PSI in us. * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. * * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. */ enum damos_quota_goal_metric { DAMOS_QUOTA_USER_INPUT, + DAMOS_QUOTA_SOME_MEM_PSI_US, NR_DAMOS_QUOTA_GOAL_METRICS, }; @@ -144,6 +146,7 @@ enum damos_quota_goal_metric { * @metric: Metric to be used for representing the goal. * @target_value: Target value of @metric to achieve with the tuning. * @current_value: Current value of @metric. + * @last_psi_total: Last measured total PSI * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. The @@ -159,6 +162,10 @@ struct damos_quota_goal { enum damos_quota_goal_metric metric; unsigned long target_value; unsigned long current_value; + /* metric-dependent fields */ + union { + u64 last_psi_total; + }; struct list_head list; }; diff --git a/mm/damon/core.c b/mm/damon/core.c index 973423166ee2..6d503c1c125e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -1125,6 +1126,25 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input, return min_input; } +#ifdef CONFIG_PSI + +static u64 damos_get_some_mem_psi_total(void) +{ + if (static_branch_likely(&psi_disabled)) + return 0; + return div_u64(psi_system.total[PSI_AVGS][PSI_MEM * 2], + NSEC_PER_USEC); +} + +#else /* CONFIG_PSI */ + +static inline u64 damos_get_some_mem_psi_total(void) +{ + return 0; +}; + +#endif /* CONFIG_PSI */ + static void damos_set_quota_goal_current_value(struct damos_quota_goal *goal) { u64 now_psi_total; @@ -1133,6 +1153,11 @@ static void damos_set_quota_goal_current_value(struct damos_quota_goal *goal) case DAMOS_QUOTA_USER_INPUT: /* User should already set goal->current_value */ break; + case DAMOS_QUOTA_SOME_MEM_PSI_US: + now_psi_total = damos_get_some_mem_psi_total(); + goal->current_value = now_psi_total - goal->last_psi_total; + goal->last_psi_total = now_psi_total; + break; default: break; } -- cgit v1.2.3 From 2a6e1a8f4cf39cd3d10c52fca639a7d6f30b7004 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 15 Feb 2024 07:36:37 +0100 Subject: writeback: remove a duplicate prototype for tag_pages_for_writeback [hch@lst.de: split from a larger patch] Link: https://lkml.kernel.org/r/20240215063649.2164017-3-hch@lst.de Signed-off-by: Christoph Hellwig Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Brian Foster Reviewed-by: Jan Kara Acked-by: Dave Chinner Cc: Christian Brauner Cc: David Howells Signed-off-by: Andrew Morton --- include/linux/writeback.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 453736fd1d23..4b8cf9e4810b 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -363,8 +363,6 @@ bool wb_over_bg_thresh(struct bdi_writeback *wb); typedef int (*writepage_t)(struct folio *folio, struct writeback_control *wbc, void *data); -void tag_pages_for_writeback(struct address_space *mapping, - pgoff_t start, pgoff_t end); int write_cache_pages(struct address_space *mapping, struct writeback_control *wbc, writepage_t writepage, void *data); -- cgit v1.2.3 From 751e0d559c62a87dc828af22c3c58dc078c734e3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 15 Feb 2024 07:36:43 +0100 Subject: writeback: factor writeback_get_batch() out of write_cache_pages() This simple helper will be the basis of the writeback iterator. To make this work, we need to remember the current index and end positions in writeback_control. [hch@lst.de: heavily rebased, add helpers to get the tag and end index, don't keep the end index in struct writeback_control] Link: https://lkml.kernel.org/r/20240215063649.2164017-9-hch@lst.de Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Jan Kara Acked-by: Dave Chinner Cc: Christian Brauner Cc: David Howells Signed-off-by: Andrew Morton --- include/linux/writeback.h | 6 +++++ mm/page-writeback.c | 60 ++++++++++++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 4b8cf9e4810b..f67b3ea866a0 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -11,6 +11,7 @@ #include #include #include +#include struct bio; @@ -40,6 +41,7 @@ enum writeback_sync_modes { * in a manner such that unspecified fields are set to zero. */ struct writeback_control { + /* public fields that can be set and/or consumed by the caller: */ long nr_to_write; /* Write this many pages, and decrement this for each page written */ long pages_skipped; /* Pages which were not written */ @@ -77,6 +79,10 @@ struct writeback_control { */ struct swap_iocb **swap_plug; + /* internal fields used by the ->writepages implementation: */ + struct folio_batch fbatch; + pgoff_t index; + #ifdef CONFIG_CGROUP_WRITEBACK struct bdi_writeback *wb; /* wb this writeback is issued under */ struct inode *inode; /* inode being written out */ diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 4bc55ce9597d..5b8dbbef7137 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2392,6 +2392,29 @@ static bool folio_prepare_writeback(struct address_space *mapping, return true; } +static xa_mark_t wbc_to_tag(struct writeback_control *wbc) +{ + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + return PAGECACHE_TAG_TOWRITE; + return PAGECACHE_TAG_DIRTY; +} + +static pgoff_t wbc_end(struct writeback_control *wbc) +{ + if (wbc->range_cyclic) + return -1; + return wbc->range_end >> PAGE_SHIFT; +} + +static void writeback_get_batch(struct address_space *mapping, + struct writeback_control *wbc) +{ + folio_batch_release(&wbc->fbatch); + cond_resched(); + filemap_get_folios_tag(mapping, &wbc->index, wbc_end(wbc), + wbc_to_tag(wbc), &wbc->fbatch); +} + /** * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. * @mapping: address space structure to write @@ -2429,38 +2452,32 @@ int write_cache_pages(struct address_space *mapping, { int ret = 0; int error; - struct folio_batch fbatch; struct folio *folio; - int nr_folios; - pgoff_t index; pgoff_t end; /* Inclusive */ - xa_mark_t tag; - folio_batch_init(&fbatch); if (wbc->range_cyclic) { - index = mapping->writeback_index; /* prev offset */ + wbc->index = mapping->writeback_index; /* prev offset */ end = -1; } else { - index = wbc->range_start >> PAGE_SHIFT; + wbc->index = wbc->range_start >> PAGE_SHIFT; end = wbc->range_end >> PAGE_SHIFT; } - if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) { - tag_pages_for_writeback(mapping, index, end); - tag = PAGECACHE_TAG_TOWRITE; - } else { - tag = PAGECACHE_TAG_DIRTY; - } - while (index <= end) { + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, wbc->index, end); + + folio_batch_init(&wbc->fbatch); + + while (wbc->index <= end) { int i; - nr_folios = filemap_get_folios_tag(mapping, &index, end, - tag, &fbatch); + writeback_get_batch(mapping, wbc); - if (nr_folios == 0) + if (wbc->fbatch.nr == 0) break; - for (i = 0; i < nr_folios; i++) { - folio = fbatch.folios[i]; + for (i = 0; i < wbc->fbatch.nr; i++) { + folio = wbc->fbatch.folios[i]; + folio_lock(folio); if (!folio_prepare_writeback(mapping, wbc, folio)) { folio_unlock(folio); @@ -2498,8 +2515,6 @@ int write_cache_pages(struct address_space *mapping, goto done; } } - folio_batch_release(&fbatch); - cond_resched(); } /* @@ -2512,6 +2527,7 @@ int write_cache_pages(struct address_space *mapping, * of the file if we are called again, which can only happen due to * -ENOMEM from the file system. */ + folio_batch_release(&wbc->fbatch); if (wbc->range_cyclic) mapping->writeback_index = 0; return ret; @@ -2519,7 +2535,7 @@ int write_cache_pages(struct address_space *mapping, done: if (wbc->range_cyclic) mapping->writeback_index = folio->index + folio_nr_pages(folio); - folio_batch_release(&fbatch); + folio_batch_release(&wbc->fbatch); return error; } EXPORT_SYMBOL(write_cache_pages); -- cgit v1.2.3 From 535c5d9dadb327bb35f6d780fa037e0d3dfcb568 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 15 Feb 2024 07:36:45 +0100 Subject: pagevec: add ability to iterate a queue Add a loop counter inside the folio_batch to let us iterate from 0-nr instead of decrementing nr and treating the batch as a stack. It would generate some very weird and suboptimal I/O patterns for page writeback to iterate over the batch as a stack. Link: https://lkml.kernel.org/r/20240215063649.2164017-11-hch@lst.de Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Jan Kara Acked-by: Dave Chinner Cc: Christian Brauner Cc: David Howells Signed-off-by: Andrew Morton --- include/linux/pagevec.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index 87cc678adc85..fcc06c300a72 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -27,6 +27,7 @@ struct folio; */ struct folio_batch { unsigned char nr; + unsigned char i; bool percpu_pvec_drained; struct folio *folios[PAGEVEC_SIZE]; }; @@ -40,12 +41,14 @@ struct folio_batch { static inline void folio_batch_init(struct folio_batch *fbatch) { fbatch->nr = 0; + fbatch->i = 0; fbatch->percpu_pvec_drained = false; } static inline void folio_batch_reinit(struct folio_batch *fbatch) { fbatch->nr = 0; + fbatch->i = 0; } static inline unsigned int folio_batch_count(struct folio_batch *fbatch) @@ -75,6 +78,21 @@ static inline unsigned folio_batch_add(struct folio_batch *fbatch, return folio_batch_space(fbatch); } +/** + * folio_batch_next - Return the next folio to process. + * @fbatch: The folio batch being processed. + * + * Use this function to implement a queue of folios. + * + * Return: The next folio in the queue, or NULL if the queue is empty. + */ +static inline struct folio *folio_batch_next(struct folio_batch *fbatch) +{ + if (fbatch->i == fbatch->nr) + return NULL; + return fbatch->folios[fbatch->i++]; +} + void __folio_batch_release(struct folio_batch *pvec); static inline void folio_batch_release(struct folio_batch *fbatch) -- cgit v1.2.3 From cdc150b575cf1176472791cfbe7738708812ea0d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 15 Feb 2024 07:36:48 +0100 Subject: writeback: add a writeback iterator Refactor the code left in write_cache_pages into an iterator that the file system can call to get the next folio for a writeback operation: struct folio *folio = NULL; while ((folio = writeback_iter(mapping, wbc, folio, &error))) { error = ; } The twist here is that the error value is passed by reference, so that the iterator can restore it when breaking out of the loop. Handling of the magic AOP_WRITEPAGE_ACTIVATE value stays outside the iterator and needs is just kept in the write_cache_pages legacy wrapper. in preparation for eventually killing it off. Heavily based on a for_each* based iterator from Matthew Wilcox. Link: https://lkml.kernel.org/r/20240215063649.2164017-14-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Jan Kara Cc: Christian Brauner Cc: Dave Chinner Cc: David Howells Cc: "Matthew Wilcox (Oracle)" Signed-off-by: Andrew Morton --- include/linux/writeback.h | 4 + mm/page-writeback.c | 192 +++++++++++++++++++++++++++------------------- 2 files changed, 118 insertions(+), 78 deletions(-) (limited to 'include/linux') diff --git a/include/linux/writeback.h b/include/linux/writeback.h index f67b3ea866a0..9845cb62e40b 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -82,6 +82,7 @@ struct writeback_control { /* internal fields used by the ->writepages implementation: */ struct folio_batch fbatch; pgoff_t index; + int saved_err; #ifdef CONFIG_CGROUP_WRITEBACK struct bdi_writeback *wb; /* wb this writeback is issued under */ @@ -366,6 +367,9 @@ int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, bool wb_over_bg_thresh(struct bdi_writeback *wb); +struct folio *writeback_iter(struct address_space *mapping, + struct writeback_control *wbc, struct folio *folio, int *error); + typedef int (*writepage_t)(struct folio *folio, struct writeback_control *wbc, void *data); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index fc421402f818..3da4345f08a3 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2325,18 +2325,18 @@ void __init page_writeback_init(void) } /** - * tag_pages_for_writeback - tag pages to be written by write_cache_pages + * tag_pages_for_writeback - tag pages to be written by writeback * @mapping: address space structure to write * @start: starting page index * @end: ending page index (inclusive) * * This function scans the page range from @start to @end (inclusive) and tags - * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is - * that write_cache_pages (or whoever calls this function) will then use - * TOWRITE tag to identify pages eligible for writeback. This mechanism is - * used to avoid livelocking of writeback by a process steadily creating new - * dirty pages in the file (thus it is important for this function to be quick - * so that it can tag pages faster than a dirtying process can create them). + * all pages that have DIRTY tag set with a special TOWRITE tag. The caller + * can then use the TOWRITE tag to identify pages eligible for writeback. + * This mechanism is used to avoid livelocking of writeback by a process + * steadily creating new dirty pages in the file (thus it is important for this + * function to be quick so that it can tag pages faster than a dirtying process + * can create them). */ void tag_pages_for_writeback(struct address_space *mapping, pgoff_t start, pgoff_t end) @@ -2434,69 +2434,68 @@ retry: } /** - * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. + * writeback_iter - iterate folio of a mapping for writeback * @mapping: address space structure to write - * @wbc: subtract the number of written pages from *@wbc->nr_to_write - * @writepage: function called for each page - * @data: data passed to writepage function + * @wbc: writeback context + * @folio: previously iterated folio (%NULL to start) + * @error: in-out pointer for writeback errors (see below) * - * If a page is already under I/O, write_cache_pages() skips it, even - * if it's dirty. This is desirable behaviour for memory-cleaning writeback, - * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() - * and msync() need to guarantee that all the data which was dirty at the time - * the call was made get new I/O started against them. If wbc->sync_mode is - * WB_SYNC_ALL then we were called for data integrity and we must wait for - * existing IO to complete. - * - * To avoid livelocks (when other process dirties new pages), we first tag - * pages which should be written back with TOWRITE tag and only then start - * writing them. For data-integrity sync we have to be careful so that we do - * not miss some pages (e.g., because some other process has cleared TOWRITE - * tag we set). The rule we follow is that TOWRITE tag can be cleared only - * by the process clearing the DIRTY tag (and submitting the page for IO). - * - * To avoid deadlocks between range_cyclic writeback and callers that hold - * pages in PageWriteback to aggregate IO until write_cache_pages() returns, - * we do not loop back to the start of the file. Doing so causes a page - * lock/page writeback access order inversion - we should only ever lock - * multiple pages in ascending page->index order, and looping back to the start - * of the file violates that rule and causes deadlocks. + * This function returns the next folio for the writeback operation described by + * @wbc on @mapping and should be called in a while loop in the ->writepages + * implementation. * - * Return: %0 on success, negative error code otherwise + * To start the writeback operation, %NULL is passed in the @folio argument, and + * for every subsequent iteration the folio returned previously should be passed + * back in. + * + * If there was an error in the per-folio writeback inside the writeback_iter() + * loop, @error should be set to the error value. + * + * Once the writeback described in @wbc has finished, this function will return + * %NULL and if there was an error in any iteration restore it to @error. + * + * Note: callers should not manually break out of the loop using break or goto + * but must keep calling writeback_iter() until it returns %NULL. + * + * Return: the folio to write or %NULL if the loop is done. */ -int write_cache_pages(struct address_space *mapping, - struct writeback_control *wbc, writepage_t writepage, - void *data) +struct folio *writeback_iter(struct address_space *mapping, + struct writeback_control *wbc, struct folio *folio, int *error) { - int ret = 0; - int error; - struct folio *folio; - pgoff_t end; /* Inclusive */ - - if (wbc->range_cyclic) { - wbc->index = mapping->writeback_index; /* prev offset */ - end = -1; - } else { - wbc->index = wbc->range_start >> PAGE_SHIFT; - end = wbc->range_end >> PAGE_SHIFT; - } - if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) - tag_pages_for_writeback(mapping, wbc->index, end); - - folio_batch_init(&wbc->fbatch); + if (!folio) { + folio_batch_init(&wbc->fbatch); + wbc->saved_err = *error = 0; - for (;;) { - folio = writeback_get_folio(mapping, wbc); - if (!folio) - break; + /* + * For range cyclic writeback we remember where we stopped so + * that we can continue where we stopped. + * + * For non-cyclic writeback we always start at the beginning of + * the passed in range. + */ + if (wbc->range_cyclic) + wbc->index = mapping->writeback_index; + else + wbc->index = wbc->range_start >> PAGE_SHIFT; - error = writepage(folio, wbc, data); + /* + * To avoid livelocks when other processes dirty new pages, we + * first tag pages which should be written back and only then + * start writing them. + * + * For data-integrity writeback we have to be careful so that we + * do not miss some pages (e.g., because some other process has + * cleared the TOWRITE tag we set). The rule we follow is that + * TOWRITE tag can be cleared only by the process clearing the + * DIRTY tag (and submitting the page for I/O). + */ + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, wbc->index, + wbc_end(wbc)); + } else { wbc->nr_to_write -= folio_nr_pages(folio); - if (error == AOP_WRITEPAGE_ACTIVATE) { - folio_unlock(folio); - error = 0; - } + WARN_ON_ONCE(*error > 0); /* * For integrity writeback we have to keep going until we have @@ -2510,33 +2509,70 @@ int write_cache_pages(struct address_space *mapping, * wbc->nr_to_write or encounter the first error. */ if (wbc->sync_mode == WB_SYNC_ALL) { - if (error && !ret) - ret = error; + if (*error && !wbc->saved_err) + wbc->saved_err = *error; } else { - if (error || wbc->nr_to_write <= 0) + if (*error || wbc->nr_to_write <= 0) goto done; } } - /* - * For range cyclic writeback we need to remember where we stopped so - * that we can continue there next time we are called. If we hit the - * last page and there is more work to be done, wrap back to the start - * of the file. - * - * For non-cyclic writeback we always start looking up at the beginning - * of the file if we are called again, which can only happen due to - * -ENOMEM from the file system. - */ - folio_batch_release(&wbc->fbatch); - if (wbc->range_cyclic) - mapping->writeback_index = 0; - return ret; + folio = writeback_get_folio(mapping, wbc); + if (!folio) { + /* + * To avoid deadlocks between range_cyclic writeback and callers + * that hold pages in PageWriteback to aggregate I/O until + * the writeback iteration finishes, we do not loop back to the + * start of the file. Doing so causes a page lock/page + * writeback access order inversion - we should only ever lock + * multiple pages in ascending page->index order, and looping + * back to the start of the file violates that rule and causes + * deadlocks. + */ + if (wbc->range_cyclic) + mapping->writeback_index = 0; + + /* + * Return the first error we encountered (if there was any) to + * the caller. + */ + *error = wbc->saved_err; + } + return folio; done: if (wbc->range_cyclic) mapping->writeback_index = folio->index + folio_nr_pages(folio); folio_batch_release(&wbc->fbatch); + return NULL; +} + +/** + * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. + * @mapping: address space structure to write + * @wbc: subtract the number of written pages from *@wbc->nr_to_write + * @writepage: function called for each page + * @data: data passed to writepage function + * + * Return: %0 on success, negative error code otherwise + * + * Note: please use writeback_iter() instead. + */ +int write_cache_pages(struct address_space *mapping, + struct writeback_control *wbc, writepage_t writepage, + void *data) +{ + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) { + error = writepage(folio, wbc, data); + if (error == AOP_WRITEPAGE_ACTIVATE) { + folio_unlock(folio); + error = 0; + } + } + return error; } EXPORT_SYMBOL(write_cache_pages); -- cgit v1.2.3 From 88debc69754f7fe5186954941bb1cc4d744f4f25 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 16:34:15 +0100 Subject: cpufreq: Remove references to 10ms min sampling rate A minimum sampling rate value of 10ms was introduced in: commit cef9615a853e ("[CPUFREQ] ondemand: Uncouple minimal sampling rate from HZ in NO_HZ case") The use of this value was removed in: commit ed4676e25463 ("cpufreq: Replace "max_transition_latency" with "dynamic_switching"") Remove: - a comment referencing this value - an unused macro associated to this value Signed-off-by: Pierre Gondois Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_ondemand.c | 1 - include/linux/cpufreq.h | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index c52d19d67557..a7c38b8b3e78 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -22,7 +22,6 @@ #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (100000) #define MICRO_FREQUENCY_UP_THRESHOLD (95) -#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) #define MIN_FREQUENCY_UP_THRESHOLD (1) #define MAX_FREQUENCY_UP_THRESHOLD (100) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 9bebeec24abb..85908b3a2f24 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -569,9 +569,7 @@ static inline unsigned long cpufreq_scale(unsigned long old, u_int div, /* * The polling frequency depends on the capability of the processor. Default - * polling frequency is 1000 times the transition latency of the processor. The - * ondemand governor will work on any processor with transition latency <= 10ms, - * using appropriate sampling rate. + * polling frequency is 1000 times the transition latency of the processor. */ #define LATENCY_MULTIPLIER (1000) -- cgit v1.2.3 From 5c656fcdd6c60f71fccb07fe7b9d8d7e6c9811ff Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 24 Feb 2024 20:35:44 +0530 Subject: mm: Introduce new flag to indicate wc safe The VM_ALLOW_ANY_UNCACHED flag is implemented for ARM64, allowing KVM stage 2 device mapping attributes to use NormalNC rather than DEVICE_nGnRE, which allows guest mappings supporting write-combining attributes (WC). ARM does not architecturally guarantee this is safe, and indeed some MMIO regions like the GICv2 VCPU interface can trigger uncontained faults if NormalNC is used. Even worse, the expectation is that there are platforms where even DEVICE_nGnRE can allow uncontained faults in corner cases. Unfortunately existing ARM IP requires platform integration to take responsibility to prevent this. To safely use VFIO in KVM the platform must guarantee full safety in the guest where no action taken against a MMIO mapping can trigger an uncontained failure. The assumption is that most VFIO PCI platforms support this for both mapping types, at least in common flows, based on some expectations of how PCI IP is integrated. This can be enabled more broadly, for instance into vfio-platform drivers, but only after the platform vendor completes auditing for safety. The VMA flag VM_ALLOW_ANY_UNCACHED was found to be the simplest and cleanest way to communicate the information from VFIO to KVM that mapping the region in S2 as NormalNC is safe. KVM consumes it to activate the code that does the S2 mapping as NormalNC. Suggested-by: Catalin Marinas Reviewed-by: Jason Gunthorpe Reviewed-by: Marc Zyngier Acked-by: David Hildenbrand Signed-off-by: Ankit Agrawal Link: https://lore.kernel.org/r/20240224150546.368-3-ankita@nvidia.com Signed-off-by: Oliver Upton --- include/linux/mm.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index f5a97dec5169..59576e56c58b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -391,6 +391,20 @@ extern unsigned int kobjsize(const void *objp); # define VM_UFFD_MINOR VM_NONE #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ +/* + * This flag is used to connect VFIO to arch specific KVM code. It + * indicates that the memory under this VMA is safe for use with any + * non-cachable memory type inside KVM. Some VFIO devices, on some + * platforms, are thought to be unsafe and can cause machine crashes + * if KVM does not lock down the memory type. + */ +#ifdef CONFIG_64BIT +#define VM_ALLOW_ANY_UNCACHED_BIT 39 +#define VM_ALLOW_ANY_UNCACHED BIT(VM_ALLOW_ANY_UNCACHED_BIT) +#else +#define VM_ALLOW_ANY_UNCACHED VM_NONE +#endif + /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) -- cgit v1.2.3 From 5270316c9fec8cc99aa0e0a258509c5c7f789d12 Mon Sep 17 00:00:00 2001 From: Petr Pavlu Date: Thu, 22 Feb 2024 14:35:00 +0100 Subject: kbuild: Use -fmin-function-alignment when available GCC recently added option -fmin-function-alignment, which should appear in GCC 14. Unlike -falign-functions, this option causes all functions to be aligned at the specified value, including the cold ones. In particular, when an arm64 kernel is built with DYNAMIC_FTRACE_WITH_CALL_OPS=y, the 8-byte function alignment is required for correct functionality. This was done by -falign-functions=8 and having workarounds in the kernel to force the compiler to follow this alignment. The new -fmin-function-alignment option directly guarantees it. Detect availability of -fmin-function-alignment and use it instead of -falign-functions when present. Introduce CC_HAS_SANE_FUNCTION_ALIGNMENT and enable __cold to work as expected when it is set. Signed-off-by: Petr Pavlu Reviewed-by: Nathan Chancellor Acked-by: Mark Rutland Signed-off-by: Masahiro Yamada --- Makefile | 7 +++++++ arch/Kconfig | 12 ++++++++++++ include/linux/compiler_types.h | 10 +++++----- 3 files changed, 24 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/Makefile b/Makefile index bc54a6cc8dc5..d84c0fb215fd 100644 --- a/Makefile +++ b/Makefile @@ -974,8 +974,15 @@ export CC_FLAGS_CFI endif ifneq ($(CONFIG_FUNCTION_ALIGNMENT),0) +# Set the minimal function alignment. Use the newer GCC option +# -fmin-function-alignment if it is available, or fall back to -falign-funtions. +# See also CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT. +ifdef CONFIG_CC_HAS_MIN_FUNCTION_ALIGNMENT +KBUILD_CFLAGS += -fmin-function-alignment=$(CONFIG_FUNCTION_ALIGNMENT) +else KBUILD_CFLAGS += -falign-functions=$(CONFIG_FUNCTION_ALIGNMENT) endif +endif # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc diff --git a/arch/Kconfig b/arch/Kconfig index a5af0edd3eb8..bd6c6335efac 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1507,4 +1507,16 @@ config FUNCTION_ALIGNMENT default 4 if FUNCTION_ALIGNMENT_4B default 0 +config CC_HAS_MIN_FUNCTION_ALIGNMENT + # Detect availability of the GCC option -fmin-function-alignment which + # guarantees minimal alignment for all functions, unlike + # -falign-functions which the compiler ignores for cold functions. + def_bool $(cc-option, -fmin-function-alignment=8) + +config CC_HAS_SANE_FUNCTION_ALIGNMENT + # Set if the guaranteed alignment with -fmin-function-alignment is + # available or extra care is required in the kernel. Clang provides + # strict alignment always, even with -falign-functions. + def_bool CC_HAS_MIN_FUNCTION_ALIGNMENT || CC_IS_CLANG + endmenu diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 0caf354cb94b..fb8888678687 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -99,17 +99,17 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } * gcc: https://gcc.gnu.org/onlinedocs/gcc/Label-Attributes.html#index-cold-label-attribute * * When -falign-functions=N is in use, we must avoid the cold attribute as - * contemporary versions of GCC drop the alignment for cold functions. Worse, - * GCC can implicitly mark callees of cold functions as cold themselves, so - * it's not sufficient to add __function_aligned here as that will not ensure - * that callees are correctly aligned. + * GCC drops the alignment for cold functions. Worse, GCC can implicitly mark + * callees of cold functions as cold themselves, so it's not sufficient to add + * __function_aligned here as that will not ensure that callees are correctly + * aligned. * * See: * * https://lore.kernel.org/lkml/Y77%2FqVgvaJidFpYt@FVFF77S0Q05N * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345#c9 */ -#if !defined(CONFIG_CC_IS_GCC) || (CONFIG_FUNCTION_ALIGNMENT == 0) +#if defined(CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT) || (CONFIG_FUNCTION_ALIGNMENT == 0) #define __cold __attribute__((__cold__)) #else #define __cold -- cgit v1.2.3 From 0b7c6075022ccff529318597dc6b165dd6a25c8f Mon Sep 17 00:00:00 2001 From: Peter Griffin Date: Tue, 20 Feb 2024 22:06:12 +0000 Subject: soc: samsung: exynos-pmu: Add regmap support for SoCs that protect PMU regs Some Exynos based SoCs like Tensor gs101 protect the PMU registers for security hardening reasons so that they are only write accessible in el3 via an SMC call. As most Exynos drivers that need to write PMU registers currently obtain a regmap via syscon (phys, pinctrl, watchdog). Support for the above usecase is implemented in this driver using a custom regmap similar to syscon to handle the SMC call. Platforms that don't secure PMU registers, get a mmio regmap like before. As regmaps abstract out the underlying register access changes to the leaf drivers are minimal. A new API exynos_get_pmu_regmap_by_phandle() is provided for leaf drivers that currently use syscon_regmap_lookup_by_phandle(). This also handles deferred probing. Tested-by: Sam Protsenko Tested-by: Alexey Klimov Reviewed-by: Sam Protsenko Signed-off-by: Peter Griffin Link: https://lore.kernel.org/r/20240220220613.797068-2-peter.griffin@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/soc/samsung/Kconfig | 1 + drivers/soc/samsung/exynos-pmu.c | 235 ++++++++++++++++++++++++++++++++- drivers/soc/samsung/exynos-pmu.h | 1 + include/linux/soc/samsung/exynos-pmu.h | 11 +- 4 files changed, 245 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/soc/samsung/Kconfig b/drivers/soc/samsung/Kconfig index 27ec99af77e3..1a5dfdc978dc 100644 --- a/drivers/soc/samsung/Kconfig +++ b/drivers/soc/samsung/Kconfig @@ -42,6 +42,7 @@ config EXYNOS_PMU depends on ARCH_EXYNOS || ((ARM || ARM64) && COMPILE_TEST) select EXYNOS_PMU_ARM_DRIVERS if ARM && ARCH_EXYNOS select MFD_CORE + select REGMAP_MMIO # There is no need to enable these drivers for ARMv8 config EXYNOS_PMU_ARM_DRIVERS diff --git a/drivers/soc/samsung/exynos-pmu.c b/drivers/soc/samsung/exynos-pmu.c index 250537d7cfd6..fd8b6ac06656 100644 --- a/drivers/soc/samsung/exynos-pmu.c +++ b/drivers/soc/samsung/exynos-pmu.c @@ -5,6 +5,7 @@ // // Exynos - CPU PMU(Power Management Unit) support +#include #include #include #include @@ -12,19 +13,134 @@ #include #include #include +#include #include #include #include "exynos-pmu.h" +#define PMUALIVE_MASK GENMASK(13, 0) +#define TENSOR_SET_BITS (BIT(15) | BIT(14)) +#define TENSOR_CLR_BITS BIT(15) +#define TENSOR_SMC_PMU_SEC_REG 0x82000504 +#define TENSOR_PMUREG_READ 0 +#define TENSOR_PMUREG_WRITE 1 +#define TENSOR_PMUREG_RMW 2 + struct exynos_pmu_context { struct device *dev; const struct exynos_pmu_data *pmu_data; + struct regmap *pmureg; }; void __iomem *pmu_base_addr; static struct exynos_pmu_context *pmu_context; +/* forward declaration */ +static struct platform_driver exynos_pmu_driver; + +/* + * Tensor SoCs are configured so that PMU_ALIVE registers can only be written + * from EL3, but are still read accessible. As Linux needs to write some of + * these registers, the following functions are provided and exposed via + * regmap. + * + * Note: This SMC interface is known to be implemented on gs101 and derivative + * SoCs. + */ + +/* Write to a protected PMU register. */ +static int tensor_sec_reg_write(void *context, unsigned int reg, + unsigned int val) +{ + struct arm_smccc_res res; + unsigned long pmu_base = (unsigned long)context; + + arm_smccc_smc(TENSOR_SMC_PMU_SEC_REG, pmu_base + reg, + TENSOR_PMUREG_WRITE, val, 0, 0, 0, 0, &res); + + /* returns -EINVAL if access isn't allowed or 0 */ + if (res.a0) + pr_warn("%s(): SMC failed: %d\n", __func__, (int)res.a0); + + return (int)res.a0; +} + +/* Read/Modify/Write a protected PMU register. */ +static int tensor_sec_reg_rmw(void *context, unsigned int reg, + unsigned int mask, unsigned int val) +{ + struct arm_smccc_res res; + unsigned long pmu_base = (unsigned long)context; + + arm_smccc_smc(TENSOR_SMC_PMU_SEC_REG, pmu_base + reg, + TENSOR_PMUREG_RMW, mask, val, 0, 0, 0, &res); + + /* returns -EINVAL if access isn't allowed or 0 */ + if (res.a0) + pr_warn("%s(): SMC failed: %d\n", __func__, (int)res.a0); + + return (int)res.a0; +} + +/* + * Read a protected PMU register. All PMU registers can be read by Linux. + * Note: The SMC read register is not used, as only registers that can be + * written are readable via SMC. + */ +static int tensor_sec_reg_read(void *context, unsigned int reg, + unsigned int *val) +{ + *val = pmu_raw_readl(reg); + return 0; +} + +/* + * For SoCs that have set/clear bit hardware this function can be used when + * the PMU register will be accessed by multiple masters. + * + * For example, to set bits 13:8 in PMU reg offset 0x3e80 + * tensor_set_bits_atomic(ctx, 0x3e80, 0x3f00, 0x3f00); + * + * Set bit 8, and clear bits 13:9 PMU reg offset 0x3e80 + * tensor_set_bits_atomic(0x3e80, 0x100, 0x3f00); + */ +static int tensor_set_bits_atomic(void *ctx, unsigned int offset, u32 val, + u32 mask) +{ + int ret; + unsigned int i; + + for (i = 0; i < 32; i++) { + if (!(mask & BIT(i))) + continue; + + offset &= ~TENSOR_SET_BITS; + + if (val & BIT(i)) + offset |= TENSOR_SET_BITS; + else + offset |= TENSOR_CLR_BITS; + + ret = tensor_sec_reg_write(ctx, offset, i); + if (ret) + return ret; + } + return ret; +} + +static int tensor_sec_update_bits(void *ctx, unsigned int reg, + unsigned int mask, unsigned int val) +{ + /* + * Use atomic operations for PMU_ALIVE registers (offset 0~0x3FFF) + * as the target registers can be accessed by multiple masters. + */ + if (reg > PMUALIVE_MASK) + return tensor_sec_reg_rmw(ctx, reg, mask, val); + + return tensor_set_bits_atomic(ctx, reg, val, mask); +} void pmu_raw_writel(u32 val, u32 offset) { @@ -75,11 +191,41 @@ void exynos_sys_powerdown_conf(enum sys_powerdown mode) #define exynos_pmu_data_arm_ptr(data) NULL #endif +static const struct regmap_config regmap_smccfg = { + .name = "pmu_regs", + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + .use_single_read = true, + .use_single_write = true, + .reg_read = tensor_sec_reg_read, + .reg_write = tensor_sec_reg_write, + .reg_update_bits = tensor_sec_update_bits, +}; + +static const struct regmap_config regmap_mmiocfg = { + .name = "pmu_regs", + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + .use_single_read = true, + .use_single_write = true, +}; + +static const struct exynos_pmu_data gs101_pmu_data = { + .pmu_secure = true +}; + /* * PMU platform driver and devicetree bindings. */ static const struct of_device_id exynos_pmu_of_device_ids[] = { { + .compatible = "google,gs101-pmu", + .data = &gs101_pmu_data, + }, { .compatible = "samsung,exynos3250-pmu", .data = exynos_pmu_data_arm_ptr(exynos3250_pmu_data), }, { @@ -113,19 +259,75 @@ static const struct mfd_cell exynos_pmu_devs[] = { { .name = "exynos-clkout", }, }; +/** + * exynos_get_pmu_regmap() - Obtain pmureg regmap + * + * Find the pmureg regmap previously configured in probe() and return regmap + * pointer. + * + * Return: A pointer to regmap if found or ERR_PTR error value. + */ struct regmap *exynos_get_pmu_regmap(void) { struct device_node *np = of_find_matching_node(NULL, exynos_pmu_of_device_ids); if (np) - return syscon_node_to_regmap(np); + return exynos_get_pmu_regmap_by_phandle(np, NULL); return ERR_PTR(-ENODEV); } EXPORT_SYMBOL_GPL(exynos_get_pmu_regmap); +/** + * exynos_get_pmu_regmap_by_phandle() - Obtain pmureg regmap via phandle + * @np: Device node holding PMU phandle property + * @propname: Name of property holding phandle value + * + * Find the pmureg regmap previously configured in probe() and return regmap + * pointer. + * + * Return: A pointer to regmap if found or ERR_PTR error value. + */ +struct regmap *exynos_get_pmu_regmap_by_phandle(struct device_node *np, + const char *propname) +{ + struct exynos_pmu_context *ctx; + struct device_node *pmu_np; + struct device *dev; + + if (propname) + pmu_np = of_parse_phandle(np, propname, 0); + else + pmu_np = np; + + if (!pmu_np) + return ERR_PTR(-ENODEV); + + /* + * Determine if exynos-pmu device has probed and therefore regmap + * has been created and can be returned to the caller. Otherwise we + * return -EPROBE_DEFER. + */ + dev = driver_find_device_by_of_node(&exynos_pmu_driver.driver, + (void *)pmu_np); + + if (propname) + of_node_put(pmu_np); + + if (!dev) + return ERR_PTR(-EPROBE_DEFER); + + ctx = dev_get_drvdata(dev); + + return ctx->pmureg; +} +EXPORT_SYMBOL_GPL(exynos_get_pmu_regmap_by_phandle); + static int exynos_pmu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct regmap_config pmu_regmcfg; + struct regmap *regmap; + struct resource *res; int ret; pmu_base_addr = devm_platform_ioremap_resource(pdev, 0); @@ -137,9 +339,38 @@ static int exynos_pmu_probe(struct platform_device *pdev) GFP_KERNEL); if (!pmu_context) return -ENOMEM; - pmu_context->dev = dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + pmu_context->pmu_data = of_device_get_match_data(dev); + /* For SoCs that secure PMU register writes use custom regmap */ + if (pmu_context->pmu_data && pmu_context->pmu_data->pmu_secure) { + pmu_regmcfg = regmap_smccfg; + pmu_regmcfg.max_register = resource_size(res) - + pmu_regmcfg.reg_stride; + /* Need physical address for SMC call */ + regmap = devm_regmap_init(dev, NULL, + (void *)(uintptr_t)res->start, + &pmu_regmcfg); + } else { + /* All other SoCs use a MMIO regmap */ + pmu_regmcfg = regmap_mmiocfg; + pmu_regmcfg.max_register = resource_size(res) - + pmu_regmcfg.reg_stride; + regmap = devm_regmap_init_mmio(dev, pmu_base_addr, + &pmu_regmcfg); + } + + if (IS_ERR(regmap)) + return dev_err_probe(&pdev->dev, PTR_ERR(regmap), + "regmap init failed\n"); + + pmu_context->pmureg = regmap; + pmu_context->dev = dev; + if (pmu_context->pmu_data && pmu_context->pmu_data->pmu_init) pmu_context->pmu_data->pmu_init(); diff --git a/drivers/soc/samsung/exynos-pmu.h b/drivers/soc/samsung/exynos-pmu.h index 1c652ffd79b4..0a49a2c9a08e 100644 --- a/drivers/soc/samsung/exynos-pmu.h +++ b/drivers/soc/samsung/exynos-pmu.h @@ -21,6 +21,7 @@ struct exynos_pmu_conf { struct exynos_pmu_data { const struct exynos_pmu_conf *pmu_config; const struct exynos_pmu_conf *pmu_config_extra; + bool pmu_secure; void (*pmu_init)(void); void (*powerdown_conf)(enum sys_powerdown); diff --git a/include/linux/soc/samsung/exynos-pmu.h b/include/linux/soc/samsung/exynos-pmu.h index a4f5516cc956..2bd9d12d9a52 100644 --- a/include/linux/soc/samsung/exynos-pmu.h +++ b/include/linux/soc/samsung/exynos-pmu.h @@ -10,6 +10,7 @@ #define __LINUX_SOC_EXYNOS_PMU_H struct regmap; +struct device_node; enum sys_powerdown { SYS_AFTR, @@ -20,12 +21,20 @@ enum sys_powerdown { extern void exynos_sys_powerdown_conf(enum sys_powerdown mode); #ifdef CONFIG_EXYNOS_PMU -extern struct regmap *exynos_get_pmu_regmap(void); +struct regmap *exynos_get_pmu_regmap(void); +struct regmap *exynos_get_pmu_regmap_by_phandle(struct device_node *np, + const char *propname); #else static inline struct regmap *exynos_get_pmu_regmap(void) { return ERR_PTR(-ENODEV); } + +static inline struct regmap *exynos_get_pmu_regmap_by_phandle(struct device_node *np, + const char *propname) +{ + return ERR_PTR(-ENODEV); +} #endif #endif /* __LINUX_SOC_EXYNOS_PMU_H */ -- cgit v1.2.3 From 5f85c4d10ef46a5f457a9d337159ff620a0a2191 Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jan 2024 22:06:45 +0100 Subject: hwmon: (core) Add support for humidity min/max alarm Add min_alarm and max_alarm attributes for humidityX to support devices that can generate these alarms. Such attributes already exist for other magnitudes such as tempX. Tested with a ChipCap 2 temperature-humidity sensor. Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240130-topic-chipcap2-v6-2-260bea05cf9b@gmail.com Signed-off-by: Guenter Roeck --- drivers/hwmon/hwmon.c | 2 ++ include/linux/hwmon.h | 4 ++++ 2 files changed, 6 insertions(+) (limited to 'include/linux') diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 18705049ad61..3b259c425ab7 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -587,6 +587,8 @@ static const char * const hwmon_humidity_attr_templates[] = { [hwmon_humidity_fault] = "humidity%d_fault", [hwmon_humidity_rated_min] = "humidity%d_rated_min", [hwmon_humidity_rated_max] = "humidity%d_rated_max", + [hwmon_humidity_min_alarm] = "humidity%d_min_alarm", + [hwmon_humidity_max_alarm] = "humidity%d_max_alarm", }; static const char * const hwmon_fan_attr_templates[] = { diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index c7885fdce88f..edf96f249eb5 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -295,6 +295,8 @@ enum hwmon_humidity_attributes { hwmon_humidity_fault, hwmon_humidity_rated_min, hwmon_humidity_rated_max, + hwmon_humidity_min_alarm, + hwmon_humidity_max_alarm, }; #define HWMON_H_ENABLE BIT(hwmon_humidity_enable) @@ -308,6 +310,8 @@ enum hwmon_humidity_attributes { #define HWMON_H_FAULT BIT(hwmon_humidity_fault) #define HWMON_H_RATED_MIN BIT(hwmon_humidity_rated_min) #define HWMON_H_RATED_MAX BIT(hwmon_humidity_rated_max) +#define HWMON_H_MIN_ALARM BIT(hwmon_humidity_min_alarm) +#define HWMON_H_MAX_ALARM BIT(hwmon_humidity_max_alarm) enum hwmon_fan_attributes { hwmon_fan_enable, -- cgit v1.2.3 From 7e84c961b2eb062d2f47037dcca52dcd1d3615b5 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Tue, 19 Dec 2023 02:33:24 +0000 Subject: mtd: ubi: introduce pre-removal notification for UBI volumes Introduce a new notification type UBI_VOLUME_SHUTDOWN to inform users that a volume is just about to be removed. This is needed because users (such as the NVMEM subsystem) expect that at the time their removal function is called, the parenting device is still available (for removal of sysfs nodes, for example, in case of NVMEM which otherwise WARNs on volume removal). Signed-off-by: Daniel Golle Signed-off-by: Richard Weinberger --- drivers/mtd/ubi/build.c | 19 ++++++++++++++----- drivers/mtd/ubi/kapi.c | 2 +- drivers/mtd/ubi/ubi.h | 2 ++ drivers/mtd/ubi/vmt.c | 17 +++++++++++++++-- include/linux/mtd/ubi.h | 2 ++ 5 files changed, 34 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index 8c3f763e4ddb..a7e3a6246c0e 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -93,7 +93,7 @@ static struct ubi_device *ubi_devices[UBI_MAX_DEVICES]; /* Serializes UBI devices creations and removals */ DEFINE_MUTEX(ubi_devices_mutex); -/* Protects @ubi_devices and @ubi->ref_count */ +/* Protects @ubi_devices, @ubi->ref_count and @ubi->is_dead */ static DEFINE_SPINLOCK(ubi_devices_lock); /* "Show" method for files in '//class/ubi/' */ @@ -261,6 +261,9 @@ struct ubi_device *ubi_get_device(int ubi_num) spin_lock(&ubi_devices_lock); ubi = ubi_devices[ubi_num]; + if (ubi && ubi->is_dead) + ubi = NULL; + if (ubi) { ubi_assert(ubi->ref_count >= 0); ubi->ref_count += 1; @@ -298,7 +301,7 @@ struct ubi_device *ubi_get_by_major(int major) spin_lock(&ubi_devices_lock); for (i = 0; i < UBI_MAX_DEVICES; i++) { ubi = ubi_devices[i]; - if (ubi && MAJOR(ubi->cdev.dev) == major) { + if (ubi && !ubi->is_dead && MAJOR(ubi->cdev.dev) == major) { ubi_assert(ubi->ref_count >= 0); ubi->ref_count += 1; get_device(&ubi->dev); @@ -327,7 +330,7 @@ int ubi_major2num(int major) for (i = 0; i < UBI_MAX_DEVICES; i++) { struct ubi_device *ubi = ubi_devices[i]; - if (ubi && MAJOR(ubi->cdev.dev) == major) { + if (ubi && !ubi->is_dead && MAJOR(ubi->cdev.dev) == major) { ubi_num = ubi->ubi_num; break; } @@ -514,7 +517,7 @@ static void ubi_free_volumes_from(struct ubi_device *ubi, int from) int i; for (i = from; i < ubi->vtbl_slots + UBI_INT_VOL_COUNT; i++) { - if (!ubi->volumes[i]) + if (!ubi->volumes[i] || ubi->volumes[i]->is_dead) continue; ubi_eba_replace_table(ubi->volumes[i], NULL); ubi_fastmap_destroy_checkmap(ubi->volumes[i]); @@ -1099,7 +1102,6 @@ int ubi_detach_mtd_dev(int ubi_num, int anyway) return -EINVAL; spin_lock(&ubi_devices_lock); - put_device(&ubi->dev); ubi->ref_count -= 1; if (ubi->ref_count) { if (!anyway) { @@ -1110,6 +1112,13 @@ int ubi_detach_mtd_dev(int ubi_num, int anyway) ubi_err(ubi, "%s reference count %d, destroy anyway", ubi->ubi_name, ubi->ref_count); } + ubi->is_dead = true; + spin_unlock(&ubi_devices_lock); + + ubi_notify_all(ubi, UBI_VOLUME_SHUTDOWN, NULL); + + spin_lock(&ubi_devices_lock); + put_device(&ubi->dev); ubi_devices[ubi_num] = NULL; spin_unlock(&ubi_devices_lock); diff --git a/drivers/mtd/ubi/kapi.c b/drivers/mtd/ubi/kapi.c index fbf3a7fe2af7..f1ea8677467f 100644 --- a/drivers/mtd/ubi/kapi.c +++ b/drivers/mtd/ubi/kapi.c @@ -152,7 +152,7 @@ struct ubi_volume_desc *ubi_open_volume(int ubi_num, int vol_id, int mode) spin_lock(&ubi->volumes_lock); vol = ubi->volumes[vol_id]; - if (!vol) + if (!vol || vol->is_dead) goto out_unlock; err = -EBUSY; diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h index a588381c50ad..32009a24869e 100644 --- a/drivers/mtd/ubi/ubi.h +++ b/drivers/mtd/ubi/ubi.h @@ -337,6 +337,7 @@ struct ubi_volume { int writers; int exclusive; int metaonly; + bool is_dead; int reserved_pebs; int vol_type; @@ -561,6 +562,7 @@ struct ubi_device { spinlock_t volumes_lock; int ref_count; int image_seq; + bool is_dead; int rsvd_pebs; int avail_pebs; diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c index 990571287e84..eaf8328f6fc3 100644 --- a/drivers/mtd/ubi/vmt.c +++ b/drivers/mtd/ubi/vmt.c @@ -59,7 +59,7 @@ static ssize_t vol_attribute_show(struct device *dev, struct ubi_device *ubi = vol->ubi; spin_lock(&ubi->volumes_lock); - if (!ubi->volumes[vol->vol_id]) { + if (!ubi->volumes[vol->vol_id] || ubi->volumes[vol->vol_id]->is_dead) { spin_unlock(&ubi->volumes_lock); return -ENODEV; } @@ -189,7 +189,7 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req) /* Ensure that the name is unique */ for (i = 0; i < ubi->vtbl_slots; i++) - if (ubi->volumes[i] && + if (ubi->volumes[i] && !ubi->volumes[i]->is_dead && ubi->volumes[i]->name_len == req->name_len && !strcmp(ubi->volumes[i]->name, req->name)) { ubi_err(ubi, "volume \"%s\" exists (ID %d)", @@ -352,6 +352,19 @@ int ubi_remove_volume(struct ubi_volume_desc *desc, int no_vtbl) err = -EBUSY; goto out_unlock; } + + /* + * Mark volume as dead at this point to prevent that anyone + * can take a reference to the volume from now on. + * This is necessary as we have to release the spinlock before + * calling ubi_volume_notify. + */ + vol->is_dead = true; + spin_unlock(&ubi->volumes_lock); + + ubi_volume_notify(ubi, vol, UBI_VOLUME_SHUTDOWN); + + spin_lock(&ubi->volumes_lock); ubi->volumes[vol_id] = NULL; spin_unlock(&ubi->volumes_lock); diff --git a/include/linux/mtd/ubi.h b/include/linux/mtd/ubi.h index a529347fd75b..562f92504f2b 100644 --- a/include/linux/mtd/ubi.h +++ b/include/linux/mtd/ubi.h @@ -192,6 +192,7 @@ struct ubi_device_info { * or a volume was removed) * @UBI_VOLUME_RESIZED: a volume has been re-sized * @UBI_VOLUME_RENAMED: a volume has been re-named + * @UBI_VOLUME_SHUTDOWN: a volume is going to removed, shutdown users * @UBI_VOLUME_UPDATED: data has been written to a volume * * These constants define which type of event has happened when a volume @@ -202,6 +203,7 @@ enum { UBI_VOLUME_REMOVED, UBI_VOLUME_RESIZED, UBI_VOLUME_RENAMED, + UBI_VOLUME_SHUTDOWN, UBI_VOLUME_UPDATED, }; -- cgit v1.2.3 From 0dc5b8abfa03e8720cb341699e3ece194058bb03 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 20 Feb 2024 08:22:13 +0100 Subject: interconnect: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Acked-by: Konrad Dybcio Acked-by: Thierry Reding # Tegra Signed-off-by: Krzysztof Kozlowski Acked-by: Alim Akhtar # Samsung Link: https://lore.kernel.org/r/20240220072213.35779-1-krzysztof.kozlowski@linaro.org Signed-off-by: Georgi Djakov --- drivers/interconnect/core.c | 4 ++-- drivers/interconnect/qcom/icc-common.c | 3 ++- drivers/interconnect/qcom/icc-common.h | 3 ++- drivers/interconnect/samsung/exynos.c | 2 +- drivers/memory/tegra/mc.c | 2 +- drivers/memory/tegra/tegra124-emc.c | 2 +- drivers/memory/tegra/tegra124.c | 2 +- drivers/memory/tegra/tegra186-emc.c | 2 +- drivers/memory/tegra/tegra20-emc.c | 2 +- drivers/memory/tegra/tegra20.c | 2 +- drivers/memory/tegra/tegra30-emc.c | 2 +- drivers/memory/tegra/tegra30.c | 2 +- include/linux/interconnect-provider.h | 11 ++++++----- include/soc/tegra/mc.h | 7 ++++--- 14 files changed, 25 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c index 50bac2d79d9b..5d1010cafed8 100644 --- a/drivers/interconnect/core.c +++ b/drivers/interconnect/core.c @@ -343,7 +343,7 @@ EXPORT_SYMBOL_GPL(icc_std_aggregate); * an array of icc nodes specified in the icc_onecell_data struct when * registering the provider. */ -struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec, +struct icc_node *of_icc_xlate_onecell(const struct of_phandle_args *spec, void *data) { struct icc_onecell_data *icc_data = data; @@ -368,7 +368,7 @@ EXPORT_SYMBOL_GPL(of_icc_xlate_onecell); * Returns a valid pointer to struct icc_node_data on success or ERR_PTR() * on failure. */ -struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec) +struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec) { struct icc_node *node = ERR_PTR(-EPROBE_DEFER); struct icc_node_data *data = NULL; diff --git a/drivers/interconnect/qcom/icc-common.c b/drivers/interconnect/qcom/icc-common.c index f27f4fdc4531..9b9ee113f172 100644 --- a/drivers/interconnect/qcom/icc-common.c +++ b/drivers/interconnect/qcom/icc-common.c @@ -9,7 +9,8 @@ #include "icc-common.h" -struct icc_node_data *qcom_icc_xlate_extended(struct of_phandle_args *spec, void *data) +struct icc_node_data *qcom_icc_xlate_extended(const struct of_phandle_args *spec, + void *data) { struct icc_node_data *ndata; struct icc_node *node; diff --git a/drivers/interconnect/qcom/icc-common.h b/drivers/interconnect/qcom/icc-common.h index 33bb2c38dff3..21c39b163948 100644 --- a/drivers/interconnect/qcom/icc-common.h +++ b/drivers/interconnect/qcom/icc-common.h @@ -8,6 +8,7 @@ #include -struct icc_node_data *qcom_icc_xlate_extended(struct of_phandle_args *spec, void *data); +struct icc_node_data *qcom_icc_xlate_extended(const struct of_phandle_args *spec, + void *data); #endif diff --git a/drivers/interconnect/samsung/exynos.c b/drivers/interconnect/samsung/exynos.c index 1ba14cb45d5a..c9e5361e17c5 100644 --- a/drivers/interconnect/samsung/exynos.c +++ b/drivers/interconnect/samsung/exynos.c @@ -82,7 +82,7 @@ static int exynos_generic_icc_set(struct icc_node *src, struct icc_node *dst) return 0; } -static struct icc_node *exynos_generic_icc_xlate(struct of_phandle_args *spec, +static struct icc_node *exynos_generic_icc_xlate(const struct of_phandle_args *spec, void *data) { struct exynos_icc_priv *priv = data; diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c index a083921a8968..224b488794e5 100644 --- a/drivers/memory/tegra/mc.c +++ b/drivers/memory/tegra/mc.c @@ -755,7 +755,7 @@ const char *const tegra_mc_error_names[8] = { [6] = "SMMU translation error", }; -struct icc_node *tegra_mc_icc_xlate(struct of_phandle_args *spec, void *data) +struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data) { struct tegra_mc *mc = icc_provider_to_tegra_mc(data); struct icc_node *node; diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c index 00ed2b6a0d1b..47c0c19e13fd 100644 --- a/drivers/memory/tegra/tegra124-emc.c +++ b/drivers/memory/tegra/tegra124-emc.c @@ -1285,7 +1285,7 @@ to_tegra_emc_provider(struct icc_provider *provider) } static struct icc_node_data * -emc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) { struct icc_provider *provider = data; struct icc_node_data *ndata; diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c index 470b7dbab2c2..9d7393e19f12 100644 --- a/drivers/memory/tegra/tegra124.c +++ b/drivers/memory/tegra/tegra124.c @@ -1170,7 +1170,7 @@ static int tegra124_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw, } static struct icc_node_data * -tegra124_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +tegra124_mc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) { struct tegra_mc *mc = icc_provider_to_tegra_mc(data); const struct tegra_mc_client *client; diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c index fcd4aea48bda..57d9ae12fcfe 100644 --- a/drivers/memory/tegra/tegra186-emc.c +++ b/drivers/memory/tegra/tegra186-emc.c @@ -236,7 +236,7 @@ static int tegra_emc_icc_set_bw(struct icc_node *src, struct icc_node *dst) } static struct icc_node * -tegra_emc_of_icc_xlate(struct of_phandle_args *spec, void *data) +tegra_emc_of_icc_xlate(const struct of_phandle_args *spec, void *data) { struct icc_provider *provider = data; struct icc_node *node; diff --git a/drivers/memory/tegra/tegra20-emc.c b/drivers/memory/tegra/tegra20-emc.c index fd595c851a27..97cf59523b0b 100644 --- a/drivers/memory/tegra/tegra20-emc.c +++ b/drivers/memory/tegra/tegra20-emc.c @@ -950,7 +950,7 @@ to_tegra_emc_provider(struct icc_provider *provider) } static struct icc_node_data * -emc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) { struct icc_provider *provider = data; struct icc_node_data *ndata; diff --git a/drivers/memory/tegra/tegra20.c b/drivers/memory/tegra/tegra20.c index aa4b97d5e732..a3022e715dee 100644 --- a/drivers/memory/tegra/tegra20.c +++ b/drivers/memory/tegra/tegra20.c @@ -390,7 +390,7 @@ static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw, } static struct icc_node_data * -tegra20_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +tegra20_mc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) { struct tegra_mc *mc = icc_provider_to_tegra_mc(data); unsigned int i, idx = spec->args[0]; diff --git a/drivers/memory/tegra/tegra30-emc.c b/drivers/memory/tegra/tegra30-emc.c index 9eae25c57ec6..d7b0a23c2d7d 100644 --- a/drivers/memory/tegra/tegra30-emc.c +++ b/drivers/memory/tegra/tegra30-emc.c @@ -1468,7 +1468,7 @@ to_tegra_emc_provider(struct icc_provider *provider) } static struct icc_node_data * -emc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) { struct icc_provider *provider = data; struct icc_node_data *ndata; diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c index 06f8b35e0a14..d3e685c8431f 100644 --- a/drivers/memory/tegra/tegra30.c +++ b/drivers/memory/tegra/tegra30.c @@ -1332,7 +1332,7 @@ static int tegra30_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw, } static struct icc_node_data * -tegra30_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data) +tegra30_mc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) { struct tegra_mc *mc = icc_provider_to_tegra_mc(data); const struct tegra_mc_client *client; diff --git a/include/linux/interconnect-provider.h b/include/linux/interconnect-provider.h index 7ba183f221f1..f5aef8784692 100644 --- a/include/linux/interconnect-provider.h +++ b/include/linux/interconnect-provider.h @@ -36,7 +36,7 @@ struct icc_onecell_data { struct icc_node *nodes[] __counted_by(num_nodes); }; -struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec, +struct icc_node *of_icc_xlate_onecell(const struct of_phandle_args *spec, void *data); /** @@ -65,8 +65,9 @@ struct icc_provider { u32 peak_bw, u32 *agg_avg, u32 *agg_peak); void (*pre_aggregate)(struct icc_node *node); int (*get_bw)(struct icc_node *node, u32 *avg, u32 *peak); - struct icc_node* (*xlate)(struct of_phandle_args *spec, void *data); - struct icc_node_data* (*xlate_extended)(struct of_phandle_args *spec, void *data); + struct icc_node* (*xlate)(const struct of_phandle_args *spec, void *data); + struct icc_node_data* (*xlate_extended)(const struct of_phandle_args *spec, + void *data); struct device *dev; int users; bool inter_set; @@ -124,7 +125,7 @@ int icc_nodes_remove(struct icc_provider *provider); void icc_provider_init(struct icc_provider *provider); int icc_provider_register(struct icc_provider *provider); void icc_provider_deregister(struct icc_provider *provider); -struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec); +struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec); void icc_sync_state(struct device *dev); #else @@ -171,7 +172,7 @@ static inline int icc_provider_register(struct icc_provider *provider) static inline void icc_provider_deregister(struct icc_provider *provider) { } -static inline struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec) +static inline struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec) { return ERR_PTR(-ENOTSUPP); } diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h index af1d73a7f0cd..6ee4c59db620 100644 --- a/include/soc/tegra/mc.h +++ b/include/soc/tegra/mc.h @@ -146,13 +146,14 @@ struct tegra_mc_icc_ops { int (*set)(struct icc_node *src, struct icc_node *dst); int (*aggregate)(struct icc_node *node, u32 tag, u32 avg_bw, u32 peak_bw, u32 *agg_avg, u32 *agg_peak); - struct icc_node* (*xlate)(struct of_phandle_args *spec, void *data); - struct icc_node_data *(*xlate_extended)(struct of_phandle_args *spec, + struct icc_node* (*xlate)(const struct of_phandle_args *spec, void *data); + struct icc_node_data *(*xlate_extended)(const struct of_phandle_args *spec, void *data); int (*get_bw)(struct icc_node *node, u32 *avg, u32 *peak); }; -struct icc_node *tegra_mc_icc_xlate(struct of_phandle_args *spec, void *data); +struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, + void *data); extern const struct tegra_mc_icc_ops tegra_mc_icc_ops; struct tegra_mc_ops { -- cgit v1.2.3 From b8a730836c6b1788ca2fbd6bcc2ac99e97ef7de9 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 19 Feb 2024 09:45:50 -0300 Subject: thunderbolt: Constify the struct device_type usage Since commit aed65af1cc2f ("drivers: make device_type const"), the driver core can properly handle constant struct device_type. Move the tb_domain_type, tb_retimer_type, tb_switch_type, usb4_port_device_type, tb_service_type and tb_xdomain_type variables to be constant structures as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Signed-off-by: Mika Westerberg --- drivers/thunderbolt/domain.c | 2 +- drivers/thunderbolt/retimer.c | 2 +- drivers/thunderbolt/switch.c | 2 +- drivers/thunderbolt/tb.h | 8 ++++---- drivers/thunderbolt/usb4_port.c | 2 +- drivers/thunderbolt/xdomain.c | 4 ++-- include/linux/thunderbolt.h | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 1d915a6c61a4..0023017299f7 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -326,7 +326,7 @@ static void tb_domain_release(struct device *dev) kfree(tb); } -struct device_type tb_domain_type = { +const struct device_type tb_domain_type = { .name = "thunderbolt_domain", .release = tb_domain_release, }; diff --git a/drivers/thunderbolt/retimer.c b/drivers/thunderbolt/retimer.c index d49d6628dbf2..6bb49bdcd6c1 100644 --- a/drivers/thunderbolt/retimer.c +++ b/drivers/thunderbolt/retimer.c @@ -356,7 +356,7 @@ static void tb_retimer_release(struct device *dev) kfree(rt); } -struct device_type tb_retimer_type = { +const struct device_type tb_retimer_type = { .name = "thunderbolt_retimer", .groups = retimer_groups, .release = tb_retimer_release, diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index bca6f28c553b..5a617ea285a7 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -2327,7 +2327,7 @@ static const struct dev_pm_ops tb_switch_pm_ops = { NULL) }; -struct device_type tb_switch_type = { +const struct device_type tb_switch_type = { .name = "thunderbolt_device", .release = tb_switch_release, .uevent = tb_switch_uevent, diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 8e87d1a0005c..feed8ecaf712 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -749,10 +749,10 @@ static inline int tb_port_write(struct tb_port *port, const void *buffer, struct tb *icm_probe(struct tb_nhi *nhi); struct tb *tb_probe(struct tb_nhi *nhi); -extern struct device_type tb_domain_type; -extern struct device_type tb_retimer_type; -extern struct device_type tb_switch_type; -extern struct device_type usb4_port_device_type; +extern const struct device_type tb_domain_type; +extern const struct device_type tb_retimer_type; +extern const struct device_type tb_switch_type; +extern const struct device_type usb4_port_device_type; int tb_domain_init(void); void tb_domain_exit(void); diff --git a/drivers/thunderbolt/usb4_port.c b/drivers/thunderbolt/usb4_port.c index e355bfd6343f..5150879888ca 100644 --- a/drivers/thunderbolt/usb4_port.c +++ b/drivers/thunderbolt/usb4_port.c @@ -243,7 +243,7 @@ static void usb4_port_device_release(struct device *dev) kfree(usb4); } -struct device_type usb4_port_device_type = { +const struct device_type usb4_port_device_type = { .name = "usb4_port", .groups = usb4_port_device_groups, .release = usb4_port_device_release, diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c index b48df88981bd..940ae97987ff 100644 --- a/drivers/thunderbolt/xdomain.c +++ b/drivers/thunderbolt/xdomain.c @@ -1002,7 +1002,7 @@ static void tb_service_release(struct device *dev) kfree(svc); } -struct device_type tb_service_type = { +const struct device_type tb_service_type = { .name = "thunderbolt_service", .groups = tb_service_attr_groups, .uevent = tb_service_uevent, @@ -1893,7 +1893,7 @@ static const struct dev_pm_ops tb_xdomain_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(tb_xdomain_suspend, tb_xdomain_resume) }; -struct device_type tb_xdomain_type = { +const struct device_type tb_xdomain_type = { .name = "thunderbolt_xdomain", .release = tb_xdomain_release, .pm = &tb_xdomain_pm_ops, diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 2c835e5c41f6..4338ea9ac4fd 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -87,8 +87,8 @@ struct tb { }; extern const struct bus_type tb_bus_type; -extern struct device_type tb_service_type; -extern struct device_type tb_xdomain_type; +extern const struct device_type tb_service_type; +extern const struct device_type tb_xdomain_type; #define TB_LINKS_PER_PHY_PORT 2 -- cgit v1.2.3 From cdeeaaba174886aa6c1ff4c0c5449c5066dbe82f Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 23 Feb 2024 19:27:17 +0100 Subject: mm, slab: deprecate SLAB_MEM_SPREAD flag The SLAB_MEM_SPREAD flag used to be implemented in SLAB, which was removed. SLUB instead relies on the page allocator's NUMA policies. Change the flag's value to 0 to free up the value it had, and mark it for full removal once all users are gone. Reported-by: Steven Rostedt Closes: https://lore.kernel.org/all/20240131172027.10f64405@gandalf.local.home/ Reviewed-and-tested-by: Xiongwei Song Reviewed-by: Chengming Zhou Reviewed-by: Roman Gushchin Acked-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 5 +++-- mm/slab.h | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index b5f5ee8308d0..b1675ff6b904 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -96,8 +96,6 @@ */ /* Defer freeing slabs to RCU */ #define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) -/* Spread some memory over cpuset */ -#define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) /* Trace allocations and frees */ #define SLAB_TRACE ((slab_flags_t __force)0x00200000U) @@ -164,6 +162,9 @@ #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ +/* Obsolete unused flag, to be removed */ +#define SLAB_MEM_SPREAD ((slab_flags_t __force)0U) + /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * diff --git a/mm/slab.h b/mm/slab.h index 54deeb0428c6..f4534eefb35d 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -469,7 +469,6 @@ static inline bool is_kmalloc_cache(struct kmem_cache *s) SLAB_STORE_USER | \ SLAB_TRACE | \ SLAB_CONSISTENCY_CHECKS | \ - SLAB_MEM_SPREAD | \ SLAB_NOLEAKTRACE | \ SLAB_RECLAIM_ACCOUNT | \ SLAB_TEMPORARY | \ -- cgit v1.2.3 From cc61eb851c9ae38546d7df6076fd883d3dbc322d Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 23 Feb 2024 19:27:18 +0100 Subject: mm, slab: use an enum to define SLAB_ cache creation flags The values of SLAB_ cache creation flags are defined by hand, which is tedious and error-prone. Use an enum to assign the bit number and a __SLAB_FLAG_BIT() macro to #define the final flags. This renumbers the flag values, which is OK as they are only used internally. Also define a __SLAB_FLAG_UNUSED macro to assign value to flags disabled by their respective config options in a unified and sparse-friendly way. Reviewed-and-tested-by: Xiongwei Song Reviewed-by: Chengming Zhou Reviewed-by: Roman Gushchin Acked-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 94 +++++++++++++++++++++++++++++++++++++--------------- mm/slub.c | 6 ++-- 2 files changed, 70 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index b1675ff6b904..f6323763cd61 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -21,29 +21,69 @@ #include #include +enum _slab_flag_bits { + _SLAB_CONSISTENCY_CHECKS, + _SLAB_RED_ZONE, + _SLAB_POISON, + _SLAB_KMALLOC, + _SLAB_HWCACHE_ALIGN, + _SLAB_CACHE_DMA, + _SLAB_CACHE_DMA32, + _SLAB_STORE_USER, + _SLAB_PANIC, + _SLAB_TYPESAFE_BY_RCU, + _SLAB_TRACE, +#ifdef CONFIG_DEBUG_OBJECTS + _SLAB_DEBUG_OBJECTS, +#endif + _SLAB_NOLEAKTRACE, + _SLAB_NO_MERGE, +#ifdef CONFIG_FAILSLAB + _SLAB_FAILSLAB, +#endif +#ifdef CONFIG_MEMCG_KMEM + _SLAB_ACCOUNT, +#endif +#ifdef CONFIG_KASAN_GENERIC + _SLAB_KASAN, +#endif + _SLAB_NO_USER_FLAGS, +#ifdef CONFIG_KFENCE + _SLAB_SKIP_KFENCE, +#endif +#ifndef CONFIG_SLUB_TINY + _SLAB_RECLAIM_ACCOUNT, +#endif + _SLAB_OBJECT_POISON, + _SLAB_CMPXCHG_DOUBLE, + _SLAB_FLAGS_LAST_BIT +}; + +#define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr))) +#define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U)) /* * Flags to pass to kmem_cache_create(). * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op */ /* DEBUG: Perform (expensive) checks on alloc/free */ -#define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) +#define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS) /* DEBUG: Red zone objs in a cache */ -#define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) +#define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE) /* DEBUG: Poison objects */ -#define SLAB_POISON ((slab_flags_t __force)0x00000800U) +#define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON) /* Indicate a kmalloc slab */ -#define SLAB_KMALLOC ((slab_flags_t __force)0x00001000U) +#define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC) /* Align objs on cache lines */ -#define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) +#define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN) /* Use GFP_DMA memory */ -#define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) +#define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA) /* Use GFP_DMA32 memory */ -#define SLAB_CACHE_DMA32 ((slab_flags_t __force)0x00008000U) +#define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32) /* DEBUG: Store the last owner for bug hunting */ -#define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) +#define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER) /* Panic if kmem_cache_create() fails */ -#define SLAB_PANIC ((slab_flags_t __force)0x00040000U) +#define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC) /* * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! * @@ -95,19 +135,19 @@ * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. */ /* Defer freeing slabs to RCU */ -#define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) +#define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU) /* Trace allocations and frees */ -#define SLAB_TRACE ((slab_flags_t __force)0x00200000U) +#define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE) /* Flag to prevent checks on free */ #ifdef CONFIG_DEBUG_OBJECTS -# define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) +# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS) #else -# define SLAB_DEBUG_OBJECTS 0 +# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED #endif /* Avoid kmemleak tracing */ -#define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) +#define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE) /* * Prevent merging with compatible kmem caches. This flag should be used @@ -119,25 +159,25 @@ * - performance critical caches, should be very rare and consulted with slab * maintainers, and not used together with CONFIG_SLUB_TINY */ -#define SLAB_NO_MERGE ((slab_flags_t __force)0x01000000U) +#define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE) /* Fault injection mark */ #ifdef CONFIG_FAILSLAB -# define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) +# define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB) #else -# define SLAB_FAILSLAB 0 +# define SLAB_FAILSLAB __SLAB_FLAG_UNUSED #endif /* Account to memcg */ #ifdef CONFIG_MEMCG_KMEM -# define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) +# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) #else -# define SLAB_ACCOUNT 0 +# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED #endif #ifdef CONFIG_KASAN_GENERIC -#define SLAB_KASAN ((slab_flags_t __force)0x08000000U) +#define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN) #else -#define SLAB_KASAN 0 +#define SLAB_KASAN __SLAB_FLAG_UNUSED #endif /* @@ -145,25 +185,25 @@ * Intended for caches created for self-tests so they have only flags * specified in the code and other flags are ignored. */ -#define SLAB_NO_USER_FLAGS ((slab_flags_t __force)0x10000000U) +#define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS) #ifdef CONFIG_KFENCE -#define SLAB_SKIP_KFENCE ((slab_flags_t __force)0x20000000U) +#define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE) #else -#define SLAB_SKIP_KFENCE 0 +#define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED #endif /* The following flags affect the page allocator grouping pages by mobility */ /* Objects are reclaimable */ #ifndef CONFIG_SLUB_TINY -#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) +#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT) #else -#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0) +#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ /* Obsolete unused flag, to be removed */ -#define SLAB_MEM_SPREAD ((slab_flags_t __force)0U) +#define SLAB_MEM_SPREAD __SLAB_FLAG_UNUSED /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. diff --git a/mm/slub.c b/mm/slub.c index 2ef88bbf56a3..2934ef5f3cff 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -306,13 +306,13 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) /* Internal SLUB flags */ /* Poison object */ -#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U) +#define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON) /* Use cmpxchg_double */ #ifdef system_has_freelist_aba -#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U) +#define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE) #else -#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0U) +#define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED #endif /* -- cgit v1.2.3 From 96d8dbb6f65041b670a79e8ae76f67cc11dee203 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 23 Feb 2024 19:27:19 +0100 Subject: mm, slab, kasan: replace kasan_never_merge() with SLAB_NO_MERGE The SLAB_KASAN flag prevents merging of caches in some configurations, which is handled in a rather complicated way via kasan_never_merge(). Since we now have a generic SLAB_NO_MERGE flag, we can instead use it for KASAN caches in addition to SLAB_KASAN in those configurations, and simplify the SLAB_NEVER_MERGE handling. Tested-by: Xiongwei Song Reviewed-by: Chengming Zhou Reviewed-by: Andrey Konovalov Tested-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/kasan.h | 6 ------ mm/kasan/generic.c | 22 ++++++---------------- mm/slab_common.c | 2 +- 3 files changed, 7 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kasan.h b/include/linux/kasan.h index dbb06d789e74..70d6a8f6e25d 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -429,7 +429,6 @@ struct kasan_cache { }; size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object); -slab_flags_t kasan_never_merge(void); void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags); @@ -446,11 +445,6 @@ static inline size_t kasan_metadata_size(struct kmem_cache *cache, { return 0; } -/* And thus nothing prevents cache merging. */ -static inline slab_flags_t kasan_never_merge(void) -{ - return 0; -} /* And no cache-related metadata initialization is required. */ static inline void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index df6627f62402..27297dc4a55b 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -334,14 +334,6 @@ DEFINE_ASAN_SET_SHADOW(f3); DEFINE_ASAN_SET_SHADOW(f5); DEFINE_ASAN_SET_SHADOW(f8); -/* Only allow cache merging when no per-object metadata is present. */ -slab_flags_t kasan_never_merge(void) -{ - if (!kasan_requires_meta()) - return 0; - return SLAB_KASAN; -} - /* * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. * For larger allocations larger redzones are used. @@ -370,15 +362,13 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, return; /* - * SLAB_KASAN is used to mark caches that are sanitized by KASAN - * and that thus have per-object metadata. - * Currently this flag is used in two places: - * 1. In slab_ksize() to account for per-object metadata when - * calculating the size of the accessible memory within the object. - * 2. In slab_common.c via kasan_never_merge() to prevent merging of - * caches with per-object metadata. + * SLAB_KASAN is used to mark caches that are sanitized by KASAN and + * that thus have per-object metadata. Currently, this flag is used in + * slab_ksize() to account for per-object metadata when calculating the + * size of the accessible memory within the object. Additionally, we use + * SLAB_NO_MERGE to prevent merging of caches with per-object metadata. */ - *flags |= SLAB_KASAN; + *flags |= SLAB_KASAN | SLAB_NO_MERGE; ok_size = *size; diff --git a/mm/slab_common.c b/mm/slab_common.c index 238293b1dbe1..7cfa2f1ce655 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -50,7 +50,7 @@ static DECLARE_WORK(slab_caches_to_rcu_destroy_work, */ #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ - SLAB_FAILSLAB | SLAB_NO_MERGE | kasan_never_merge()) + SLAB_FAILSLAB | SLAB_NO_MERGE) #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ SLAB_CACHE_DMA32 | SLAB_ACCOUNT) -- cgit v1.2.3 From f1d00496a15bcdfe54bdb54cf1e7f207d6ad9b79 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:13 +0100 Subject: mtd: lpc32xx: use typedef for dma_filter_fn Use existing typedef for dma_filter_fn to avoid duplicating type definition. Signed-off-by: Krzysztof Kozlowski Acked-by: Vladimir Zapolskiy Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20240208202113.630190-1-krzysztof.kozlowski@linaro.org --- include/linux/mtd/lpc32xx_mlc.h | 2 +- include/linux/mtd/lpc32xx_slc.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mtd/lpc32xx_mlc.h b/include/linux/mtd/lpc32xx_mlc.h index d168c628c0d5..35e971be0950 100644 --- a/include/linux/mtd/lpc32xx_mlc.h +++ b/include/linux/mtd/lpc32xx_mlc.h @@ -11,7 +11,7 @@ #include struct lpc32xx_mlc_platform_data { - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); + dma_filter_fn dma_filter; }; #endif /* __LINUX_MTD_LPC32XX_MLC_H */ diff --git a/include/linux/mtd/lpc32xx_slc.h b/include/linux/mtd/lpc32xx_slc.h index cf54a9f80460..a044b806566b 100644 --- a/include/linux/mtd/lpc32xx_slc.h +++ b/include/linux/mtd/lpc32xx_slc.h @@ -11,7 +11,7 @@ #include struct lpc32xx_slc_platform_data { - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); + dma_filter_fn dma_filter; }; #endif /* __LINUX_MTD_LPC32XX_SLC_H */ -- cgit v1.2.3 From 081064cc103929c8a0d7d5fd057110c739c62ee1 Mon Sep 17 00:00:00 2001 From: Marcel Hamer Date: Fri, 16 Feb 2024 16:50:22 +0100 Subject: mtd: fix minor comment typo for struct mtd_master Minor typo in the suspend description. Signed-off-by: Marcel Hamer Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20240216155022.79371-1-marcel.hamer@windriver.com --- include/linux/mtd/mtd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mtd/mtd.h b/include/linux/mtd/mtd.h index 914a9f974baa..8d10d9d2e830 100644 --- a/include/linux/mtd/mtd.h +++ b/include/linux/mtd/mtd.h @@ -223,7 +223,7 @@ struct mtd_part { * @partitions_lock: lock protecting accesses to the partition list. Protects * not only the master partition list, but also all * sub-partitions. - * @suspended: et to 1 when the device is suspended, 0 otherwise + * @suspended: set to 1 when the device is suspended, 0 otherwise * * This struct is embedded in mtd_info and contains master-specific * properties/fields. The master is the root MTD device from the MTD partition -- cgit v1.2.3 From 4a5917cd504c7afd5e9de7166eb710687a9b026f Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 13 Feb 2024 12:48:52 +0200 Subject: clk: ti: Improve clksel clock bit parsing for reg property Because of legacy reasons, the TI clksel composite clocks can have overlapping reg properties, and use a custom ti,bit-shift property. For the clksel clocks we can start using of the standard reg property instead of the custom ti,bit-shift property. To do this, let's add a ti_clk_get_legacy_bit_shift() helper, and make ti_clk_get_reg_addr() populate the clock bit offset. This makes it possible to update the devicetree files to use the reg property one clock at a time. Acked-by: Stephen Boyd Signed-off-by: Tony Lindgren --- drivers/clk/ti/apll.c | 11 +++------ drivers/clk/ti/clk.c | 57 ++++++++++++++++++++++++++++++++++++++++------ drivers/clk/ti/clock.h | 1 + drivers/clk/ti/divider.c | 5 +--- drivers/clk/ti/gate.c | 9 ++------ drivers/clk/ti/interface.c | 4 +--- drivers/clk/ti/mux.c | 6 ++--- include/linux/clk/ti.h | 3 +++ 8 files changed, 63 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/drivers/clk/ti/apll.c b/drivers/clk/ti/apll.c index 93183287c58d..43514e6f3b78 100644 --- a/drivers/clk/ti/apll.c +++ b/drivers/clk/ti/apll.c @@ -376,14 +376,9 @@ static void __init of_omap2_apll_setup(struct device_node *node) } clk_hw->fixed_rate = val; - if (of_property_read_u32(node, "ti,bit-shift", &val)) { - pr_err("%pOFn missing bit-shift\n", node); - goto cleanup; - } - - clk_hw->enable_bit = val; - ad->enable_mask = 0x3 << val; - ad->autoidle_mask = 0x3 << val; + clk_hw->enable_bit = ti_clk_get_legacy_bit_shift(node); + ad->enable_mask = 0x3 << clk_hw->enable_bit; + ad->autoidle_mask = 0x3 << clk_hw->enable_bit; if (of_property_read_u32(node, "ti,idlest-shift", &val)) { pr_err("%pOFn missing idlest-shift\n", node); diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c index 53173bb250da..f2117fef7c7d 100644 --- a/drivers/clk/ti/clk.c +++ b/drivers/clk/ti/clk.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -307,8 +308,9 @@ int __init ti_clk_retry_init(struct device_node *node, void *user, int ti_clk_get_reg_addr(struct device_node *node, int index, struct clk_omap_reg *reg) { - u32 val; - int i; + u32 clksel_addr, val; + bool is_clksel = false; + int i, err; for (i = 0; i < CLK_MAX_MEMMAPS; i++) { if (clocks_node_ptr[i] == node->parent) @@ -324,21 +326,62 @@ int ti_clk_get_reg_addr(struct device_node *node, int index, reg->index = i; - if (of_property_read_u32_index(node, "reg", index, &val)) { - if (of_property_read_u32_index(node->parent, "reg", - index, &val)) { - pr_err("%pOFn or parent must have reg[%d]!\n", - node, index); + if (of_device_is_compatible(node->parent, "ti,clksel")) { + err = of_property_read_u32_index(node->parent, "reg", index, &clksel_addr); + if (err) { + pr_err("%pOFn parent clksel must have reg[%d]!\n", node, index); return -EINVAL; } + is_clksel = true; + } + + err = of_property_read_u32_index(node, "reg", index, &val); + if (err && is_clksel) { + /* Legacy clksel with no reg and a possible ti,bit-shift property */ + reg->offset = clksel_addr; + reg->bit = ti_clk_get_legacy_bit_shift(node); + reg->ptr = NULL; + + return 0; } + /* Updated clksel clock with a proper reg property */ + if (is_clksel) { + reg->offset = clksel_addr; + reg->bit = val; + reg->ptr = NULL; + return 0; + } + + /* Other clocks that may or may not have ti,bit-shift property */ reg->offset = val; + reg->bit = ti_clk_get_legacy_bit_shift(node); reg->ptr = NULL; return 0; } +/** + * ti_clk_get_legacy_bit_shift - get bit shift for a clock register + * @node: device node for the clock + * + * Gets the clock register bit shift using the legacy ti,bit-shift + * property. Only needed for legacy clock, and can be eventually + * dropped once all the composite clocks use a clksel node with a + * proper reg property. + */ +int ti_clk_get_legacy_bit_shift(struct device_node *node) +{ + int err; + u32 val; + + err = of_property_read_u32(node, "ti,bit-shift", &val); + if (!err && in_range(val, 0, 32)) + return val; + + return 0; +} + void ti_clk_latch(struct clk_omap_reg *reg, s8 shift) { u32 latch; diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h index 16a9f7c2280a..2de7acea1ea0 100644 --- a/drivers/clk/ti/clock.h +++ b/drivers/clk/ti/clock.h @@ -216,6 +216,7 @@ int ti_clk_parse_divider_data(int *div_table, int num_dividers, int max_div, int ti_clk_get_reg_addr(struct device_node *node, int index, struct clk_omap_reg *reg); +int ti_clk_get_legacy_bit_shift(struct device_node *node); void ti_dt_clocks_register(struct ti_dt_clk *oclks); int ti_clk_retry_init(struct device_node *node, void *user, ti_of_clk_init_cb_t func); diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index 5d5bb123ba94..ade99ab6cfa9 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -477,10 +477,7 @@ static int __init ti_clk_divider_populate(struct device_node *node, if (ret) return ret; - if (!of_property_read_u32(node, "ti,bit-shift", &val)) - div->shift = val; - else - div->shift = 0; + div->shift = div->reg.bit; if (!of_property_read_u32(node, "ti,latch-bit", &val)) div->latch = val; diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c index 8e477d50d0fd..a9febd6356b8 100644 --- a/drivers/clk/ti/gate.c +++ b/drivers/clk/ti/gate.c @@ -132,7 +132,6 @@ static void __init _of_ti_gate_clk_setup(struct device_node *node, struct clk_omap_reg reg; const char *name; u8 enable_bit = 0; - u32 val; u32 flags = 0; u8 clk_gate_flags = 0; @@ -140,8 +139,7 @@ static void __init _of_ti_gate_clk_setup(struct device_node *node, if (ti_clk_get_reg_addr(node, 0, ®)) return; - if (!of_property_read_u32(node, "ti,bit-shift", &val)) - enable_bit = val; + enable_bit = reg.bit; } if (of_clk_get_parent_count(node) != 1) { @@ -170,7 +168,6 @@ _of_ti_composite_gate_clk_setup(struct device_node *node, const struct clk_hw_omap_ops *hw_ops) { struct clk_hw_omap *gate; - u32 val = 0; gate = kzalloc(sizeof(*gate), GFP_KERNEL); if (!gate) @@ -179,9 +176,7 @@ _of_ti_composite_gate_clk_setup(struct device_node *node, if (ti_clk_get_reg_addr(node, 0, &gate->enable_reg)) goto cleanup; - of_property_read_u32(node, "ti,bit-shift", &val); - - gate->enable_bit = val; + gate->enable_bit = gate->enable_reg.bit; gate->ops = hw_ops; if (!ti_clk_add_component(node, &gate->hw, CLK_COMPONENT_TYPE_GATE)) diff --git a/drivers/clk/ti/interface.c b/drivers/clk/ti/interface.c index 172301c646f8..3eb35c87c0ed 100644 --- a/drivers/clk/ti/interface.c +++ b/drivers/clk/ti/interface.c @@ -66,13 +66,11 @@ static void __init _of_ti_interface_clk_setup(struct device_node *node, struct clk_omap_reg reg; u8 enable_bit = 0; const char *name; - u32 val; if (ti_clk_get_reg_addr(node, 0, ®)) return; - if (!of_property_read_u32(node, "ti,bit-shift", &val)) - enable_bit = val; + enable_bit = reg.bit; parent_name = of_clk_get_parent_name(node, 0); if (!parent_name) { diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index 1ebafa386be6..216d85d6aac6 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -189,7 +189,7 @@ static void of_mux_clk_setup(struct device_node *node) if (ti_clk_get_reg_addr(node, 0, ®)) goto cleanup; - of_property_read_u32(node, "ti,bit-shift", &shift); + shift = reg.bit; of_property_read_u32(node, "ti,latch-bit", &latch); @@ -252,7 +252,6 @@ static void __init of_ti_composite_mux_clk_setup(struct device_node *node) { struct clk_omap_mux *mux; unsigned int num_parents; - u32 val; mux = kzalloc(sizeof(*mux), GFP_KERNEL); if (!mux) @@ -261,8 +260,7 @@ static void __init of_ti_composite_mux_clk_setup(struct device_node *node) if (ti_clk_get_reg_addr(node, 0, &mux->reg)) goto cleanup; - if (!of_property_read_u32(node, "ti,bit-shift", &val)) - mux->shift = val; + mux->shift = mux->reg.bit; if (of_property_read_bool(node, "ti,index-starts-at-one")) mux->flags |= CLK_MUX_INDEX_ONE; diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h index cbfcbf186ce3..e656f63efdce 100644 --- a/include/linux/clk/ti.h +++ b/include/linux/clk/ti.h @@ -13,11 +13,14 @@ /** * struct clk_omap_reg - OMAP register declaration * @offset: offset from the master IP module base address + * @bit: register bit offset * @index: index of the master IP module + * @flags: flags */ struct clk_omap_reg { void __iomem *ptr; u16 offset; + u8 bit; u8 index; u8 flags; }; -- cgit v1.2.3 From 8afc7a78d55de726b2747d7775c54def79509ec5 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 22 Feb 2024 10:50:10 +0000 Subject: ipv6: prepare inet6_fill_ifinfo() for RCU protection We want to use RCU protection instead of RTNL for inet6_fill_ifinfo(). Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 6 ++++-- net/core/dev.c | 4 ++-- net/ipv6/addrconf.c | 11 +++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index f07c8374f29c..09023e44db4e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4354,8 +4354,10 @@ static inline bool netif_testing(const struct net_device *dev) */ static inline bool netif_oper_up(const struct net_device *dev) { - return (dev->operstate == IF_OPER_UP || - dev->operstate == IF_OPER_UNKNOWN /* backward compat */); + unsigned int operstate = READ_ONCE(dev->operstate); + + return operstate == IF_OPER_UP || + operstate == IF_OPER_UNKNOWN /* backward compat */; } /** diff --git a/net/core/dev.c b/net/core/dev.c index 0628d8ff1ed9..275fd5259a4a 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -8632,12 +8632,12 @@ unsigned int dev_get_flags(const struct net_device *dev) { unsigned int flags; - flags = (dev->flags & ~(IFF_PROMISC | + flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | IFF_ALLMULTI | IFF_RUNNING | IFF_LOWER_UP | IFF_DORMANT)) | - (dev->gflags & (IFF_PROMISC | + (READ_ONCE(dev->gflags) & (IFF_PROMISC | IFF_ALLMULTI)); if (netif_running(dev)) { diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index a56dad307fe3..daa81556d118 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -6062,6 +6062,7 @@ static int inet6_fill_ifinfo(struct sk_buff *skb, struct inet6_dev *idev, struct net_device *dev = idev->dev; struct ifinfomsg *hdr; struct nlmsghdr *nlh; + int ifindex, iflink; void *protoinfo; nlh = nlmsg_put(skb, portid, seq, event, sizeof(*hdr), flags); @@ -6072,16 +6073,18 @@ static int inet6_fill_ifinfo(struct sk_buff *skb, struct inet6_dev *idev, hdr->ifi_family = AF_INET6; hdr->__ifi_pad = 0; hdr->ifi_type = dev->type; - hdr->ifi_index = dev->ifindex; + ifindex = READ_ONCE(dev->ifindex); + hdr->ifi_index = ifindex; hdr->ifi_flags = dev_get_flags(dev); hdr->ifi_change = 0; + iflink = dev_get_iflink(dev); if (nla_put_string(skb, IFLA_IFNAME, dev->name) || (dev->addr_len && nla_put(skb, IFLA_ADDRESS, dev->addr_len, dev->dev_addr)) || - nla_put_u32(skb, IFLA_MTU, dev->mtu) || - (dev->ifindex != dev_get_iflink(dev) && - nla_put_u32(skb, IFLA_LINK, dev_get_iflink(dev))) || + nla_put_u32(skb, IFLA_MTU, READ_ONCE(dev->mtu)) || + (ifindex != iflink && + nla_put_u32(skb, IFLA_LINK, iflink)) || nla_put_u8(skb, IFLA_OPERSTATE, netif_running(dev) ? READ_ONCE(dev->operstate) : IF_OPER_DOWN)) goto nla_put_failure; -- cgit v1.2.3 From 386520e0ecc01004d3a29c70c5a77d4bbf8a8420 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 22 Feb 2024 10:50:15 +0000 Subject: rtnetlink: add RTNL_FLAG_DUMP_UNLOCKED flag Similarly to RTNL_FLAG_DOIT_UNLOCKED, this new flag allows dump operations registered via rtnl_register() or rtnl_register_module() to opt-out from RTNL protection. Signed-off-by: Eric Dumazet Reviewed-by: Donald Hunter Signed-off-by: David S. Miller --- include/linux/netlink.h | 2 ++ include/net/rtnetlink.h | 1 + net/core/rtnetlink.c | 2 ++ net/netlink/af_netlink.c | 3 +++ 4 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netlink.h b/include/linux/netlink.h index 1a4445bf2ab9..5df7340d4dab 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -291,6 +291,7 @@ struct netlink_callback { u16 answer_flags; u32 min_dump_alloc; unsigned int prev_seq, seq; + int flags; bool strict_check; union { u8 ctx[48]; @@ -323,6 +324,7 @@ struct netlink_dump_control { void *data; struct module *module; u32 min_dump_alloc; + int flags; }; int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb, diff --git a/include/net/rtnetlink.h b/include/net/rtnetlink.h index 6506221c5fe3..3bfb80bad173 100644 --- a/include/net/rtnetlink.h +++ b/include/net/rtnetlink.h @@ -12,6 +12,7 @@ typedef int (*rtnl_dumpit_func)(struct sk_buff *, struct netlink_callback *); enum rtnl_link_flags { RTNL_FLAG_DOIT_UNLOCKED = BIT(0), RTNL_FLAG_BULK_DEL_SUPPORTED = BIT(1), + RTNL_FLAG_DUMP_UNLOCKED = BIT(2), }; enum rtnl_kinds { diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 060543fe7919..1b26dfa5668d 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -6532,6 +6532,7 @@ static int rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, } owner = link->owner; dumpit = link->dumpit; + flags = link->flags; if (type == RTM_GETLINK - RTM_BASE) min_dump_alloc = rtnl_calcit(skb, nlh); @@ -6549,6 +6550,7 @@ static int rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, .dump = dumpit, .min_dump_alloc = min_dump_alloc, .module = owner, + .flags = flags, }; err = netlink_dump_start(rtnl, skb, nlh, &c); /* netlink_dump_start() will keep a reference on diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 84cad7be6d43..be5792b638aa 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -2261,6 +2261,8 @@ static int netlink_dump(struct sock *sk, bool lock_taken) cb->extack = &extack; + if (cb->flags & RTNL_FLAG_DUMP_UNLOCKED) + extra_mutex = NULL; if (extra_mutex) mutex_lock(extra_mutex); nlk->dump_done_errno = cb->dump(skb, cb); @@ -2355,6 +2357,7 @@ int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb, cb->data = control->data; cb->module = control->module; cb->min_dump_alloc = control->min_dump_alloc; + cb->flags = control->flags; cb->skb = skb; cb->strict_check = nlk_test_bit(STRICT_CHK, NETLINK_CB(skb).sk); -- cgit v1.2.3 From 4d52f575e258c6f93f4180c21afda8634b0d2af5 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:24 +0100 Subject: regulator: max8973: Finalize switch to GPIO descriptors The dvs gpio was still using a legacy number passed from the platform data. There are no in-tree users of the platform data so just switch it to a gpio descriptor and obtain it in probe(), the device tree users will work just as fine with this. Drop the entirely unused enable_gpio from the platform data as well. The device tree bindings mentions this but the driver does not look for it and makes no use of it: it should probably be implemented properly in a separate patch. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-1-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/max8973-regulator.c | 36 ++++++++++------------------- include/linux/regulator/max8973-regulator.h | 6 ----- 2 files changed, 12 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c index 8d5193207552..f8bb6828feef 100644 --- a/drivers/regulator/max8973-regulator.c +++ b/drivers/regulator/max8973-regulator.c @@ -20,9 +20,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -102,7 +100,7 @@ struct max8973_chip { struct regulator_desc desc; struct regmap *regmap; bool enable_external_control; - int dvs_gpio; + struct gpio_desc *dvs_gpiod; int lru_index[MAX8973_MAX_VOUT_REG]; int curr_vout_val[MAX8973_MAX_VOUT_REG]; int curr_vout_reg; @@ -184,7 +182,7 @@ static int max8973_dcdc_set_voltage_sel(struct regulator_dev *rdev, * If gpios are available to select the VOUT register then least * recently used register for new configuration. */ - if (gpio_is_valid(max->dvs_gpio)) + if (max->dvs_gpiod) found = find_voltage_set_register(max, vsel, &vout_reg, &gpio_val); @@ -201,8 +199,8 @@ static int max8973_dcdc_set_voltage_sel(struct regulator_dev *rdev, } /* Select proper VOUT register vio gpios */ - if (gpio_is_valid(max->dvs_gpio)) { - gpio_set_value_cansleep(max->dvs_gpio, gpio_val & 0x1); + if (max->dvs_gpiod) { + gpiod_set_value_cansleep(max->dvs_gpiod, gpio_val & 0x1); max->curr_gpio_val = gpio_val; } return 0; @@ -531,7 +529,6 @@ static struct max8973_regulator_platform_data *max8973_parse_dt( pdata->enable_ext_control = of_property_read_bool(np, "maxim,externally-enable"); - pdata->dvs_gpio = of_get_named_gpio(np, "maxim,dvs-gpio", 0); ret = of_property_read_u32(np, "maxim,dvs-default-state", &pval); if (!ret) @@ -612,13 +609,17 @@ static int max8973_probe(struct i2c_client *client) return -EIO; } - if (pdata->dvs_gpio == -EPROBE_DEFER) - return -EPROBE_DEFER; - max = devm_kzalloc(&client->dev, sizeof(*max), GFP_KERNEL); if (!max) return -ENOMEM; + max->dvs_gpiod = devm_gpiod_get_optional(&client->dev, "maxim,dvs", + (pdata->dvs_def_state) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW); + if (IS_ERR(max->dvs_gpiod)) + return dev_err_probe(&client->dev, PTR_ERR(max->dvs_gpiod), + "failed to obtain dvs gpio\n"); + gpiod_set_consumer_name(max->dvs_gpiod, "max8973-dvs"); + max->regmap = devm_regmap_init_i2c(client, &max8973_regmap_config); if (IS_ERR(max->regmap)) { ret = PTR_ERR(max->regmap); @@ -663,7 +664,6 @@ static int max8973_probe(struct i2c_client *client) max->desc.ramp_delay_table = max8973_buck_ramp_table; max->desc.n_ramp_values = ARRAY_SIZE(max8973_buck_ramp_table); - max->dvs_gpio = (pdata->dvs_gpio) ? pdata->dvs_gpio : -EINVAL; max->enable_external_control = pdata->enable_ext_control; max->curr_gpio_val = pdata->dvs_def_state; max->curr_vout_reg = MAX8973_VOUT + pdata->dvs_def_state; @@ -671,21 +671,9 @@ static int max8973_probe(struct i2c_client *client) max->lru_index[0] = max->curr_vout_reg; - if (gpio_is_valid(max->dvs_gpio)) { - int gpio_flags; + if (max->dvs_gpiod) { int i; - gpio_flags = (pdata->dvs_def_state) ? - GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW; - ret = devm_gpio_request_one(&client->dev, max->dvs_gpio, - gpio_flags, "max8973-dvs"); - if (ret) { - dev_err(&client->dev, - "gpio_request for gpio %d failed, err = %d\n", - max->dvs_gpio, ret); - return ret; - } - /* * Initialize the lru index with vout_reg id * The index 0 will be most recently used and diff --git a/include/linux/regulator/max8973-regulator.h b/include/linux/regulator/max8973-regulator.h index 8313e7ed6aec..a225e9eeb30d 100644 --- a/include/linux/regulator/max8973-regulator.h +++ b/include/linux/regulator/max8973-regulator.h @@ -48,10 +48,6 @@ * control signal from EN input pin. If it is false then * voltage output will be enabled/disabled through EN bit of * device register. - * @enable_gpio: Enable GPIO. If EN pin is controlled through GPIO from host - * then GPIO number can be provided. If no GPIO controlled then - * it should be -1. - * @dvs_gpio: GPIO for dvs. It should be -1 if this is tied with fixed logic. * @dvs_def_state: Default state of dvs. 1 if it is high else 0. */ struct max8973_regulator_platform_data { @@ -59,8 +55,6 @@ struct max8973_regulator_platform_data { unsigned long control_flags; unsigned long junction_temp_warning; bool enable_ext_control; - int enable_gpio; - int dvs_gpio; unsigned dvs_def_state:1; }; -- cgit v1.2.3 From e450a2b3a335332d4a51fe10c9fff8150c6e2364 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:25 +0100 Subject: regulator: da9055: Fully convert to GPIO descriptors The DA9055 regulator was touched before, requireing enable GPIOs to be passed from pdata. As we have a device for each regulator, obtain the three gpios ren ("regulator enable"), rsel ("regulator select") and the ena ("enable") GPIO associated with the regulator enable directly from the device and cut down on the amount of GPIO numbers passed as platform data. The ren and rsel are just requested as inputs: these are actually handled by hardware. The ena gpios are driven actively by the regulator core. There are no in-tree users, but the regulators are instantiated from the (undocumed) device tree nodes with "dlg,da9055-regulator" as compatible, and by simply adding regulator-enable-gpios, regulator-select-gpios and enable-gpios to this DT node, all will work as before. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-2-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/da9055-regulator.c | 48 ++++++++++++++++++------------------ include/linux/mfd/da9055/pdata.h | 13 ---------- 2 files changed, 24 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/drivers/regulator/da9055-regulator.c b/drivers/regulator/da9055-regulator.c index 8fd9ac787588..352547c375bd 100644 --- a/drivers/regulator/da9055-regulator.c +++ b/drivers/regulator/da9055-regulator.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -413,31 +412,35 @@ static struct da9055_regulator_info da9055_regulator_info[] = { * GPIO can control regulator state and/or select the regulator register * set A/B for voltage ramping. */ -static int da9055_gpio_init(struct da9055_regulator *regulator, +static int da9055_gpio_init(struct device *dev, + struct da9055_regulator *regulator, struct regulator_config *config, struct da9055_pdata *pdata, int id) { struct da9055_regulator_info *info = regulator->info; + struct gpio_desc *ren; + struct gpio_desc *ena; + struct gpio_desc *rsel; int ret = 0; - if (!pdata) - return 0; + /* Look for "regulator-enable-gpios" GPIOs in the regulator node */ + ren = devm_gpiod_get_optional(dev, "regulator-enable", GPIOD_IN); + if (IS_ERR(ren)) + return PTR_ERR(ren); - if (pdata->gpio_ren && pdata->gpio_ren[id]) { - char name[18]; - int gpio_mux = pdata->gpio_ren[id]; + if (ren) { + /* This GPIO is not optional at this point */ + ena = devm_gpiod_get(dev, "enable", GPIOD_OUT_HIGH); + if (IS_ERR(ena)) + return PTR_ERR(ena); - config->ena_gpiod = pdata->ena_gpiods[id]; + config->ena_gpiod = ena; /* * GPI pin is muxed with regulator to control the * regulator state. */ - sprintf(name, "DA9055 GPI %d", gpio_mux); - ret = devm_gpio_request_one(config->dev, gpio_mux, GPIOF_DIR_IN, - name); - if (ret < 0) - goto err; + gpiod_set_consumer_name(ren, "DA9055 ren GPI"); /* * Let the regulator know that its state is controlled @@ -448,24 +451,22 @@ static int da9055_gpio_init(struct da9055_regulator *regulator, pdata->reg_ren[id] << DA9055_E_GPI_SHIFT); if (ret < 0) - goto err; + return ret; } - if (pdata->gpio_rsel && pdata->gpio_rsel[id]) { - char name[18]; - int gpio_mux = pdata->gpio_rsel[id]; + /* Look for "regulator-select-gpios" GPIOs in the regulator node */ + rsel = devm_gpiod_get_optional(dev, "regulator-select", GPIOD_IN); + if (IS_ERR(rsel)) + return PTR_ERR(rsel); + if (rsel) { regulator->reg_rselect = pdata->reg_rsel[id]; /* * GPI pin is muxed with regulator to select the * regulator register set A/B for voltage ramping. */ - sprintf(name, "DA9055 GPI %d", gpio_mux); - ret = devm_gpio_request_one(config->dev, gpio_mux, GPIOF_DIR_IN, - name); - if (ret < 0) - goto err; + gpiod_set_consumer_name(rsel, "DA9055 rsel GPI"); /* * Let the regulator know that its register set A/B @@ -477,7 +478,6 @@ static int da9055_gpio_init(struct da9055_regulator *regulator, << DA9055_V_GPI_SHIFT); } -err: return ret; } @@ -532,7 +532,7 @@ static int da9055_regulator_probe(struct platform_device *pdev) if (pdata) config.init_data = pdata->regulators[pdev->id]; - ret = da9055_gpio_init(regulator, &config, pdata, pdev->id); + ret = da9055_gpio_init(&pdev->dev, regulator, &config, pdata, pdev->id); if (ret < 0) return ret; diff --git a/include/linux/mfd/da9055/pdata.h b/include/linux/mfd/da9055/pdata.h index d3f126990ad0..137a2b067512 100644 --- a/include/linux/mfd/da9055/pdata.h +++ b/include/linux/mfd/da9055/pdata.h @@ -7,7 +7,6 @@ #define DA9055_MAX_REGULATORS 8 struct da9055; -struct gpio_desc; enum gpio_select { NO_GPIO = 0, @@ -23,16 +22,6 @@ struct da9055_pdata { struct regulator_init_data *regulators[DA9055_MAX_REGULATORS]; /* Enable RTC in RESET Mode */ bool reset_enable; - /* - * GPI muxed pin to control - * regulator state A/B, 0 if not available. - */ - int *gpio_ren; - /* - * GPI muxed pin to control - * regulator set, 0 if not available. - */ - int *gpio_rsel; /* * Regulator mode control bits value (GPI offset) that * controls the regulator state, 0 if not available. @@ -43,7 +32,5 @@ struct da9055_pdata { * controls the regulator set A/B, 0 if not available. */ enum gpio_select *reg_rsel; - /* GPIO descriptors to enable regulator, NULL if not available */ - struct gpio_desc **ena_gpiods; }; #endif /* __DA9055_PDATA_H */ -- cgit v1.2.3 From 95daa868f22b509ad641bf003d9d441d6a2fa505 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:26 +0100 Subject: regulator: lp8788-buck: Fully convert to GPIO descriptors This converts the LP8788 BUCK regulator driver to use GPIO descriptors. BUCK1 can use one DVS GPIO and BUCK2 can use two DVS GPIOS, and no more so just hardcode two GPIO descriptors into the per-DVS state containers. Obtain the descriptors from each regulators subdevice. As there are no in-tree users, board files need to populate descriptor tables for the buck regulator devices when they want to use this driver. BUCK1 need a GPIO descriptor at index 0 and BUCK2 needs two GPIO descriptors at indices 0 and 1. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-3-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/lp8788-buck.c | 64 +++++++++++++++++------------------------ include/linux/mfd/lp8788.h | 9 ++---- 2 files changed, 28 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/regulator/lp8788-buck.c b/drivers/regulator/lp8788-buck.c index e97ade09dede..712eaa6ff8ab 100644 --- a/drivers/regulator/lp8788-buck.c +++ b/drivers/regulator/lp8788-buck.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include /* register address */ #define LP8788_EN_BUCK 0x0C @@ -69,8 +69,8 @@ #define BUCK_FPWM_SHIFT(x) (x) enum lp8788_dvs_state { - DVS_LOW = GPIOF_OUT_INIT_LOW, - DVS_HIGH = GPIOF_OUT_INIT_HIGH, + DVS_LOW = 0, + DVS_HIGH = 1, }; enum lp8788_dvs_mode { @@ -89,6 +89,8 @@ struct lp8788_buck { struct lp8788 *lp; struct regulator_dev *regulator; void *dvs; + struct gpio_desc *gpio1; + struct gpio_desc *gpio2; /* Only used on BUCK2 */ }; /* BUCK 1 ~ 4 voltage ranges */ @@ -106,8 +108,7 @@ static void lp8788_buck1_set_dvs(struct lp8788_buck *buck) return; pinstate = dvs->vsel == DVS_SEL_V0 ? DVS_LOW : DVS_HIGH; - if (gpio_is_valid(dvs->gpio)) - gpio_set_value(dvs->gpio, pinstate); + gpiod_set_value(buck->gpio1, pinstate); } static void lp8788_buck2_set_dvs(struct lp8788_buck *buck) @@ -139,11 +140,8 @@ static void lp8788_buck2_set_dvs(struct lp8788_buck *buck) return; } - if (gpio_is_valid(dvs->gpio[0])) - gpio_set_value(dvs->gpio[0], pin1); - - if (gpio_is_valid(dvs->gpio[1])) - gpio_set_value(dvs->gpio[1], pin2); + gpiod_set_value(buck->gpio1, pin1); + gpiod_set_value(buck->gpio2, pin2); } static void lp8788_set_dvs(struct lp8788_buck *buck, enum lp8788_buck_id id) @@ -202,19 +200,13 @@ static u8 lp8788_select_buck_vout_addr(struct lp8788_buck *buck, enum lp8788_buck_id id) { enum lp8788_dvs_mode mode = lp8788_get_buck_dvs_ctrl_mode(buck, id); - struct lp8788_buck1_dvs *b1_dvs; - struct lp8788_buck2_dvs *b2_dvs; u8 val, idx, addr; int pin1, pin2; switch (id) { case BUCK1: if (mode == EXTPIN) { - b1_dvs = (struct lp8788_buck1_dvs *)buck->dvs; - if (!b1_dvs) - goto err; - - idx = gpio_get_value(b1_dvs->gpio) ? 1 : 0; + idx = gpiod_get_value(buck->gpio1); } else { lp8788_read_byte(buck->lp, LP8788_BUCK_DVS_SEL, &val); idx = (val & LP8788_BUCK1_DVS_M) >> LP8788_BUCK1_DVS_S; @@ -223,12 +215,8 @@ static u8 lp8788_select_buck_vout_addr(struct lp8788_buck *buck, break; case BUCK2: if (mode == EXTPIN) { - b2_dvs = (struct lp8788_buck2_dvs *)buck->dvs; - if (!b2_dvs) - goto err; - - pin1 = gpio_get_value(b2_dvs->gpio[0]); - pin2 = gpio_get_value(b2_dvs->gpio[1]); + pin1 = gpiod_get_value(buck->gpio1); + pin2 = gpiod_get_value(buck->gpio2); if (pin1 == PIN_LOW && pin2 == PIN_LOW) idx = 0; @@ -424,28 +412,28 @@ static int lp8788_dvs_gpio_request(struct platform_device *pdev, enum lp8788_buck_id id) { struct lp8788_platform_data *pdata = buck->lp->pdata; - char *b1_name = "LP8788_B1_DVS"; - char *b2_name[] = { "LP8788_B2_DVS1", "LP8788_B2_DVS2" }; - int i, gpio, ret; + struct device *dev = &pdev->dev; switch (id) { case BUCK1: - gpio = pdata->buck1_dvs->gpio; - ret = devm_gpio_request_one(&pdev->dev, gpio, DVS_LOW, - b1_name); - if (ret) - return ret; + buck->gpio1 = devm_gpiod_get(dev, "dvs", GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B1_DVS"); buck->dvs = pdata->buck1_dvs; break; case BUCK2: - for (i = 0; i < LP8788_NUM_BUCK2_DVS; i++) { - gpio = pdata->buck2_dvs->gpio[i]; - ret = devm_gpio_request_one(&pdev->dev, gpio, - DVS_LOW, b2_name[i]); - if (ret) - return ret; - } + buck->gpio1 = devm_gpiod_get_index(dev, "dvs", 0, GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS1"); + + buck->gpio2 = devm_gpiod_get_index(dev, "dvs", 1, GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS2"); + buck->dvs = pdata->buck2_dvs; break; default: diff --git a/include/linux/mfd/lp8788.h b/include/linux/mfd/lp8788.h index 3d5c480d58ea..51b47966a04d 100644 --- a/include/linux/mfd/lp8788.h +++ b/include/linux/mfd/lp8788.h @@ -10,7 +10,6 @@ #ifndef __MFD_LP8788_H__ #define __MFD_LP8788_H__ -#include #include #include #include @@ -159,21 +158,17 @@ struct lp8788; /* * lp8788_buck1_dvs - * @gpio : gpio pin number for dvs control * @vsel : dvs selector for buck v1 register */ struct lp8788_buck1_dvs { - int gpio; enum lp8788_dvs_sel vsel; }; /* * lp8788_buck2_dvs - * @gpio : two gpio pin numbers are used for dvs * @vsel : dvs selector for buck v2 register */ struct lp8788_buck2_dvs { - int gpio[LP8788_NUM_BUCK2_DVS]; enum lp8788_dvs_sel vsel; }; @@ -268,8 +263,8 @@ struct lp8788_vib_platform_data { * @buck_data : regulator initial data for buck * @dldo_data : regulator initial data for digital ldo * @aldo_data : regulator initial data for analog ldo - * @buck1_dvs : gpio configurations for buck1 dvs - * @buck2_dvs : gpio configurations for buck2 dvs + * @buck1_dvs : configurations for buck1 dvs + * @buck2_dvs : configurations for buck2 dvs * @chg_pdata : platform data for charger driver * @alarm_sel : rtc alarm selection (1 or 2) * @bl_pdata : configurable data for backlight driver -- cgit v1.2.3 From 84618d5e31cfd01fc3f53a8c2ebb68bc43d8b760 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:27 +0100 Subject: regulator: max8997: Convert to GPIO descriptors This rewrites the max8997 regulator driver to fetch the dvs regulators as descriptors. This will likely mostly come from the device tree since there are no in-tree users of the platform data, but supplying GPIO descriptor tables from board files is also possible if needed. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-4-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/max8997-regulator.c | 85 +++++++++++------------------------ include/linux/mfd/max8997.h | 1 - 2 files changed, 25 insertions(+), 61 deletions(-) (limited to 'include/linux') diff --git a/drivers/regulator/max8997-regulator.c b/drivers/regulator/max8997-regulator.c index 0b38eaa73597..5f201ee9a5b8 100644 --- a/drivers/regulator/max8997-regulator.c +++ b/drivers/regulator/max8997-regulator.c @@ -9,8 +9,7 @@ #include #include -#include -#include +#include #include #include #include @@ -32,7 +31,7 @@ struct max8997_data { u8 buck1_vol[8]; u8 buck2_vol[8]; u8 buck5_vol[8]; - int buck125_gpios[3]; + struct gpio_desc *buck125_gpiods[3]; int buck125_gpioindex; bool ignore_gpiodvs_side_effect; @@ -52,9 +51,9 @@ static inline void max8997_set_gpio(struct max8997_data *max8997) int set2 = ((max8997->buck125_gpioindex) >> 1) & 0x1; int set1 = ((max8997->buck125_gpioindex) >> 2) & 0x1; - gpio_set_value(max8997->buck125_gpios[0], set1); - gpio_set_value(max8997->buck125_gpios[1], set2); - gpio_set_value(max8997->buck125_gpios[2], set3); + gpiod_set_value(max8997->buck125_gpiods[0], set1); + gpiod_set_value(max8997->buck125_gpiods[1], set2); + gpiod_set_value(max8997->buck125_gpiods[2], set3); } struct voltage_map_desc { @@ -873,31 +872,13 @@ static struct regulator_desc regulators[] = { }; #ifdef CONFIG_OF -static int max8997_pmic_dt_parse_dvs_gpio(struct platform_device *pdev, - struct max8997_platform_data *pdata, - struct device_node *pmic_np) -{ - int i, gpio; - - for (i = 0; i < 3; i++) { - gpio = of_get_named_gpio(pmic_np, - "max8997,pmic-buck125-dvs-gpios", i); - if (!gpio_is_valid(gpio)) { - dev_err(&pdev->dev, "invalid gpio[%d]: %d\n", i, gpio); - return -EINVAL; - } - pdata->buck125_gpios[i] = gpio; - } - return 0; -} - static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev, struct max8997_platform_data *pdata) { struct max8997_dev *iodev = dev_get_drvdata(pdev->dev.parent); struct device_node *pmic_np, *regulators_np, *reg_np; struct max8997_regulator_data *rdata; - unsigned int i, dvs_voltage_nr = 1, ret; + unsigned int i, dvs_voltage_nr = 1; pmic_np = iodev->dev->of_node; if (!pmic_np) { @@ -949,10 +930,6 @@ static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev, if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || pdata->buck5_gpiodvs) { - ret = max8997_pmic_dt_parse_dvs_gpio(pdev, pdata, pmic_np); - if (ret) - return -EINVAL; - if (of_property_read_u32(pmic_np, "max8997,pmic-buck125-default-dvs-idx", &pdata->buck125_default_idx)) { @@ -1039,7 +1016,6 @@ static int max8997_pmic_probe(struct platform_device *pdev) max8997->buck1_gpiodvs = pdata->buck1_gpiodvs; max8997->buck2_gpiodvs = pdata->buck2_gpiodvs; max8997->buck5_gpiodvs = pdata->buck5_gpiodvs; - memcpy(max8997->buck125_gpios, pdata->buck125_gpios, sizeof(int) * 3); max8997->ignore_gpiodvs_side_effect = pdata->ignore_gpiodvs_side_effect; nr_dvs = (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || @@ -1110,38 +1086,27 @@ static int max8997_pmic_probe(struct platform_device *pdev) */ if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || pdata->buck5_gpiodvs) { + const char *gpio_names[3] = {"MAX8997 SET1", "MAX8997 SET2", "MAX8997 SET3"}; - if (!gpio_is_valid(pdata->buck125_gpios[0]) || - !gpio_is_valid(pdata->buck125_gpios[1]) || - !gpio_is_valid(pdata->buck125_gpios[2])) { - dev_err(&pdev->dev, "GPIO NOT VALID\n"); - return -EINVAL; - } + for (i = 0; i < 3; i++) { + enum gpiod_flags flags; - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[0], - "MAX8997 SET1"); - if (ret) - return ret; - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[1], - "MAX8997 SET2"); - if (ret) - return ret; - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[2], - "MAX8997 SET3"); - if (ret) - return ret; - - gpio_direction_output(pdata->buck125_gpios[0], - (max8997->buck125_gpioindex >> 2) - & 0x1); /* SET1 */ - gpio_direction_output(pdata->buck125_gpios[1], - (max8997->buck125_gpioindex >> 1) - & 0x1); /* SET2 */ - gpio_direction_output(pdata->buck125_gpios[2], - (max8997->buck125_gpioindex >> 0) - & 0x1); /* SET3 */ + if (max8997->buck125_gpioindex & BIT(2 - i)) + flags = GPIOD_OUT_HIGH; + else + flags = GPIOD_OUT_LOW; + + max8997->buck125_gpiods[i] = devm_gpiod_get_index(iodev->dev, + "max8997,pmic-buck125-dvs", + i, + flags); + if (IS_ERR(max8997->buck125_gpiods[i])) { + ret = PTR_ERR(max8997->buck125_gpiods[i]); + return dev_err_probe(iodev->dev, ret, "cant get GPIO %d (%d)\n", + i, ret); + } + gpiod_set_consumer_name(max8997->buck125_gpiods[i], gpio_names[i]); + } } /* DVS-GPIO disabled */ diff --git a/include/linux/mfd/max8997.h b/include/linux/mfd/max8997.h index 6193905abbb5..5c2cc1103437 100644 --- a/include/linux/mfd/max8997.h +++ b/include/linux/mfd/max8997.h @@ -178,7 +178,6 @@ struct max8997_platform_data { * */ bool ignore_gpiodvs_side_effect; - int buck125_gpios[3]; /* GPIO of [0]SET1, [1]SET2, [2]SET3 */ int buck125_default_idx; /* Default value of SET1, 2, 3 */ unsigned int buck1_voltage[8]; /* buckx_voltage in uV */ bool buck1_gpiodvs; -- cgit v1.2.3 From f25828a1eae1ee1a9257e2818b237b8208bd383e Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:28 +0100 Subject: regulator: max8998: Convert to GPIO descriptors This rewrites the max8998 regulator driver to fetch the dvs regulators as descriptors. This will likely mostly come from the device tree since there are no in-tree users of the platform data, but supplying GPIO descriptor tables from board files is also possible if needed. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-5-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/max8998.c | 150 ++++++++++++++++---------------------------- include/linux/mfd/max8998.h | 6 -- 2 files changed, 54 insertions(+), 102 deletions(-) (limited to 'include/linux') diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c index fadb4717384a..254a77887f66 100644 --- a/drivers/regulator/max8998.c +++ b/drivers/regulator/max8998.c @@ -10,12 +10,12 @@ #include #include #include -#include +#include +#include #include #include #include #include -#include #include #include #include @@ -31,6 +31,9 @@ struct max8998_data { unsigned int buck1_idx; /* index to last changed voltage */ /* value in a set */ unsigned int buck2_idx; + struct gpio_desc *buck1_gpio1; + struct gpio_desc *buck1_gpio2; + struct gpio_desc *buck2_gpio; }; static const unsigned int charger_current_table[] = { @@ -227,15 +230,15 @@ static int max8998_set_voltage_ldo_sel(struct regulator_dev *rdev, return ret; } -static inline void buck1_gpio_set(int gpio1, int gpio2, int v) +static inline void buck1_gpio_set(struct gpio_desc *gpio1, struct gpio_desc *gpio2, int v) { - gpio_set_value(gpio1, v & 0x1); - gpio_set_value(gpio2, (v >> 1) & 0x1); + gpiod_set_value(gpio1, v & 0x1); + gpiod_set_value(gpio2, (v >> 1) & 0x1); } -static inline void buck2_gpio_set(int gpio, int v) +static inline void buck2_gpio_set(struct gpio_desc *gpio, int v) { - gpio_set_value(gpio, v & 0x1); + gpiod_set_value(gpio, v & 0x1); } static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, @@ -260,16 +263,15 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, selector, max8998->buck1_vol[0], max8998->buck1_vol[1], max8998->buck1_vol[2], max8998->buck1_vol[3]); - if (gpio_is_valid(pdata->buck1_set1) && - gpio_is_valid(pdata->buck1_set2)) { + if (max8998->buck1_gpio1 && max8998->buck1_gpio2) { /* check if requested voltage */ /* value is already defined */ for (j = 0; j < ARRAY_SIZE(max8998->buck1_vol); j++) { if (max8998->buck1_vol[j] == selector) { max8998->buck1_idx = j; - buck1_gpio_set(pdata->buck1_set1, - pdata->buck1_set2, j); + buck1_gpio_set(max8998->buck1_gpio1, + max8998->buck1_gpio2, j); goto buck1_exit; } } @@ -286,13 +288,13 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, &shift, &mask); ret = max8998_write_reg(i2c, reg, selector); - buck1_gpio_set(pdata->buck1_set1, - pdata->buck1_set2, max8998->buck1_idx); + buck1_gpio_set(max8998->buck1_gpio1, + max8998->buck1_gpio2, max8998->buck1_idx); buck1_last_val++; buck1_exit: dev_dbg(max8998->dev, "%s: SET1:%d, SET2:%d\n", - i2c->name, gpio_get_value(pdata->buck1_set1), - gpio_get_value(pdata->buck1_set2)); + i2c->name, gpiod_get_value(max8998->buck1_gpio1), + gpiod_get_value(max8998->buck1_gpio2)); break; } else { ret = max8998_write_reg(i2c, reg, selector); @@ -303,14 +305,13 @@ buck1_exit: dev_dbg(max8998->dev, "BUCK2, selector:%d buck2_vol1:%d, buck2_vol2:%d\n", selector, max8998->buck2_vol[0], max8998->buck2_vol[1]); - if (gpio_is_valid(pdata->buck2_set3)) { - + if (max8998->buck2_gpio) { /* check if requested voltage */ /* value is already defined */ for (j = 0; j < ARRAY_SIZE(max8998->buck2_vol); j++) { if (max8998->buck2_vol[j] == selector) { max8998->buck2_idx = j; - buck2_gpio_set(pdata->buck2_set3, j); + buck2_gpio_set(max8998->buck2_gpio, j); goto buck2_exit; } } @@ -322,10 +323,10 @@ buck1_exit: ®, &shift, &mask); ret = max8998_write_reg(i2c, reg, selector); max8998->buck2_vol[max8998->buck2_idx] = selector; - buck2_gpio_set(pdata->buck2_set3, max8998->buck2_idx); + buck2_gpio_set(max8998->buck2_gpio, max8998->buck2_idx); buck2_exit: dev_dbg(max8998->dev, "%s: SET3:%d\n", i2c->name, - gpio_get_value(pdata->buck2_set3)); + gpiod_get_value(max8998->buck2_gpio)); } else { ret = max8998_write_reg(i2c, reg, selector); } @@ -539,36 +540,6 @@ static const struct regulator_desc regulators[] = { charger_current_table, MAX8998_REG_CHGR1, 0x7), }; -static int max8998_pmic_dt_parse_dvs_gpio(struct max8998_dev *iodev, - struct max8998_platform_data *pdata, - struct device_node *pmic_np) -{ - int gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck1-dvs-gpios", 0); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck1 gpio[0]: %d\n", gpio); - return -EINVAL; - } - pdata->buck1_set1 = gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck1-dvs-gpios", 1); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck1 gpio[1]: %d\n", gpio); - return -EINVAL; - } - pdata->buck1_set2 = gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck2-dvs-gpio", 0); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck 2 gpio: %d\n", gpio); - return -EINVAL; - } - pdata->buck2_set3 = gpio; - - return 0; -} - static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, struct max8998_platform_data *pdata) { @@ -614,10 +585,6 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, of_node_put(reg_np); of_node_put(regulators_np); - ret = max8998_pmic_dt_parse_dvs_gpio(iodev, pdata, pmic_np); - if (ret) - return -EINVAL; - pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock"); ret = of_property_read_u32(pmic_np, @@ -665,6 +632,7 @@ static int max8998_pmic_probe(struct platform_device *pdev) struct regulator_dev *rdev; struct max8998_data *max8998; struct i2c_client *i2c; + enum gpiod_flags flags; int i, ret; unsigned int v; @@ -693,37 +661,38 @@ static int max8998_pmic_probe(struct platform_device *pdev) max8998->buck1_idx = pdata->buck1_default_idx; max8998->buck2_idx = pdata->buck2_default_idx; - /* NOTE: */ - /* For unused GPIO NOT marked as -1 (thereof equal to 0) WARN_ON */ - /* will be displayed */ - /* Check if MAX8998 voltage selection GPIOs are defined */ - if (gpio_is_valid(pdata->buck1_set1) && - gpio_is_valid(pdata->buck1_set2)) { - /* Check if SET1 is not equal to 0 */ - if (!pdata->buck1_set1) { - dev_err(&pdev->dev, - "MAX8998 SET1 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck1_set1); - return -EIO; - } - /* Check if SET2 is not equal to 0 */ - if (!pdata->buck1_set2) { - dev_err(&pdev->dev, - "MAX8998 SET2 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck1_set2); - return -EIO; - } - - gpio_request(pdata->buck1_set1, "MAX8998 BUCK1_SET1"); - gpio_direction_output(pdata->buck1_set1, - max8998->buck1_idx & 0x1); - - - gpio_request(pdata->buck1_set2, "MAX8998 BUCK1_SET2"); - gpio_direction_output(pdata->buck1_set2, - (max8998->buck1_idx >> 1) & 0x1); - + flags = (max8998->buck1_idx & BIT(0)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck1_gpio1 = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck1-dvs", + 0, + flags); + if (IS_ERR(max8998->buck1_gpio1)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck1_gpio1), + "could not get BUCK1 GPIO1\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK1_SET1"); + + flags = (max8998->buck1_idx & BIT(1)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck1_gpio2 = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck1-dvs", + 1, + flags); + if (IS_ERR(max8998->buck1_gpio2)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck1_gpio2), + "could not get BUCK1 GPIO2\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK1_SET2"); + + flags = (max8998->buck2_idx & BIT(0)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck2_gpio = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck2-dvs", + 0, + flags); + if (IS_ERR(max8998->buck2_gpio)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck2_gpio), + "could not get BUCK2 GPIO\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK2_SET3"); + + if (max8998->buck1_gpio1 && max8998->buck1_gpio2) { /* Set predefined values for BUCK1 registers */ for (v = 0; v < ARRAY_SIZE(pdata->buck1_voltage); ++v) { int index = MAX8998_BUCK1 - MAX8998_LDO2; @@ -742,18 +711,7 @@ static int max8998_pmic_probe(struct platform_device *pdev) } } - if (gpio_is_valid(pdata->buck2_set3)) { - /* Check if SET3 is not equal to 0 */ - if (!pdata->buck2_set3) { - dev_err(&pdev->dev, - "MAX8998 SET3 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck2_set3); - return -EIO; - } - gpio_request(pdata->buck2_set3, "MAX8998 BUCK2_SET3"); - gpio_direction_output(pdata->buck2_set3, - max8998->buck2_idx & 0x1); - + if (max8998->buck2_gpio) { /* Set predefined values for BUCK2 registers */ for (v = 0; v < ARRAY_SIZE(pdata->buck2_voltage); ++v) { int index = MAX8998_BUCK2 - MAX8998_LDO2; diff --git a/include/linux/mfd/max8998.h b/include/linux/mfd/max8998.h index 79c020bd0c70..a054e55c8646 100644 --- a/include/linux/mfd/max8998.h +++ b/include/linux/mfd/max8998.h @@ -65,10 +65,7 @@ struct max8998_regulator_data { * be other than the preset values. * @buck1_voltage: BUCK1 DVS mode 1 voltage registers * @buck2_voltage: BUCK2 DVS mode 2 voltage registers - * @buck1_set1: BUCK1 gpio pin 1 to set output voltage - * @buck1_set2: BUCK1 gpio pin 2 to set output voltage * @buck1_default_idx: Default for BUCK1 gpio pin 1, 2 - * @buck2_set3: BUCK2 gpio pin to set output voltage * @buck2_default_idx: Default for BUCK2 gpio pin. * @wakeup: Allow to wake up from suspend * @rtc_delay: LP3974 RTC chip bug that requires delay after a register @@ -91,10 +88,7 @@ struct max8998_platform_data { bool buck_voltage_lock; int buck1_voltage[4]; int buck2_voltage[2]; - int buck1_set1; - int buck1_set2; int buck1_default_idx; - int buck2_set3; int buck2_default_idx; bool wakeup; bool rtc_delay; -- cgit v1.2.3 From 7b1d87af14d9ae902ed0c5dc5fabf4eea5abdf02 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 19 Feb 2024 16:33:18 -0600 Subject: spi: add spi_optimize_message() APIs This adds a new spi_optimize_message() function that can be used to optimize SPI messages that are used more than once. Peripheral drivers that use the same message multiple times can use this API to perform SPI message validation and controller-specific optimizations once and then reuse the message while avoiding the overhead of revalidating the message on each spi_(a)sync() call. Internally, the SPI core will also call this function for each message if the peripheral driver did not explicitly call it. This is done to so that controller drivers don't have to have multiple code paths for optimized and non-optimized messages. A hook is provided for controller drivers to perform controller-specific optimizations. Suggested-by: Martin Sperl Link: https://lore.kernel.org/linux-spi/39DEC004-10A1-47EF-9D77-276188D2580C@martin.sperl.org/ Signed-off-by: David Lechner Link: https://msgid.link/r/20240219-mainline-spi-precook-message-v2-1-4a762c6701b9@baylibre.com Reviewed-by: Jonathan Cameron Signed-off-by: Mark Brown --- drivers/spi/spi.c | 151 ++++++++++++++++++++++++++++++++++++++++++++++-- include/linux/spi/spi.h | 20 +++++++ 2 files changed, 167 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index c2b10e2c75f0..f68d92b57543 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -2106,6 +2106,41 @@ struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr) } EXPORT_SYMBOL_GPL(spi_get_next_queued_message); +/* + * __spi_unoptimize_message - shared implementation of spi_unoptimize_message() + * and spi_maybe_unoptimize_message() + * @msg: the message to unoptimize + * + * Peripheral drivers should use spi_unoptimize_message() and callers inside + * core should use spi_maybe_unoptimize_message() rather than calling this + * function directly. + * + * It is not valid to call this on a message that is not currently optimized. + */ +static void __spi_unoptimize_message(struct spi_message *msg) +{ + struct spi_controller *ctlr = msg->spi->controller; + + if (ctlr->unoptimize_message) + ctlr->unoptimize_message(msg); + + msg->optimized = false; + msg->opt_state = NULL; +} + +/* + * spi_maybe_unoptimize_message - unoptimize msg not managed by a peripheral + * @msg: the message to unoptimize + * + * This function is used to unoptimize a message if and only if it was + * optimized by the core (via spi_maybe_optimize_message()). + */ +static void spi_maybe_unoptimize_message(struct spi_message *msg) +{ + if (!msg->pre_optimized && msg->optimized) + __spi_unoptimize_message(msg); +} + /** * spi_finalize_current_message() - the current message is complete * @ctlr: the controller to return the message to @@ -2153,6 +2188,8 @@ void spi_finalize_current_message(struct spi_controller *ctlr) mesg->prepared = false; + spi_maybe_unoptimize_message(mesg); + WRITE_ONCE(ctlr->cur_msg_incomplete, false); smp_mb(); /* See __spi_pump_transfer_message()... */ if (READ_ONCE(ctlr->cur_msg_need_completion)) @@ -4194,6 +4231,110 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) return 0; } +/* + * __spi_optimize_message - shared implementation for spi_optimize_message() + * and spi_maybe_optimize_message() + * @spi: the device that will be used for the message + * @msg: the message to optimize + * + * Peripheral drivers will call spi_optimize_message() and the spi core will + * call spi_maybe_optimize_message() instead of calling this directly. + * + * It is not valid to call this on a message that has already been optimized. + * + * Return: zero on success, else a negative error code + */ +static int __spi_optimize_message(struct spi_device *spi, + struct spi_message *msg) +{ + struct spi_controller *ctlr = spi->controller; + int ret; + + ret = __spi_validate(spi, msg); + if (ret) + return ret; + + if (ctlr->optimize_message) { + ret = ctlr->optimize_message(msg); + if (ret) + return ret; + } + + msg->optimized = true; + + return 0; +} + +/* + * spi_maybe_optimize_message - optimize message if it isn't already pre-optimized + * @spi: the device that will be used for the message + * @msg: the message to optimize + * Return: zero on success, else a negative error code + */ +static int spi_maybe_optimize_message(struct spi_device *spi, + struct spi_message *msg) +{ + if (msg->pre_optimized) + return 0; + + return __spi_optimize_message(spi, msg); +} + +/** + * spi_optimize_message - do any one-time validation and setup for a SPI message + * @spi: the device that will be used for the message + * @msg: the message to optimize + * + * Peripheral drivers that reuse the same message repeatedly may call this to + * perform as much message prep as possible once, rather than repeating it each + * time a message transfer is performed to improve throughput and reduce CPU + * usage. + * + * Once a message has been optimized, it cannot be modified with the exception + * of updating the contents of any xfer->tx_buf (the pointer can't be changed, + * only the data in the memory it points to). + * + * Calls to this function must be balanced with calls to spi_unoptimize_message() + * to avoid leaking resources. + * + * Context: can sleep + * Return: zero on success, else a negative error code + */ +int spi_optimize_message(struct spi_device *spi, struct spi_message *msg) +{ + int ret; + + ret = __spi_optimize_message(spi, msg); + if (ret) + return ret; + + /* + * This flag indicates that the peripheral driver called spi_optimize_message() + * and therefore we shouldn't unoptimize message automatically when finalizing + * the message but rather wait until spi_unoptimize_message() is called + * by the peripheral driver. + */ + msg->pre_optimized = true; + + return 0; +} +EXPORT_SYMBOL_GPL(spi_optimize_message); + +/** + * spi_unoptimize_message - releases any resources allocated by spi_optimize_message() + * @msg: the message to unoptimize + * + * Calls to this function must be balanced with calls to spi_optimize_message(). + * + * Context: can sleep + */ +void spi_unoptimize_message(struct spi_message *msg) +{ + __spi_unoptimize_message(msg); + msg->pre_optimized = false; +} +EXPORT_SYMBOL_GPL(spi_unoptimize_message); + static int __spi_async(struct spi_device *spi, struct spi_message *message) { struct spi_controller *ctlr = spi->controller; @@ -4258,8 +4399,8 @@ int spi_async(struct spi_device *spi, struct spi_message *message) int ret; unsigned long flags; - ret = __spi_validate(spi, message); - if (ret != 0) + ret = spi_maybe_optimize_message(spi, message); + if (ret) return ret; spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); @@ -4271,6 +4412,8 @@ int spi_async(struct spi_device *spi, struct spi_message *message) spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); + spi_maybe_unoptimize_message(message); + return ret; } EXPORT_SYMBOL_GPL(spi_async); @@ -4331,8 +4474,8 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) return -ESHUTDOWN; } - status = __spi_validate(spi, message); - if (status != 0) + status = spi_maybe_optimize_message(spi, message); + if (status) return status; SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync); diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 2b8e2746769a..ddfb66dd4caf 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -475,6 +475,8 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * * @set_cs: set the logic level of the chip select line. May be called * from interrupt context. + * @optimize_message: optimize the message for reuse + * @unoptimize_message: release resources allocated by optimize_message * @prepare_message: set up the controller to transfer a single message, * for example doing DMA mapping. Called from threaded * context. @@ -715,6 +717,8 @@ struct spi_controller { struct completion xfer_completion; size_t max_dma_len; + int (*optimize_message)(struct spi_message *msg); + int (*unoptimize_message)(struct spi_message *msg); int (*prepare_transfer_hardware)(struct spi_controller *ctlr); int (*transfer_one_message)(struct spi_controller *ctlr, struct spi_message *mesg); @@ -1111,6 +1115,8 @@ struct spi_transfer { * @spi: SPI device to which the transaction is queued * @is_dma_mapped: if true, the caller provided both DMA and CPU virtual * addresses for each transfer buffer + * @pre_optimized: peripheral driver pre-optimized the message + * @optimized: the message is in the optimized state * @prepared: spi_prepare_message was called for the this message * @status: zero for success, else negative errno * @complete: called to report transaction completions @@ -1120,6 +1126,7 @@ struct spi_transfer { * successful segments * @queue: for use by whichever driver currently owns the message * @state: for use by whichever driver currently owns the message + * @opt_state: for use by whichever driver currently owns the message * @resources: for resource management when the SPI message is processed * * A @spi_message is used to execute an atomic sequence of data transfers, @@ -1143,6 +1150,11 @@ struct spi_message { unsigned is_dma_mapped:1; + /* spi_optimize_message() was called for this message */ + bool pre_optimized; + /* __spi_optimize_message() was called for this message */ + bool optimized; + /* spi_prepare_message() was called for this message */ bool prepared; @@ -1172,6 +1184,11 @@ struct spi_message { */ struct list_head queue; void *state; + /* + * Optional state for use by controller driver between calls to + * __spi_optimize_message() and __spi_unoptimize_message(). + */ + void *opt_state; /* List of spi_res resources when the SPI message is processed */ struct list_head resources; @@ -1255,6 +1272,9 @@ static inline void spi_message_free(struct spi_message *m) kfree(m); } +extern int spi_optimize_message(struct spi_device *spi, struct spi_message *msg); +extern void spi_unoptimize_message(struct spi_message *msg); + extern int spi_setup(struct spi_device *spi); extern int spi_async(struct spi_device *spi, struct spi_message *message); extern int spi_slave_abort(struct spi_device *spi); -- cgit v1.2.3 From 3d8bb3d3080d1609c2a6bef007ede2d8f8ffea5b Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 22 Feb 2024 11:25:12 +0100 Subject: gpio: provide for_each_hwgpio() We only provide iterators for requested GPIOs to provider drivers. In order to allow them to display debug information about all GPIOs, let's provide a variant for iterating over all GPIOs. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij --- include/linux/gpio/driver.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 9c1fbfaebaa8..175129a92656 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -551,6 +551,21 @@ DEFINE_CLASS(_gpiochip_for_each_data, }), const char **label, int *i) +/** + * for_each_hwgpio - Iterates over all GPIOs for given chip. + * @_chip: Chip to iterate over. + * @_i: Loop counter. + * @_label: Place to store the address of the label if the GPIO is requested. + * Set to NULL for unused GPIOs. + */ +#define for_each_hwgpio(_chip, _i, _label) \ + for (CLASS(_gpiochip_for_each_data, _data)(&_label, &_i); \ + *_data.i < _chip->ngpio; \ + (*_data.i)++, kfree(*(_data.label)), *_data.label = NULL) \ + if (IS_ERR(*_data.label = \ + gpiochip_dup_line_label(_chip, *_data.i))) {} \ + else + /** * for_each_requested_gpio_in_range - iterates over requested GPIOs in a given range * @_chip: the chip to query -- cgit v1.2.3 From 5340f7647294fa8ff8cf5a1bee326b2bd8340e27 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:30:49 +0100 Subject: thermal: core: Add flags to struct thermal_trip In order to allow thermal zone creators to specify the writability of trip point temperature and hysteresis on a per-trip basis, add a flags field to struct thermal_trip and define flags to represent the desired trip properties. Also make thermal_zone_device_register_with_trips() set the THERMAL_TRIP_FLAG_RW_TEMP flag for all trips covered by the writable trips mask passed to it and modify the thermal sysfs code to look at the trip flags instead of using the writable trips mask directly or checking the presence of the .set_trip_hyst() zone callback. Additionally, make trip_point_temp_store() and trip_point_hyst_store() fail with an error code if the trip passed to one of them has THERMAL_TRIP_FLAG_RW_TEMP or THERMAL_TRIP_FLAG_RW_HYST, respectively, clear in its flags. No intentional functional impact. Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 9 ++++++++- drivers/thermal/thermal_core.h | 2 +- drivers/thermal/thermal_sysfs.c | 18 +++++++++--------- include/linux/thermal.h | 8 ++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 1eabc8ebe27d..2ab495220c6e 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1278,6 +1278,7 @@ thermal_zone_device_register_with_trips(const char *type, int passive_delay, int polling_delay) { struct thermal_zone_device *tz; + struct thermal_trip *trip; int id; int result; struct thermal_governor *governor; @@ -1356,13 +1357,19 @@ thermal_zone_device_register_with_trips(const char *type, tz->devdata = devdata; tz->num_trips = num_trips; memcpy(tz->trips, trips, num_trips * sizeof(*trips)); + for_each_trip(tz, trip) { + if (mask & 1) + trip->flags |= THERMAL_TRIP_FLAG_RW_TEMP; + + mask >>= 1; + } thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); /* sys I/F */ /* Add nodes that are always present via .groups */ - result = thermal_zone_create_device_groups(tz, mask); + result = thermal_zone_create_device_groups(tz); if (result) goto remove_id; diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index e9c099ecdd0f..0d8a42bb7ce8 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -131,7 +131,7 @@ void thermal_zone_trip_updated(struct thermal_zone_device *tz, int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp); /* sysfs I/F */ -int thermal_zone_create_device_groups(struct thermal_zone_device *, int); +int thermal_zone_create_device_groups(struct thermal_zone_device *tz); void thermal_zone_destroy_device_groups(struct thermal_zone_device *); void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *); void thermal_cooling_device_destroy_sysfs(struct thermal_cooling_device *cdev); diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 83914aa2248b..39dbbdac75d0 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -392,17 +392,16 @@ static const struct attribute_group *thermal_zone_attribute_groups[] = { /** * create_trip_attrs() - create attributes for trip points * @tz: the thermal zone device - * @mask: Writeable trip point bitmap. * * helper function to instantiate sysfs entries for every trip * point and its properties of a struct thermal_zone_device. * * Return: 0 on success, the proper error value otherwise. */ -static int create_trip_attrs(struct thermal_zone_device *tz, int mask) +static int create_trip_attrs(struct thermal_zone_device *tz) { + const struct thermal_trip *trip; struct attribute **attrs; - int indx; /* This function works only for zones with at least one trip */ if (tz->num_trips <= 0) @@ -437,7 +436,9 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) return -ENOMEM; } - for (indx = 0; indx < tz->num_trips; indx++) { + for_each_trip(tz, trip) { + int indx = thermal_zone_trip_id(tz, trip); + /* create trip type attribute */ snprintf(tz->trip_type_attrs[indx].name, THERMAL_NAME_LENGTH, "trip_point_%d_type", indx); @@ -458,7 +459,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_temp_attrs[indx].name; tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show; - if (mask & (1 << indx)) { + if (trip->flags & THERMAL_TRIP_FLAG_RW_TEMP) { tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_temp_attrs[indx].attr.store = trip_point_temp_store; @@ -473,7 +474,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_hyst_attrs[indx].name; tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show; - if (tz->ops.set_trip_hyst) { + if (trip->flags & THERMAL_TRIP_FLAG_RW_HYST) { tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_hyst_attrs[indx].attr.store = trip_point_hyst_store; @@ -505,8 +506,7 @@ static void destroy_trip_attrs(struct thermal_zone_device *tz) kfree(tz->trips_attribute_group.attrs); } -int thermal_zone_create_device_groups(struct thermal_zone_device *tz, - int mask) +int thermal_zone_create_device_groups(struct thermal_zone_device *tz) { const struct attribute_group **groups; int i, size, result; @@ -522,7 +522,7 @@ int thermal_zone_create_device_groups(struct thermal_zone_device *tz, groups[i] = thermal_zone_attribute_groups[i]; if (tz->num_trips) { - result = create_trip_attrs(tz, mask); + result = create_trip_attrs(tz); if (result) { kfree(groups); diff --git a/include/linux/thermal.h b/include/linux/thermal.h index ec0559e98d6f..6eb6f3297ea0 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -64,15 +64,23 @@ enum thermal_notify_event { * @threshold: trip crossing notification threshold miliCelsius * @type: trip point type * @priv: pointer to driver data associated with this trip + * @flags: flags representing binary properties of the trip */ struct thermal_trip { int temperature; int hysteresis; int threshold; enum thermal_trip_type type; + u8 flags; void *priv; }; +#define THERMAL_TRIP_FLAG_RW_TEMP BIT(0) +#define THERMAL_TRIP_FLAG_RW_HYST BIT(1) + +#define THERMAL_TRIP_FLAG_RW (THERMAL_TRIP_FLAG_RW_TEMP | \ + THERMAL_TRIP_FLAG_RW_HYST) + struct thermal_zone_device_ops { int (*bind) (struct thermal_zone_device *, struct thermal_cooling_device *); -- cgit v1.2.3 From 46f5bef8ec2e1e05ad2fda0bcf5ac32e191ec694 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:32:03 +0100 Subject: thermal: core: Drop the .set_trip_hyst() thermal zone operation None of the users of the thermal core provides a .set_trip_hyst() thermal zone operation, so drop that callback from struct thermal_zone_device_ops and update trip_point_hyst_store() accordingly. No functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_sysfs.c | 7 ------- include/linux/thermal.h | 1 - 2 files changed, 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 39dbbdac75d0..7c02d35384ce 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -174,18 +174,11 @@ trip_point_hyst_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (hyst != trip->hysteresis) { - if (tz->ops.set_trip_hyst) { - ret = tz->ops.set_trip_hyst(tz, trip_id, hyst); - if (ret) - goto unlock; - } - trip->hysteresis = hyst; thermal_zone_trip_updated(tz, trip); } -unlock: mutex_unlock(&tz->lock); return ret ? ret : count; diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 6eb6f3297ea0..eb4145a64c7e 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -91,7 +91,6 @@ struct thermal_zone_device_ops { int (*change_mode) (struct thermal_zone_device *, enum thermal_device_mode); int (*set_trip_temp) (struct thermal_zone_device *, int, int); - int (*set_trip_hyst) (struct thermal_zone_device *, int, int); int (*get_crit_temp) (struct thermal_zone_device *, int *); int (*set_emul_temp) (struct thermal_zone_device *, int); int (*get_trend) (struct thermal_zone_device *, -- cgit v1.2.3 From 4a62d588a84e13c68017bd16bc9c2531a2cde08f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 19:09:16 +0100 Subject: thermal: core: Eliminate writable trip points masks All of the thermal_zone_device_register_with_trips() callers pass zero writable trip points masks to it, so drop the mask argument from that function and update all of its callers accordingly. This also removes the artificial trip points per zone limit of 32, related to using writable trip points masks. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/acpi/thermal.c | 2 +- drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c | 2 +- drivers/net/ethernet/mellanox/mlxsw/core_thermal.c | 3 --- drivers/net/wireless/intel/iwlwifi/mvm/tt.c | 1 - drivers/platform/x86/acerhdf.c | 2 +- drivers/thermal/da9062-thermal.c | 2 +- drivers/thermal/imx_thermal.c | 2 +- .../intel/int340x_thermal/int340x_thermal_zone.c | 2 +- .../int340x_thermal/processor_thermal_device_pci.c | 2 +- drivers/thermal/intel/intel_pch_thermal.c | 2 +- drivers/thermal/intel/intel_quark_dts_thermal.c | 2 +- drivers/thermal/intel/intel_soc_dts_iosf.c | 2 +- drivers/thermal/intel/x86_pkg_temp_thermal.c | 2 +- drivers/thermal/rcar_thermal.c | 2 +- drivers/thermal/st/st_thermal.c | 2 +- drivers/thermal/thermal_core.c | 28 +++------------------- drivers/thermal/thermal_of.c | 2 +- include/linux/thermal.h | 6 ++--- 18 files changed, 19 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 8042e96016e6..302dce0b2b50 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -665,7 +665,7 @@ static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz, tz->thermal_zone = thermal_zone_device_register_with_trips("acpitz", trip_table, trip_count, - 0, tz, + tz, &acpi_thermal_zone_ops, NULL, passive_delay, diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c index dea9d2907666..b08356060fb4 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c @@ -60,7 +60,7 @@ int cxgb4_thermal_init(struct adapter *adap) snprintf(ch_tz_name, sizeof(ch_tz_name), "cxgb4_%s", adap->name); ch_thermal->tzdev = thermal_zone_device_register_with_trips(ch_tz_name, &trip, num_trip, - 0, adap, + adap, &cxgb4_thermal_ops, NULL, 0, 0); if (IS_ERR(ch_thermal->tzdev)) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index 29ec0c122139..5c511e1a8efa 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -423,7 +423,6 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) module_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, module_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - 0, module_tz, &mlxsw_thermal_module_ops, &mlxsw_thermal_params, @@ -551,7 +550,6 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) gearbox_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, gearbox_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - 0, gearbox_tz, &mlxsw_thermal_gearbox_ops, &mlxsw_thermal_params, 0, @@ -776,7 +774,6 @@ int mlxsw_thermal_init(struct mlxsw_core *core, thermal->tzdev = thermal_zone_device_register_with_trips("mlxsw", thermal->trips, MLXSW_THERMAL_NUM_TRIPS, - 0, thermal, &mlxsw_thermal_ops, &mlxsw_thermal_params, 0, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index 2353177f2dfa..61a4638d1be2 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -694,7 +694,6 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) mvm->tz_device.tzone = thermal_zone_device_register_with_trips(name, mvm->tz_device.trips, IWL_MAX_DTS_TRIPS, - 0, mvm, &tzone_ops, NULL, 0, 0); if (IS_ERR(mvm->tz_device.tzone)) { diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c index 74bcb3d13104..018c48429616 100644 --- a/drivers/platform/x86/acerhdf.c +++ b/drivers/platform/x86/acerhdf.c @@ -678,7 +678,7 @@ static int __init acerhdf_register_thermal(void) return -EINVAL; thz_dev = thermal_zone_device_register_with_trips("acerhdf", trips, ARRAY_SIZE(trips), - 0, NULL, &acerhdf_dev_ops, + NULL, &acerhdf_dev_ops, &acerhdf_zone_params, 0, (kernelmode) ? interval*1000 : 0); if (IS_ERR(thz_dev)) diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c index 160d64913057..a27aff88cd96 100644 --- a/drivers/thermal/da9062-thermal.c +++ b/drivers/thermal/da9062-thermal.c @@ -197,7 +197,7 @@ static int da9062_thermal_probe(struct platform_device *pdev) mutex_init(&thermal->lock); thermal->zone = thermal_zone_device_register_with_trips(thermal->config->name, - trips, ARRAY_SIZE(trips), 0, thermal, + trips, ARRAY_SIZE(trips), thermal, &da9062_thermal_ops, NULL, pp_tmp, 0); if (IS_ERR(thermal->zone)) { diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 0dbede6bec54..83eaae5ca3b8 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -701,7 +701,7 @@ static int imx_thermal_probe(struct platform_device *pdev) data->tz = thermal_zone_device_register_with_trips("imx_thermal_zone", trips, ARRAY_SIZE(trips), - 0, data, + data, &imx_tz_ops, NULL, IMX_PASSIVE_DELAY, IMX_POLLING_DELAY); diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index 0e8148afb490..e8182f0929d3 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -171,7 +171,7 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->zone = thermal_zone_device_register_with_trips( acpi_device_bid(adev), zone_trips, trip_cnt, - 0, int34x_zone, + int34x_zone, &zone_ops, &int340x_thermal_params, 0, 0); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index 7572f3a73c54..7dcf6006816e 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -290,7 +290,7 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ psv_trip.temperature = get_trip_temp(pci_info); pci_info->tzone = thermal_zone_device_register_with_trips("TCPU_PCI", &psv_trip, - 1, 0, pci_info, + 1, pci_info, &tzone_ops, &tzone_params, 0, 0); if (IS_ERR(pci_info->tzone)) { diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c index 8bfb221e57e7..f5be2c389351 100644 --- a/drivers/thermal/intel/intel_pch_thermal.c +++ b/drivers/thermal/intel/intel_pch_thermal.c @@ -237,7 +237,7 @@ read_trips: ptd->tzd = thermal_zone_device_register_with_trips(board_names[board_id], ptd_trips, nr_trips, - 0, ptd, &tzd_ops, + ptd, &tzd_ops, NULL, 0, 0); if (IS_ERR(ptd->tzd)) { dev_err(&pdev->dev, "Failed to register thermal zone %s\n", diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c index 55a031fc3a9a..ec6ad26027bc 100644 --- a/drivers/thermal/intel/intel_quark_dts_thermal.c +++ b/drivers/thermal/intel/intel_quark_dts_thermal.c @@ -363,7 +363,7 @@ static struct soc_sensor_entry *alloc_soc_dts(void) aux_entry->tzone = thermal_zone_device_register_with_trips("quark_dts", trips, QRK_MAX_DTS_TRIPS, - 0, aux_entry, + aux_entry, &tzone_ops, NULL, 0, polling_delay); if (IS_ERR(aux_entry->tzone)) { diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c index 5a84c9166274..2ab943b66f7a 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.c +++ b/drivers/thermal/intel/intel_soc_dts_iosf.c @@ -230,7 +230,7 @@ static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, snprintf(name, sizeof(name), "soc_dts%d", id); dts->tzone = thermal_zone_device_register_with_trips(name, trips, SOC_MAX_DTS_TRIPS, - 0, dts, &tzone_ops, + dts, &tzone_ops, NULL, 0, 0); if (IS_ERR(dts->tzone)) { ret = PTR_ERR(dts->tzone); diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 5a48b73a0951..22a1e3268a98 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -338,7 +338,7 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn); zonedev->cpu = cpu; zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp", - trips, thres_count, 0, + trips, thres_count, zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0); if (IS_ERR(zonedev->tzone)) { err = PTR_ERR(zonedev->tzone); diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index feb848d595fa..925183753fcb 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -489,7 +489,7 @@ static int rcar_thermal_probe(struct platform_device *pdev) &rcar_thermal_zone_ops); } else { priv->zone = thermal_zone_device_register_with_trips( - "rcar_thermal", trips, ARRAY_SIZE(trips), 0, priv, + "rcar_thermal", trips, ARRAY_SIZE(trips), priv, &rcar_thermal_zone_ops, NULL, 0, idle); diff --git a/drivers/thermal/st/st_thermal.c b/drivers/thermal/st/st_thermal.c index 0d6249b36609..2a105409864e 100644 --- a/drivers/thermal/st/st_thermal.c +++ b/drivers/thermal/st/st_thermal.c @@ -203,7 +203,7 @@ int st_thermal_register(struct platform_device *pdev, trip.type = THERMAL_TRIP_CRITICAL; sensor->thermal_dev = - thermal_zone_device_register_with_trips(dev_name(dev), &trip, 1, 0, sensor, + thermal_zone_device_register_with_trips(dev_name(dev), &trip, 1, sensor, &st_tz_ops, NULL, 0, polling_delay); if (IS_ERR(sensor->thermal_dev)) { dev_err(dev, "failed to register thermal zone device\n"); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 2ab495220c6e..f7a7d43809e7 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1248,7 +1248,6 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); * @type: the thermal zone device type * @trips: a pointer to an array of thermal trips * @num_trips: the number of trip points the thermal zone support - * @mask: a bit string indicating the writeablility of trip points * @devdata: private device data * @ops: standard thermal zone device callbacks * @tzp: thermal zone platform parameters @@ -1271,14 +1270,12 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); struct thermal_zone_device * thermal_zone_device_register_with_trips(const char *type, const struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, + int num_trips, void *devdata, const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { struct thermal_zone_device *tz; - struct thermal_trip *trip; int id; int result; struct thermal_governor *governor; @@ -1294,20 +1291,7 @@ thermal_zone_device_register_with_trips(const char *type, return ERR_PTR(-EINVAL); } - /* - * Max trip count can't exceed 31 as the "mask >> num_trips" condition. - * For example, shifting by 32 will result in compiler warning: - * warning: right shift count >= width of type [-Wshift-count- overflow] - * - * Also "mask >> num_trips" will always be true with 32 bit shift. - * E.g. mask = 0x80000000 for trip id 31 to be RW. Then - * mask >> 32 = 0x80000000 - * This will result in failure for the below condition. - * - * Check will be true when the bit 31 of the mask is set. - * 32 bit shift will cause overflow of 4 byte integer. - */ - if (num_trips > (BITS_PER_TYPE(int) - 1) || num_trips < 0 || mask >> num_trips) { + if (num_trips < 0) { pr_err("Incorrect number of thermal trips\n"); return ERR_PTR(-EINVAL); } @@ -1357,12 +1341,6 @@ thermal_zone_device_register_with_trips(const char *type, tz->devdata = devdata; tz->num_trips = num_trips; memcpy(tz->trips, trips, num_trips * sizeof(*trips)); - for_each_trip(tz, trip) { - if (mask & 1) - trip->flags |= THERMAL_TRIP_FLAG_RW_TEMP; - - mask >>= 1; - } thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); @@ -1447,7 +1425,7 @@ struct thermal_zone_device *thermal_tripless_zone_device_register( const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp) { - return thermal_zone_device_register_with_trips(type, NULL, 0, 0, devdata, + return thermal_zone_device_register_with_trips(type, NULL, 0, devdata, ops, tzp, 0, 0); } EXPORT_SYMBOL_GPL(thermal_tripless_zone_device_register); diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index c3bd5fb15c7e..f1cbf9aa62cf 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -507,7 +507,7 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * of_ops.critical = thermal_zone_device_critical_reboot; tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, - 0, data, &of_ops, &tzp, + data, &of_ops, &tzp, pdelay, delay); if (IS_ERR(tz)) { ret = PTR_ERR(tz); diff --git a/include/linux/thermal.h b/include/linux/thermal.h index eb4145a64c7e..c33f50177f51 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -323,8 +323,7 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp); struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, const struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, + int num_trips, void *devdata, const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay); @@ -383,8 +382,7 @@ void thermal_zone_device_critical(struct thermal_zone_device *tz); static inline struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, const struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, + int num_trips, void *devdata, const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) -- cgit v1.2.3 From 4f299135d5668f56be270d224d41eb83d2002038 Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Mon, 19 Feb 2024 12:59:15 +0100 Subject: platform/x86: wmi: Prevent incompatible event driver from probing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a WMI event driver has no_notify_data set, then it indicates support for WMI events which provide no notify data, otherwise the notify() callback expects a valid ACPI object as notify data. However if a WMI event driver which requires notify data is bound to a WMI event device which cannot retrieve such data due to the _WED ACPI method being absent, then the driver will be dysfunctional since all WMI events will be dropped due to the missing notify data. Fix this by not allowing such WMI event drivers to bind to WMI event devices which do not support retrieving of notify data. Also reword the description of no_notify_data a bit. Signed-off-by: Armin Wolf Link: https://lore.kernel.org/r/20240219115919.16526-2-W_Armin@gmx.de Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen --- drivers/platform/x86/wmi.c | 10 ++++++++++ include/linux/wmi.h | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c index 5a613b06b269..8fb90b726f50 100644 --- a/drivers/platform/x86/wmi.c +++ b/drivers/platform/x86/wmi.c @@ -57,6 +57,7 @@ static_assert(__alignof__(struct guid_block) == 1); enum { /* wmi_block flags */ WMI_READ_TAKES_NO_ARGS, + WMI_NO_EVENT_DATA, }; struct wmi_block { @@ -869,6 +870,11 @@ static int wmi_dev_probe(struct device *dev) struct wmi_driver *wdriver = drv_to_wdrv(dev->driver); int ret = 0; + if (wdriver->notify) { + if (test_bit(WMI_NO_EVENT_DATA, &wblock->flags) && !wdriver->no_notify_data) + return -ENODEV; + } + if (ACPI_FAILURE(wmi_method_enable(wblock, true))) dev_warn(dev, "failed to enable device -- probing anyway\n"); @@ -1094,6 +1100,7 @@ static int parse_wdg(struct device *wmi_bus_dev, struct platform_device *pdev) struct acpi_device *device = ACPI_COMPANION(&pdev->dev); struct acpi_buffer out = {ACPI_ALLOCATE_BUFFER, NULL}; const struct guid_block *gblock; + bool event_data_available; struct wmi_block *wblock; union acpi_object *obj; acpi_status status; @@ -1113,6 +1120,7 @@ static int parse_wdg(struct device *wmi_bus_dev, struct platform_device *pdev) return -ENXIO; } + event_data_available = acpi_has_method(device->handle, "_WED"); gblock = (const struct guid_block *)obj->buffer.pointer; total = obj->buffer.length / sizeof(struct guid_block); @@ -1131,6 +1139,8 @@ static int parse_wdg(struct device *wmi_bus_dev, struct platform_device *pdev) wblock->acpi_device = device; wblock->gblock = gblock[i]; + if (gblock[i].flags & ACPI_WMI_EVENT && !event_data_available) + set_bit(WMI_NO_EVENT_DATA, &wblock->flags); retval = wmi_create_device(wmi_bus_dev, wblock, device); if (retval) { diff --git a/include/linux/wmi.h b/include/linux/wmi.h index 686291b87852..781958310bfb 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -48,7 +48,7 @@ u8 wmidev_instance_count(struct wmi_device *wdev); * struct wmi_driver - WMI driver structure * @driver: Driver model structure * @id_table: List of WMI GUIDs supported by this driver - * @no_notify_data: WMI events provide no event data + * @no_notify_data: Driver supports WMI events which provide no event data * @probe: Callback for device binding * @remove: Callback for device unbinding * @notify: Callback for receiving WMI events -- cgit v1.2.3 From 11b4eedfc87de394ed8cc54dea87c745d37ff9dc Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 19 Feb 2024 10:37:29 +0100 Subject: fbdev: Do not include in header Forward declare struct backlight_device and remove the include statement. Signed-off-by: Thomas Zimmermann Reviewed-by: Jani Nikula Acked-by: Helge Deller Link: https://patchwork.freedesktop.org/patch/msgid/20240219093941.3684-5-tzimmermann@suse.de --- include/linux/fb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 2ce2f5c2fca9..7380d959c5d5 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -13,11 +13,11 @@ #include #include #include -#include #include #include +struct backlight_device; struct vm_area_struct; struct fb_info; struct device; -- cgit v1.2.3 From 183c81569ddef5b7cc9b0403a9bdf9090e54c4f2 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 19 Feb 2024 10:37:30 +0100 Subject: fbdev: Do not include in header Forward declare struct inode and remove the include statement. Signed-off-by: Thomas Zimmermann Reviewed-by: Jani Nikula Acked-by: Helge Deller Link: https://patchwork.freedesktop.org/patch/msgid/20240219093941.3684-6-tzimmermann@suse.de --- include/linux/fb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 7380d959c5d5..f269ba520280 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -8,7 +8,6 @@ #define FBIO_CURSOR _IOWR('F', 0x08, struct fb_cursor_user) -#include #include #include #include @@ -22,6 +21,7 @@ struct vm_area_struct; struct fb_info; struct device; struct file; +struct inode; struct videomode; struct device_node; -- cgit v1.2.3 From 0f115335cff5caa53738e0240d6f7f0b85c72e14 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 19 Feb 2024 10:37:31 +0100 Subject: fbdev: Do not include in header Forward declare struct notifier_block and remove the include statement. Signed-off-by: Thomas Zimmermann Reviewed-by: Jani Nikula Acked-by: Helge Deller Link: https://patchwork.freedesktop.org/patch/msgid/20240219093941.3684-7-tzimmermann@suse.de --- include/linux/fb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index f269ba520280..90f348f14a49 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -10,7 +10,6 @@ #include #include -#include #include #include @@ -22,6 +21,7 @@ struct fb_info; struct device; struct file; struct inode; +struct notifier_block; struct videomode; struct device_node; -- cgit v1.2.3 From 7a46212f2a15cd8f041d2b4243ab64649c394260 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 19 Feb 2024 10:37:32 +0100 Subject: fbdev: Do not include in header Forward declare struct page and remove the include statement. Signed-off-by: Thomas Zimmermann Reviewed-by: Jani Nikula Acked-by: Helge Deller Link: https://patchwork.freedesktop.org/patch/msgid/20240219093941.3684-8-tzimmermann@suse.de --- include/linux/fb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 90f348f14a49..42155898374b 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -11,7 +11,6 @@ #include #include #include -#include #include @@ -22,6 +21,7 @@ struct device; struct file; struct inode; struct notifier_block; +struct page; struct videomode; struct device_node; -- cgit v1.2.3 From f6d520783a087fad2c5196f1231876312996cf07 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 19 Feb 2024 10:37:33 +0100 Subject: fbdev: Clean up forward declarations in header file Add forward declarations for struct i2c_adapter and struct module, and sort the list alphabetically. Signed-off-by: Thomas Zimmermann Reviewed-by: Jani Nikula Acked-by: Helge Deller Link: https://patchwork.freedesktop.org/patch/msgid/20240219093941.3684-9-tzimmermann@suse.de --- include/linux/fb.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 42155898374b..8f70ca727a30 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -15,15 +15,17 @@ #include struct backlight_device; -struct vm_area_struct; -struct fb_info; struct device; +struct device_node; +struct fb_info; struct file; +struct i2c_adapter; struct inode; +struct module; struct notifier_block; struct page; struct videomode; -struct device_node; +struct vm_area_struct; /* Definitions below are used in the parsed monitor specs */ #define FB_DPMS_ACTIVE_OFF 1 -- cgit v1.2.3 From 0c591381e4462005234f942d9fc36a369c0f5998 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Mon, 19 Feb 2024 10:37:34 +0100 Subject: fbdev: Clean up include statements in header file Include mutex.h, printk.h and types.h, remove several unnecessary include statements, and sort the list alphabetically. Signed-off-by: Thomas Zimmermann Reviewed-by: Jani Nikula Acked-by: Helge Deller Link: https://patchwork.freedesktop.org/patch/msgid/20240219093941.3684-10-tzimmermann@suse.de --- include/linux/fb.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 8f70ca727a30..708e6a177b1b 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -2,15 +2,15 @@ #ifndef _LINUX_FB_H #define _LINUX_FB_H -#include -#include #include #define FBIO_CURSOR _IOWR('F', 0x08, struct fb_cursor_user) -#include +#include +#include +#include +#include #include -#include #include -- cgit v1.2.3 From 416eb60317c64676d158dffea150762930ec008f Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 23 Feb 2024 10:01:46 +0100 Subject: bitfield: suppress "dubious: x & !y" sparse warning There's a somewhat common pattern of using FIELD_PREP() even for single bits, e.g. cmd->info1 |= FIELD_PREP(HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP, !!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP)); which might as well be written as if (params.flags & HAL_SRNG_FLAGS_MSI_SWAP) cmd->info1 |= HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP; (since info1 is fully initialized to start with), but in a long chain of FIELD_PREP() this really seems fine. However, it triggers a sparse warning, in the check in the macro for whether a constant value fits into the mask, as this contains a "& (_val)". In this case, this really is always intentional, so just suppress the warning by adding "0+" to the expression, indicating explicitly that this is correct. Signed-off-by: Johannes Berg Signed-off-by: Kalle Valo Link: https://msgid.link/20240223100146.d243b6b1a9a1.I033828b1187c6bccf086e31400f7e933bb8373e7@changeid --- include/linux/bitfield.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index ebfa12f69501..63928f173223 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -66,7 +66,8 @@ _pfx "mask is not constant"); \ BUILD_BUG_ON_MSG((_mask) == 0, _pfx "mask is zero"); \ BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \ - ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \ + ~((_mask) >> __bf_shf(_mask)) & \ + (0 + (_val)) : 0, \ _pfx "value too large for the field"); \ BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \ __bf_cast_unsigned(_reg, ~0ull), \ -- cgit v1.2.3 From 292fac464b012200c4e99d08974fed3bc087b848 Mon Sep 17 00:00:00 2001 From: Andrew Lunn Date: Mon, 26 Feb 2024 19:29:15 -0600 Subject: net: ethtool: eee: Remove legacy _u32 from keee All MAC drivers have been converted to use the link mode members of keee. So remove the _u32 values, and the code in the ethtool core to convert the legacy _u32 values to link modes. Reviewed-by: Simon Horman Signed-off-by: Andrew Lunn Signed-off-by: David S. Miller --- include/linux/ethtool.h | 3 --- net/ethtool/eee.c | 31 ++++--------------------------- net/ethtool/ioctl.c | 29 ++++++++++------------------- 3 files changed, 14 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index b90c33607594..9901e563f706 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -226,9 +226,6 @@ struct ethtool_keee { __ETHTOOL_DECLARE_LINK_MODE_MASK(supported); __ETHTOOL_DECLARE_LINK_MODE_MASK(advertised); __ETHTOOL_DECLARE_LINK_MODE_MASK(lp_advertised); - u32 supported_u32; - u32 advertised_u32; - u32 lp_advertised_u32; u32 tx_lpi_timer; bool tx_lpi_enabled; bool eee_active; diff --git a/net/ethtool/eee.c b/net/ethtool/eee.c index db6faa18fe41..bf398973eb8a 100644 --- a/net/ethtool/eee.c +++ b/net/ethtool/eee.c @@ -4,9 +4,6 @@ #include "common.h" #include "bitset.h" -#define EEE_MODES_COUNT \ - (sizeof_field(struct ethtool_keee, supported_u32) * BITS_PER_BYTE) - struct eee_req_info { struct ethnl_req_info base; }; @@ -41,15 +38,6 @@ static int eee_prepare_data(const struct ethnl_req_info *req_base, ret = dev->ethtool_ops->get_eee(dev, eee); ethnl_ops_complete(dev); - if (!ret && !ethtool_eee_use_linkmodes(eee)) { - ethtool_convert_legacy_u32_to_link_mode(eee->supported, - eee->supported_u32); - ethtool_convert_legacy_u32_to_link_mode(eee->advertised, - eee->advertised_u32); - ethtool_convert_legacy_u32_to_link_mode(eee->lp_advertised, - eee->lp_advertised_u32); - } - return ret; } @@ -62,11 +50,6 @@ static int eee_reply_size(const struct ethnl_req_info *req_base, int len = 0; int ret; - BUILD_BUG_ON(sizeof(eee->advertised_u32) * BITS_PER_BYTE != - EEE_MODES_COUNT); - BUILD_BUG_ON(sizeof(eee->lp_advertised_u32) * BITS_PER_BYTE != - EEE_MODES_COUNT); - /* MODES_OURS */ ret = ethnl_bitset_size(eee->advertised, eee->supported, __ETHTOOL_LINK_MODE_MASK_NBITS, @@ -154,16 +137,10 @@ ethnl_set_eee(struct ethnl_req_info *req_info, struct genl_info *info) if (ret < 0) return ret; - if (ethtool_eee_use_linkmodes(&eee)) { - ret = ethnl_update_bitset(eee.advertised, - __ETHTOOL_LINK_MODE_MASK_NBITS, - tb[ETHTOOL_A_EEE_MODES_OURS], - link_mode_names, info->extack, &mod); - } else { - ret = ethnl_update_bitset32(&eee.advertised_u32, EEE_MODES_COUNT, - tb[ETHTOOL_A_EEE_MODES_OURS], - link_mode_names, info->extack, &mod); - } + ret = ethnl_update_bitset(eee.advertised, + __ETHTOOL_LINK_MODE_MASK_NBITS, + tb[ETHTOOL_A_EEE_MODES_OURS], + link_mode_names, info->extack, &mod); if (ret < 0) return ret; ethnl_update_bool(&eee.eee_enabled, tb[ETHTOOL_A_EEE_ENABLED], &mod); diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index b419969c0dcb..317308bdbda9 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1514,9 +1514,6 @@ static void eee_to_keee(struct ethtool_keee *keee, { memset(keee, 0, sizeof(*keee)); - keee->supported_u32 = eee->supported; - keee->advertised_u32 = eee->advertised; - keee->lp_advertised_u32 = eee->lp_advertised; keee->eee_active = eee->eee_active; keee->eee_enabled = eee->eee_enabled; keee->tx_lpi_enabled = eee->tx_lpi_enabled; @@ -1533,6 +1530,8 @@ static void eee_to_keee(struct ethtool_keee *keee, static void keee_to_eee(struct ethtool_eee *eee, const struct ethtool_keee *keee) { + bool overflow; + memset(eee, 0, sizeof(*eee)); eee->eee_active = keee->eee_active; @@ -1540,22 +1539,14 @@ static void keee_to_eee(struct ethtool_eee *eee, eee->tx_lpi_enabled = keee->tx_lpi_enabled; eee->tx_lpi_timer = keee->tx_lpi_timer; - if (ethtool_eee_use_linkmodes(keee)) { - bool overflow; - - overflow = !ethtool_convert_link_mode_to_legacy_u32(&eee->supported, - keee->supported); - ethtool_convert_link_mode_to_legacy_u32(&eee->advertised, - keee->advertised); - ethtool_convert_link_mode_to_legacy_u32(&eee->lp_advertised, - keee->lp_advertised); - if (overflow) - pr_warn("Ethtool ioctl interface doesn't support passing EEE linkmodes beyond bit 32\n"); - } else { - eee->supported = keee->supported_u32; - eee->advertised = keee->advertised_u32; - eee->lp_advertised = keee->lp_advertised_u32; - } + overflow = !ethtool_convert_link_mode_to_legacy_u32(&eee->supported, + keee->supported); + ethtool_convert_link_mode_to_legacy_u32(&eee->advertised, + keee->advertised); + ethtool_convert_link_mode_to_legacy_u32(&eee->lp_advertised, + keee->lp_advertised); + if (overflow) + pr_warn("Ethtool ioctl interface doesn't support passing EEE linkmodes beyond bit 32\n"); } static int ethtool_get_eee(struct net_device *dev, char __user *useraddr) -- cgit v1.2.3 From 02e765697038c596dc4a1126a13b018608365d81 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Tue, 9 Jan 2024 15:04:56 +0800 Subject: swiotlb: add debugfs to track swiotlb transient pool usage Introduce a new debugfs interface io_tlb_transient_nslabs. The device driver can create a new swiotlb transient memory pool once default memory pool is full. To export the swiotlb transient memory pool usage via debugfs would help the user estimate the size of transient swiotlb memory pool or analyze device driver memory leak issue. Signed-off-by: ZhangPeng Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 3 +++ kernel/dma/swiotlb.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index ecde0312dd52..ea23097e351f 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -120,6 +120,8 @@ struct io_tlb_pool { * debugfs. * @used_hiwater: The high water mark for total_used. Used only for reporting * in debugfs. + * @transient_nslabs: The total number of slots in all transient pools that + * are currently used across all areas. */ struct io_tlb_mem { struct io_tlb_pool defpool; @@ -137,6 +139,7 @@ struct io_tlb_mem { #ifdef CONFIG_DEBUG_FS atomic_long_t total_used; atomic_long_t used_hiwater; + atomic_long_t transient_nslabs; #endif }; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index b079a9a8e087..77974cea3e69 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -956,6 +956,28 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) } #endif /* CONFIG_DEBUG_FS */ +#ifdef CONFIG_SWIOTLB_DYNAMIC +#ifdef CONFIG_DEBUG_FS +static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_add(nslots, &mem->transient_nslabs); +} + +static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_sub(nslots, &mem->transient_nslabs); +} + +#else /* !CONFIG_DEBUG_FS */ +static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_SWIOTLB_DYNAMIC */ + /** * swiotlb_search_pool_area() - search one memory area in one pool * @dev: Device which maps the buffer. @@ -1170,6 +1192,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); list_add_rcu(&pool->node, &dev->dma_io_tlb_pools); spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); + inc_transient_used(mem, pool->nslabs); found: WRITE_ONCE(dev->dma_uses_io_tlb, true); @@ -1415,6 +1438,7 @@ static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr) dec_used(dev->dma_io_tlb_mem, pool->nslabs); swiotlb_del_pool(dev, pool); + dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs); return true; } @@ -1557,6 +1581,23 @@ phys_addr_t default_swiotlb_limit(void) } #ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_SWIOTLB_DYNAMIC +static unsigned long mem_transient_used(struct io_tlb_mem *mem) +{ + return atomic_long_read(&mem->transient_nslabs); +} + +static int io_tlb_transient_used_get(void *data, u64 *val) +{ + struct io_tlb_mem *mem = data; + + *val = mem_transient_used(mem); + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get, + NULL, "%llu\n"); +#endif /* CONFIG_SWIOTLB_DYNAMIC */ static int io_tlb_used_get(void *data, u64 *val) { @@ -1605,6 +1646,11 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, &fops_io_tlb_used); debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem, &fops_io_tlb_hiwater); +#ifdef CONFIG_SWIOTLB_DYNAMIC + atomic_long_set(&mem->transient_nslabs, 0); + debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs, + mem, &fops_io_tlb_transient_used); +#endif } static int __init swiotlb_create_default_debugfs(void) -- cgit v1.2.3 From 3f7edeac0bbb301a07d7ea2abd28727aaa7fdab0 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Tue, 12 Dec 2023 16:13:22 +0100 Subject: SUNRPC: Add a transport callback to handle dequeuing of an RPC request Add a transport level callback to allow it to handle the consequences of dequeuing the request that was in the process of being transmitted. For something like a TCP connection, we may need to disconnect if the request was partially transmitted. Signed-off-by: Trond Myklebust --- include/linux/sunrpc/xprt.h | 1 + net/sunrpc/xprt.c | 6 ++++++ net/sunrpc/xprtsock.c | 13 +++++++++++++ 3 files changed, 20 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index 464f6a9492ab..81b952649d35 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -152,6 +152,7 @@ struct rpc_xprt_ops { int (*prepare_request)(struct rpc_rqst *req, struct xdr_buf *buf); int (*send_request)(struct rpc_rqst *req); + void (*abort_send_request)(struct rpc_rqst *req); void (*wait_for_reply_request)(struct rpc_task *task); void (*timer)(struct rpc_xprt *xprt, struct rpc_task *task); void (*release_request)(struct rpc_task *task); diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index db3754a3298f..09f245cda526 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -1398,6 +1398,12 @@ xprt_request_dequeue_transmit_locked(struct rpc_task *task) if (!test_and_clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) return; if (!list_empty(&req->rq_xmit)) { + struct rpc_xprt *xprt = req->rq_xprt; + + if (list_is_first(&req->rq_xmit, &xprt->xmit_queue) && + xprt->ops->abort_send_request) + xprt->ops->abort_send_request(req); + list_del(&req->rq_xmit); if (!list_empty(&req->rq_xmit2)) { struct rpc_rqst *next = list_first_entry(&req->rq_xmit2, diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 1a961314b737..bb81050c870e 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -884,6 +884,17 @@ static int xs_stream_prepare_request(struct rpc_rqst *req, struct xdr_buf *buf) return xdr_alloc_bvec(buf, rpc_task_gfp_mask()); } +static void xs_stream_abort_send_request(struct rpc_rqst *req) +{ + struct rpc_xprt *xprt = req->rq_xprt; + struct sock_xprt *transport = + container_of(xprt, struct sock_xprt, xprt); + + if (transport->xmit.offset != 0 && + !test_bit(XPRT_CLOSE_WAIT, &xprt->state)) + xprt_force_disconnect(xprt); +} + /* * Determine if the previous message in the stream was aborted before it * could complete transmission. @@ -3029,6 +3040,7 @@ static const struct rpc_xprt_ops xs_local_ops = { .buf_free = rpc_free, .prepare_request = xs_stream_prepare_request, .send_request = xs_local_send_request, + .abort_send_request = xs_stream_abort_send_request, .wait_for_reply_request = xprt_wait_for_reply_request_def, .close = xs_close, .destroy = xs_destroy, @@ -3076,6 +3088,7 @@ static const struct rpc_xprt_ops xs_tcp_ops = { .buf_free = rpc_free, .prepare_request = xs_stream_prepare_request, .send_request = xs_tcp_send_request, + .abort_send_request = xs_stream_abort_send_request, .wait_for_reply_request = xprt_wait_for_reply_request_def, .close = xs_tcp_shutdown, .destroy = xs_destroy, -- cgit v1.2.3 From 2c35f43b5a4b9cdfaa6fdd946f5a212615dac8eb Mon Sep 17 00:00:00 2001 From: Dai Ngo Date: Tue, 30 Jan 2024 11:38:25 -0800 Subject: SUNRPC: increase size of rpc_wait_queue.qlen from unsigned short to unsigned int When the NFS client is under extreme load the rpc_wait_queue.qlen counter can be overflowed. Here is an instant of the backlog queue overflow in a real world environment shown by drgn helper: rpc_task_stats(rpc_clnt): ------------------------- rpc_clnt: 0xffff92b65d2bae00 rpc_xprt: 0xffff9275db64f000 Queue: sending[64887] pending[524] backlog[30441] binding[0] XMIT task: 0xffff925c6b1d8e98 WRITE: 750654 __dta_call_status_580: 65463 __dta_call_transmit_status_579: 1 call_reserveresult: 685189 nfs_client_init_is_complete: 1 COMMIT: 584 call_reserveresult: 573 __dta_call_status_580: 11 ACCESS: 1 __dta_call_status_580: 1 GETATTR: 10 __dta_call_status_580: 4 call_reserveresult: 6 751249 tasks for server 111.222.333.444 Total tasks: 751249 count_rpc_wait_queues(xprt): ---------------------------- **** rpc_xprt: 0xffff9275db64f000 num_reqs: 65511 wait_queue: xprt_binding[0] cnt: 0 wait_queue: xprt_binding[1] cnt: 0 wait_queue: xprt_binding[2] cnt: 0 wait_queue: xprt_binding[3] cnt: 0 rpc_wait_queue[xprt_binding].qlen: 0 maxpriority: 0 wait_queue: xprt_sending[0] cnt: 0 wait_queue: xprt_sending[1] cnt: 64887 wait_queue: xprt_sending[2] cnt: 0 wait_queue: xprt_sending[3] cnt: 0 rpc_wait_queue[xprt_sending].qlen: 64887 maxpriority: 3 wait_queue: xprt_pending[0] cnt: 524 wait_queue: xprt_pending[1] cnt: 0 wait_queue: xprt_pending[2] cnt: 0 wait_queue: xprt_pending[3] cnt: 0 rpc_wait_queue[xprt_pending].qlen: 524 maxpriority: 0 wait_queue: xprt_backlog[0] cnt: 0 wait_queue: xprt_backlog[1] cnt: 685801 wait_queue: xprt_backlog[2] cnt: 0 wait_queue: xprt_backlog[3] cnt: 0 rpc_wait_queue[xprt_backlog].qlen: 30441 maxpriority: 3 [task cnt mismatch] There is no effect on operations when this overflow occurs. However it causes confusion when trying to diagnose the performance problem. Signed-off-by: Dai Ngo Reviewed-by: Jeff Layton Signed-off-by: Trond Myklebust --- include/linux/sunrpc/sched.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index 2d61987b3545..0c77ba488bba 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -197,7 +197,7 @@ struct rpc_wait_queue { unsigned char maxpriority; /* maximum priority (0 if queue is not a priority queue) */ unsigned char priority; /* current priority */ unsigned char nr; /* # tasks remaining for cookie */ - unsigned short qlen; /* total # tasks waiting in queue */ + unsigned int qlen; /* total # tasks waiting in queue */ struct rpc_timer timer_list; #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) const char * name; -- cgit v1.2.3 From b0cde62e4c548b2e7cb535caa6eb0df135888601 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 4 Jan 2024 23:55:11 +0100 Subject: clk: Add a devm variant of clk_rate_exclusive_get() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to simplify drivers that use clk_rate_exclusive_get() in their probe routine as calling clk_rate_exclusive_put() is cared for automatically. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20240104225512.1124519-2-u.kleine-koenig@pengutronix.de Acked-by: Russell King (Oracle) Signed-off-by: Stephen Boyd --- drivers/clk/clk.c | 19 +++++++++++++++++++ include/linux/clk.h | 12 ++++++++++++ 2 files changed, 31 insertions(+) (limited to 'include/linux') diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 2253c154a824..a3bc7fb90d0f 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -939,6 +939,25 @@ int clk_rate_exclusive_get(struct clk *clk) } EXPORT_SYMBOL_GPL(clk_rate_exclusive_get); +static void devm_clk_rate_exclusive_put(void *data) +{ + struct clk *clk = data; + + clk_rate_exclusive_put(clk); +} + +int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk) +{ + int ret; + + ret = clk_rate_exclusive_get(clk); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_clk_rate_exclusive_put, clk); +} +EXPORT_SYMBOL_GPL(devm_clk_rate_exclusive_get); + static void clk_core_unprepare(struct clk_core *core) { lockdep_assert_held(&prepare_lock); diff --git a/include/linux/clk.h b/include/linux/clk.h index 06f1b292f8a0..24c49b01c25d 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -201,6 +201,18 @@ bool clk_is_match(const struct clk *p, const struct clk *q); */ int clk_rate_exclusive_get(struct clk *clk); +/** + * devm_clk_rate_exclusive_get - devm variant of clk_rate_exclusive_get + * @dev: device the exclusivity is bound to + * @clk: clock source + * + * Calls clk_rate_exclusive_get() on @clk and registers a devm cleanup handler + * on @dev to call clk_rate_exclusive_put(). + * + * Must not be called from within atomic context. + */ +int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk); + /** * clk_rate_exclusive_put - release exclusivity over the rate control of a * producer -- cgit v1.2.3 From 0598f8f3bb77893a13105d47bb7dfe42f1dc1f4e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 27 Feb 2024 09:24:09 +0000 Subject: inet: annotate devconf data-races Add READ_ONCE() in ipv4_devconf_get() and corresponding WRITE_ONCE() in ipv4_devconf_set() Add IPV4_DEVCONF_RO() and IPV4_DEVCONF_ALL_RO() macros, and use them when reading devconf fields. Signed-off-by: Eric Dumazet Reviewed-by: Jiri Pirko Link: https://lore.kernel.org/r/20240227092411.2315725-2-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/inetdevice.h | 14 ++++++++------ net/ipv4/devinet.c | 21 +++++++++++---------- net/ipv4/igmp.c | 4 ++-- net/ipv4/proc.c | 2 +- net/ipv4/route.c | 4 ++-- 5 files changed, 24 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index ddb27fc0ee8c..cb5280e6cc21 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -53,13 +53,15 @@ struct in_device { }; #define IPV4_DEVCONF(cnf, attr) ((cnf).data[IPV4_DEVCONF_ ## attr - 1]) +#define IPV4_DEVCONF_RO(cnf, attr) READ_ONCE(IPV4_DEVCONF(cnf, attr)) #define IPV4_DEVCONF_ALL(net, attr) \ IPV4_DEVCONF((*(net)->ipv4.devconf_all), attr) +#define IPV4_DEVCONF_ALL_RO(net, attr) READ_ONCE(IPV4_DEVCONF_ALL(net, attr)) -static inline int ipv4_devconf_get(struct in_device *in_dev, int index) +static inline int ipv4_devconf_get(const struct in_device *in_dev, int index) { index--; - return in_dev->cnf.data[index]; + return READ_ONCE(in_dev->cnf.data[index]); } static inline void ipv4_devconf_set(struct in_device *in_dev, int index, @@ -67,7 +69,7 @@ static inline void ipv4_devconf_set(struct in_device *in_dev, int index, { index--; set_bit(index, in_dev->cnf.state); - in_dev->cnf.data[index] = val; + WRITE_ONCE(in_dev->cnf.data[index], val); } static inline void ipv4_devconf_setall(struct in_device *in_dev) @@ -81,18 +83,18 @@ static inline void ipv4_devconf_setall(struct in_device *in_dev) ipv4_devconf_set((in_dev), IPV4_DEVCONF_ ## attr, (val)) #define IN_DEV_ANDCONF(in_dev, attr) \ - (IPV4_DEVCONF_ALL(dev_net(in_dev->dev), attr) && \ + (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), attr) && \ IN_DEV_CONF_GET((in_dev), attr)) #define IN_DEV_NET_ORCONF(in_dev, net, attr) \ - (IPV4_DEVCONF_ALL(net, attr) || \ + (IPV4_DEVCONF_ALL_RO(net, attr) || \ IN_DEV_CONF_GET((in_dev), attr)) #define IN_DEV_ORCONF(in_dev, attr) \ IN_DEV_NET_ORCONF(in_dev, dev_net(in_dev->dev), attr) #define IN_DEV_MAXCONF(in_dev, attr) \ - (max(IPV4_DEVCONF_ALL(dev_net(in_dev->dev), attr), \ + (max(IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), attr), \ IN_DEV_CONF_GET((in_dev), attr))) #define IN_DEV_FORWARD(in_dev) IN_DEV_CONF_GET((in_dev), FORWARDING) diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index bc74f131fe4d..ca75d0fff1d1 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -1982,7 +1982,7 @@ static int inet_fill_link_af(struct sk_buff *skb, const struct net_device *dev, return -EMSGSIZE; for (i = 0; i < IPV4_DEVCONF_MAX; i++) - ((u32 *) nla_data(nla))[i] = in_dev->cnf.data[i]; + ((u32 *) nla_data(nla))[i] = READ_ONCE(in_dev->cnf.data[i]); return 0; } @@ -2068,9 +2068,9 @@ static int inet_netconf_msgsize_devconf(int type) } static int inet_netconf_fill_devconf(struct sk_buff *skb, int ifindex, - struct ipv4_devconf *devconf, u32 portid, - u32 seq, int event, unsigned int flags, - int type) + const struct ipv4_devconf *devconf, + u32 portid, u32 seq, int event, + unsigned int flags, int type) { struct nlmsghdr *nlh; struct netconfmsg *ncm; @@ -2095,27 +2095,28 @@ static int inet_netconf_fill_devconf(struct sk_buff *skb, int ifindex, if ((all || type == NETCONFA_FORWARDING) && nla_put_s32(skb, NETCONFA_FORWARDING, - IPV4_DEVCONF(*devconf, FORWARDING)) < 0) + IPV4_DEVCONF_RO(*devconf, FORWARDING)) < 0) goto nla_put_failure; if ((all || type == NETCONFA_RP_FILTER) && nla_put_s32(skb, NETCONFA_RP_FILTER, - IPV4_DEVCONF(*devconf, RP_FILTER)) < 0) + IPV4_DEVCONF_RO(*devconf, RP_FILTER)) < 0) goto nla_put_failure; if ((all || type == NETCONFA_MC_FORWARDING) && nla_put_s32(skb, NETCONFA_MC_FORWARDING, - IPV4_DEVCONF(*devconf, MC_FORWARDING)) < 0) + IPV4_DEVCONF_RO(*devconf, MC_FORWARDING)) < 0) goto nla_put_failure; if ((all || type == NETCONFA_BC_FORWARDING) && nla_put_s32(skb, NETCONFA_BC_FORWARDING, - IPV4_DEVCONF(*devconf, BC_FORWARDING)) < 0) + IPV4_DEVCONF_RO(*devconf, BC_FORWARDING)) < 0) goto nla_put_failure; if ((all || type == NETCONFA_PROXY_NEIGH) && nla_put_s32(skb, NETCONFA_PROXY_NEIGH, - IPV4_DEVCONF(*devconf, PROXY_ARP)) < 0) + IPV4_DEVCONF_RO(*devconf, PROXY_ARP)) < 0) goto nla_put_failure; if ((all || type == NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN) && nla_put_s32(skb, NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, - IPV4_DEVCONF(*devconf, IGNORE_ROUTES_WITH_LINKDOWN)) < 0) + IPV4_DEVCONF_RO(*devconf, + IGNORE_ROUTES_WITH_LINKDOWN)) < 0) goto nla_put_failure; out: diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index efeeca2b1328..717e97a389a8 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -120,12 +120,12 @@ */ #define IGMP_V1_SEEN(in_dev) \ - (IPV4_DEVCONF_ALL(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 1 || \ + (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 1 || \ IN_DEV_CONF_GET((in_dev), FORCE_IGMP_VERSION) == 1 || \ ((in_dev)->mr_v1_seen && \ time_before(jiffies, (in_dev)->mr_v1_seen))) #define IGMP_V2_SEEN(in_dev) \ - (IPV4_DEVCONF_ALL(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 2 || \ + (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 2 || \ IN_DEV_CONF_GET((in_dev), FORCE_IGMP_VERSION) == 2 || \ ((in_dev)->mr_v2_seen && \ time_before(jiffies, (in_dev)->mr_v2_seen))) diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c index 5f4654ebff48..914bc9c35cc7 100644 --- a/net/ipv4/proc.c +++ b/net/ipv4/proc.c @@ -395,7 +395,7 @@ static int snmp_seq_show_ipstats(struct seq_file *seq, void *v) seq_printf(seq, " %s", snmp4_ipstats_list[i].name); seq_printf(seq, "\nIp: %d %d", - IPV4_DEVCONF_ALL(net, FORWARDING) ? 1 : 2, + IPV4_DEVCONF_ALL_RO(net, FORWARDING) ? 1 : 2, READ_ONCE(net->ipv4.sysctl_ip_default_ttl)); BUILD_BUG_ON(offsetof(struct ipstats_mib, mibs) != 0); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index b512288d6fcc..c8f76f56dc16 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -2313,7 +2313,7 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, if (IN_DEV_BFORWARD(in_dev)) goto make_route; /* not do cache if bc_forwarding is enabled */ - if (IPV4_DEVCONF_ALL(net, BC_FORWARDING)) + if (IPV4_DEVCONF_ALL_RO(net, BC_FORWARDING)) do_cache = false; goto brd_input; } @@ -2993,7 +2993,7 @@ static int rt_fill_info(struct net *net, __be32 dst, __be32 src, #ifdef CONFIG_IP_MROUTE if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) && - IPV4_DEVCONF_ALL(net, MC_FORWARDING)) { + IPV4_DEVCONF_ALL_RO(net, MC_FORWARDING)) { int err = ipmr_get_route(net, skb, fl4->saddr, fl4->daddr, r, portid); -- cgit v1.2.3 From 99123622050f10ca9148a0fffba2de0afd6cdfff Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 27 Feb 2024 19:27:21 +0000 Subject: tcp: remove some holes in struct tcp_sock By moving some fields around, this patch shrinks holes size from 56 to 32, saving 24 bytes on 64bit arches. After the patch pahole gives the following for 'struct tcp_sock': /* size: 2304, cachelines: 36, members: 162 */ /* sum members: 2234, holes: 6, sum holes: 32 */ /* sum bitfield members: 34 bits, bit holes: 5, sum bit holes: 14 bits */ /* padding: 32 */ /* paddings: 3, sum paddings: 10 */ /* forced alignments: 1, forced holes: 1, sum forced holes: 12 */ Signed-off-by: Eric Dumazet Reviewed-by: Jiri Pirko Link: https://lore.kernel.org/r/20240227192721.3558982-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/tcp.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tcp.h b/include/linux/tcp.h index a1c47a6d69b0..988a30ef6bfe 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -264,10 +264,10 @@ struct tcp_sock { u32 pushed_seq; /* Last pushed seq, required to talk to windows */ u32 lsndtime; u32 mdev_us; /* medium deviation */ + u32 rtt_seq; /* sequence number to update rttvar */ u64 tcp_wstamp_ns; /* departure time for next sent data packet */ u64 tcp_clock_cache; /* cache last tcp_clock_ns() (see tcp_mstamp_refresh()) */ u64 tcp_mstamp; /* most recent packet received/sent */ - u32 rtt_seq; /* sequence number to update rttvar */ struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */ struct sk_buff *highest_sack; /* skb just after the highest * skb with SACKed bit set @@ -350,7 +350,6 @@ struct tcp_sock { u32 dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups * total number of DSACK blocks received */ - u32 last_oow_ack_time; /* timestamp of last out-of-window ACK */ u32 compressed_ack_rcv_nxt; struct list_head tsq_node; /* anchor in tsq_tasklet.head list */ @@ -384,12 +383,12 @@ struct tcp_sock { syn_fastopen_ch:1, /* Active TFO re-enabling probe */ syn_data_acked:1;/* data in SYN is acked by SYN-ACK */ + u8 keepalive_probes; /* num of allowed keep alive probes */ u32 tcp_tx_delay; /* delay (in usec) added to TX packets */ /* RTT measurement */ u32 mdev_max_us; /* maximal mdev for the last rtt period */ - u8 keepalive_probes; /* num of allowed keep alive probes */ u32 reord_seen; /* number of data packet reordering events */ /* @@ -402,6 +401,7 @@ struct tcp_sock { u32 prior_cwnd; /* cwnd right before starting loss recovery */ u32 prr_delivered; /* Number of newly delivered packets to * receiver in Recovery. */ + u32 last_oow_ack_time; /* timestamp of last out-of-window ACK */ struct hrtimer pacing_timer; struct hrtimer compressed_ack_timer; @@ -477,8 +477,8 @@ struct tcp_sock { bool is_mptcp; #endif #if IS_ENABLED(CONFIG_SMC) - bool (*smc_hs_congested)(const struct sock *sk); bool syn_smc; /* SYN includes SMC */ + bool (*smc_hs_congested)(const struct sock *sk); #endif #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) -- cgit v1.2.3 From 966942ae493650210b9514f3d4bfc95f78ef0129 Mon Sep 17 00:00:00 2001 From: Théo Lebrun Date: Wed, 28 Feb 2024 12:28:04 +0100 Subject: gpio: nomadik: extract GPIO platform driver from drivers/pinctrl/nomadik/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, drivers/pinctrl/nomadik/pinctrl-nomadik.c registered two platform drivers: pinctrl & GPIO. Move the GPIO aspect to the drivers/gpio/ folder, as would be expected. Both drivers are intertwined for a reason; pinctrl requires access to GPIO registers for pinmuxing, pull-disable, disabling interrupts while setting the muxing and wakeup control. Information sharing is done through a shared array containing GPIO chips and a few helper functions. That shared array is not touched from gpio-nomadik when CONFIG_PINCTRL_NOMADIK is not defined. Make no change to the code that moved into gpio-nomadik; there should be no behavior change following. A few functions are shared and header comments are added. Checkpatch warnings are addressed. NUM_BANKS is renamed to NMK_MAX_BANKS. It is supported to compile gpio-nomadik without pinctrl-nomadik. The opposite is not true. Signed-off-by: Théo Lebrun Link: https://lore.kernel.org/r/20240228-mbly-gpio-v2-6-3ba757474006@bootlin.com Signed-off-by: Linus Walleij --- MAINTAINERS | 1 + drivers/gpio/Kconfig | 12 + drivers/gpio/Makefile | 1 + drivers/gpio/gpio-nomadik.c | 660 ++++++++++++++++++++ drivers/pinctrl/nomadik/Kconfig | 5 +- drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c | 3 +- drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c | 3 +- drivers/pinctrl/nomadik/pinctrl-nomadik.c | 722 +--------------------- drivers/pinctrl/nomadik/pinctrl-nomadik.h | 180 ------ include/linux/gpio/gpio-nomadik.h | 276 +++++++++ 10 files changed, 971 insertions(+), 892 deletions(-) create mode 100644 drivers/gpio/gpio-nomadik.c delete mode 100644 drivers/pinctrl/nomadik/pinctrl-nomadik.h create mode 100644 include/linux/gpio/gpio-nomadik.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index c03718d39384..c6a9ae28a783 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2474,6 +2474,7 @@ F: drivers/clk/clk-nomadik.c F: drivers/clocksource/clksrc-dbx500-prcmu.c F: drivers/dma/ste_dma40* F: drivers/pmdomain/st/ste-ux500-pm-domain.c +F: drivers/gpio/gpio-nomadik.c F: drivers/hwspinlock/u8500_hsem.c F: drivers/i2c/busses/i2c-nomadik.c F: drivers/iio/adc/ab8500-gpadc.c diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 1301cec94f12..ff83371251c1 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -478,6 +478,18 @@ config GPIO_MXS select GPIO_GENERIC select GENERIC_IRQ_CHIP +config GPIO_NOMADIK + bool "Nomadik GPIO driver" + depends on ARCH_U8500 || ARCH_NOMADIK || COMPILE_TEST + select OF_GPIO + select GPIOLIB_IRQCHIP + help + Say yes here to support the Nomadik SoC GPIO block. + + It handles up to 32 GPIOs per bank, that can all be interrupt sources. + It is deeply interconnected with the associated pinctrl driver as GPIO + registers handle muxing ("alternate functions") as well. + config GPIO_NPCM_SGPIO bool "Nuvoton SGPIO support" depends on ARCH_NPCM || COMPILE_TEST diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 9e40af196aae..9fc2f5931b22 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -116,6 +116,7 @@ obj-$(CONFIG_GPIO_MT7621) += gpio-mt7621.o obj-$(CONFIG_GPIO_MVEBU) += gpio-mvebu.o obj-$(CONFIG_GPIO_MXC) += gpio-mxc.o obj-$(CONFIG_GPIO_MXS) += gpio-mxs.o +obj-$(CONFIG_GPIO_NOMADIK) += gpio-nomadik.o obj-$(CONFIG_GPIO_NPCM_SGPIO) += gpio-npcm-sgpio.o obj-$(CONFIG_GPIO_OCTEON) += gpio-octeon.o obj-$(CONFIG_GPIO_OMAP) += gpio-omap.o diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c new file mode 100644 index 000000000000..e39477e1a58f --- /dev/null +++ b/drivers/gpio/gpio-nomadik.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * GPIO driver for the IP block found in the Nomadik SoC; it is an AMBA device, + * managing 32 pins with alternate functions. It can also handle the STA2X11 + * block from ST. + * + * The GPIO chips are shared with pinctrl-nomadik if used; it needs access for + * pinmuxing functionality and others. + * + * Copyright (C) 2008,2009 STMicroelectronics + * Copyright (C) 2009 Alessandro Rubini + * Rewritten based on work by Prafulla WADASKAR + * Copyright (C) 2011-2013 Linus Walleij + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifndef CONFIG_PINCTRL_NOMADIK +static DEFINE_SPINLOCK(nmk_gpio_slpm_lock); +#endif + +void __nmk_gpio_set_slpm(struct nmk_gpio_chip *nmk_chip, unsigned int offset, + enum nmk_gpio_slpm mode) +{ + u32 slpm; + + slpm = readl(nmk_chip->addr + NMK_GPIO_SLPC); + if (mode == NMK_GPIO_SLPM_NOCHANGE) + slpm |= BIT(offset); + else + slpm &= ~BIT(offset); + writel(slpm, nmk_chip->addr + NMK_GPIO_SLPC); +} + +static void __nmk_gpio_set_output(struct nmk_gpio_chip *nmk_chip, + unsigned int offset, int val) +{ + if (val) + writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DATS); + else + writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DATC); +} + +void __nmk_gpio_make_output(struct nmk_gpio_chip *nmk_chip, + unsigned int offset, int val) +{ + writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DIRS); + __nmk_gpio_set_output(nmk_chip, offset, val); +} + +/* IRQ functions */ + +static void nmk_gpio_irq_ack(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + clk_enable(nmk_chip->clk); + writel(BIT(d->hwirq), nmk_chip->addr + NMK_GPIO_IC); + clk_disable(nmk_chip->clk); +} + +enum nmk_gpio_irq_type { + NORMAL, + WAKE, +}; + +static void __nmk_gpio_irq_modify(struct nmk_gpio_chip *nmk_chip, + int offset, enum nmk_gpio_irq_type which, + bool enable) +{ + u32 *rimscval; + u32 *fimscval; + u32 rimscreg; + u32 fimscreg; + + if (which == NORMAL) { + rimscreg = NMK_GPIO_RIMSC; + fimscreg = NMK_GPIO_FIMSC; + rimscval = &nmk_chip->rimsc; + fimscval = &nmk_chip->fimsc; + } else { + rimscreg = NMK_GPIO_RWIMSC; + fimscreg = NMK_GPIO_FWIMSC; + rimscval = &nmk_chip->rwimsc; + fimscval = &nmk_chip->fwimsc; + } + + /* we must individually set/clear the two edges */ + if (nmk_chip->edge_rising & BIT(offset)) { + if (enable) + *rimscval |= BIT(offset); + else + *rimscval &= ~BIT(offset); + writel(*rimscval, nmk_chip->addr + rimscreg); + } + if (nmk_chip->edge_falling & BIT(offset)) { + if (enable) + *fimscval |= BIT(offset); + else + *fimscval &= ~BIT(offset); + writel(*fimscval, nmk_chip->addr + fimscreg); + } +} + +static void __nmk_gpio_set_wake(struct nmk_gpio_chip *nmk_chip, + int offset, bool on) +{ + /* + * Ensure WAKEUP_ENABLE is on. No need to disable it if wakeup is + * disabled, since setting SLPM to 1 increases power consumption, and + * wakeup is anyhow controlled by the RIMSC and FIMSC registers. + */ + if (nmk_chip->sleepmode && on) { + __nmk_gpio_set_slpm(nmk_chip, offset, + NMK_GPIO_SLPM_WAKEUP_ENABLE); + } + + __nmk_gpio_irq_modify(nmk_chip, offset, WAKE, on); +} + +static void nmk_gpio_irq_maskunmask(struct nmk_gpio_chip *nmk_chip, + struct irq_data *d, bool enable) +{ + unsigned long flags; + + clk_enable(nmk_chip->clk); + spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); + spin_lock(&nmk_chip->lock); + + __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, enable); + + if (!(nmk_chip->real_wake & BIT(d->hwirq))) + __nmk_gpio_set_wake(nmk_chip, d->hwirq, enable); + + spin_unlock(&nmk_chip->lock); + spin_unlock_irqrestore(&nmk_gpio_slpm_lock, flags); + clk_disable(nmk_chip->clk); +} + +static void nmk_gpio_irq_mask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + nmk_gpio_irq_maskunmask(nmk_chip, d, false); + gpiochip_disable_irq(gc, irqd_to_hwirq(d)); +} + +static void nmk_gpio_irq_unmask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + gpiochip_enable_irq(gc, irqd_to_hwirq(d)); + nmk_gpio_irq_maskunmask(nmk_chip, d, true); +} + +static int nmk_gpio_irq_set_wake(struct irq_data *d, unsigned int on) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + unsigned long flags; + + clk_enable(nmk_chip->clk); + spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); + spin_lock(&nmk_chip->lock); + + if (irqd_irq_disabled(d)) + __nmk_gpio_set_wake(nmk_chip, d->hwirq, on); + + if (on) + nmk_chip->real_wake |= BIT(d->hwirq); + else + nmk_chip->real_wake &= ~BIT(d->hwirq); + + spin_unlock(&nmk_chip->lock); + spin_unlock_irqrestore(&nmk_gpio_slpm_lock, flags); + clk_disable(nmk_chip->clk); + + return 0; +} + +static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + bool enabled = !irqd_irq_disabled(d); + bool wake = irqd_is_wakeup_set(d); + unsigned long flags; + + if (type & IRQ_TYPE_LEVEL_HIGH) + return -EINVAL; + if (type & IRQ_TYPE_LEVEL_LOW) + return -EINVAL; + + clk_enable(nmk_chip->clk); + spin_lock_irqsave(&nmk_chip->lock, flags); + + if (enabled) + __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, false); + + if (enabled || wake) + __nmk_gpio_irq_modify(nmk_chip, d->hwirq, WAKE, false); + + nmk_chip->edge_rising &= ~BIT(d->hwirq); + if (type & IRQ_TYPE_EDGE_RISING) + nmk_chip->edge_rising |= BIT(d->hwirq); + + nmk_chip->edge_falling &= ~BIT(d->hwirq); + if (type & IRQ_TYPE_EDGE_FALLING) + nmk_chip->edge_falling |= BIT(d->hwirq); + + if (enabled) + __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, true); + + if (enabled || wake) + __nmk_gpio_irq_modify(nmk_chip, d->hwirq, WAKE, true); + + spin_unlock_irqrestore(&nmk_chip->lock, flags); + clk_disable(nmk_chip->clk); + + return 0; +} + +static unsigned int nmk_gpio_irq_startup(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + clk_enable(nmk_chip->clk); + nmk_gpio_irq_unmask(d); + return 0; +} + +static void nmk_gpio_irq_shutdown(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + nmk_gpio_irq_mask(d); + clk_disable(nmk_chip->clk); +} + +static void nmk_gpio_irq_handler(struct irq_desc *desc) +{ + struct irq_chip *host_chip = irq_desc_get_chip(desc); + struct gpio_chip *chip = irq_desc_get_handler_data(desc); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + u32 status; + + chained_irq_enter(host_chip, desc); + + clk_enable(nmk_chip->clk); + status = readl(nmk_chip->addr + NMK_GPIO_IS); + clk_disable(nmk_chip->clk); + + while (status) { + int bit = __ffs(status); + + generic_handle_domain_irq(chip->irq.domain, bit); + status &= ~BIT(bit); + } + + chained_irq_exit(host_chip, desc); +} + +/* I/O Functions */ + +static int nmk_gpio_get_dir(struct gpio_chip *chip, unsigned int offset) +{ + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + int dir; + + clk_enable(nmk_chip->clk); + + dir = readl(nmk_chip->addr + NMK_GPIO_DIR) & BIT(offset); + + clk_disable(nmk_chip->clk); + + if (dir) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + +static int nmk_gpio_make_input(struct gpio_chip *chip, unsigned int offset) +{ + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + + clk_enable(nmk_chip->clk); + + writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DIRC); + + clk_disable(nmk_chip->clk); + + return 0; +} + +static int nmk_gpio_get_input(struct gpio_chip *chip, unsigned int offset) +{ + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + int value; + + clk_enable(nmk_chip->clk); + + value = !!(readl(nmk_chip->addr + NMK_GPIO_DAT) & BIT(offset)); + + clk_disable(nmk_chip->clk); + + return value; +} + +static void nmk_gpio_set_output(struct gpio_chip *chip, unsigned int offset, + int val) +{ + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + + clk_enable(nmk_chip->clk); + + __nmk_gpio_set_output(nmk_chip, offset, val); + + clk_disable(nmk_chip->clk); +} + +static int nmk_gpio_make_output(struct gpio_chip *chip, unsigned int offset, + int val) +{ + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + + clk_enable(nmk_chip->clk); + + __nmk_gpio_make_output(nmk_chip, offset, val); + + clk_disable(nmk_chip->clk); + + return 0; +} + +#ifdef CONFIG_DEBUG_FS + +static int nmk_gpio_get_mode(struct nmk_gpio_chip *nmk_chip, int offset) +{ + u32 afunc, bfunc; + + clk_enable(nmk_chip->clk); + + afunc = readl(nmk_chip->addr + NMK_GPIO_AFSLA) & BIT(offset); + bfunc = readl(nmk_chip->addr + NMK_GPIO_AFSLB) & BIT(offset); + + clk_disable(nmk_chip->clk); + + return (afunc ? NMK_GPIO_ALT_A : 0) | (bfunc ? NMK_GPIO_ALT_B : 0); +} + +void nmk_gpio_dbg_show_one(struct seq_file *s, struct pinctrl_dev *pctldev, + struct gpio_chip *chip, unsigned int offset, + unsigned int gpio) +{ + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + int mode; + bool is_out; + bool data_out; + bool pull; + static const char * const modes[] = { + [NMK_GPIO_ALT_GPIO] = "gpio", + [NMK_GPIO_ALT_A] = "altA", + [NMK_GPIO_ALT_B] = "altB", + [NMK_GPIO_ALT_C] = "altC", + [NMK_GPIO_ALT_C + 1] = "altC1", + [NMK_GPIO_ALT_C + 2] = "altC2", + [NMK_GPIO_ALT_C + 3] = "altC3", + [NMK_GPIO_ALT_C + 4] = "altC4", + }; + + char *label = gpiochip_dup_line_label(chip, offset); + if (IS_ERR(label)) + return; + + clk_enable(nmk_chip->clk); + is_out = !!(readl(nmk_chip->addr + NMK_GPIO_DIR) & BIT(offset)); + pull = !(readl(nmk_chip->addr + NMK_GPIO_PDIS) & BIT(offset)); + data_out = !!(readl(nmk_chip->addr + NMK_GPIO_DAT) & BIT(offset)); + mode = nmk_gpio_get_mode(nmk_chip, offset); +#ifdef CONFIG_PINCTRL_NOMADIK + if (mode == NMK_GPIO_ALT_C && pctldev) + mode = nmk_prcm_gpiocr_get_mode(pctldev, gpio); +#endif + + if (is_out) { + seq_printf(s, " gpio-%-3d (%-20.20s) out %s %s", + gpio, + label ?: "(none)", + data_out ? "hi" : "lo", + (mode < 0) ? "unknown" : modes[mode]); + } else { + int irq = chip->to_irq(chip, offset); + const int pullidx = pull ? 1 : 0; + int val; + static const char * const pulls[] = { + "none ", + "pull enabled", + }; + + seq_printf(s, " gpio-%-3d (%-20.20s) in %s %s", + gpio, + label ?: "(none)", + pulls[pullidx], + (mode < 0) ? "unknown" : modes[mode]); + + val = nmk_gpio_get_input(chip, offset); + seq_printf(s, " VAL %d", val); + + /* + * This races with request_irq(), set_irq_type(), + * and set_irq_wake() ... but those are "rare". + */ + if (irq > 0 && irq_has_action(irq)) { + char *trigger; + bool wake; + + if (nmk_chip->edge_rising & BIT(offset)) + trigger = "edge-rising"; + else if (nmk_chip->edge_falling & BIT(offset)) + trigger = "edge-falling"; + else + trigger = "edge-undefined"; + + wake = !!(nmk_chip->real_wake & BIT(offset)); + + seq_printf(s, " irq-%d %s%s", + irq, trigger, wake ? " wakeup" : ""); + } + } + clk_disable(nmk_chip->clk); +} + +static void nmk_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) +{ + unsigned int i, gpio = chip->base; + + for (i = 0; i < chip->ngpio; i++, gpio++) { + nmk_gpio_dbg_show_one(s, NULL, chip, i, gpio); + seq_puts(s, "\n"); + } +} + +#else + +static inline void nmk_gpio_dbg_show_one(struct seq_file *s, + struct pinctrl_dev *pctldev, + struct gpio_chip *chip, + unsigned int offset, + unsigned int gpio) +{ +} + +#define nmk_gpio_dbg_show NULL + +#endif + +/* + * We will allocate memory for the state container using devm* allocators + * binding to the first device reaching this point, it doesn't matter if + * it is the pin controller or GPIO driver. However we need to use the right + * platform device when looking up resources so pay attention to pdev. + */ +struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, + struct platform_device *pdev) +{ + struct nmk_gpio_chip *nmk_chip; + struct platform_device *gpio_pdev; + struct gpio_chip *chip; + struct resource *res; + struct clk *clk; + void __iomem *base; + u32 id; + + gpio_pdev = of_find_device_by_node(np); + if (!gpio_pdev) { + pr_err("populate \"%pOFn\": device not found\n", np); + return ERR_PTR(-ENODEV); + } + if (of_property_read_u32(np, "gpio-bank", &id)) { + dev_err(&pdev->dev, "populate: gpio-bank property not found\n"); + platform_device_put(gpio_pdev); + return ERR_PTR(-EINVAL); + } + +#ifdef CONFIG_PINCTRL_NOMADIK + /* Already populated? */ + nmk_chip = nmk_gpio_chips[id]; + if (nmk_chip) { + platform_device_put(gpio_pdev); + return nmk_chip; + } +#endif + + nmk_chip = devm_kzalloc(&pdev->dev, sizeof(*nmk_chip), GFP_KERNEL); + if (!nmk_chip) { + platform_device_put(gpio_pdev); + return ERR_PTR(-ENOMEM); + } + + nmk_chip->bank = id; + chip = &nmk_chip->chip; + chip->base = id * NMK_GPIO_PER_CHIP; + chip->ngpio = NMK_GPIO_PER_CHIP; + chip->label = dev_name(&gpio_pdev->dev); + chip->parent = &gpio_pdev->dev; + + res = platform_get_resource(gpio_pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) { + platform_device_put(gpio_pdev); + return ERR_CAST(base); + } + nmk_chip->addr = base; + + clk = clk_get(&gpio_pdev->dev, NULL); + if (IS_ERR(clk)) { + platform_device_put(gpio_pdev); + return (void *)clk; + } + clk_prepare(clk); + nmk_chip->clk = clk; + +#ifdef CONFIG_PINCTRL_NOMADIK + BUG_ON(nmk_chip->bank >= ARRAY_SIZE(nmk_gpio_chips)); + nmk_gpio_chips[id] = nmk_chip; +#endif + return nmk_chip; +} + +static void nmk_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + seq_printf(p, "nmk%u-%u-%u", nmk_chip->bank, + gc->base, gc->base + gc->ngpio - 1); +} + +static const struct irq_chip nmk_irq_chip = { + .irq_ack = nmk_gpio_irq_ack, + .irq_mask = nmk_gpio_irq_mask, + .irq_unmask = nmk_gpio_irq_unmask, + .irq_set_type = nmk_gpio_irq_set_type, + .irq_set_wake = nmk_gpio_irq_set_wake, + .irq_startup = nmk_gpio_irq_startup, + .irq_shutdown = nmk_gpio_irq_shutdown, + .irq_print_chip = nmk_gpio_irq_print_chip, + .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + +static int nmk_gpio_probe(struct platform_device *dev) +{ + struct device_node *np = dev->dev.of_node; + struct nmk_gpio_chip *nmk_chip; + struct gpio_chip *chip; + struct gpio_irq_chip *girq; + bool supports_sleepmode; + int irq; + int ret; + + nmk_chip = nmk_gpio_populate_chip(np, dev); + if (IS_ERR(nmk_chip)) { + dev_err(&dev->dev, "could not populate nmk chip struct\n"); + return PTR_ERR(nmk_chip); + } + + supports_sleepmode = + of_property_read_bool(np, "st,supports-sleepmode"); + + /* Correct platform device ID */ + dev->id = nmk_chip->bank; + + irq = platform_get_irq(dev, 0); + if (irq < 0) + return irq; + + /* + * The virt address in nmk_chip->addr is in the nomadik register space, + * so we can simply convert the resource address, without remapping + */ + nmk_chip->sleepmode = supports_sleepmode; + spin_lock_init(&nmk_chip->lock); + + chip = &nmk_chip->chip; + chip->parent = &dev->dev; + chip->request = gpiochip_generic_request; + chip->free = gpiochip_generic_free; + chip->get_direction = nmk_gpio_get_dir; + chip->direction_input = nmk_gpio_make_input; + chip->get = nmk_gpio_get_input; + chip->direction_output = nmk_gpio_make_output; + chip->set = nmk_gpio_set_output; + chip->dbg_show = nmk_gpio_dbg_show; + chip->can_sleep = false; + chip->owner = THIS_MODULE; + + girq = &chip->irq; + gpio_irq_chip_set_chip(girq, &nmk_irq_chip); + girq->parent_handler = nmk_gpio_irq_handler; + girq->num_parents = 1; + girq->parents = devm_kcalloc(&dev->dev, 1, + sizeof(*girq->parents), + GFP_KERNEL); + if (!girq->parents) + return -ENOMEM; + girq->parents[0] = irq; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_edge_irq; + + clk_enable(nmk_chip->clk); + nmk_chip->lowemi = readl_relaxed(nmk_chip->addr + NMK_GPIO_LOWEMI); + clk_disable(nmk_chip->clk); + + ret = gpiochip_add_data(chip, nmk_chip); + if (ret) + return ret; + + platform_set_drvdata(dev, nmk_chip); + + dev_info(&dev->dev, "chip registered\n"); + + return 0; +} + +static const struct of_device_id nmk_gpio_match[] = { + { .compatible = "st,nomadik-gpio", }, + {} +}; + +static struct platform_driver nmk_gpio_driver = { + .driver = { + .name = "gpio", + .of_match_table = nmk_gpio_match, + }, + .probe = nmk_gpio_probe, +}; + +static int __init nmk_gpio_init(void) +{ + return platform_driver_register(&nmk_gpio_driver); +} +subsys_initcall(nmk_gpio_init); diff --git a/drivers/pinctrl/nomadik/Kconfig b/drivers/pinctrl/nomadik/Kconfig index 0fea167c283f..f47f0755a835 100644 --- a/drivers/pinctrl/nomadik/Kconfig +++ b/drivers/pinctrl/nomadik/Kconfig @@ -22,11 +22,10 @@ if (ARCH_U8500 || ARCH_NOMADIK) config PINCTRL_NOMADIK bool "Nomadik pin controller driver" - depends on OF && GPIOLIB + depends on OF select PINMUX select PINCONF - select OF_GPIO - select GPIOLIB_IRQCHIP + select GPIO_NOMADIK config PINCTRL_STN8815 bool "STN8815 pin controller driver" diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c index 490e0959e8be..0b4a3dd9d8c7 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c @@ -3,8 +3,9 @@ #include #include +#include -#include "pinctrl-nomadik.h" +#include /* All the pins that can be used for GPIO and some other functions */ #define _GPIO(offset) (offset) diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c b/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c index 1552222ac68e..c5a52fcaba30 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c @@ -3,8 +3,9 @@ #include #include +#include -#include "pinctrl-nomadik.h" +#include /* All the pins that can be used for GPIO and some other functions */ #define _GPIO(offset) (offset) diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c index 4f7c4af4f93c..16d836157f74 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Generic GPIO driver for logic cells found in the Nomadik SoC + * Pinmux & pinconf driver for the IP block found in the Nomadik SoC. This + * depends on gpio-nomadik and some handling is intertwined; see nmk_gpio_chips + * which is used by this driver to access the GPIO banks array. * * Copyright (C) 2008,2009 STMicroelectronics * Copyright (C) 2009 Alessandro Rubini @@ -25,6 +27,7 @@ #include #include #include +#include /* Since we request GPIOs from ourself */ #include @@ -36,15 +39,7 @@ #include "../core.h" #include "../pinctrl-utils.h" -#include "pinctrl-nomadik.h" - -/* - * The GPIO module in the Nomadik family of Systems-on-Chip is an - * AMBA device, managing 32 pins and alternate functions. The logic block - * is currently used in the Nomadik and ux500. - * - * Symbols in this file are called "nmk_gpio" for "nomadik gpio" - */ +#include /* * pin configurations are represented by 32-bit integers: @@ -200,75 +195,6 @@ typedef unsigned long pin_cfg_t; (PIN_CFG_DEFAULT |\ (PIN_NUM(num) | PIN_##alt | PIN_OUTPUT_##val)) -/* - * "nmk_gpio" and "NMK_GPIO" stand for "Nomadik GPIO", leaving - * the "gpio" namespace for generic and cross-machine functions - */ - -#define GPIO_BLOCK_SHIFT 5 -#define NMK_GPIO_PER_CHIP (1 << GPIO_BLOCK_SHIFT) -#define NMK_MAX_BANKS DIV_ROUND_UP(512, NMK_GPIO_PER_CHIP) - -/* Register in the logic block */ -#define NMK_GPIO_DAT 0x00 -#define NMK_GPIO_DATS 0x04 -#define NMK_GPIO_DATC 0x08 -#define NMK_GPIO_PDIS 0x0c -#define NMK_GPIO_DIR 0x10 -#define NMK_GPIO_DIRS 0x14 -#define NMK_GPIO_DIRC 0x18 -#define NMK_GPIO_SLPC 0x1c -#define NMK_GPIO_AFSLA 0x20 -#define NMK_GPIO_AFSLB 0x24 -#define NMK_GPIO_LOWEMI 0x28 - -#define NMK_GPIO_RIMSC 0x40 -#define NMK_GPIO_FIMSC 0x44 -#define NMK_GPIO_IS 0x48 -#define NMK_GPIO_IC 0x4c -#define NMK_GPIO_RWIMSC 0x50 -#define NMK_GPIO_FWIMSC 0x54 -#define NMK_GPIO_WKS 0x58 -/* These appear in DB8540 and later ASICs */ -#define NMK_GPIO_EDGELEVEL 0x5C -#define NMK_GPIO_LEVEL 0x60 - - -/* Pull up/down values */ -enum nmk_gpio_pull { - NMK_GPIO_PULL_NONE, - NMK_GPIO_PULL_UP, - NMK_GPIO_PULL_DOWN, -}; - -/* Sleep mode */ -enum nmk_gpio_slpm { - NMK_GPIO_SLPM_INPUT, - NMK_GPIO_SLPM_WAKEUP_ENABLE = NMK_GPIO_SLPM_INPUT, - NMK_GPIO_SLPM_NOCHANGE, - NMK_GPIO_SLPM_WAKEUP_DISABLE = NMK_GPIO_SLPM_NOCHANGE, -}; - -struct nmk_gpio_chip { - struct gpio_chip chip; - void __iomem *addr; - struct clk *clk; - unsigned int bank; - void (*set_ioforce)(bool enable); - spinlock_t lock; - bool sleepmode; - /* Keep track of configured edges */ - u32 edge_rising; - u32 edge_falling; - u32 real_wake; - u32 rwimsc; - u32 fwimsc; - u32 rimsc; - u32 fimsc; - u32 pull_up; - u32 lowemi; -}; - /** * struct nmk_pinctrl - state container for the Nomadik pin controller * @dev: containing device pointer @@ -283,11 +209,10 @@ struct nmk_pinctrl { void __iomem *prcm_base; }; -static struct nmk_gpio_chip *nmk_gpio_chips[NMK_MAX_BANKS]; +/* See nmk_gpio_populate_chip() that fills this array. */ +struct nmk_gpio_chip *nmk_gpio_chips[NMK_MAX_BANKS]; -static DEFINE_SPINLOCK(nmk_gpio_slpm_lock); - -#define NUM_BANKS ARRAY_SIZE(nmk_gpio_chips) +DEFINE_SPINLOCK(nmk_gpio_slpm_lock); static void __nmk_gpio_set_mode(struct nmk_gpio_chip *nmk_chip, unsigned offset, int gpio_mode) @@ -304,19 +229,6 @@ static void __nmk_gpio_set_mode(struct nmk_gpio_chip *nmk_chip, writel(bfunc, nmk_chip->addr + NMK_GPIO_AFSLB); } -static void __nmk_gpio_set_slpm(struct nmk_gpio_chip *nmk_chip, - unsigned offset, enum nmk_gpio_slpm mode) -{ - u32 slpm; - - slpm = readl(nmk_chip->addr + NMK_GPIO_SLPC); - if (mode == NMK_GPIO_SLPM_NOCHANGE) - slpm |= BIT(offset); - else - slpm &= ~BIT(offset); - writel(slpm, nmk_chip->addr + NMK_GPIO_SLPC); -} - static void __nmk_gpio_set_pull(struct nmk_gpio_chip *nmk_chip, unsigned offset, enum nmk_gpio_pull pull) { @@ -364,22 +276,6 @@ static void __nmk_gpio_make_input(struct nmk_gpio_chip *nmk_chip, writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DIRC); } -static void __nmk_gpio_set_output(struct nmk_gpio_chip *nmk_chip, - unsigned offset, int val) -{ - if (val) - writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DATS); - else - writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DATC); -} - -static void __nmk_gpio_make_output(struct nmk_gpio_chip *nmk_chip, - unsigned offset, int val) -{ - writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DIRS); - __nmk_gpio_set_output(nmk_chip, offset, val); -} - static void __nmk_gpio_set_mode_safe(struct nmk_gpio_chip *nmk_chip, unsigned offset, int gpio_mode, bool glitch) @@ -548,7 +444,7 @@ static void nmk_gpio_glitch_slpm_init(unsigned int *slpm) { int i; - for (i = 0; i < NUM_BANKS; i++) { + for (i = 0; i < NMK_MAX_BANKS; i++) { struct nmk_gpio_chip *chip = nmk_gpio_chips[i]; unsigned int temp = slpm[i]; @@ -566,7 +462,7 @@ static void nmk_gpio_glitch_slpm_restore(unsigned int *slpm) { int i; - for (i = 0; i < NUM_BANKS; i++) { + for (i = 0; i < NMK_MAX_BANKS; i++) { struct nmk_gpio_chip *chip = nmk_gpio_chips[i]; if (!chip) @@ -578,7 +474,8 @@ static void nmk_gpio_glitch_slpm_restore(unsigned int *slpm) } } -static int __maybe_unused nmk_prcm_gpiocr_get_mode(struct pinctrl_dev *pctldev, int gpio) +/* Only called by gpio-nomadik but requires knowledge of struct nmk_pinctrl. */ +int __maybe_unused nmk_prcm_gpiocr_get_mode(struct pinctrl_dev *pctldev, int gpio) { int i; u16 reg; @@ -610,576 +507,6 @@ static int __maybe_unused nmk_prcm_gpiocr_get_mode(struct pinctrl_dev *pctldev, return NMK_GPIO_ALT_C; } -/* IRQ functions */ - -static void nmk_gpio_irq_ack(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - - clk_enable(nmk_chip->clk); - writel(BIT(d->hwirq), nmk_chip->addr + NMK_GPIO_IC); - clk_disable(nmk_chip->clk); -} - -enum nmk_gpio_irq_type { - NORMAL, - WAKE, -}; - -static void __nmk_gpio_irq_modify(struct nmk_gpio_chip *nmk_chip, - int offset, enum nmk_gpio_irq_type which, - bool enable) -{ - u32 *rimscval; - u32 *fimscval; - u32 rimscreg; - u32 fimscreg; - - if (which == NORMAL) { - rimscreg = NMK_GPIO_RIMSC; - fimscreg = NMK_GPIO_FIMSC; - rimscval = &nmk_chip->rimsc; - fimscval = &nmk_chip->fimsc; - } else { - rimscreg = NMK_GPIO_RWIMSC; - fimscreg = NMK_GPIO_FWIMSC; - rimscval = &nmk_chip->rwimsc; - fimscval = &nmk_chip->fwimsc; - } - - /* we must individually set/clear the two edges */ - if (nmk_chip->edge_rising & BIT(offset)) { - if (enable) - *rimscval |= BIT(offset); - else - *rimscval &= ~BIT(offset); - writel(*rimscval, nmk_chip->addr + rimscreg); - } - if (nmk_chip->edge_falling & BIT(offset)) { - if (enable) - *fimscval |= BIT(offset); - else - *fimscval &= ~BIT(offset); - writel(*fimscval, nmk_chip->addr + fimscreg); - } -} - -static void __nmk_gpio_set_wake(struct nmk_gpio_chip *nmk_chip, - int offset, bool on) -{ - /* - * Ensure WAKEUP_ENABLE is on. No need to disable it if wakeup is - * disabled, since setting SLPM to 1 increases power consumption, and - * wakeup is anyhow controlled by the RIMSC and FIMSC registers. - */ - if (nmk_chip->sleepmode && on) { - __nmk_gpio_set_slpm(nmk_chip, offset, - NMK_GPIO_SLPM_WAKEUP_ENABLE); - } - - __nmk_gpio_irq_modify(nmk_chip, offset, WAKE, on); -} - -static void nmk_gpio_irq_maskunmask(struct nmk_gpio_chip *nmk_chip, - struct irq_data *d, bool enable) -{ - unsigned long flags; - - clk_enable(nmk_chip->clk); - spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); - spin_lock(&nmk_chip->lock); - - __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, enable); - - if (!(nmk_chip->real_wake & BIT(d->hwirq))) - __nmk_gpio_set_wake(nmk_chip, d->hwirq, enable); - - spin_unlock(&nmk_chip->lock); - spin_unlock_irqrestore(&nmk_gpio_slpm_lock, flags); - clk_disable(nmk_chip->clk); -} - -static void nmk_gpio_irq_mask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - - nmk_gpio_irq_maskunmask(nmk_chip, d, false); - gpiochip_disable_irq(gc, irqd_to_hwirq(d)); -} - -static void nmk_gpio_irq_unmask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - - gpiochip_enable_irq(gc, irqd_to_hwirq(d)); - nmk_gpio_irq_maskunmask(nmk_chip, d, true); -} - -static int nmk_gpio_irq_set_wake(struct irq_data *d, unsigned int on) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - unsigned long flags; - - clk_enable(nmk_chip->clk); - spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); - spin_lock(&nmk_chip->lock); - - if (irqd_irq_disabled(d)) - __nmk_gpio_set_wake(nmk_chip, d->hwirq, on); - - if (on) - nmk_chip->real_wake |= BIT(d->hwirq); - else - nmk_chip->real_wake &= ~BIT(d->hwirq); - - spin_unlock(&nmk_chip->lock); - spin_unlock_irqrestore(&nmk_gpio_slpm_lock, flags); - clk_disable(nmk_chip->clk); - - return 0; -} - -static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - bool enabled = !irqd_irq_disabled(d); - bool wake = irqd_is_wakeup_set(d); - unsigned long flags; - - if (type & IRQ_TYPE_LEVEL_HIGH) - return -EINVAL; - if (type & IRQ_TYPE_LEVEL_LOW) - return -EINVAL; - - clk_enable(nmk_chip->clk); - spin_lock_irqsave(&nmk_chip->lock, flags); - - if (enabled) - __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, false); - - if (enabled || wake) - __nmk_gpio_irq_modify(nmk_chip, d->hwirq, WAKE, false); - - nmk_chip->edge_rising &= ~BIT(d->hwirq); - if (type & IRQ_TYPE_EDGE_RISING) - nmk_chip->edge_rising |= BIT(d->hwirq); - - nmk_chip->edge_falling &= ~BIT(d->hwirq); - if (type & IRQ_TYPE_EDGE_FALLING) - nmk_chip->edge_falling |= BIT(d->hwirq); - - if (enabled) - __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, true); - - if (enabled || wake) - __nmk_gpio_irq_modify(nmk_chip, d->hwirq, WAKE, true); - - spin_unlock_irqrestore(&nmk_chip->lock, flags); - clk_disable(nmk_chip->clk); - - return 0; -} - -static unsigned int nmk_gpio_irq_startup(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - - clk_enable(nmk_chip->clk); - nmk_gpio_irq_unmask(d); - return 0; -} - -static void nmk_gpio_irq_shutdown(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - - nmk_gpio_irq_mask(d); - clk_disable(nmk_chip->clk); -} - -static void nmk_gpio_irq_handler(struct irq_desc *desc) -{ - struct irq_chip *host_chip = irq_desc_get_chip(desc); - struct gpio_chip *chip = irq_desc_get_handler_data(desc); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - u32 status; - - chained_irq_enter(host_chip, desc); - - clk_enable(nmk_chip->clk); - status = readl(nmk_chip->addr + NMK_GPIO_IS); - clk_disable(nmk_chip->clk); - - while (status) { - int bit = __ffs(status); - - generic_handle_domain_irq(chip->irq.domain, bit); - status &= ~BIT(bit); - } - - chained_irq_exit(host_chip, desc); -} - -/* I/O Functions */ - -static int nmk_gpio_get_dir(struct gpio_chip *chip, unsigned offset) -{ - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - int dir; - - clk_enable(nmk_chip->clk); - - dir = readl(nmk_chip->addr + NMK_GPIO_DIR) & BIT(offset); - - clk_disable(nmk_chip->clk); - - if (dir) - return GPIO_LINE_DIRECTION_OUT; - - return GPIO_LINE_DIRECTION_IN; -} - -static int nmk_gpio_make_input(struct gpio_chip *chip, unsigned offset) -{ - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - - clk_enable(nmk_chip->clk); - - writel(BIT(offset), nmk_chip->addr + NMK_GPIO_DIRC); - - clk_disable(nmk_chip->clk); - - return 0; -} - -static int nmk_gpio_get_input(struct gpio_chip *chip, unsigned offset) -{ - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - int value; - - clk_enable(nmk_chip->clk); - - value = !!(readl(nmk_chip->addr + NMK_GPIO_DAT) & BIT(offset)); - - clk_disable(nmk_chip->clk); - - return value; -} - -static void nmk_gpio_set_output(struct gpio_chip *chip, unsigned offset, - int val) -{ - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - - clk_enable(nmk_chip->clk); - - __nmk_gpio_set_output(nmk_chip, offset, val); - - clk_disable(nmk_chip->clk); -} - -static int nmk_gpio_make_output(struct gpio_chip *chip, unsigned offset, - int val) -{ - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - - clk_enable(nmk_chip->clk); - - __nmk_gpio_make_output(nmk_chip, offset, val); - - clk_disable(nmk_chip->clk); - - return 0; -} - -#ifdef CONFIG_DEBUG_FS -static int nmk_gpio_get_mode(struct nmk_gpio_chip *nmk_chip, int offset) -{ - u32 afunc, bfunc; - - clk_enable(nmk_chip->clk); - - afunc = readl(nmk_chip->addr + NMK_GPIO_AFSLA) & BIT(offset); - bfunc = readl(nmk_chip->addr + NMK_GPIO_AFSLB) & BIT(offset); - - clk_disable(nmk_chip->clk); - - return (afunc ? NMK_GPIO_ALT_A : 0) | (bfunc ? NMK_GPIO_ALT_B : 0); -} - -static void nmk_gpio_dbg_show_one(struct seq_file *s, - struct pinctrl_dev *pctldev, struct gpio_chip *chip, - unsigned offset, unsigned gpio) -{ - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); - int mode; - bool is_out; - bool data_out; - bool pull; - const char *modes[] = { - [NMK_GPIO_ALT_GPIO] = "gpio", - [NMK_GPIO_ALT_A] = "altA", - [NMK_GPIO_ALT_B] = "altB", - [NMK_GPIO_ALT_C] = "altC", - [NMK_GPIO_ALT_C+1] = "altC1", - [NMK_GPIO_ALT_C+2] = "altC2", - [NMK_GPIO_ALT_C+3] = "altC3", - [NMK_GPIO_ALT_C+4] = "altC4", - }; - - char *label = gpiochip_dup_line_label(chip, offset); - if (IS_ERR(label)) - return; - - clk_enable(nmk_chip->clk); - is_out = !!(readl(nmk_chip->addr + NMK_GPIO_DIR) & BIT(offset)); - pull = !(readl(nmk_chip->addr + NMK_GPIO_PDIS) & BIT(offset)); - data_out = !!(readl(nmk_chip->addr + NMK_GPIO_DAT) & BIT(offset)); - mode = nmk_gpio_get_mode(nmk_chip, offset); - if ((mode == NMK_GPIO_ALT_C) && pctldev) - mode = nmk_prcm_gpiocr_get_mode(pctldev, gpio); - - if (is_out) { - seq_printf(s, " gpio-%-3d (%-20.20s) out %s %s", - gpio, - label ?: "(none)", - data_out ? "hi" : "lo", - (mode < 0) ? "unknown" : modes[mode]); - } else { - int irq = chip->to_irq(chip, offset); - const int pullidx = pull ? 1 : 0; - int val; - static const char * const pulls[] = { - "none ", - "pull enabled", - }; - - seq_printf(s, " gpio-%-3d (%-20.20s) in %s %s", - gpio, - label ?: "(none)", - pulls[pullidx], - (mode < 0) ? "unknown" : modes[mode]); - - val = nmk_gpio_get_input(chip, offset); - seq_printf(s, " VAL %d", val); - - /* - * This races with request_irq(), set_irq_type(), - * and set_irq_wake() ... but those are "rare". - */ - if (irq > 0 && irq_has_action(irq)) { - char *trigger; - bool wake; - - if (nmk_chip->edge_rising & BIT(offset)) - trigger = "edge-rising"; - else if (nmk_chip->edge_falling & BIT(offset)) - trigger = "edge-falling"; - else - trigger = "edge-undefined"; - - wake = !!(nmk_chip->real_wake & BIT(offset)); - - seq_printf(s, " irq-%d %s%s", - irq, trigger, wake ? " wakeup" : ""); - } - } - clk_disable(nmk_chip->clk); -} - -static void nmk_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) -{ - unsigned i; - unsigned gpio = chip->base; - - for (i = 0; i < chip->ngpio; i++, gpio++) { - nmk_gpio_dbg_show_one(s, NULL, chip, i, gpio); - seq_printf(s, "\n"); - } -} - -#else -static inline void nmk_gpio_dbg_show_one(struct seq_file *s, - struct pinctrl_dev *pctldev, - struct gpio_chip *chip, - unsigned offset, unsigned gpio) -{ -} -#define nmk_gpio_dbg_show NULL -#endif - -/* - * We will allocate memory for the state container using devm* allocators - * binding to the first device reaching this point, it doesn't matter if - * it is the pin controller or GPIO driver. However we need to use the right - * platform device when looking up resources so pay attention to pdev. - */ -static struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, - struct platform_device *pdev) -{ - struct nmk_gpio_chip *nmk_chip; - struct platform_device *gpio_pdev; - struct gpio_chip *chip; - struct resource *res; - struct clk *clk; - void __iomem *base; - u32 id; - - gpio_pdev = of_find_device_by_node(np); - if (!gpio_pdev) { - pr_err("populate \"%pOFn\": device not found\n", np); - return ERR_PTR(-ENODEV); - } - if (of_property_read_u32(np, "gpio-bank", &id)) { - dev_err(&pdev->dev, "populate: gpio-bank property not found\n"); - platform_device_put(gpio_pdev); - return ERR_PTR(-EINVAL); - } - - /* Already populated? */ - nmk_chip = nmk_gpio_chips[id]; - if (nmk_chip) { - platform_device_put(gpio_pdev); - return nmk_chip; - } - - nmk_chip = devm_kzalloc(&pdev->dev, sizeof(*nmk_chip), GFP_KERNEL); - if (!nmk_chip) { - platform_device_put(gpio_pdev); - return ERR_PTR(-ENOMEM); - } - - nmk_chip->bank = id; - chip = &nmk_chip->chip; - chip->base = id * NMK_GPIO_PER_CHIP; - chip->ngpio = NMK_GPIO_PER_CHIP; - chip->label = dev_name(&gpio_pdev->dev); - chip->parent = &gpio_pdev->dev; - - res = platform_get_resource(gpio_pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(base)) { - platform_device_put(gpio_pdev); - return ERR_CAST(base); - } - nmk_chip->addr = base; - - clk = clk_get(&gpio_pdev->dev, NULL); - if (IS_ERR(clk)) { - platform_device_put(gpio_pdev); - return (void *) clk; - } - clk_prepare(clk); - nmk_chip->clk = clk; - - BUG_ON(nmk_chip->bank >= ARRAY_SIZE(nmk_gpio_chips)); - nmk_gpio_chips[id] = nmk_chip; - return nmk_chip; -} - -static void nmk_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); - - seq_printf(p, "nmk%u-%u-%u", nmk_chip->bank, - gc->base, gc->base + gc->ngpio - 1); -} - -static const struct irq_chip nmk_irq_chip = { - .irq_ack = nmk_gpio_irq_ack, - .irq_mask = nmk_gpio_irq_mask, - .irq_unmask = nmk_gpio_irq_unmask, - .irq_set_type = nmk_gpio_irq_set_type, - .irq_set_wake = nmk_gpio_irq_set_wake, - .irq_startup = nmk_gpio_irq_startup, - .irq_shutdown = nmk_gpio_irq_shutdown, - .irq_print_chip = nmk_gpio_irq_print_chip, - .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_IMMUTABLE, - GPIOCHIP_IRQ_RESOURCE_HELPERS, -}; - -static int nmk_gpio_probe(struct platform_device *dev) -{ - struct device_node *np = dev->dev.of_node; - struct nmk_gpio_chip *nmk_chip; - struct gpio_chip *chip; - struct gpio_irq_chip *girq; - bool supports_sleepmode; - int irq; - int ret; - - nmk_chip = nmk_gpio_populate_chip(np, dev); - if (IS_ERR(nmk_chip)) { - dev_err(&dev->dev, "could not populate nmk chip struct\n"); - return PTR_ERR(nmk_chip); - } - - supports_sleepmode = - of_property_read_bool(np, "st,supports-sleepmode"); - - /* Correct platform device ID */ - dev->id = nmk_chip->bank; - - irq = platform_get_irq(dev, 0); - if (irq < 0) - return irq; - - /* - * The virt address in nmk_chip->addr is in the nomadik register space, - * so we can simply convert the resource address, without remapping - */ - nmk_chip->sleepmode = supports_sleepmode; - spin_lock_init(&nmk_chip->lock); - - chip = &nmk_chip->chip; - chip->parent = &dev->dev; - chip->request = gpiochip_generic_request; - chip->free = gpiochip_generic_free; - chip->get_direction = nmk_gpio_get_dir; - chip->direction_input = nmk_gpio_make_input; - chip->get = nmk_gpio_get_input; - chip->direction_output = nmk_gpio_make_output; - chip->set = nmk_gpio_set_output; - chip->dbg_show = nmk_gpio_dbg_show; - chip->can_sleep = false; - chip->owner = THIS_MODULE; - - girq = &chip->irq; - gpio_irq_chip_set_chip(girq, &nmk_irq_chip); - girq->parent_handler = nmk_gpio_irq_handler; - girq->num_parents = 1; - girq->parents = devm_kcalloc(&dev->dev, 1, - sizeof(*girq->parents), - GFP_KERNEL); - if (!girq->parents) - return -ENOMEM; - girq->parents[0] = irq; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_edge_irq; - - clk_enable(nmk_chip->clk); - nmk_chip->lowemi = readl_relaxed(nmk_chip->addr + NMK_GPIO_LOWEMI); - clk_disable(nmk_chip->clk); - - ret = gpiochip_add_data(chip, nmk_chip); - if (ret) - return ret; - - platform_set_drvdata(dev, nmk_chip); - - dev_info(&dev->dev, "chip registered\n"); - - return 0; -} - static int nmk_get_groups_cnt(struct pinctrl_dev *pctldev) { struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev); @@ -1197,12 +524,12 @@ static const char *nmk_get_group_name(struct pinctrl_dev *pctldev, static int nmk_get_group_pins(struct pinctrl_dev *pctldev, unsigned selector, const unsigned **pins, - unsigned *npins) + unsigned int *num_pins) { struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev); *pins = npct->soc->groups[selector].grp.pins; - *npins = npct->soc->groups[selector].grp.npins; + *num_pins = npct->soc->groups[selector].grp.npins; return 0; } @@ -1533,7 +860,7 @@ static int nmk_pmx_set(struct pinctrl_dev *pctldev, unsigned function, { struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev); const struct nmk_pingroup *g; - static unsigned int slpm[NUM_BANKS]; + static unsigned int slpm[NMK_MAX_BANKS]; unsigned long flags = 0; bool glitch; int ret = -EINVAL; @@ -1921,19 +1248,6 @@ static int nmk_pinctrl_probe(struct platform_device *pdev) return 0; } -static const struct of_device_id nmk_gpio_match[] = { - { .compatible = "st,nomadik-gpio", }, - {} -}; - -static struct platform_driver nmk_gpio_driver = { - .driver = { - .name = "gpio", - .of_match_table = nmk_gpio_match, - }, - .probe = nmk_gpio_probe, -}; - static SIMPLE_DEV_PM_OPS(nmk_pinctrl_pm_ops, nmk_pinctrl_suspend, nmk_pinctrl_resume); @@ -1947,12 +1261,6 @@ static struct platform_driver nmk_pinctrl_driver = { .probe = nmk_pinctrl_probe, }; -static int __init nmk_gpio_init(void) -{ - return platform_driver_register(&nmk_gpio_driver); -} -subsys_initcall(nmk_gpio_init); - static int __init nmk_pinctrl_init(void) { return platform_driver_register(&nmk_pinctrl_driver); diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.h b/drivers/pinctrl/nomadik/pinctrl-nomadik.h deleted file mode 100644 index 1ef2559bc571..000000000000 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.h +++ /dev/null @@ -1,180 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef PINCTRL_PINCTRL_NOMADIK_H -#define PINCTRL_PINCTRL_NOMADIK_H - -#include -#include - -#include - -/* Package definitions */ -#define PINCTRL_NMK_STN8815 0 -#define PINCTRL_NMK_DB8500 1 - -/* Alternate functions: function C is set in hw by setting both A and B */ -#define NMK_GPIO_ALT_GPIO 0 -#define NMK_GPIO_ALT_A 1 -#define NMK_GPIO_ALT_B 2 -#define NMK_GPIO_ALT_C (NMK_GPIO_ALT_A | NMK_GPIO_ALT_B) - -#define NMK_GPIO_ALT_CX_SHIFT 2 -#define NMK_GPIO_ALT_C1 ((1< Date: Wed, 28 Feb 2024 12:28:22 +0100 Subject: gpio: nomadik: support mobileye,eyeq5-gpio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We create a custom compatible for the STA2X11 IP block as integrated into the Mobileye EyeQ5 platform. Its wake and alternate functions have been disabled, we want to avoid touching those registers. We both do: (1) early return in functions that do not support the platform, but with warnings, and (2) avoid calling those functions in the first place. We ensure that pinctrl-nomadik is not used with this STA2X11 variant. Reviewed-by: Linus Walleij Signed-off-by: Théo Lebrun Link: https://lore.kernel.org/r/20240228-mbly-gpio-v2-24-3ba757474006@bootlin.com Signed-off-by: Linus Walleij --- drivers/gpio/Kconfig | 5 ++-- drivers/gpio/gpio-nomadik.c | 42 ++++++++++++++++++++++++++----- drivers/pinctrl/nomadik/pinctrl-nomadik.c | 2 ++ include/linux/gpio/gpio-nomadik.h | 1 + 4 files changed, 42 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index ff83371251c1..fe6112abb73a 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -480,11 +480,12 @@ config GPIO_MXS config GPIO_NOMADIK bool "Nomadik GPIO driver" - depends on ARCH_U8500 || ARCH_NOMADIK || COMPILE_TEST + depends on ARCH_U8500 || ARCH_NOMADIK || MACH_EYEQ5 || COMPILE_TEST select OF_GPIO select GPIOLIB_IRQCHIP help - Say yes here to support the Nomadik SoC GPIO block. + Say yes here to support the Nomadik SoC GPIO block. This block is also + used by the Mobileye EyeQ5 SoC. It handles up to 32 GPIOs per bank, that can all be interrupt sources. It is deeply interconnected with the associated pinctrl driver as GPIO diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c index 19394dc7e34a..6adb3c4417d3 100644 --- a/drivers/gpio/gpio-nomadik.c +++ b/drivers/gpio/gpio-nomadik.c @@ -7,6 +7,12 @@ * The GPIO chips are shared with pinctrl-nomadik if used; it needs access for * pinmuxing functionality and others. * + * This driver also handles the mobileye,eyeq5-gpio compatible. It is an STA2X11 + * but with only data, direction and interrupts register active. We want to + * avoid touching SLPM, RWIMSC, FWIMSC, AFSLA and AFSLB registers; that is, + * wake and alternate function registers. It is NOT compatible with + * pinctrl-nomadik. + * * Copyright (C) 2008,2009 STMicroelectronics * Copyright (C) 2009 Alessandro Rubini * Rewritten based on work by Prafulla WADASKAR @@ -37,6 +43,10 @@ void __nmk_gpio_set_slpm(struct nmk_gpio_chip *nmk_chip, unsigned int offset, { u32 slpm; + /* We should NOT have been called. */ + if (WARN_ON(nmk_chip->is_mobileye_soc)) + return; + slpm = readl(nmk_chip->addr + NMK_GPIO_SLPC); if (mode == NMK_GPIO_SLPM_NOCHANGE) slpm |= BIT(offset); @@ -93,6 +103,9 @@ static void __nmk_gpio_irq_modify(struct nmk_gpio_chip *nmk_chip, rimscval = &nmk_chip->rimsc; fimscval = &nmk_chip->fimsc; } else { + /* We should NOT have been called. */ + if (WARN_ON(nmk_chip->is_mobileye_soc)) + return; rimscreg = NMK_GPIO_RWIMSC; fimscreg = NMK_GPIO_FWIMSC; rimscval = &nmk_chip->rwimsc; @@ -119,6 +132,10 @@ static void __nmk_gpio_irq_modify(struct nmk_gpio_chip *nmk_chip, static void __nmk_gpio_set_wake(struct nmk_gpio_chip *nmk_chip, int offset, bool on) { + /* We should NOT have been called. */ + if (WARN_ON(nmk_chip->is_mobileye_soc)) + return; + /* * Ensure WAKEUP_ENABLE is on. No need to disable it if wakeup is * disabled, since setting SLPM to 1 increases power consumption, and @@ -143,7 +160,7 @@ static void nmk_gpio_irq_maskunmask(struct nmk_gpio_chip *nmk_chip, __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, enable); - if (!(nmk_chip->real_wake & BIT(d->hwirq))) + if (!nmk_chip->is_mobileye_soc && !(nmk_chip->real_wake & BIT(d->hwirq))) __nmk_gpio_set_wake(nmk_chip, d->hwirq, enable); spin_unlock(&nmk_chip->lock); @@ -175,6 +192,10 @@ static int nmk_gpio_irq_set_wake(struct irq_data *d, unsigned int on) struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); unsigned long flags; + /* Handler is registered in all cases. */ + if (nmk_chip->is_mobileye_soc) + return -ENXIO; + clk_enable(nmk_chip->clk); spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); spin_lock(&nmk_chip->lock); @@ -213,7 +234,7 @@ static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type) if (enabled) __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, false); - if (enabled || wake) + if (!nmk_chip->is_mobileye_soc && (enabled || wake)) __nmk_gpio_irq_modify(nmk_chip, d->hwirq, WAKE, false); nmk_chip->edge_rising &= ~BIT(d->hwirq); @@ -227,7 +248,7 @@ static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type) if (enabled) __nmk_gpio_irq_modify(nmk_chip, d->hwirq, NORMAL, true); - if (enabled || wake) + if (!nmk_chip->is_mobileye_soc && (enabled || wake)) __nmk_gpio_irq_modify(nmk_chip, d->hwirq, WAKE, true); spin_unlock_irqrestore(&nmk_chip->lock, flags); @@ -357,6 +378,10 @@ static int nmk_gpio_get_mode(struct nmk_gpio_chip *nmk_chip, int offset) { u32 afunc, bfunc; + /* We don't support modes. */ + if (nmk_chip->is_mobileye_soc) + return NMK_GPIO_ALT_GPIO; + clk_enable(nmk_chip->clk); afunc = readl(nmk_chip->addr + NMK_GPIO_AFSLA) & BIT(offset); @@ -523,6 +548,8 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, dev_dbg(&pdev->dev, "populate: using default ngpio (%d)\n", ngpio); } + nmk_chip->is_mobileye_soc = device_is_compatible(gpio_dev, + "mobileye,eyeq5-gpio"); nmk_chip->bank = id; chip = &nmk_chip->chip; chip->base = -1; @@ -636,9 +663,11 @@ static int nmk_gpio_probe(struct platform_device *pdev) return ret; } - clk_enable(nmk_chip->clk); - nmk_chip->lowemi = readl_relaxed(nmk_chip->addr + NMK_GPIO_LOWEMI); - clk_disable(nmk_chip->clk); + if (!nmk_chip->is_mobileye_soc) { + clk_enable(nmk_chip->clk); + nmk_chip->lowemi = readl_relaxed(nmk_chip->addr + NMK_GPIO_LOWEMI); + clk_disable(nmk_chip->clk); + } ret = gpiochip_add_data(chip, nmk_chip); if (ret) @@ -653,6 +682,7 @@ static int nmk_gpio_probe(struct platform_device *pdev) static const struct of_device_id nmk_gpio_match[] = { { .compatible = "st,nomadik-gpio", }, + { .compatible = "mobileye,eyeq5-gpio", }, {} }; diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c index 5633f0ec2715..7849144b3b80 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c @@ -1230,6 +1230,8 @@ static int nmk_pinctrl_probe(struct platform_device *pdev) dev_err(&pdev->dev, "could not populate nmk chip struct - continue anyway\n"); of_node_put(gpio_np); + /* We are NOT compatible with mobileye,eyeq5-gpio. */ + BUG_ON(nmk_chip->is_mobileye_soc); } prcm_np = of_parse_phandle(np, "prcm", 0); diff --git a/include/linux/gpio/gpio-nomadik.h b/include/linux/gpio/gpio-nomadik.h index 0166ddb71f43..9bdb09fda4c9 100644 --- a/include/linux/gpio/gpio-nomadik.h +++ b/include/linux/gpio/gpio-nomadik.h @@ -57,6 +57,7 @@ struct nmk_gpio_chip { void (*set_ioforce)(bool enable); spinlock_t lock; bool sleepmode; + bool is_mobileye_soc; /* Keep track of configured edges */ u32 edge_rising; u32 edge_falling; -- cgit v1.2.3 From f9e28904e6442019043a8e94ec6747a064d06003 Mon Sep 17 00:00:00 2001 From: Zhiguo Niu Date: Tue, 20 Feb 2024 14:11:24 +0800 Subject: f2fs: stop checkpoint when get a out-of-bounds segment There is low probability that an out-of-bounds segment will be got on a small-capacity device. In order to prevent subsequent write requests allocating block address from this invalid segment, which may cause unexpected issue, stop checkpoint should be performed. Also introduce a new stop cp reason: STOP_CP_REASON_NO_SEGMENT. Note, f2fs_stop_checkpoint(, false) is complex and it may sleep, so we should move it outside segmap_lock spinlock coverage in get_new_segment(). Signed-off-by: Zhiguo Niu Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/segment.c | 12 +++++++++++- include/linux/f2fs_fs.h | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 01067254ec40..dc511630eaa5 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -2649,6 +2649,7 @@ static void get_new_segment(struct f2fs_sb_info *sbi, unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); bool init = true; int i; + int ret = 0; spin_lock(&free_i->segmap_lock); @@ -2673,7 +2674,10 @@ find_other_zone: if (secno >= MAIN_SECS(sbi)) { secno = find_first_zero_bit(free_i->free_secmap, MAIN_SECS(sbi)); - f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); + if (secno >= MAIN_SECS(sbi)) { + ret = -ENOSPC; + goto out_unlock; + } } segno = GET_SEG_FROM_SEC(sbi, secno); zoneno = GET_ZONE_FROM_SEC(sbi, secno); @@ -2703,7 +2707,13 @@ got_it: f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); __set_inuse(sbi, segno); *newseg = segno; +out_unlock: spin_unlock(&free_i->segmap_lock); + + if (ret) { + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT); + f2fs_bug_on(sbi, 1); + } } static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 9b69c50255b2..755e9a41b196 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -75,6 +75,7 @@ enum stop_cp_reason { STOP_CP_REASON_CORRUPTED_SUMMARY, STOP_CP_REASON_UPDATE_INODE, STOP_CP_REASON_FLUSH_FAIL, + STOP_CP_REASON_NO_SEGMENT, STOP_CP_REASON_MAX, }; -- cgit v1.2.3 From d394abcb12bb1a6f309c1221fdb8e73594ecf1b4 Mon Sep 17 00:00:00 2001 From: Shivnandan Kumar Date: Tue, 27 Feb 2024 14:43:51 +0530 Subject: cpufreq: Limit resolving a frequency to policy min/max Resolving a frequency to an efficient one should not transgress policy->max (which can be set for thermal reason) and policy->min. Currently, there is possibility where scaling_cur_freq can exceed scaling_max_freq when scaling_max_freq is an inefficient frequency. Add a check to ensure that resolving a frequency will respect policy->min/max. Cc: All applicable Fixes: 1f39fa0dccff ("cpufreq: Introducing CPUFREQ_RELATION_E") Signed-off-by: Shivnandan Kumar [ rjw: Whitespace adjustment, changelog edits ] Signed-off-by: Rafael J. Wysocki --- include/linux/cpufreq.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 85908b3a2f24..692ea6e55129 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -1020,6 +1020,18 @@ static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy, efficiencies); } +static inline bool cpufreq_is_in_limits(struct cpufreq_policy *policy, int idx) +{ + unsigned int freq; + + if (idx < 0) + return false; + + freq = policy->freq_table[idx].frequency; + + return freq == clamp_val(freq, policy->min, policy->max); +} + static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) @@ -1053,7 +1065,8 @@ retry: return 0; } - if (idx < 0 && efficiencies) { + /* Limit frequency index to honor policy->min/max */ + if (!cpufreq_is_in_limits(policy, idx) && efficiencies) { efficiencies = false; goto retry; } -- cgit v1.2.3 From adeb04362d74188c1e22ccb824b15a0a7b3de2f4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 14 Feb 2024 19:26:32 +0200 Subject: kernel.h: Move upper_*_bits() and lower_*_bits() to wordpart.h The wordpart.h header is collecting APIs related to the handling parts of the word (usually in byte granularity). The upper_*_bits() and lower_*_bits() are good candidates to be moved to there. This helps to clean up header dependency hell with regard to kernel.h as the latter gathers completely unrelated stuff together and slows down compilation (especially when it's included into other header). Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240214172752.3605073-1-andriy.shevchenko@linux.intel.com Reviewed-by: Randy Dunlap Signed-off-by: Kees Cook --- include/linux/kernel.h | 30 ++---------------------------- include/linux/wordpart.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index f4a1d582b79d..86dd8939c2cd 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -33,6 +33,8 @@ #include #include #include +#include + #include #include @@ -52,34 +54,6 @@ } \ ) -/** - * upper_32_bits - return bits 32-63 of a number - * @n: the number we're accessing - * - * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress - * the "right shift count >= width of type" warning when that quantity is - * 32-bits. - */ -#define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) - -/** - * lower_32_bits - return bits 0-31 of a number - * @n: the number we're accessing - */ -#define lower_32_bits(n) ((u32)((n) & 0xffffffff)) - -/** - * upper_16_bits - return bits 16-31 of a number - * @n: the number we're accessing - */ -#define upper_16_bits(n) ((u16)((n) >> 16)) - -/** - * lower_16_bits - return bits 0-15 of a number - * @n: the number we're accessing - */ -#define lower_16_bits(n) ((u16)((n) & 0xffff)) - struct completion; struct user; diff --git a/include/linux/wordpart.h b/include/linux/wordpart.h index c9e6bd773ebd..f6f8f83b15b0 100644 --- a/include/linux/wordpart.h +++ b/include/linux/wordpart.h @@ -2,6 +2,35 @@ #ifndef _LINUX_WORDPART_H #define _LINUX_WORDPART_H + +/** + * upper_32_bits - return bits 32-63 of a number + * @n: the number we're accessing + * + * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress + * the "right shift count >= width of type" warning when that quantity is + * 32-bits. + */ +#define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) + +/** + * lower_32_bits - return bits 0-31 of a number + * @n: the number we're accessing + */ +#define lower_32_bits(n) ((u32)((n) & 0xffffffff)) + +/** + * upper_16_bits - return bits 16-31 of a number + * @n: the number we're accessing + */ +#define upper_16_bits(n) ((u16)((n) >> 16)) + +/** + * lower_16_bits - return bits 0-15 of a number + * @n: the number we're accessing + */ +#define lower_16_bits(n) ((u16)((n) & 0xffff)) + /** * REPEAT_BYTE - repeat the value @x multiple times as an unsigned long value * @x: value to repeat -- cgit v1.2.3 From 3e19086fb5a9079611de426e8cb2f4503e28757e Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 29 Jan 2024 10:21:58 -0800 Subject: overflow: Adjust check_*_overflow() kern-doc to reflect results The check_*_overflow() helpers will return results with potentially wrapped-around values. These values have always been checked by the selftests, so avoid the confusing language in the kern-doc. The idea of "safe for use" was relative to the expectation of whether or not the caller wants a wrapped value -- the calculation itself will always follow arithmetic wrapping rules. Reviewed-by: Gustavo A. R. Silva Acked-by: Mark Rutland Signed-off-by: Kees Cook --- include/linux/overflow.h | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/overflow.h b/include/linux/overflow.h index 7b5cf4a5cd19..ad64d810c8aa 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -57,11 +57,10 @@ static inline bool __must_check __must_check_overflow(bool overflow) * @b: second addend * @d: pointer to store sum * - * Returns 0 on success. + * Returns true on wrap-around, false otherwise. * - * *@d holds the results of the attempted addition, but is not considered - * "safe for use" on a non-zero return value, which indicates that the - * sum has overflowed or been truncated. + * *@d holds the results of the attempted addition, regardless of whether + * wrap-around occurred. */ #define check_add_overflow(a, b, d) \ __must_check_overflow(__builtin_add_overflow(a, b, d)) @@ -72,11 +71,10 @@ static inline bool __must_check __must_check_overflow(bool overflow) * @b: subtrahend; value to subtract from @a * @d: pointer to store difference * - * Returns 0 on success. + * Returns true on wrap-around, false otherwise. * - * *@d holds the results of the attempted subtraction, but is not considered - * "safe for use" on a non-zero return value, which indicates that the - * difference has underflowed or been truncated. + * *@d holds the results of the attempted subtraction, regardless of whether + * wrap-around occurred. */ #define check_sub_overflow(a, b, d) \ __must_check_overflow(__builtin_sub_overflow(a, b, d)) @@ -87,11 +85,10 @@ static inline bool __must_check __must_check_overflow(bool overflow) * @b: second factor * @d: pointer to store product * - * Returns 0 on success. + * Returns true on wrap-around, false otherwise. * - * *@d holds the results of the attempted multiplication, but is not - * considered "safe for use" on a non-zero return value, which indicates - * that the product has overflowed or been truncated. + * *@d holds the results of the attempted multiplication, regardless of whether + * wrap-around occurred. */ #define check_mul_overflow(a, b, d) \ __must_check_overflow(__builtin_mul_overflow(a, b, d)) -- cgit v1.2.3 From d70de8054c58d7bd9a4654c9f4797d29fa46d545 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 18 Jan 2024 16:05:52 -0800 Subject: overflow: Introduce wrapping_add(), wrapping_sub(), and wrapping_mul() Provide helpers that will perform wrapping addition, subtraction, or multiplication without tripping the arithmetic wrap-around sanitizers. The first argument is the type under which the wrap-around should happen with. In other words, these two calls will get very different results: wrapping_mul(int, 50, 50) == 2500 wrapping_mul(u8, 50, 50) == 196 Add to the selftests to validate behavior and lack of side-effects. Reviewed-by: Gustavo A. R. Silva Reviewed-by: Marco Elver Acked-by: Mark Rutland Signed-off-by: Kees Cook --- include/linux/overflow.h | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/overflow_kunit.c | 24 ++++++++++++++++++++---- 2 files changed, 68 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/overflow.h b/include/linux/overflow.h index ad64d810c8aa..d3ff8e2bec29 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -65,6 +65,22 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define check_add_overflow(a, b, d) \ __must_check_overflow(__builtin_add_overflow(a, b, d)) +/** + * wrapping_add() - Intentionally perform a wrapping addition + * @type: type for result of calculation + * @a: first addend + * @b: second addend + * + * Return the potentially wrapped-around addition without + * tripping any wrap-around sanitizers that may be enabled. + */ +#define wrapping_add(type, a, b) \ + ({ \ + type __val; \ + __builtin_add_overflow(a, b, &__val); \ + __val; \ + }) + /** * check_sub_overflow() - Calculate subtraction with overflow checking * @a: minuend; value to subtract from @@ -79,6 +95,22 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define check_sub_overflow(a, b, d) \ __must_check_overflow(__builtin_sub_overflow(a, b, d)) +/** + * wrapping_sub() - Intentionally perform a wrapping subtraction + * @type: type for result of calculation + * @a: minuend; value to subtract from + * @b: subtrahend; value to subtract from @a + * + * Return the potentially wrapped-around subtraction without + * tripping any wrap-around sanitizers that may be enabled. + */ +#define wrapping_sub(type, a, b) \ + ({ \ + type __val; \ + __builtin_sub_overflow(a, b, &__val); \ + __val; \ + }) + /** * check_mul_overflow() - Calculate multiplication with overflow checking * @a: first factor @@ -93,6 +125,22 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define check_mul_overflow(a, b, d) \ __must_check_overflow(__builtin_mul_overflow(a, b, d)) +/** + * wrapping_mul() - Intentionally perform a wrapping multiplication + * @type: type for result of calculation + * @a: first factor + * @b: second factor + * + * Return the potentially wrapped-around multiplication without + * tripping any wrap-around sanitizers that may be enabled. + */ +#define wrapping_mul(type, a, b) \ + ({ \ + type __val; \ + __builtin_mul_overflow(a, b, &__val); \ + __val; \ + }) + /** * check_shl_overflow() - Calculate a left-shifted value and check overflow * @a: Value to be shifted diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c index c527f6b75789..d3fdb906d3fe 100644 --- a/lib/overflow_kunit.c +++ b/lib/overflow_kunit.c @@ -258,20 +258,36 @@ DEFINE_TEST_ARRAY(s64) = { \ _of = check_ ## op ## _overflow(a, b, &_r); \ KUNIT_EXPECT_EQ_MSG(test, _of, of, \ - "expected "fmt" "sym" "fmt" to%s overflow (type %s)\n", \ + "expected check "fmt" "sym" "fmt" to%s overflow (type %s)\n", \ a, b, of ? "" : " not", #t); \ KUNIT_EXPECT_EQ_MSG(test, _r, r, \ - "expected "fmt" "sym" "fmt" == "fmt", got "fmt" (type %s)\n", \ + "expected check "fmt" "sym" "fmt" == "fmt", got "fmt" (type %s)\n", \ a, b, r, _r, #t); \ /* Check for internal macro side-effects. */ \ _of = check_ ## op ## _overflow(_a_orig++, _b_orig++, &_r); \ - KUNIT_EXPECT_EQ_MSG(test, _a_orig, _a_bump, "Unexpected " #op " macro side-effect!\n"); \ - KUNIT_EXPECT_EQ_MSG(test, _b_orig, _b_bump, "Unexpected " #op " macro side-effect!\n"); \ + KUNIT_EXPECT_EQ_MSG(test, _a_orig, _a_bump, \ + "Unexpected check " #op " macro side-effect!\n"); \ + KUNIT_EXPECT_EQ_MSG(test, _b_orig, _b_bump, \ + "Unexpected check " #op " macro side-effect!\n"); \ + \ + _r = wrapping_ ## op(t, a, b); \ + KUNIT_EXPECT_TRUE_MSG(test, _r == r, \ + "expected wrap "fmt" "sym" "fmt" == "fmt", got "fmt" (type %s)\n", \ + a, b, r, _r, #t); \ + /* Check for internal macro side-effects. */ \ + _a_orig = a; \ + _b_orig = b; \ + _r = wrapping_ ## op(t, _a_orig++, _b_orig++); \ + KUNIT_EXPECT_EQ_MSG(test, _a_orig, _a_bump, \ + "Unexpected wrap " #op " macro side-effect!\n"); \ + KUNIT_EXPECT_EQ_MSG(test, _b_orig, _b_bump, \ + "Unexpected wrap " #op " macro side-effect!\n"); \ } while (0) #define DEFINE_TEST_FUNC_TYPED(n, t, fmt) \ static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ { \ + /* check_{add,sub,mul}_overflow() and wrapping_{add,sub,mul} */ \ check_one_op(t, fmt, add, "+", p->a, p->b, p->sum, p->s_of); \ check_one_op(t, fmt, add, "+", p->b, p->a, p->sum, p->s_of); \ check_one_op(t, fmt, sub, "-", p->a, p->b, p->diff, p->d_of); \ -- cgit v1.2.3 From 08d45ee84bb2650e237e150caca87cc4ded9b3e2 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 26 Jan 2024 22:09:50 -0800 Subject: overflow: Introduce wrapping_assign_add() and wrapping_assign_sub() This allows replacements of the idioms "var += offset" and "var -= offset" with the wrapping_assign_add() and wrapping_assign_sub() helpers respectively. They will avoid wrap-around sanitizer instrumentation. Add to the selftests to validate behavior and lack of side-effects. Reviewed-by: Marco Elver Acked-by: Mark Rutland Signed-off-by: Kees Cook --- include/linux/overflow.h | 32 ++++++++++++++++++++++++++++++++ lib/overflow_kunit.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) (limited to 'include/linux') diff --git a/include/linux/overflow.h b/include/linux/overflow.h index d3ff8e2bec29..dede374832c9 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -81,6 +81,22 @@ static inline bool __must_check __must_check_overflow(bool overflow) __val; \ }) +/** + * wrapping_assign_add() - Intentionally perform a wrapping increment assignment + * @var: variable to be incremented + * @offset: amount to add + * + * Increments @var by @offset with wrap-around. Returns the resulting + * value of @var. Will not trip any wrap-around sanitizers. + * + * Returns the new value of @var. + */ +#define wrapping_assign_add(var, offset) \ + ({ \ + typeof(var) *__ptr = &(var); \ + *__ptr = wrapping_add(typeof(var), *__ptr, offset); \ + }) + /** * check_sub_overflow() - Calculate subtraction with overflow checking * @a: minuend; value to subtract from @@ -111,6 +127,22 @@ static inline bool __must_check __must_check_overflow(bool overflow) __val; \ }) +/** + * wrapping_assign_sub() - Intentionally perform a wrapping decrement assign + * @var: variable to be decremented + * @offset: amount to subtract + * + * Decrements @var by @offset with wrap-around. Returns the resulting + * value of @var. Will not trip any wrap-around sanitizers. + * + * Returns the new value of @var. + */ +#define wrapping_assign_sub(var, offset) \ + ({ \ + typeof(var) *__ptr = &(var); \ + *__ptr = wrapping_sub(typeof(var), *__ptr, offset); \ + }) + /** * check_mul_overflow() - Calculate multiplication with overflow checking * @a: first factor diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c index d3fdb906d3fe..65e8a72a83bf 100644 --- a/lib/overflow_kunit.c +++ b/lib/overflow_kunit.c @@ -284,6 +284,45 @@ DEFINE_TEST_ARRAY(s64) = { "Unexpected wrap " #op " macro side-effect!\n"); \ } while (0) +static int global_counter; +static void bump_counter(void) +{ + global_counter++; +} + +static int get_index(void) +{ + volatile int index = 0; + bump_counter(); + return index; +} + +#define check_self_op(fmt, op, sym, a, b) do { \ + typeof(a + 0) _a = a; \ + typeof(b + 0) _b = b; \ + typeof(a + 0) _a_sym = a; \ + typeof(a + 0) _a_orig[1] = { a }; \ + typeof(b + 0) _b_orig = b; \ + typeof(b + 0) _b_bump = b + 1; \ + typeof(a + 0) _r; \ + \ + _a_sym sym _b; \ + _r = wrapping_ ## op(_a, _b); \ + KUNIT_EXPECT_TRUE_MSG(test, _r == _a_sym, \ + "expected "fmt" "#op" "fmt" == "fmt", got "fmt"\n", \ + a, b, _a_sym, _r); \ + KUNIT_EXPECT_TRUE_MSG(test, _a == _a_sym, \ + "expected "fmt" "#op" "fmt" == "fmt", got "fmt"\n", \ + a, b, _a_sym, _a); \ + /* Check for internal macro side-effects. */ \ + global_counter = 0; \ + wrapping_ ## op(_a_orig[get_index()], _b_orig++); \ + KUNIT_EXPECT_EQ_MSG(test, global_counter, 1, \ + "Unexpected wrapping_" #op " macro side-effect on arg1!\n"); \ + KUNIT_EXPECT_EQ_MSG(test, _b_orig, _b_bump, \ + "Unexpected wrapping_" #op " macro side-effect on arg2!\n"); \ +} while (0) + #define DEFINE_TEST_FUNC_TYPED(n, t, fmt) \ static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ { \ @@ -293,6 +332,10 @@ static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ check_one_op(t, fmt, sub, "-", p->a, p->b, p->diff, p->d_of); \ check_one_op(t, fmt, mul, "*", p->a, p->b, p->prod, p->p_of); \ check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of); \ + /* wrapping_assign_{add,sub}() */ \ + check_self_op(fmt, assign_add, +=, p->a, p->b); \ + check_self_op(fmt, assign_add, +=, p->b, p->a); \ + check_self_op(fmt, assign_sub, -=, p->a, p->b); \ } \ \ static void n ## _overflow_test(struct kunit *test) { \ -- cgit v1.2.3 From 9ca5facd0400f610f3f7f71aeb7fc0b949a48c67 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 14 Feb 2024 17:50:15 +0100 Subject: lib/string_choices: Add str_plural() helper Add str_plural() helper to replace existing open implementations used by many drivers and help improve future user facing messages. Signed-off-by: Michal Wajdeczko Link: https://lore.kernel.org/r/20240214165015.1656-1-michal.wajdeczko@intel.com Signed-off-by: Kees Cook --- include/linux/string_choices.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include/linux') diff --git a/include/linux/string_choices.h b/include/linux/string_choices.h index 3c1091941eb8..d9ebe20229f8 100644 --- a/include/linux/string_choices.h +++ b/include/linux/string_choices.h @@ -42,4 +42,15 @@ static inline const char *str_yes_no(bool v) return v ? "yes" : "no"; } +/** + * str_plural - Return the simple pluralization based on English counts + * @num: Number used for deciding pluralization + * + * If @num is 1, returns empty string, otherwise returns "s". + */ +static inline const char *str_plural(size_t num) +{ + return num == 1 ? "" : "s"; +} + #endif -- cgit v1.2.3 From 99db710f768e988e70f1164537bf533a017be24d Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 20 Feb 2024 21:16:38 -0800 Subject: refcount: Annotated intentional signed integer wrap-around Mark the various refcount_t functions with __signed_wrap, as we depend on the wrapping behavior to detect the overflow and perform saturation. Silences warnings seen with the LKDTM REFCOUNT_* tests: UBSAN: signed-integer-overflow in ../include/linux/refcount.h:189:11 2147483647 + 1 cannot be represented in type 'int' Reviewed-by: Miguel Ojeda Link: https://lore.kernel.org/r/20240221051634.work.287-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/refcount.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/refcount.h b/include/linux/refcount.h index 85c6df0d1bef..59b3b752394d 100644 --- a/include/linux/refcount.h +++ b/include/linux/refcount.h @@ -136,7 +136,8 @@ static inline unsigned int refcount_read(const refcount_t *r) return atomic_read(&r->refs); } -static inline __must_check bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp) +static inline __must_check __signed_wrap +bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp) { int old = refcount_read(r); @@ -177,7 +178,8 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) return __refcount_add_not_zero(i, r, NULL); } -static inline void __refcount_add(int i, refcount_t *r, int *oldp) +static inline __signed_wrap +void __refcount_add(int i, refcount_t *r, int *oldp) { int old = atomic_fetch_add_relaxed(i, &r->refs); @@ -256,7 +258,8 @@ static inline void refcount_inc(refcount_t *r) __refcount_inc(r, NULL); } -static inline __must_check bool __refcount_sub_and_test(int i, refcount_t *r, int *oldp) +static inline __must_check __signed_wrap +bool __refcount_sub_and_test(int i, refcount_t *r, int *oldp) { int old = atomic_fetch_sub_release(i, &r->refs); -- cgit v1.2.3 From 475ddf1fce1ec4826c8dda40ec59f7f83a7aadb8 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 7 Apr 2023 12:27:13 -0700 Subject: fortify: Split reporting and avoid passing string pointer In preparation for KUnit testing and further improvements in fortify failure reporting, split out the report and encode the function and access failure (read or write overflow) into a single u8 argument. This mainly ends up saving a tiny bit of space in the data segment. For a defconfig with FORTIFY_SOURCE enabled: $ size gcc/vmlinux.before gcc/vmlinux.after text data bss dec hex filename 26132309 9760658 2195460 38088427 2452eeb gcc/vmlinux.before 26132386 9748382 2195460 38076228 244ff44 gcc/vmlinux.after Reviewed-by: Alexander Lobakin Signed-off-by: Kees Cook --- arch/arm/boot/compressed/misc.c | 2 +- arch/arm/boot/compressed/misc.h | 2 +- arch/x86/boot/compressed/misc.c | 2 +- include/linux/fortify-string.h | 81 ++++++++++++++++++++++++++++++----------- lib/string_helpers.c | 23 ++++++++++-- tools/objtool/noreturns.h | 2 +- 6 files changed, 84 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c index 6b4baa6a9a50..d93e2e466f6a 100644 --- a/arch/arm/boot/compressed/misc.c +++ b/arch/arm/boot/compressed/misc.c @@ -154,7 +154,7 @@ decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p, putstr(" done, booting the kernel.\n"); } -void fortify_panic(const char *name) +void __fortify_panic(const u8 reason) { error("detected buffer overflow"); } diff --git a/arch/arm/boot/compressed/misc.h b/arch/arm/boot/compressed/misc.h index 6da00a26ac08..4d59c427253c 100644 --- a/arch/arm/boot/compressed/misc.h +++ b/arch/arm/boot/compressed/misc.h @@ -10,7 +10,7 @@ void __div0(void); void decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p, unsigned long free_mem_ptr_end_p, int arch_id); -void fortify_panic(const char *name); +void __fortify_panic(const u8 reason); int atags_to_fdt(void *atag_list, void *fdt, int total_space); uint32_t fdt_check_mem_start(uint32_t mem_start, const void *fdt); int do_decompress(u8 *input, int len, u8 *output, void (*error)(char *x)); diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index b99e08e6815b..c9971b9dbb09 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -496,7 +496,7 @@ asmlinkage __visible void *extract_kernel(void *rmode, unsigned char *output) return output + entry_offset; } -void fortify_panic(const char *name) +void __fortify_panic(const u8 reason) { error("detected buffer overflow"); } diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 06b3aaa63724..4f6767dcd933 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -2,6 +2,7 @@ #ifndef _LINUX_FORTIFY_STRING_H_ #define _LINUX_FORTIFY_STRING_H_ +#include #include #include #include @@ -9,7 +10,44 @@ #define __FORTIFY_INLINE extern __always_inline __gnu_inline __overloadable #define __RENAME(x) __asm__(#x) -void fortify_panic(const char *name) __noreturn __cold; +#define FORTIFY_REASON_DIR(r) FIELD_GET(BIT(0), r) +#define FORTIFY_REASON_FUNC(r) FIELD_GET(GENMASK(7, 1), r) +#define FORTIFY_REASON(func, write) (FIELD_PREP(BIT(0), write) | \ + FIELD_PREP(GENMASK(7, 1), func)) + +#define fortify_panic(func, write) \ + __fortify_panic(FORTIFY_REASON(func, write)) + +#define FORTIFY_READ 0 +#define FORTIFY_WRITE 1 + +#define EACH_FORTIFY_FUNC(macro) \ + macro(strncpy), \ + macro(strnlen), \ + macro(strlen), \ + macro(strscpy), \ + macro(strlcat), \ + macro(strcat), \ + macro(strncat), \ + macro(memset), \ + macro(memcpy), \ + macro(memmove), \ + macro(memscan), \ + macro(memcmp), \ + macro(memchr), \ + macro(memchr_inv), \ + macro(kmemdup), \ + macro(strcpy), \ + macro(UNKNOWN), + +#define MAKE_FORTIFY_FUNC(func) FORTIFY_FUNC_##func + +enum fortify_func { + EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC) +}; + +void __fortify_report(const u8 reason); +void __fortify_panic(const u8 reason) __cold __noreturn; void __read_overflow(void) __compiletime_error("detected read beyond size of object (1st parameter)"); void __read_overflow2(void) __compiletime_error("detected read beyond size of object (2nd parameter)"); void __read_overflow2_field(size_t avail, size_t wanted) __compiletime_warning("detected read beyond size of field (2nd parameter); maybe use struct_group()?"); @@ -143,7 +181,7 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __write_overflow(); if (p_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE); return __underlying_strncpy(p, q, size); } @@ -174,7 +212,7 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size /* Do not check characters beyond the end of p. */ ret = __real_strnlen(p, maxlen < p_size ? maxlen : p_size); if (p_size <= ret && maxlen != ret) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ); return ret; } @@ -210,7 +248,7 @@ __kernel_size_t __fortify_strlen(const char * const POS p) return __underlying_strlen(p); ret = strnlen(p, p_size); if (p_size <= ret) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ); return ret; } @@ -261,7 +299,7 @@ __FORTIFY_INLINE ssize_t sized_strscpy(char * const POS p, const char * const PO * p_size. */ if (len > p_size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE); /* * We can now safely call vanilla strscpy because we are protected from: @@ -319,7 +357,7 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) /* Give up if string is already overflowed. */ if (p_size <= p_len) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ); if (actual >= avail) { copy_len = avail - p_len - 1; @@ -328,7 +366,7 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) /* Give up if copy will overflow. */ if (p_size <= actual) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE); __underlying_memcpy(p + p_len, q, copy_len); p[actual] = '\0'; @@ -357,7 +395,7 @@ char *strcat(char * const POS p, const char *q) const size_t p_size = __member_size(p); if (strlcat(p, q, p_size) >= p_size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE); return p; } @@ -393,7 +431,7 @@ char *strncat(char * const POS p, const char * const POS q, __kernel_size_t coun p_len = strlen(p); copy_len = strnlen(q, count); if (p_size < p_len + copy_len + 1) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE); __underlying_memcpy(p + p_len, q, copy_len); p[p_len + copy_len] = '\0'; return p; @@ -434,7 +472,7 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, * lengths are unknown.) */ if (p_size != SIZE_MAX && p_size < size) - fortify_panic("memset"); + fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE); } #define __fortify_memset_chk(p, c, size, p_size, p_size_field) ({ \ @@ -488,7 +526,7 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, const size_t q_size, const size_t p_size_field, const size_t q_size_field, - const char *func) + const u8 func) { if (__builtin_constant_p(size)) { /* @@ -532,9 +570,10 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, * (The SIZE_MAX test is to optimize away checks where the buffer * lengths are unknown.) */ - if ((p_size != SIZE_MAX && p_size < size) || - (q_size != SIZE_MAX && q_size < size)) - fortify_panic(func); + if (p_size != SIZE_MAX && p_size < size) + fortify_panic(func, FORTIFY_WRITE); + else if (q_size != SIZE_MAX && q_size < size) + fortify_panic(func, FORTIFY_READ); /* * Warn when writing beyond destination field size. @@ -567,7 +606,7 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, const size_t __q_size_field = (q_size_field); \ WARN_ONCE(fortify_memcpy_chk(__fortify_size, __p_size, \ __q_size, __p_size_field, \ - __q_size_field, #op), \ + __q_size_field, FORTIFY_FUNC_ ##op), \ #op ": detected field-spanning write (size %zu) of single %s (size %zu)\n", \ __fortify_size, \ "field \"" #p "\" at " FILE_LINE, \ @@ -634,7 +673,7 @@ __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ); return __real_memscan(p, c, size); } @@ -651,7 +690,7 @@ int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t __read_overflow2(); } if (p_size < size || q_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ); return __underlying_memcmp(p, q, size); } @@ -663,7 +702,7 @@ void *memchr(const void * const POS0 p, int c, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ); return __underlying_memchr(p, c, size); } @@ -675,7 +714,7 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ); return __real_memchr_inv(p, c, size); } @@ -688,7 +727,7 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ); return __real_kmemdup(p, size, gfp); } @@ -725,7 +764,7 @@ char *strcpy(char * const POS p, const char * const POS q) __write_overflow(); /* Run-time check for dynamic size overflow. */ if (p_size < size) - fortify_panic(__func__); + fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE); __underlying_memcpy(p, q, size); return p; } diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 606c3099013f..9291dc74ae01 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -1008,10 +1008,27 @@ EXPORT_SYMBOL(__read_overflow2_field); void __write_overflow_field(size_t avail, size_t wanted) { } EXPORT_SYMBOL(__write_overflow_field); -void fortify_panic(const char *name) +static const char * const fortify_func_name[] = { +#define MAKE_FORTIFY_FUNC_NAME(func) [MAKE_FORTIFY_FUNC(func)] = #func + EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC_NAME) +#undef MAKE_FORTIFY_FUNC_NAME +}; + +void __fortify_report(const u8 reason) +{ + const u8 func = FORTIFY_REASON_FUNC(reason); + const bool write = FORTIFY_REASON_DIR(reason); + const char *name; + + name = fortify_func_name[umin(func, FORTIFY_FUNC_UNKNOWN)]; + WARN(1, "%s: detected buffer %s overflow\n", name, str_read_write(!write)); +} +EXPORT_SYMBOL(__fortify_report); + +void __fortify_panic(const u8 reason) { - pr_emerg("detected buffer overflow in %s\n", name); + __fortify_report(reason); BUG(); } -EXPORT_SYMBOL(fortify_panic); +EXPORT_SYMBOL(__fortify_panic); #endif /* CONFIG_FORTIFY_SOURCE */ diff --git a/tools/objtool/noreturns.h b/tools/objtool/noreturns.h index 1685d7ea6a9f..3a301696f005 100644 --- a/tools/objtool/noreturns.h +++ b/tools/objtool/noreturns.h @@ -6,6 +6,7 @@ * * Yes, this is unfortunate. A better solution is in the works. */ +NORETURN(__fortify_panic) NORETURN(__kunit_abort) NORETURN(__module_put_and_kthread_exit) NORETURN(__reiserfs_panic) @@ -22,7 +23,6 @@ NORETURN(do_exit) NORETURN(do_group_exit) NORETURN(do_task_dead) NORETURN(ex_handler_msr_mce) -NORETURN(fortify_panic) NORETURN(hlt_play_dead) NORETURN(hv_ghcb_terminate) NORETURN(kthread_complete_and_exit) -- cgit v1.2.3 From 4ce615e798a752d4431fcc52960478906dec2f0e Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 7 Apr 2023 12:27:14 -0700 Subject: fortify: Provide KUnit counters for failure testing The standard C string APIs were not designed to have a failure mode; they were expected to always succeed without memory safety issues. Normally, CONFIG_FORTIFY_SOURCE will use fortify_panic() to stop processing, as truncating a read or write may provide an even worse system state. However, this creates a problem for testing under things like KUnit, which needs a way to survive failures. When building with CONFIG_KUNIT, provide a failure path for all users of fortify_panic, and track whether the failure was a read overflow or a write overflow, for KUnit tests to examine. Inspired by similar logic in the slab tests. Signed-off-by: Kees Cook --- include/linux/fortify-string.h | 43 ++++++++++++++++++++++-------------------- lib/fortify_kunit.c | 41 ++++++++++++++++++++++++++++++++++++++++ lib/string_helpers.c | 2 ++ 3 files changed, 66 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 4f6767dcd933..fbfb90479b8f 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -15,8 +15,10 @@ #define FORTIFY_REASON(func, write) (FIELD_PREP(BIT(0), write) | \ FIELD_PREP(GENMASK(7, 1), func)) -#define fortify_panic(func, write) \ - __fortify_panic(FORTIFY_REASON(func, write)) +#ifndef fortify_panic +# define fortify_panic(func, write, retfail) \ + __fortify_panic(FORTIFY_REASON(func, write)) +#endif #define FORTIFY_READ 0 #define FORTIFY_WRITE 1 @@ -181,7 +183,7 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __write_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE, p); return __underlying_strncpy(p, q, size); } @@ -212,7 +214,7 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size /* Do not check characters beyond the end of p. */ ret = __real_strnlen(p, maxlen < p_size ? maxlen : p_size); if (p_size <= ret && maxlen != ret) - fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ, ret); return ret; } @@ -248,7 +250,7 @@ __kernel_size_t __fortify_strlen(const char * const POS p) return __underlying_strlen(p); ret = strnlen(p, p_size); if (p_size <= ret) - fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ, ret); return ret; } @@ -299,7 +301,7 @@ __FORTIFY_INLINE ssize_t sized_strscpy(char * const POS p, const char * const PO * p_size. */ if (len > p_size) - fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE, -E2BIG); /* * We can now safely call vanilla strscpy because we are protected from: @@ -357,7 +359,7 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) /* Give up if string is already overflowed. */ if (p_size <= p_len) - fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ, wanted); if (actual >= avail) { copy_len = avail - p_len - 1; @@ -366,7 +368,7 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) /* Give up if copy will overflow. */ if (p_size <= actual) - fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE, wanted); __underlying_memcpy(p + p_len, q, copy_len); p[actual] = '\0'; @@ -395,7 +397,7 @@ char *strcat(char * const POS p, const char *q) const size_t p_size = __member_size(p); if (strlcat(p, q, p_size) >= p_size) - fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE, p); return p; } @@ -431,13 +433,13 @@ char *strncat(char * const POS p, const char * const POS q, __kernel_size_t coun p_len = strlen(p); copy_len = strnlen(q, count); if (p_size < p_len + copy_len + 1) - fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE, p); __underlying_memcpy(p + p_len, q, copy_len); p[p_len + copy_len] = '\0'; return p; } -__FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, +__FORTIFY_INLINE bool fortify_memset_chk(__kernel_size_t size, const size_t p_size, const size_t p_size_field) { @@ -472,7 +474,8 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, * lengths are unknown.) */ if (p_size != SIZE_MAX && p_size < size) - fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE, true); + return false; } #define __fortify_memset_chk(p, c, size, p_size, p_size_field) ({ \ @@ -571,9 +574,9 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, * lengths are unknown.) */ if (p_size != SIZE_MAX && p_size < size) - fortify_panic(func, FORTIFY_WRITE); + fortify_panic(func, FORTIFY_WRITE, true); else if (q_size != SIZE_MAX && q_size < size) - fortify_panic(func, FORTIFY_READ); + fortify_panic(func, FORTIFY_READ, true); /* * Warn when writing beyond destination field size. @@ -673,7 +676,7 @@ __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ, NULL); return __real_memscan(p, c, size); } @@ -690,7 +693,7 @@ int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t __read_overflow2(); } if (p_size < size || q_size < size) - fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, INT_MIN); return __underlying_memcmp(p, q, size); } @@ -702,7 +705,7 @@ void *memchr(const void * const POS0 p, int c, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ, NULL); return __underlying_memchr(p, c, size); } @@ -714,7 +717,7 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ, NULL); return __real_memchr_inv(p, c, size); } @@ -727,7 +730,7 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ); + fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, NULL); return __real_kmemdup(p, size, gfp); } @@ -764,7 +767,7 @@ char *strcpy(char * const POS p, const char * const POS q) __write_overflow(); /* Run-time check for dynamic size overflow. */ if (p_size < size) - fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE); + fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE, p); __underlying_memcpy(p, q, size); return p; } diff --git a/lib/fortify_kunit.c b/lib/fortify_kunit.c index 7a88b5dd3d27..4ba7d02fdd78 100644 --- a/lib/fortify_kunit.c +++ b/lib/fortify_kunit.c @@ -15,8 +15,17 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +/* Redefine fortify_panic() to track failures. */ +void fortify_add_kunit_error(int write); +#define fortify_panic(func, write, retfail) do { \ + __fortify_report(FORTIFY_REASON(func, write)); \ + fortify_add_kunit_error(write); \ + return (retfail); \ +} while (0) + #include #include +#include #include #include #include @@ -27,10 +36,34 @@ # define __compiletime_strlen __builtin_strlen #endif +static struct kunit_resource read_resource; +static struct kunit_resource write_resource; +static int fortify_read_overflows; +static int fortify_write_overflows; + static const char array_of_10[] = "this is 10"; static const char *ptr_of_11 = "this is 11!"; static char array_unknown[] = "compiler thinks I might change"; +void fortify_add_kunit_error(int write) +{ + struct kunit_resource *resource; + struct kunit *current_test; + + current_test = kunit_get_current_test(); + if (!current_test) + return; + + resource = kunit_find_named_resource(current_test, + write ? "fortify_write_overflows" + : "fortify_read_overflows"); + if (!resource) + return; + + (*(int *)resource->data)++; + kunit_put_resource(resource); +} + static void known_sizes_test(struct kunit *test) { KUNIT_EXPECT_EQ(test, __compiletime_strlen("88888888"), 8); @@ -318,6 +351,14 @@ static int fortify_test_init(struct kunit *test) if (!IS_ENABLED(CONFIG_FORTIFY_SOURCE)) kunit_skip(test, "Not built with CONFIG_FORTIFY_SOURCE=y"); + fortify_read_overflows = 0; + kunit_add_named_resource(test, NULL, NULL, &read_resource, + "fortify_read_overflows", + &fortify_read_overflows); + fortify_write_overflows = 0; + kunit_add_named_resource(test, NULL, NULL, &write_resource, + "fortify_write_overflows", + &fortify_write_overflows); return 0; } diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 9291dc74ae01..5e53d42e32bb 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include /** * string_get_size - get the size in the specified units -- cgit v1.2.3 From 3d965b33e40d973b450cb0212913f039476c16f4 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 7 Apr 2023 12:27:16 -0700 Subject: fortify: Improve buffer overflow reporting Improve the reporting of buffer overflows under CONFIG_FORTIFY_SOURCE to help accelerate debugging efforts. The calculations are all just sitting in registers anyway, so pass them along to the function to be reported. For example, before: detected buffer overflow in memcpy and after: memcpy: detected buffer overflow: 4096 byte read of buffer size 1 Link: https://lore.kernel.org/r/20230407192717.636137-10-keescook@chromium.org Signed-off-by: Kees Cook --- arch/arm/boot/compressed/misc.c | 2 +- arch/arm/boot/compressed/misc.h | 2 +- arch/x86/boot/compressed/misc.c | 2 +- include/linux/fortify-string.h | 56 ++++++++++++++++++++++------------------- lib/fortify_kunit.c | 4 +-- lib/string_helpers.c | 9 ++++--- 6 files changed, 40 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c index d93e2e466f6a..6c41b270560e 100644 --- a/arch/arm/boot/compressed/misc.c +++ b/arch/arm/boot/compressed/misc.c @@ -154,7 +154,7 @@ decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p, putstr(" done, booting the kernel.\n"); } -void __fortify_panic(const u8 reason) +void __fortify_panic(const u8 reason, size_t avail, size_t size) { error("detected buffer overflow"); } diff --git a/arch/arm/boot/compressed/misc.h b/arch/arm/boot/compressed/misc.h index 4d59c427253c..8c73940b5fe4 100644 --- a/arch/arm/boot/compressed/misc.h +++ b/arch/arm/boot/compressed/misc.h @@ -10,7 +10,7 @@ void __div0(void); void decompress_kernel(unsigned long output_start, unsigned long free_mem_ptr_p, unsigned long free_mem_ptr_end_p, int arch_id); -void __fortify_panic(const u8 reason); +void __fortify_panic(const u8 reason, size_t avail, size_t size); int atags_to_fdt(void *atag_list, void *fdt, int total_space); uint32_t fdt_check_mem_start(uint32_t mem_start, const void *fdt); int do_decompress(u8 *input, int len, u8 *output, void (*error)(char *x)); diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index c9971b9dbb09..1844da203da9 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -496,7 +496,7 @@ asmlinkage __visible void *extract_kernel(void *rmode, unsigned char *output) return output + entry_offset; } -void __fortify_panic(const u8 reason) +void __fortify_panic(const u8 reason, size_t avail, size_t size) { error("detected buffer overflow"); } diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index fbfb90479b8f..6aeebe0a6777 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -16,8 +16,8 @@ FIELD_PREP(GENMASK(7, 1), func)) #ifndef fortify_panic -# define fortify_panic(func, write, retfail) \ - __fortify_panic(FORTIFY_REASON(func, write)) +# define fortify_panic(func, write, avail, size, retfail) \ + __fortify_panic(FORTIFY_REASON(func, write), avail, size) #endif #define FORTIFY_READ 0 @@ -48,8 +48,8 @@ enum fortify_func { EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC) }; -void __fortify_report(const u8 reason); -void __fortify_panic(const u8 reason) __cold __noreturn; +void __fortify_report(const u8 reason, const size_t avail, const size_t size); +void __fortify_panic(const u8 reason, const size_t avail, const size_t size) __cold __noreturn; void __read_overflow(void) __compiletime_error("detected read beyond size of object (1st parameter)"); void __read_overflow2(void) __compiletime_error("detected read beyond size of object (2nd parameter)"); void __read_overflow2_field(size_t avail, size_t wanted) __compiletime_warning("detected read beyond size of field (2nd parameter); maybe use struct_group()?"); @@ -183,7 +183,7 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __write_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE, p); + fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE, p_size, size, p); return __underlying_strncpy(p, q, size); } @@ -214,7 +214,7 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size /* Do not check characters beyond the end of p. */ ret = __real_strnlen(p, maxlen < p_size ? maxlen : p_size); if (p_size <= ret && maxlen != ret) - fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ, ret); + fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ, p_size, ret + 1, ret); return ret; } @@ -250,7 +250,7 @@ __kernel_size_t __fortify_strlen(const char * const POS p) return __underlying_strlen(p); ret = strnlen(p, p_size); if (p_size <= ret) - fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ, ret); + fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ, p_size, ret + 1, ret); return ret; } @@ -300,8 +300,8 @@ __FORTIFY_INLINE ssize_t sized_strscpy(char * const POS p, const char * const PO * Generate a runtime write overflow error if len is greater than * p_size. */ - if (len > p_size) - fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE, -E2BIG); + if (p_size < len) + fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE, p_size, len, -E2BIG); /* * We can now safely call vanilla strscpy because we are protected from: @@ -359,7 +359,7 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) /* Give up if string is already overflowed. */ if (p_size <= p_len) - fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ, wanted); + fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ, p_size, p_len + 1, wanted); if (actual >= avail) { copy_len = avail - p_len - 1; @@ -368,7 +368,7 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) /* Give up if copy will overflow. */ if (p_size <= actual) - fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE, wanted); + fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE, p_size, actual + 1, wanted); __underlying_memcpy(p + p_len, q, copy_len); p[actual] = '\0'; @@ -395,9 +395,10 @@ __FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2) char *strcat(char * const POS p, const char *q) { const size_t p_size = __member_size(p); + const size_t wanted = strlcat(p, q, p_size); - if (strlcat(p, q, p_size) >= p_size) - fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE, p); + if (p_size <= wanted) + fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE, p_size, wanted + 1, p); return p; } @@ -426,14 +427,15 @@ char *strncat(char * const POS p, const char * const POS q, __kernel_size_t coun { const size_t p_size = __member_size(p); const size_t q_size = __member_size(q); - size_t p_len, copy_len; + size_t p_len, copy_len, total; if (p_size == SIZE_MAX && q_size == SIZE_MAX) return __underlying_strncat(p, q, count); p_len = strlen(p); copy_len = strnlen(q, count); - if (p_size < p_len + copy_len + 1) - fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE, p); + total = p_len + copy_len + 1; + if (p_size < total) + fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE, p_size, total, p); __underlying_memcpy(p + p_len, q, copy_len); p[p_len + copy_len] = '\0'; return p; @@ -474,7 +476,7 @@ __FORTIFY_INLINE bool fortify_memset_chk(__kernel_size_t size, * lengths are unknown.) */ if (p_size != SIZE_MAX && p_size < size) - fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE, true); + fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE, p_size, size, true); return false; } @@ -574,9 +576,9 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, * lengths are unknown.) */ if (p_size != SIZE_MAX && p_size < size) - fortify_panic(func, FORTIFY_WRITE, true); + fortify_panic(func, FORTIFY_WRITE, p_size, size, true); else if (q_size != SIZE_MAX && q_size < size) - fortify_panic(func, FORTIFY_READ, true); + fortify_panic(func, FORTIFY_READ, p_size, size, true); /* * Warn when writing beyond destination field size. @@ -676,7 +678,7 @@ __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ, NULL); + fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ, p_size, size, NULL); return __real_memscan(p, c, size); } @@ -692,8 +694,10 @@ int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t if (__compiletime_lessthan(q_size, size)) __read_overflow2(); } - if (p_size < size || q_size < size) - fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, INT_MIN); + if (p_size < size) + fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, p_size, size, INT_MIN); + else if (q_size < size) + fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, q_size, size, INT_MIN); return __underlying_memcmp(p, q, size); } @@ -705,7 +709,7 @@ void *memchr(const void * const POS0 p, int c, __kernel_size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ, NULL); + fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ, p_size, size, NULL); return __underlying_memchr(p, c, size); } @@ -717,7 +721,7 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ, NULL); + fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ, p_size, size, NULL); return __real_memchr_inv(p, c, size); } @@ -730,7 +734,7 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) - fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, NULL); + fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL); return __real_kmemdup(p, size, gfp); } @@ -767,7 +771,7 @@ char *strcpy(char * const POS p, const char * const POS q) __write_overflow(); /* Run-time check for dynamic size overflow. */ if (p_size < size) - fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE, p); + fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE, p_size, size, p); __underlying_memcpy(p, q, size); return p; } diff --git a/lib/fortify_kunit.c b/lib/fortify_kunit.c index f0accebeca02..493ec02dd5b3 100644 --- a/lib/fortify_kunit.c +++ b/lib/fortify_kunit.c @@ -17,8 +17,8 @@ /* Redefine fortify_panic() to track failures. */ void fortify_add_kunit_error(int write); -#define fortify_panic(func, write, retfail) do { \ - __fortify_report(FORTIFY_REASON(func, write)); \ +#define fortify_panic(func, write, avail, size, retfail) do { \ + __fortify_report(FORTIFY_REASON(func, write), avail, size); \ fortify_add_kunit_error(write); \ return (retfail); \ } while (0) diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 5e53d42e32bb..6bbafd6a10d9 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -1016,20 +1016,21 @@ static const char * const fortify_func_name[] = { #undef MAKE_FORTIFY_FUNC_NAME }; -void __fortify_report(const u8 reason) +void __fortify_report(const u8 reason, const size_t avail, const size_t size) { const u8 func = FORTIFY_REASON_FUNC(reason); const bool write = FORTIFY_REASON_DIR(reason); const char *name; name = fortify_func_name[umin(func, FORTIFY_FUNC_UNKNOWN)]; - WARN(1, "%s: detected buffer %s overflow\n", name, str_read_write(!write)); + WARN(1, "%s: detected buffer overflow: %zu byte %s of buffer size %zu\n", + name, size, str_read_write(!write), avail); } EXPORT_SYMBOL(__fortify_report); -void __fortify_panic(const u8 reason) +void __fortify_panic(const u8 reason, const size_t avail, const size_t size) { - __fortify_report(reason); + __fortify_report(reason, avail, size); BUG(); } EXPORT_SYMBOL(__fortify_panic); -- cgit v1.2.3 From 57914905f3ff2212a949e7191d52d9994c2c6215 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 3 Oct 2023 16:01:42 +0300 Subject: kernel.h: Move lib/cmdline.c prototypes to string.h The lib/cmdline.c is basically a set of some small string parsers which are wide used in the kernel. Their prototypes belong to the string.h rather then kernel.h. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20231003130142.2936503-1-andriy.shevchenko@linux.intel.com Signed-off-by: Kees Cook --- include/linux/kernel.h | 6 ------ include/linux/string.h | 8 ++++++++ 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 86dd8939c2cd..d718fbec72dd 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -165,12 +165,6 @@ static inline void might_fault(void) { } void do_exit(long error_code) __noreturn; -extern int get_option(char **str, int *pint); -extern char *get_options(const char *str, int nints, int *ints); -extern unsigned long long memparse(const char *ptr, char **retptr); -extern bool parse_option_str(const char *str, const char *option); -extern char *next_arg(char *args, char **param, char **val); - extern int core_kernel_text(unsigned long addr); extern int __kernel_text_address(unsigned long addr); extern int kernel_text_address(unsigned long addr); diff --git a/include/linux/string.h b/include/linux/string.h index 96e6b1af86b5..adf3b3eb0ab7 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -286,9 +286,17 @@ extern void *kmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp); +/* lib/argv_split.c */ extern char **argv_split(gfp_t gfp, const char *str, int *argcp); extern void argv_free(char **argv); +/* lib/cmdline.c */ +extern int get_option(char **str, int *pint); +extern char *get_options(const char *str, int nints, int *ints); +extern unsigned long long memparse(const char *ptr, char **retptr); +extern bool parse_option_str(const char *str, const char *option); +extern char *next_arg(char *args, char **param, char **val); + extern bool sysfs_streq(const char *s1, const char *s2); int match_string(const char * const *array, size_t n, const char *string); int __sysfs_match_string(const char * const *array, size_t n, const char *s); -- cgit v1.2.3 From c5e6d3d85efa7451590edd94725b4b280e2fd8a3 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 28 Feb 2024 22:41:31 +0200 Subject: overflow: Use POD in check_shl_overflow() The check_shl_overflow() uses u64 type that is defined in types.h. Instead of including that header, just switch to use POD type directly. Signed-off-by: Andy Shevchenko Acked-by: Kees Cook Link: https://lore.kernel.org/r/20240228204919.3680786-2-andriy.shevchenko@linux.intel.com Signed-off-by: Kees Cook --- include/linux/overflow.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/overflow.h b/include/linux/overflow.h index dede374832c9..bc390f026128 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -197,7 +197,7 @@ static inline bool __must_check __must_check_overflow(bool overflow) typeof(a) _a = a; \ typeof(s) _s = s; \ typeof(d) _d = d; \ - u64 _a_full = _a; \ + unsigned long long _a_full = _a; \ unsigned int _to_shift = \ is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0; \ *_d = (_a_full << _to_shift); \ -- cgit v1.2.3 From f0b7f8ade9d2532a7d7da40eb297570d48dd2147 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 29 Feb 2024 22:52:30 +0200 Subject: lib/string_helpers: Add flags param to string_get_size() The new flags parameter allows controlling - Whether or not the units suffix is separated by a space, for compatibility with sort -h - Whether or not to append a B suffix - we're not always printing bytes. Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Signed-off-by: Andy Shevchenko Reviewed-by: Kent Overstreet Link: https://lore.kernel.org/r/20240229205345.93902-1-andriy.shevchenko@linux.intel.com Signed-off-by: Kees Cook --- include/linux/string_helpers.h | 10 +++++-- lib/string_helpers.c | 29 ++++++++++--------- lib/test-string_helpers.c | 63 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/string_helpers.h b/include/linux/string_helpers.h index 58fb1f90eda5..e93fbb5b0c01 100644 --- a/include/linux/string_helpers.h +++ b/include/linux/string_helpers.h @@ -17,14 +17,18 @@ static inline bool string_is_terminated(const char *s, int len) return memchr(s, '\0', len) ? true : false; } -/* Descriptions of the types of units to - * print in */ +/* Descriptions of the types of units to print in */ enum string_size_units { STRING_UNITS_10, /* use powers of 10^3 (standard SI) */ STRING_UNITS_2, /* use binary powers of 2^10 */ + STRING_UNITS_MASK = BIT(0), + + /* Modifiers */ + STRING_UNITS_NO_SPACE = BIT(30), + STRING_UNITS_NO_BYTES = BIT(31), }; -int string_get_size(u64 size, u64 blk_size, enum string_size_units units, +int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, char *buf, int len); int parse_int_array_user(const char __user *from, size_t count, int **array); diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 6bbafd6a10d9..69ba49b853c7 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -25,7 +25,7 @@ * string_get_size - get the size in the specified units * @size: The size to be converted in blocks * @blk_size: Size of the block (use 1 for size in bytes) - * @units: units to use (powers of 1000 or 1024) + * @units: Units to use (powers of 1000 or 1024), whether to include space separator * @buf: buffer to format to * @len: length of buffer * @@ -39,11 +39,12 @@ int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, char *buf, int len) { + enum string_size_units units_base = units & STRING_UNITS_MASK; static const char *const units_10[] = { - "B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" + "", "k", "M", "G", "T", "P", "E", "Z", "Y", }; static const char *const units_2[] = { - "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB" + "", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi", }; static const char *const *const units_str[] = { [STRING_UNITS_10] = units_10, @@ -68,7 +69,7 @@ int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, /* This is Napier's algorithm. Reduce the original block size to * - * coefficient * divisor[units]^i + * coefficient * divisor[units_base]^i * * we do the reduction so both coefficients are just under 32 bits so * that multiplying them together won't overflow 64 bits and we keep @@ -78,12 +79,12 @@ int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, * precision is in the coefficients. */ while (blk_size >> 32) { - do_div(blk_size, divisor[units]); + do_div(blk_size, divisor[units_base]); i++; } while (size >> 32) { - do_div(size, divisor[units]); + do_div(size, divisor[units_base]); i++; } @@ -92,8 +93,8 @@ int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, size *= blk_size; /* and logarithmically reduce it until it's just under the divisor */ - while (size >= divisor[units]) { - remainder = do_div(size, divisor[units]); + while (size >= divisor[units_base]) { + remainder = do_div(size, divisor[units_base]); i++; } @@ -103,10 +104,10 @@ int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, for (j = 0; sf_cap*10 < 1000; j++) sf_cap *= 10; - if (units == STRING_UNITS_2) { + if (units_base == STRING_UNITS_2) { /* express the remainder as a decimal. It's currently the * numerator of a fraction whose denominator is - * divisor[units], which is 1 << 10 for STRING_UNITS_2 */ + * divisor[units_base], which is 1 << 10 for STRING_UNITS_2 */ remainder *= 1000; remainder >>= 10; } @@ -128,10 +129,12 @@ int string_get_size(u64 size, u64 blk_size, const enum string_size_units units, if (i >= ARRAY_SIZE(units_2)) unit = "UNK"; else - unit = units_str[units][i]; + unit = units_str[units_base][i]; - return snprintf(buf, len, "%u%s %s", (u32)size, - tmp, unit); + return snprintf(buf, len, "%u%s%s%s%s", (u32)size, tmp, + (units & STRING_UNITS_NO_SPACE) ? "" : " ", + unit, + (units & STRING_UNITS_NO_BYTES) ? "" : "B"); } EXPORT_SYMBOL(string_get_size); diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 9a68849a5d55..dce67698297b 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -3,6 +3,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -500,21 +501,65 @@ static __init void test_string_get_size_check(const char *units, pr_warn("expected: '%s', got '%s'\n", exp, res); } -static __init void __test_string_get_size(const u64 size, const u64 blk_size, - const char *exp_result10, - const char *exp_result2) +static __init void __strchrcut(char *dst, const char *src, const char *cut) +{ + const char *from = src; + size_t len; + + do { + len = strcspn(from, cut); + memcpy(dst, from, len); + dst += len; + from += len; + } while (*from++); + *dst = '\0'; +} + +static __init void __test_string_get_size_one(const u64 size, const u64 blk_size, + const char *exp_result10, + const char *exp_result2, + enum string_size_units units, + const char *cut) { char buf10[string_get_size_maxbuf]; char buf2[string_get_size_maxbuf]; + char exp10[string_get_size_maxbuf]; + char exp2[string_get_size_maxbuf]; + char prefix10[64]; + char prefix2[64]; + + sprintf(prefix10, "STRING_UNITS_10 [%s]", cut); + sprintf(prefix2, "STRING_UNITS_2 [%s]", cut); + + __strchrcut(exp10, exp_result10, cut); + __strchrcut(exp2, exp_result2, cut); - string_get_size(size, blk_size, STRING_UNITS_10, buf10, sizeof(buf10)); - string_get_size(size, blk_size, STRING_UNITS_2, buf2, sizeof(buf2)); + string_get_size(size, blk_size, STRING_UNITS_10 | units, buf10, sizeof(buf10)); + string_get_size(size, blk_size, STRING_UNITS_2 | units, buf2, sizeof(buf2)); - test_string_get_size_check("STRING_UNITS_10", exp_result10, buf10, - size, blk_size); + test_string_get_size_check(prefix10, exp10, buf10, size, blk_size); + test_string_get_size_check(prefix2, exp2, buf2, size, blk_size); +} + +static __init void __test_string_get_size(const u64 size, const u64 blk_size, + const char *exp_result10, + const char *exp_result2) +{ + struct { + enum string_size_units units; + const char *cut; + } get_size_test_cases[] = { + { 0, "" }, + { STRING_UNITS_NO_SPACE, " " }, + { STRING_UNITS_NO_SPACE | STRING_UNITS_NO_BYTES, " B" }, + { STRING_UNITS_NO_BYTES, "B" }, + }; + int i; - test_string_get_size_check("STRING_UNITS_2", exp_result2, buf2, - size, blk_size); + for (i = 0; i < ARRAY_SIZE(get_size_test_cases); i++) + __test_string_get_size_one(size, blk_size, exp_result10, exp_result2, + get_size_test_cases[i].units, + get_size_test_cases[i].cut); } static __init void test_string_get_size(void) -- cgit v1.2.3 From 096361b15577a583afcc28179a08c75cf95e9dae Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 28 Feb 2024 13:54:25 +0000 Subject: ipv6: add ipv6_devconf_read_txrx cacheline_group IPv6 TX and RX fast path use the following fields: - disable_ipv6 - hop_limit - mtu6 - forwarding - disable_policy - proxy_ndp Place them in a group to increase data locality. Signed-off-by: Eric Dumazet Reviewed-by: Jiri Pirko Signed-off-by: David S. Miller --- include/linux/ipv6.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index ef3aa060a289..383a0ea2ab91 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -3,6 +3,7 @@ #define _IPV6_H #include +#include #define ipv6_optlen(p) (((p)->hdrlen+1) << 3) #define ipv6_authlen(p) (((p)->hdrlen+2) << 2) @@ -10,9 +11,16 @@ * This structure contains configuration options per IPv6 link. */ struct ipv6_devconf { - __s32 forwarding; + /* RX & TX fastpath fields. */ + __cacheline_group_begin(ipv6_devconf_read_txrx); + __s32 disable_ipv6; __s32 hop_limit; __s32 mtu6; + __s32 forwarding; + __s32 disable_policy; + __s32 proxy_ndp; + __cacheline_group_end(ipv6_devconf_read_txrx); + __s32 accept_ra; __s32 accept_redirects; __s32 autoconf; @@ -45,7 +53,6 @@ struct ipv6_devconf { __s32 accept_ra_rt_info_max_plen; #endif #endif - __s32 proxy_ndp; __s32 accept_source_route; __s32 accept_ra_from_local; #ifdef CONFIG_IPV6_OPTIMISTIC_DAD @@ -55,7 +62,6 @@ struct ipv6_devconf { #ifdef CONFIG_IPV6_MROUTE atomic_t mc_forwarding; #endif - __s32 disable_ipv6; __s32 drop_unicast_in_l2_multicast; __s32 accept_dad; __s32 force_tllao; @@ -76,7 +82,6 @@ struct ipv6_devconf { #endif __u32 enhanced_dad; __u32 addr_gen_mode; - __s32 disable_policy; __s32 ndisc_tclass; __s32 rpl_seg_enabled; __u32 ioam6_id; -- cgit v1.2.3 From f29f9199c2d2b3c258f577f438885288016847ed Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 28 Feb 2024 15:05:29 +0100 Subject: Simplify net_dbg_ratelimited() dummy There is no need to wrap calls to the no_printk() helper inside an always-false check, as no_printk() already does that internally. Signed-off-by: Geert Uytterhoeven Reviewed-by: Jiri Pirko Signed-off-by: David S. Miller --- include/linux/net.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/net.h b/include/linux/net.h index c9b4a63791a4..15df6d5f27a7 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -299,10 +299,7 @@ do { \ net_ratelimited_function(pr_debug, fmt, ##__VA_ARGS__) #else #define net_dbg_ratelimited(fmt, ...) \ - do { \ - if (0) \ - no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ - } while (0) + no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) #endif #define net_get_random_once(buf, nbytes) \ -- cgit v1.2.3 From b42a905b6aad40c092cf17f4b295a4c389bc7206 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 15:40:25 +0100 Subject: iommu: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216144027.185959-2-krzysztof.kozlowski@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/apple-dart.c | 3 ++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 ++- drivers/iommu/arm/arm-smmu/arm-smmu.c | 3 ++- drivers/iommu/arm/arm-smmu/qcom_iommu.c | 3 ++- drivers/iommu/exynos-iommu.c | 2 +- drivers/iommu/iommu.c | 2 +- drivers/iommu/ipmmu-vmsa.c | 4 ++-- drivers/iommu/msm_iommu.c | 4 ++-- drivers/iommu/mtk_iommu.c | 3 ++- drivers/iommu/mtk_iommu_v1.c | 3 ++- drivers/iommu/rockchip-iommu.c | 2 +- drivers/iommu/sprd-iommu.c | 3 ++- drivers/iommu/sun50i-iommu.c | 2 +- drivers/iommu/tegra-smmu.c | 4 ++-- drivers/iommu/virtio-iommu.c | 3 ++- include/linux/iommu.h | 4 ++-- 16 files changed, 28 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ef3ee95706da..eb1e62cd499a 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -779,7 +779,8 @@ static void apple_dart_domain_free(struct iommu_domain *domain) kfree(dart_domain); } -static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args) +static int apple_dart_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct platform_device *iommu_pdev = of_find_device_by_node(args->np); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 02580364acda..0234ddb39c22 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2704,7 +2704,8 @@ static int arm_smmu_enable_nesting(struct iommu_domain *domain) return ret; } -static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int arm_smmu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); } diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 68b6bc5e7c71..8e5e4ab5fad3 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1551,7 +1551,8 @@ static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, return ret; } -static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int arm_smmu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { u32 mask, fwid = 0; diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index 17a1c163fef6..e079bb7a993e 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -546,7 +546,8 @@ static struct iommu_device *qcom_iommu_probe_device(struct device *dev) return &qcom_iommu->iommu; } -static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int qcom_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct qcom_iommu_dev *qcom_iommu; struct platform_device *iommu_pdev; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 2c6e9094f1e9..d98c9161948a 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1431,7 +1431,7 @@ static void exynos_iommu_release_device(struct device *dev) } static int exynos_iommu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct platform_device *sysmmu = of_find_device_by_node(spec->np); struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index fa329bcd60e6..b0a41f598dcf 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2825,7 +2825,7 @@ void iommu_fwspec_free(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_fwspec_free); -int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) +int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i, new_num; diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index ace1fc4bd34b..cd7219319c8b 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -709,7 +709,7 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, } static int ipmmu_init_platform_device(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *ipmmu_pdev; @@ -773,7 +773,7 @@ static bool ipmmu_device_is_allowed(struct device *dev) } static int ipmmu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { if (!ipmmu_device_is_allowed(dev)) return -ENODEV; diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index f86af9815d6f..989e0869d805 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -598,7 +598,7 @@ static void print_ctx_regs(void __iomem *base, int ctx) static int insert_iommu_master(struct device *dev, struct msm_iommu_dev **iommu, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct msm_iommu_ctx_dev *master = dev_iommu_priv_get(dev); int sid; @@ -626,7 +626,7 @@ static int insert_iommu_master(struct device *dev, } static int qcom_iommu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct msm_iommu_dev *iommu = NULL, *iter; unsigned long flags; diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7abe9e85a570..955de8fb7732 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -957,7 +957,8 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) return group; } -static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *m4updev; diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 25b41222abae..ae16d8238d84 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -398,7 +398,8 @@ static const struct iommu_ops mtk_iommu_v1_ops; * MTK generation one iommu HW only support one iommu domain, and all the client * sharing the same iova address space. */ -static int mtk_iommu_v1_create_mapping(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_v1_create_mapping(struct device *dev, + const struct of_phandle_args *args) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_v1_data *data; diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 2685861c0a12..da79d9f4cf63 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1140,7 +1140,7 @@ static void rk_iommu_release_device(struct device *dev) } static int rk_iommu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_dev; struct rk_iommudata *data; diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 537359f10997..ba53571a8239 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -390,7 +390,8 @@ static struct iommu_device *sprd_iommu_probe_device(struct device *dev) return &sdev->iommu; } -static int sprd_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int sprd_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *pdev; diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index 41484a5a399b..decd52cba998 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -819,7 +819,7 @@ static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) } static int sun50i_iommu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); unsigned id = args->args[0]; diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 310871728ab4..14e525bd0d9b 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -830,7 +830,7 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np) } static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { const struct iommu_ops *ops = smmu->iommu.ops; int err; @@ -959,7 +959,7 @@ static struct iommu_group *tegra_smmu_device_group(struct device *dev) } static int tegra_smmu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); struct tegra_mc *mc = platform_get_drvdata(iommu_pdev); diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 34db37fd9675..04048f64a2c0 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1051,7 +1051,8 @@ static struct iommu_group *viommu_device_group(struct device *dev) return generic_device_group(dev); } -static int viommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int viommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 7cc56cfe98dd..98a958621dcb 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -567,7 +567,7 @@ struct iommu_ops { /* Request/Free a list of reserved regions for a device */ void (*get_resv_regions)(struct device *dev, struct list_head *list); - int (*of_xlate)(struct device *dev, struct of_phandle_args *args); + int (*of_xlate)(struct device *dev, const struct of_phandle_args *args); bool (*is_attach_deferred)(struct device *dev); /* Per device IOMMU features */ @@ -985,7 +985,7 @@ struct iommu_mm_data { int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops); void iommu_fwspec_free(struct device *dev); -int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); +int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids); const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) -- cgit v1.2.3 From 5896e6e39b86c1d820b3ccf5caea9aef40c2eacd Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 15:40:26 +0100 Subject: iommu: constify fwnode in iommu_ops_from_fwnode() Make pointer to fwnode_handle a pointer to const for code safety. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216144027.185959-3-krzysztof.kozlowski@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 2 +- include/linux/iommu.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b0a41f598dcf..28b18ed83273 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2774,7 +2774,7 @@ bool iommu_default_passthrough(void) } EXPORT_SYMBOL_GPL(iommu_default_passthrough); -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) { const struct iommu_ops *ops = NULL; struct iommu_device *iommu; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 98a958621dcb..7c8032202457 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -986,7 +986,7 @@ int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops); void iommu_fwspec_free(struct device *dev); int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids); -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode); static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) { @@ -1309,7 +1309,7 @@ static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, } static inline -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) { return NULL; } -- cgit v1.2.3 From f094323867668d50124886ad884b665de7319537 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 26 Jan 2024 10:39:42 -0500 Subject: sunrpc: pass in the sv_stats struct through svc_create_pooled Since only one service actually reports the rpc stats there's not much of a reason to have a pointer to it in the svc_program struct. Adjust the svc_create_pooled function to take the sv_stats as an argument and pass the struct through there as desired instead of getting it from the svc_program->pg_stats. Signed-off-by: Josef Bacik Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfssvc.c | 3 ++- include/linux/sunrpc/svc.h | 4 +++- net/sunrpc/svc.c | 12 +++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index a0b117107e86..d640f893021a 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -661,7 +661,8 @@ int nfsd_create_serv(struct net *net) if (nfsd_max_blksize == 0) nfsd_max_blksize = nfsd_get_default_max_blksize(); nfsd_reset_versions(nn); - serv = svc_create_pooled(&nfsd_program, nfsd_max_blksize, nfsd); + serv = svc_create_pooled(&nfsd_program, &nfsd_svcstats, + nfsd_max_blksize, nfsd); if (serv == NULL) return -ENOMEM; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 67cf1c9efd80..91a653eb3a50 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -411,7 +411,9 @@ bool svc_rqst_replace_page(struct svc_rqst *rqstp, void svc_rqst_release_pages(struct svc_rqst *rqstp); void svc_rqst_free(struct svc_rqst *); void svc_exit_thread(struct svc_rqst *); -struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, +struct svc_serv * svc_create_pooled(struct svc_program *prog, + struct svc_stat *stats, + unsigned int bufsize, int (*threadfn)(void *data)); int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); int svc_pool_stats_open(struct svc_info *si, struct file *file); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 62afc3dbc837..1ce6a3b7175c 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -451,8 +451,8 @@ __svc_init_bc(struct svc_serv *serv) * Create an RPC service */ static struct svc_serv * -__svc_create(struct svc_program *prog, unsigned int bufsize, int npools, - int (*threadfn)(void *data)) +__svc_create(struct svc_program *prog, struct svc_stat *stats, + unsigned int bufsize, int npools, int (*threadfn)(void *data)) { struct svc_serv *serv; unsigned int vers; @@ -463,7 +463,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, return NULL; serv->sv_name = prog->pg_name; serv->sv_program = prog; - serv->sv_stats = prog->pg_stats; + serv->sv_stats = stats; if (bufsize > RPCSVC_MAXPAYLOAD) bufsize = RPCSVC_MAXPAYLOAD; serv->sv_max_payload = bufsize? bufsize : 4096; @@ -529,26 +529,28 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, struct svc_serv *svc_create(struct svc_program *prog, unsigned int bufsize, int (*threadfn)(void *data)) { - return __svc_create(prog, bufsize, 1, threadfn); + return __svc_create(prog, NULL, bufsize, 1, threadfn); } EXPORT_SYMBOL_GPL(svc_create); /** * svc_create_pooled - Create an RPC service with pooled threads * @prog: the RPC program the new service will handle + * @stats: the stats struct if desired * @bufsize: maximum message size for @prog * @threadfn: a function to service RPC requests for @prog * * Returns an instantiated struct svc_serv object or NULL. */ struct svc_serv *svc_create_pooled(struct svc_program *prog, + struct svc_stat *stats, unsigned int bufsize, int (*threadfn)(void *data)) { struct svc_serv *serv; unsigned int npools = svc_pool_map_get(); - serv = __svc_create(prog, bufsize, npools, threadfn); + serv = __svc_create(prog, stats, bufsize, npools, threadfn); if (!serv) goto out_err; return serv; -- cgit v1.2.3 From 3f6ef182f144dcc9a4d942f97b6a8ed969f13c95 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 26 Jan 2024 10:39:43 -0500 Subject: sunrpc: remove ->pg_stats from svc_program Now that this isn't used anywhere, remove it. Signed-off-by: Josef Bacik Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfssvc.c | 1 - include/linux/sunrpc/svc.h | 1 - 2 files changed, 2 deletions(-) (limited to 'include/linux') diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index d640f893021a..d98a6abad990 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -127,7 +127,6 @@ struct svc_program nfsd_program = { .pg_vers = nfsd_version, /* version table */ .pg_name = "nfsd", /* program name */ .pg_class = "nfsd", /* authentication class */ - .pg_stats = &nfsd_svcstats, /* version table */ .pg_authenticate = &svc_set_client, /* export authentication */ .pg_init_request = nfsd_init_request, .pg_rpcbind_set = nfsd_rpcbind_set, diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 91a653eb3a50..23617da0e565 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -339,7 +339,6 @@ struct svc_program { const struct svc_version **pg_vers; /* version array */ char * pg_name; /* service name */ char * pg_class; /* class name: services sharing authentication */ - struct svc_stat * pg_stats; /* rpc statistics */ enum svc_auth_status (*pg_authenticate)(struct svc_rqst *rqstp); __be32 (*pg_init_request)(struct svc_rqst *, const struct svc_program *, -- cgit v1.2.3 From 71b43531ee0be6dcaa406132ebd540022dcb12ea Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 4 Feb 2024 18:17:22 -0500 Subject: svcrdma: Post Send WR chain Eventually I'd like the server to post the reply's Send WR along with any Write WRs using only a single call to ib_post_send(), in order to reduce the NIC's doorbell rate. To do this, add an anchor for a WR chain to svc_rdma_send_ctxt, and refactor svc_rdma_send() to post this WR chain to the Send Queue. For the moment, the posted chain will continue to contain a single Send WR. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 6 ++-- net/sunrpc/xprtrdma/svc_rdma_backchannel.c | 2 +- net/sunrpc/xprtrdma/svc_rdma_sendto.c | 49 ++++++++++++++++++++---------- 3 files changed, 38 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index e7595ae62fe2..ee05087d6499 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -210,6 +210,8 @@ struct svc_rdma_send_ctxt { struct svcxprt_rdma *sc_rdma; struct ib_send_wr sc_send_wr; + struct ib_send_wr *sc_wr_chain; + int sc_sqecount; struct ib_cqe sc_cqe; struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; @@ -258,8 +260,8 @@ extern struct svc_rdma_send_ctxt * svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma); extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt); -extern int svc_rdma_send(struct svcxprt_rdma *rdma, - struct svc_rdma_send_ctxt *ctxt); +extern int svc_rdma_post_send(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt); extern int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *sctxt, const struct svc_rdma_pcl *write_pcl, diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c index c9be6778643b..e5a78b761012 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c +++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c @@ -90,7 +90,7 @@ static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma, */ get_page(virt_to_page(rqst->rq_buffer)); sctxt->sc_send_wr.opcode = IB_WR_SEND; - return svc_rdma_send(rdma, sctxt); + return svc_rdma_post_send(rdma, sctxt); } /* Server-side transport endpoint wants a whole page for its send diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 0ee9185f5f3f..0f02fb09d5b0 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -208,6 +208,9 @@ out: ctxt->sc_send_wr.num_sge = 0; ctxt->sc_cur_sge_no = 0; ctxt->sc_page_count = 0; + ctxt->sc_wr_chain = &ctxt->sc_send_wr; + ctxt->sc_sqecount = 1; + return ctxt; out_empty: @@ -293,7 +296,7 @@ static void svc_rdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) struct svc_rdma_send_ctxt *ctxt = container_of(cqe, struct svc_rdma_send_ctxt, sc_cqe); - svc_rdma_wake_send_waiters(rdma, 1); + svc_rdma_wake_send_waiters(rdma, ctxt->sc_sqecount); if (unlikely(wc->status != IB_WC_SUCCESS)) goto flushed; @@ -312,36 +315,44 @@ flushed: } /** - * svc_rdma_send - Post a single Send WR - * @rdma: transport on which to post the WR - * @ctxt: send ctxt with a Send WR ready to post + * svc_rdma_post_send - Post a WR chain to the Send Queue + * @rdma: transport context + * @ctxt: WR chain to post * * Copy fields in @ctxt to stack variables in order to guarantee * that these values remain available after the ib_post_send() call. * In some error flow cases, svc_rdma_wc_send() releases @ctxt. * + * Note there is potential for starvation when the Send Queue is + * full because there is no order to when waiting threads are + * awoken. The transport is typically provisioned with a deep + * enough Send Queue that SQ exhaustion should be a rare event. + * * Return values: * %0: @ctxt's WR chain was posted successfully * %-ENOTCONN: The connection was lost */ -int svc_rdma_send(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt) +int svc_rdma_post_send(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt) { - struct ib_send_wr *wr = &ctxt->sc_send_wr; + struct ib_send_wr *first_wr = ctxt->sc_wr_chain; + struct ib_send_wr *send_wr = &ctxt->sc_send_wr; + const struct ib_send_wr *bad_wr = first_wr; struct rpc_rdma_cid cid = ctxt->sc_cid; - int ret; + int ret, sqecount = ctxt->sc_sqecount; might_sleep(); /* Sync the transport header buffer */ ib_dma_sync_single_for_device(rdma->sc_pd->device, - wr->sg_list[0].addr, - wr->sg_list[0].length, + send_wr->sg_list[0].addr, + send_wr->sg_list[0].length, DMA_TO_DEVICE); /* If the SQ is full, wait until an SQ entry is available */ while (!test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags)) { - if ((atomic_dec_return(&rdma->sc_sq_avail) < 0)) { - svc_rdma_wake_send_waiters(rdma, 1); + if (atomic_sub_return(sqecount, &rdma->sc_sq_avail) < 0) { + svc_rdma_wake_send_waiters(rdma, sqecount); /* When the transport is torn down, assume * ib_drain_sq() will trigger enough Send @@ -358,12 +369,18 @@ int svc_rdma_send(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt) } trace_svcrdma_post_send(ctxt); - ret = ib_post_send(rdma->sc_qp, wr, NULL); + ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); if (ret) { trace_svcrdma_sq_post_err(rdma, &cid, ret); svc_xprt_deferred_close(&rdma->sc_xprt); - svc_rdma_wake_send_waiters(rdma, 1); - break; + + /* If even one WR was posted, there will be a + * Send completion that bumps sc_sq_avail. + */ + if (bad_wr == first_wr) { + svc_rdma_wake_send_waiters(rdma, sqecount); + break; + } } return 0; } @@ -884,7 +901,7 @@ static int svc_rdma_send_reply_msg(struct svcxprt_rdma *rdma, sctxt->sc_send_wr.opcode = IB_WR_SEND; } - return svc_rdma_send(rdma, sctxt); + return svc_rdma_post_send(rdma, sctxt); } /** @@ -948,7 +965,7 @@ void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma, sctxt->sc_send_wr.num_sge = 1; sctxt->sc_send_wr.opcode = IB_WR_SEND; sctxt->sc_sges[0].length = sctxt->sc_hdrbuf.len; - if (svc_rdma_send(rdma, sctxt)) + if (svc_rdma_post_send(rdma, sctxt)) goto put_ctxt; return; -- cgit v1.2.3 From a1f5788a0c250c87d3007d59d11a00ab98e66f01 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 4 Feb 2024 18:17:28 -0500 Subject: svcrdma: Move write_info for Reply chunks into struct svc_rdma_send_ctxt Since the RPC transaction's svc_rdma_send_ctxt will stay around for the duration of the RDMA Write operation, the write_info structure for the Reply chunk can reside in the request's svc_rdma_send_ctxt instead of being allocated separately. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 25 ++++++++++ include/trace/events/rpcrdma.h | 4 ++ net/sunrpc/xprtrdma/svc_rdma_rw.c | 91 ++++++++++++++++++++--------------- net/sunrpc/xprtrdma/svc_rdma_sendto.c | 2 +- 4 files changed, 82 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index ee05087d6499..918cf4fda728 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -203,6 +203,29 @@ struct svc_rdma_recv_ctxt { struct page *rc_pages[RPCSVC_MAXPAGES]; }; +/* + * State for sending a Write chunk. + * - Tracks progress of writing one chunk over all its segments + * - Stores arguments for the SGL constructor functions + */ +struct svc_rdma_write_info { + struct svcxprt_rdma *wi_rdma; + + const struct svc_rdma_chunk *wi_chunk; + + /* write state of this chunk */ + unsigned int wi_seg_off; + unsigned int wi_seg_no; + + /* SGL constructor arguments */ + const struct xdr_buf *wi_xdr; + unsigned char *wi_base; + unsigned int wi_next_off; + + struct svc_rdma_chunk_ctxt wi_cc; + struct work_struct wi_work; +}; + struct svc_rdma_send_ctxt { struct llist_node sc_node; struct rpc_rdma_cid sc_cid; @@ -215,6 +238,7 @@ struct svc_rdma_send_ctxt { struct ib_cqe sc_cqe; struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; + struct svc_rdma_write_info sc_reply_info; void *sc_xprt_buf; int sc_page_count; int sc_cur_sge_no; @@ -249,6 +273,7 @@ extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, const struct xdr_buf *xdr); extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_recv_ctxt *rctxt, + struct svc_rdma_send_ctxt *sctxt, const struct xdr_buf *xdr); extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index 110c1475c527..027ac3ab457d 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -2118,6 +2118,10 @@ DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_write); DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_flush); DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_err); +DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_reply); +DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_reply_flush); +DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_reply_err); + TRACE_EVENT(svcrdma_qp_error, TP_PROTO( const struct ib_event *event, diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index c00fcce61d1e..2ca3c6311c5e 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -197,28 +197,6 @@ void svc_rdma_cc_release(struct svcxprt_rdma *rdma, llist_add_batch(first, last, &rdma->sc_rw_ctxts); } -/* State for sending a Write or Reply chunk. - * - Tracks progress of writing one chunk over all its segments - * - Stores arguments for the SGL constructor functions - */ -struct svc_rdma_write_info { - struct svcxprt_rdma *wi_rdma; - - const struct svc_rdma_chunk *wi_chunk; - - /* write state of this chunk */ - unsigned int wi_seg_off; - unsigned int wi_seg_no; - - /* SGL constructor arguments */ - const struct xdr_buf *wi_xdr; - unsigned char *wi_base; - unsigned int wi_next_off; - - struct svc_rdma_chunk_ctxt wi_cc; - struct work_struct wi_work; -}; - static struct svc_rdma_write_info * svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, const struct svc_rdma_chunk *chunk) @@ -252,6 +230,43 @@ static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) queue_work(svcrdma_wq, &info->wi_work); } +static void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_chunk_ctxt *cc) +{ + svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); + svc_rdma_cc_release(rdma, cc, DMA_TO_DEVICE); +} + +/** + * svc_rdma_reply_done - Reply chunk Write completion handler + * @cq: controlling Completion Queue + * @wc: Work Completion report + * + * Pages under I/O are released by a subsequent Send completion. + */ +static void svc_rdma_reply_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct ib_cqe *cqe = wc->wr_cqe; + struct svc_rdma_chunk_ctxt *cc = + container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); + struct svcxprt_rdma *rdma = cq->cq_context; + + switch (wc->status) { + case IB_WC_SUCCESS: + trace_svcrdma_wc_reply(&cc->cc_cid); + svc_rdma_reply_chunk_release(rdma, cc); + return; + case IB_WC_WR_FLUSH_ERR: + trace_svcrdma_wc_reply_flush(wc, &cc->cc_cid); + break; + default: + trace_svcrdma_wc_reply_err(wc, &cc->cc_cid); + } + + svc_rdma_reply_chunk_release(rdma, cc); + svc_xprt_deferred_close(&rdma->sc_xprt); +} + /** * svc_rdma_write_done - Write chunk completion * @cq: controlling Completion Queue @@ -624,7 +639,8 @@ out_err: /** * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk * @rdma: controlling RDMA transport - * @rctxt: Write and Reply chunks from client + * @rctxt: Write and Reply chunks provisioned by the client + * @sctxt: Send WR resources * @xdr: xdr_buf containing an RPC Reply * * Returns a non-negative number of bytes the chunk consumed, or @@ -636,37 +652,34 @@ out_err: */ int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_recv_ctxt *rctxt, + struct svc_rdma_send_ctxt *sctxt, const struct xdr_buf *xdr) { - struct svc_rdma_write_info *info; - struct svc_rdma_chunk_ctxt *cc; - struct svc_rdma_chunk *chunk; + struct svc_rdma_write_info *info = &sctxt->sc_reply_info; + struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; int ret; - if (pcl_is_empty(&rctxt->rc_reply_pcl)) - return 0; + if (likely(pcl_is_empty(&rctxt->rc_reply_pcl))) + return 0; /* client provided no Reply chunk */ - chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); - info = svc_rdma_write_info_alloc(rdma, chunk); - if (!info) - return -ENOMEM; - cc = &info->wi_cc; + info->wi_rdma = rdma; + info->wi_chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); + info->wi_seg_off = 0; + info->wi_seg_no = 0; + svc_rdma_cc_init(rdma, &info->wi_cc); + info->wi_cc.cc_cqe.done = svc_rdma_reply_done; ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr, svc_rdma_xb_write, info); if (ret < 0) - goto out_err; + return ret; trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); ret = svc_rdma_post_chunk_ctxt(rdma, cc); if (ret < 0) - goto out_err; + return ret; return xdr->len; - -out_err: - svc_rdma_write_info_free(info); - return ret; } /** diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 0f02fb09d5b0..d8e079be36e2 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -1012,7 +1012,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) if (!p) goto put_ctxt; - ret = svc_rdma_send_reply_chunk(rdma, rctxt, &rqstp->rq_res); + ret = svc_rdma_send_reply_chunk(rdma, rctxt, sctxt, &rqstp->rq_res); if (ret < 0) goto reply_chunk; rc_size = ret; -- cgit v1.2.3 From 10e6fc1054d900a205e5233f186ebce9c50e1d1d Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 4 Feb 2024 18:17:34 -0500 Subject: svcrdma: Post the Reply chunk and Send WR together Reduce the doorbell and Send completion rates when sending RPC/RDMA replies that have Reply chunks. NFS READDIR procedures typically return their result in a Reply chunk, for example. Instead of calling ib_post_send() to post the Write WRs for the Reply chunk, and then calling it again to post the Send WR that conveys the transport header, chain the Write WRs to the Send WR and call ib_post_send() only once. Thanks to the Send Queue completion ordering rules, when the Send WR completes, that guarantees that Write WRs posted before it have also completed successfully. Thus all Write WRs for the Reply chunk can remain unsignaled. Instead of handling a Write completion and then a Send completion, only the Send completion is seen, and it handles clean up for both the Writes and the Send. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 13 +++++--- net/sunrpc/xprtrdma/svc_rdma_rw.c | 58 ++++++++++++++++++++++------------- net/sunrpc/xprtrdma/svc_rdma_sendto.c | 34 +++++++++++--------- 3 files changed, 66 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 918cf4fda728..ac882bd23ca2 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -262,19 +262,24 @@ extern void svc_rdma_release_ctxt(struct svc_xprt *xprt, void *ctxt); extern int svc_rdma_recvfrom(struct svc_rqst *); /* svc_rdma_rw.c */ +extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma, + struct svc_rdma_chunk_ctxt *cc); extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma); extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc); extern void svc_rdma_cc_release(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc, enum dma_data_direction dir); +extern void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt); extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_chunk *chunk, const struct xdr_buf *xdr); -extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - struct svc_rdma_send_ctxt *sctxt, - const struct xdr_buf *xdr); +extern int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, + const struct svc_rdma_pcl *write_pcl, + const struct svc_rdma_pcl *reply_pcl, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr); extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head); diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 2ca3c6311c5e..2b25edc6c73c 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -230,10 +230,18 @@ static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) queue_work(svcrdma_wq, &info->wi_work); } -static void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, - struct svc_rdma_chunk_ctxt *cc) +/** + * svc_rdma_reply_chunk_release - Release Reply chunk I/O resources + * @rdma: controlling transport + * @ctxt: Send context that is being released + */ +void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt) { - svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); + struct svc_rdma_chunk_ctxt *cc = &ctxt->sc_reply_info.wi_cc; + + if (!cc->cc_sqecount) + return; svc_rdma_cc_release(rdma, cc, DMA_TO_DEVICE); } @@ -254,7 +262,6 @@ static void svc_rdma_reply_done(struct ib_cq *cq, struct ib_wc *wc) switch (wc->status) { case IB_WC_SUCCESS: trace_svcrdma_wc_reply(&cc->cc_cid); - svc_rdma_reply_chunk_release(rdma, cc); return; case IB_WC_WR_FLUSH_ERR: trace_svcrdma_wc_reply_flush(wc, &cc->cc_cid); @@ -263,7 +270,6 @@ static void svc_rdma_reply_done(struct ib_cq *cq, struct ib_wc *wc) trace_svcrdma_wc_reply_err(wc, &cc->cc_cid); } - svc_rdma_reply_chunk_release(rdma, cc); svc_xprt_deferred_close(&rdma->sc_xprt); } @@ -637,9 +643,10 @@ out_err: } /** - * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk + * svc_rdma_prepare_reply_chunk - Construct WR chain for writing the Reply chunk * @rdma: controlling RDMA transport - * @rctxt: Write and Reply chunks provisioned by the client + * @write_pcl: Write chunk list provided by client + * @reply_pcl: Reply chunk provided by client * @sctxt: Send WR resources * @xdr: xdr_buf containing an RPC Reply * @@ -650,35 +657,44 @@ out_err: * %-ENOTCONN if posting failed (connection is lost), * %-EIO if rdma_rw initialization failed (DMA mapping, etc). */ -int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - struct svc_rdma_send_ctxt *sctxt, - const struct xdr_buf *xdr) +int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, + const struct svc_rdma_pcl *write_pcl, + const struct svc_rdma_pcl *reply_pcl, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr) { struct svc_rdma_write_info *info = &sctxt->sc_reply_info; struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; + struct ib_send_wr *first_wr; + struct list_head *pos; + struct ib_cqe *cqe; int ret; - if (likely(pcl_is_empty(&rctxt->rc_reply_pcl))) - return 0; /* client provided no Reply chunk */ - info->wi_rdma = rdma; - info->wi_chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); + info->wi_chunk = pcl_first_chunk(reply_pcl); info->wi_seg_off = 0; info->wi_seg_no = 0; - svc_rdma_cc_init(rdma, &info->wi_cc); info->wi_cc.cc_cqe.done = svc_rdma_reply_done; - ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr, + ret = pcl_process_nonpayloads(write_pcl, xdr, svc_rdma_xb_write, info); if (ret < 0) return ret; - trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); - ret = svc_rdma_post_chunk_ctxt(rdma, cc); - if (ret < 0) - return ret; + first_wr = sctxt->sc_wr_chain; + cqe = &cc->cc_cqe; + list_for_each(pos, &cc->cc_rwctxts) { + struct svc_rdma_rw_ctxt *rwc; + rwc = list_entry(pos, struct svc_rdma_rw_ctxt, rw_list); + first_wr = rdma_rw_ctx_wrs(&rwc->rw_ctx, rdma->sc_qp, + rdma->sc_port_num, cqe, first_wr); + cqe = NULL; + } + sctxt->sc_wr_chain = first_wr; + sctxt->sc_sqecount += cc->cc_sqecount; + + trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); return xdr->len; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index d8e079be36e2..6dfd2232ce5b 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -205,6 +205,7 @@ out: xdr_init_encode(&ctxt->sc_stream, &ctxt->sc_hdrbuf, ctxt->sc_xprt_buf, NULL); + svc_rdma_cc_init(rdma, &ctxt->sc_reply_info.wi_cc); ctxt->sc_send_wr.num_sge = 0; ctxt->sc_cur_sge_no = 0; ctxt->sc_page_count = 0; @@ -226,6 +227,8 @@ static void svc_rdma_send_ctxt_release(struct svcxprt_rdma *rdma, struct ib_device *device = rdma->sc_cm_id->device; unsigned int i; + svc_rdma_reply_chunk_release(rdma, ctxt); + if (ctxt->sc_page_count) release_pages(ctxt->sc_pages, ctxt->sc_page_count); @@ -867,16 +870,10 @@ static void svc_rdma_save_io_pages(struct svc_rqst *rqstp, * in sc_sges[0], and the RPC xdr_buf is prepared in following sges. * * Depending on whether a Write list or Reply chunk is present, - * the server may send all, a portion of, or none of the xdr_buf. + * the server may Send all, a portion of, or none of the xdr_buf. * In the latter case, only the transport header (sc_sges[0]) is * transmitted. * - * RDMA Send is the last step of transmitting an RPC reply. Pages - * involved in the earlier RDMA Writes are here transferred out - * of the rqstp and into the sctxt's page array. These pages are - * DMA unmapped by each Write completion, but the subsequent Send - * completion finally releases these pages. - * * Assumptions: * - The Reply's transport header will never be larger than a page. */ @@ -885,6 +882,7 @@ static int svc_rdma_send_reply_msg(struct svcxprt_rdma *rdma, const struct svc_rdma_recv_ctxt *rctxt, struct svc_rqst *rqstp) { + struct ib_send_wr *send_wr = &sctxt->sc_send_wr; int ret; ret = svc_rdma_map_reply_msg(rdma, sctxt, &rctxt->rc_write_pcl, @@ -892,13 +890,16 @@ static int svc_rdma_send_reply_msg(struct svcxprt_rdma *rdma, if (ret < 0) return ret; + /* Transfer pages involved in RDMA Writes to the sctxt's + * page array. Completion handling releases these pages. + */ svc_rdma_save_io_pages(rqstp, sctxt); if (rctxt->rc_inv_rkey) { - sctxt->sc_send_wr.opcode = IB_WR_SEND_WITH_INV; - sctxt->sc_send_wr.ex.invalidate_rkey = rctxt->rc_inv_rkey; + send_wr->opcode = IB_WR_SEND_WITH_INV; + send_wr->ex.invalidate_rkey = rctxt->rc_inv_rkey; } else { - sctxt->sc_send_wr.opcode = IB_WR_SEND; + send_wr->opcode = IB_WR_SEND; } return svc_rdma_post_send(rdma, sctxt); @@ -1012,10 +1013,15 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) if (!p) goto put_ctxt; - ret = svc_rdma_send_reply_chunk(rdma, rctxt, sctxt, &rqstp->rq_res); - if (ret < 0) - goto reply_chunk; - rc_size = ret; + rc_size = 0; + if (!pcl_is_empty(&rctxt->rc_reply_pcl)) { + ret = svc_rdma_prepare_reply_chunk(rdma, &rctxt->rc_write_pcl, + &rctxt->rc_reply_pcl, sctxt, + &rqstp->rq_res); + if (ret < 0) + goto reply_chunk; + rc_size = ret; + } *p++ = *rdma_argp; *p++ = *(rdma_argp + 1); -- cgit v1.2.3 From d2727cefff0204c3074a10e3ad2e1b5c9dfb986c Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 4 Feb 2024 18:17:41 -0500 Subject: svcrdma: Post WRs for Write chunks in svc_rdma_sendto() Refactor to eventually enable svcrdma to post the Write WRs for each RPC response using the same ib_post_send() as the Send WR (ie, as a single WR chain). svc_rdma_result_payload (originally svc_rdma_read_payload) was added so that the upper layer XDR encoder could identify a range of bytes to be possibly conveyed by RDMA (if a Write chunk was provided by the client). The purpose of commit f6ad77590a5d ("svcrdma: Post RDMA Writes while XDR encoding replies") was to post as much of the result payload outside of svc_rdma_sendto() as possible because svc_rdma_sendto() used to be called with the xpt_mutex held. However, since commit ca4faf543a33 ("SUNRPC: Move xpt_mutex into socket xpo_sendto methods"), the xpt_mutex is no longer held when calling svc_rdma_sendto(). Thus, that benefit is no longer an issue. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 6 ++-- net/sunrpc/xprtrdma/svc_rdma_rw.c | 56 +++++++++++++++++++++++------------ net/sunrpc/xprtrdma/svc_rdma_sendto.c | 30 +++++++------------ 3 files changed, 51 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index ac882bd23ca2..d33bab33099a 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -272,9 +272,9 @@ extern void svc_rdma_cc_release(struct svcxprt_rdma *rdma, enum dma_data_direction dir); extern void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt); -extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, - const struct svc_rdma_chunk *chunk, - const struct xdr_buf *xdr); +extern int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_recv_ctxt *rctxt, + const struct xdr_buf *xdr); extern int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_pcl *write_pcl, const struct svc_rdma_pcl *reply_pcl, diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 2b25edc6c73c..40797114d50a 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -601,47 +601,65 @@ static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) return xdr->len; } -/** - * svc_rdma_send_write_chunk - Write all segments in a Write chunk - * @rdma: controlling RDMA transport - * @chunk: Write chunk provided by the client - * @xdr: xdr_buf containing the data payload - * - * Returns a non-negative number of bytes the chunk consumed, or - * %-E2BIG if the payload was larger than the Write chunk, - * %-EINVAL if client provided too many segments, - * %-ENOMEM if rdma_rw context pool was exhausted, - * %-ENOTCONN if posting failed (connection is lost), - * %-EIO if rdma_rw initialization failed (DMA mapping, etc). - */ -int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, - const struct svc_rdma_chunk *chunk, - const struct xdr_buf *xdr) +static int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, + const struct svc_rdma_chunk *chunk, + const struct xdr_buf *xdr) { struct svc_rdma_write_info *info; struct svc_rdma_chunk_ctxt *cc; + struct xdr_buf payload; int ret; + if (xdr_buf_subsegment(xdr, &payload, chunk->ch_position, + chunk->ch_payload_length)) + return -EMSGSIZE; + info = svc_rdma_write_info_alloc(rdma, chunk); if (!info) return -ENOMEM; cc = &info->wi_cc; - ret = svc_rdma_xb_write(xdr, info); - if (ret != xdr->len) + ret = svc_rdma_xb_write(&payload, info); + if (ret != payload.len) goto out_err; trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); ret = svc_rdma_post_chunk_ctxt(rdma, cc); if (ret < 0) goto out_err; - return xdr->len; + return 0; out_err: svc_rdma_write_info_free(info); return ret; } +/** + * svc_rdma_send_write_list - Send all chunks on the Write list + * @rdma: controlling RDMA transport + * @rctxt: Write list provisioned by the client + * @xdr: xdr_buf containing an RPC Reply message + * + * Returns zero on success, or a negative errno if one or more + * Write chunks could not be sent. + */ +int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_recv_ctxt *rctxt, + const struct xdr_buf *xdr) +{ + struct svc_rdma_chunk *chunk; + int ret; + + pcl_for_each_chunk(chunk, &rctxt->rc_write_pcl) { + if (!chunk->ch_payload_length) + break; + ret = svc_rdma_send_write_chunk(rdma, chunk, xdr); + if (ret < 0) + return ret; + } + return 0; +} + /** * svc_rdma_prepare_reply_chunk - Construct WR chain for writing the Reply chunk * @rdma: controlling RDMA transport diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 6dfd2232ce5b..bb5436b719e0 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -1013,6 +1013,10 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) if (!p) goto put_ctxt; + ret = svc_rdma_send_write_list(rdma, rctxt, &rqstp->rq_res); + if (ret < 0) + goto put_ctxt; + rc_size = 0; if (!pcl_is_empty(&rctxt->rc_reply_pcl)) { ret = svc_rdma_prepare_reply_chunk(rdma, &rctxt->rc_write_pcl, @@ -1064,45 +1068,33 @@ drop_connection: /** * svc_rdma_result_payload - special processing for a result payload - * @rqstp: svc_rqst to operate on - * @offset: payload's byte offset in @xdr + * @rqstp: RPC transaction context + * @offset: payload's byte offset in @rqstp->rq_res * @length: size of payload, in bytes * + * Assign the passed-in result payload to the current Write chunk, + * and advance to cur_result_payload to the next Write chunk, if + * there is one. + * * Return values: * %0 if successful or nothing needed to be done - * %-EMSGSIZE on XDR buffer overflow * %-E2BIG if the payload was larger than the Write chunk - * %-EINVAL if client provided too many segments - * %-ENOMEM if rdma_rw context pool was exhausted - * %-ENOTCONN if posting failed (connection is lost) - * %-EIO if rdma_rw initialization failed (DMA mapping, etc) */ int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset, unsigned int length) { struct svc_rdma_recv_ctxt *rctxt = rqstp->rq_xprt_ctxt; struct svc_rdma_chunk *chunk; - struct svcxprt_rdma *rdma; - struct xdr_buf subbuf; - int ret; chunk = rctxt->rc_cur_result_payload; if (!length || !chunk) return 0; rctxt->rc_cur_result_payload = pcl_next_chunk(&rctxt->rc_write_pcl, chunk); + if (length > chunk->ch_length) return -E2BIG; - chunk->ch_position = offset; chunk->ch_payload_length = length; - - if (xdr_buf_subsegment(&rqstp->rq_res, &subbuf, offset, length)) - return -EMSGSIZE; - - rdma = container_of(rqstp->rq_xprt, struct svcxprt_rdma, sc_xprt); - ret = svc_rdma_send_write_chunk(rdma, chunk, &subbuf); - if (ret < 0) - return ret; return 0; } -- cgit v1.2.3 From e084ee673c77cade06ab4c2e36b5624c82608b8c Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 4 Feb 2024 18:17:47 -0500 Subject: svcrdma: Add Write chunk WRs to the RPC's Send WR chain Chain RDMA Writes that convey Write chunks onto the local Send chain. This means all WRs for an RPC Reply are now posted with a single ib_post_send() call, and there is a single Send completion when all of these are done. That reduces both the per-transport doorbell rate and completion rate. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 13 ++++-- net/sunrpc/xprtrdma/svc_rdma_rw.c | 86 ++++++++++++++++++++++++++--------- net/sunrpc/xprtrdma/svc_rdma_sendto.c | 5 +- 3 files changed, 78 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index d33bab33099a..24cd199dd6f3 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -210,6 +210,7 @@ struct svc_rdma_recv_ctxt { */ struct svc_rdma_write_info { struct svcxprt_rdma *wi_rdma; + struct list_head wi_list; const struct svc_rdma_chunk *wi_chunk; @@ -238,7 +239,10 @@ struct svc_rdma_send_ctxt { struct ib_cqe sc_cqe; struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; + + struct list_head sc_write_info_list; struct svc_rdma_write_info sc_reply_info; + void *sc_xprt_buf; int sc_page_count; int sc_cur_sge_no; @@ -270,11 +274,14 @@ extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma, extern void svc_rdma_cc_release(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc, enum dma_data_direction dir); +extern void svc_rdma_write_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt); extern void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt); -extern int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - const struct xdr_buf *xdr); +extern int svc_rdma_prepare_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_pcl *write_pcl, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr); extern int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_pcl *write_pcl, const struct svc_rdma_pcl *reply_pcl, diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 40797114d50a..f2a100c4c81f 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -230,6 +230,28 @@ static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) queue_work(svcrdma_wq, &info->wi_work); } +/** + * svc_rdma_write_chunk_release - Release Write chunk I/O resources + * @rdma: controlling transport + * @ctxt: Send context that is being released + */ +void svc_rdma_write_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt) +{ + struct svc_rdma_write_info *info; + struct svc_rdma_chunk_ctxt *cc; + + while (!list_empty(&ctxt->sc_write_info_list)) { + info = list_first_entry(&ctxt->sc_write_info_list, + struct svc_rdma_write_info, wi_list); + list_del(&info->wi_list); + + cc = &info->wi_cc; + svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); + svc_rdma_write_info_free(info); + } +} + /** * svc_rdma_reply_chunk_release - Release Reply chunk I/O resources * @rdma: controlling transport @@ -286,13 +308,11 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) struct ib_cqe *cqe = wc->wr_cqe; struct svc_rdma_chunk_ctxt *cc = container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); - struct svc_rdma_write_info *info = - container_of(cc, struct svc_rdma_write_info, wi_cc); switch (wc->status) { case IB_WC_SUCCESS: trace_svcrdma_wc_write(&cc->cc_cid); - break; + return; case IB_WC_WR_FLUSH_ERR: trace_svcrdma_wc_write_flush(wc, &cc->cc_cid); break; @@ -300,12 +320,11 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) trace_svcrdma_wc_write_err(wc, &cc->cc_cid); } - svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); - - if (unlikely(wc->status != IB_WC_SUCCESS)) - svc_xprt_deferred_close(&rdma->sc_xprt); - - svc_rdma_write_info_free(info); + /* The RDMA Write has flushed, so the client won't get + * some of the outgoing RPC message. Signal the loss + * to the client by closing the connection. + */ + svc_xprt_deferred_close(&rdma->sc_xprt); } /** @@ -601,13 +620,19 @@ static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) return xdr->len; } -static int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, - const struct svc_rdma_chunk *chunk, - const struct xdr_buf *xdr) +/* Link Write WRs for @chunk onto @sctxt's WR chain. + */ +static int svc_rdma_prepare_write_chunk(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *sctxt, + const struct svc_rdma_chunk *chunk, + const struct xdr_buf *xdr) { struct svc_rdma_write_info *info; struct svc_rdma_chunk_ctxt *cc; + struct ib_send_wr *first_wr; struct xdr_buf payload; + struct list_head *pos; + struct ib_cqe *cqe; int ret; if (xdr_buf_subsegment(xdr, &payload, chunk->ch_position, @@ -623,10 +648,25 @@ static int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, if (ret != payload.len) goto out_err; - trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); - ret = svc_rdma_post_chunk_ctxt(rdma, cc); - if (ret < 0) + ret = -EINVAL; + if (unlikely(cc->cc_sqecount > rdma->sc_sq_depth)) goto out_err; + + first_wr = sctxt->sc_wr_chain; + cqe = &cc->cc_cqe; + list_for_each(pos, &cc->cc_rwctxts) { + struct svc_rdma_rw_ctxt *rwc; + + rwc = list_entry(pos, struct svc_rdma_rw_ctxt, rw_list); + first_wr = rdma_rw_ctx_wrs(&rwc->rw_ctx, rdma->sc_qp, + rdma->sc_port_num, cqe, first_wr); + cqe = NULL; + } + sctxt->sc_wr_chain = first_wr; + sctxt->sc_sqecount += cc->cc_sqecount; + list_add(&info->wi_list, &sctxt->sc_write_info_list); + + trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); return 0; out_err: @@ -635,25 +675,27 @@ out_err: } /** - * svc_rdma_send_write_list - Send all chunks on the Write list + * svc_rdma_prepare_write_list - Construct WR chain for sending Write list * @rdma: controlling RDMA transport - * @rctxt: Write list provisioned by the client + * @write_pcl: Write list provisioned by the client + * @sctxt: Send WR resources * @xdr: xdr_buf containing an RPC Reply message * * Returns zero on success, or a negative errno if one or more * Write chunks could not be sent. */ -int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - const struct xdr_buf *xdr) +int svc_rdma_prepare_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_pcl *write_pcl, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr) { struct svc_rdma_chunk *chunk; int ret; - pcl_for_each_chunk(chunk, &rctxt->rc_write_pcl) { + pcl_for_each_chunk(chunk, write_pcl) { if (!chunk->ch_payload_length) break; - ret = svc_rdma_send_write_chunk(rdma, chunk, xdr); + ret = svc_rdma_prepare_write_chunk(rdma, sctxt, chunk, xdr); if (ret < 0) return ret; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index bb5436b719e0..dfca39abd16c 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -142,6 +142,7 @@ svc_rdma_send_ctxt_alloc(struct svcxprt_rdma *rdma) ctxt->sc_send_wr.sg_list = ctxt->sc_sges; ctxt->sc_send_wr.send_flags = IB_SEND_SIGNALED; ctxt->sc_cqe.done = svc_rdma_wc_send; + INIT_LIST_HEAD(&ctxt->sc_write_info_list); ctxt->sc_xprt_buf = buffer; xdr_buf_init(&ctxt->sc_hdrbuf, ctxt->sc_xprt_buf, rdma->sc_max_req_size); @@ -227,6 +228,7 @@ static void svc_rdma_send_ctxt_release(struct svcxprt_rdma *rdma, struct ib_device *device = rdma->sc_cm_id->device; unsigned int i; + svc_rdma_write_chunk_release(rdma, ctxt); svc_rdma_reply_chunk_release(rdma, ctxt); if (ctxt->sc_page_count) @@ -1013,7 +1015,8 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) if (!p) goto put_ctxt; - ret = svc_rdma_send_write_list(rdma, rctxt, &rqstp->rq_res); + ret = svc_rdma_prepare_write_list(rdma, &rctxt->rc_write_pcl, sctxt, + &rqstp->rq_res); if (ret < 0) goto put_ctxt; -- cgit v1.2.3 From 71c2cc5cbf686c2397f43cbcb51a31589bdcee7b Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Fri, 1 Mar 2024 14:46:15 -0300 Subject: power: supply: core: make power_supply_class constant Since commit 43a7206b0963 ("driver core: class: make class_register() take a const *"), the driver core allows for struct class to be in read-only memory, so move the power_supply_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20240301-class_cleanup-power-v1-1-97e0b7bf9c94@marliere.net Signed-off-by: Sebastian Reichel --- drivers/power/supply/ab8500_btemp.c | 2 +- drivers/power/supply/ab8500_chargalg.c | 2 +- drivers/power/supply/ab8500_charger.c | 2 +- drivers/power/supply/ab8500_fg.c | 2 +- drivers/power/supply/apm_power.c | 2 +- drivers/power/supply/power_supply_core.c | 39 +++++++++++++++++--------------- include/linux/power_supply.h | 2 +- 7 files changed, 27 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/ab8500_btemp.c b/drivers/power/supply/ab8500_btemp.c index 7905eba93dea..41dba40fffdf 100644 --- a/drivers/power/supply/ab8500_btemp.c +++ b/drivers/power/supply/ab8500_btemp.c @@ -617,7 +617,7 @@ static int ab8500_btemp_get_ext_psy_data(struct device *dev, void *data) */ static void ab8500_btemp_external_power_changed(struct power_supply *psy) { - class_for_each_device(power_supply_class, NULL, psy, + class_for_each_device(&power_supply_class, NULL, psy, ab8500_btemp_get_ext_psy_data); } diff --git a/drivers/power/supply/ab8500_chargalg.c b/drivers/power/supply/ab8500_chargalg.c index de912658facb..329ae784a72d 100644 --- a/drivers/power/supply/ab8500_chargalg.c +++ b/drivers/power/supply/ab8500_chargalg.c @@ -1231,7 +1231,7 @@ static void ab8500_chargalg_algorithm(struct ab8500_chargalg *di) int ret; /* Collect data from all power_supply class devices */ - class_for_each_device(power_supply_class, NULL, + class_for_each_device(&power_supply_class, NULL, di->chargalg_psy, ab8500_chargalg_get_ext_psy_data); ab8500_chargalg_end_of_charge(di); diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c index d72f32c663bc..1c2b69bbed17 100644 --- a/drivers/power/supply/ab8500_charger.c +++ b/drivers/power/supply/ab8500_charger.c @@ -1949,7 +1949,7 @@ static void ab8500_charger_check_vbat_work(struct work_struct *work) struct ab8500_charger *di = container_of(work, struct ab8500_charger, check_vbat_work.work); - class_for_each_device(power_supply_class, NULL, + class_for_each_device(&power_supply_class, NULL, &di->usb_chg, ab8500_charger_get_ext_psy_data); /* First run old_vbat is 0. */ diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index 8c593fbdd45a..e49e704023e1 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -2407,7 +2407,7 @@ out: */ static void ab8500_fg_external_power_changed(struct power_supply *psy) { - class_for_each_device(power_supply_class, NULL, psy, + class_for_each_device(&power_supply_class, NULL, psy, ab8500_fg_get_ext_psy_data); } diff --git a/drivers/power/supply/apm_power.c b/drivers/power/supply/apm_power.c index 9d1a7fbcaed4..034f28699977 100644 --- a/drivers/power/supply/apm_power.c +++ b/drivers/power/supply/apm_power.c @@ -79,7 +79,7 @@ static void find_main_battery(void) main_battery = NULL; bp.main = main_battery; - error = class_for_each_device(power_supply_class, NULL, &bp, + error = class_for_each_device(&power_supply_class, NULL, &bp, __find_main_battery); if (error) { main_battery = bp.main; diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index 37dac7669090..4f27f17f8741 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -26,7 +26,10 @@ #include "samsung-sdi-battery.h" /* exported for the APM Power driver, APM emulation */ -struct class *power_supply_class; +const struct class power_supply_class = { + .name = "power_supply", + .dev_uevent = power_supply_uevent, +}; EXPORT_SYMBOL_GPL(power_supply_class); static BLOCKING_NOTIFIER_HEAD(power_supply_notifier); @@ -97,7 +100,7 @@ static void power_supply_changed_work(struct work_struct *work) if (likely(psy->changed)) { psy->changed = false; spin_unlock_irqrestore(&psy->changed_lock, flags); - class_for_each_device(power_supply_class, NULL, psy, + class_for_each_device(&power_supply_class, NULL, psy, __power_supply_changed_work); power_supply_update_leds(psy); blocking_notifier_call_chain(&power_supply_notifier, @@ -191,7 +194,7 @@ static int power_supply_populate_supplied_from(struct power_supply *psy) { int error; - error = class_for_each_device(power_supply_class, NULL, psy, + error = class_for_each_device(&power_supply_class, NULL, psy, __power_supply_populate_supplied_from); dev_dbg(&psy->dev, "%s %d\n", __func__, error); @@ -226,8 +229,8 @@ static int power_supply_find_supply_from_node(struct device_node *supply_node) * We return 0 if class_for_each_device() returned 1, -EPROBE_DEFER if * it returned 0, or error as returned by it. */ - error = class_for_each_device(power_supply_class, NULL, supply_node, - __power_supply_find_supply_from_node); + error = class_for_each_device(&power_supply_class, NULL, supply_node, + __power_supply_find_supply_from_node); return error ? (error == 1 ? 0 : error) : -EPROBE_DEFER; } @@ -333,7 +336,7 @@ int power_supply_am_i_supplied(struct power_supply *psy) struct psy_am_i_supplied_data data = { psy, 0 }; int error; - error = class_for_each_device(power_supply_class, NULL, &data, + error = class_for_each_device(&power_supply_class, NULL, &data, __power_supply_am_i_supplied); dev_dbg(&psy->dev, "%s count %u err %d\n", __func__, data.count, error); @@ -369,7 +372,7 @@ int power_supply_is_system_supplied(void) int error; unsigned int count = 0; - error = class_for_each_device(power_supply_class, NULL, &count, + error = class_for_each_device(&power_supply_class, NULL, &count, __power_supply_is_system_supplied); /* @@ -416,7 +419,7 @@ int power_supply_get_property_from_supplier(struct power_supply *psy, * This function is not intended for use with a supply with multiple * suppliers, we simply pick the first supply to report the psp. */ - ret = class_for_each_device(power_supply_class, NULL, &data, + ret = class_for_each_device(&power_supply_class, NULL, &data, __power_supply_get_supplier_property); if (ret < 0) return ret; @@ -462,8 +465,8 @@ static int power_supply_match_device_by_name(struct device *dev, const void *dat struct power_supply *power_supply_get_by_name(const char *name) { struct power_supply *psy = NULL; - struct device *dev = class_find_device(power_supply_class, NULL, name, - power_supply_match_device_by_name); + struct device *dev = class_find_device(&power_supply_class, NULL, name, + power_supply_match_device_by_name); if (dev) { psy = dev_get_drvdata(dev); @@ -519,8 +522,8 @@ struct power_supply *power_supply_get_by_phandle(struct device_node *np, if (!power_supply_np) return ERR_PTR(-ENODEV); - dev = class_find_device(power_supply_class, NULL, power_supply_np, - power_supply_match_device_node); + dev = class_find_device(&power_supply_class, NULL, power_supply_np, + power_supply_match_device_node); of_node_put(power_supply_np); @@ -1373,7 +1376,7 @@ __power_supply_register(struct device *parent, device_initialize(dev); - dev->class = power_supply_class; + dev->class = &power_supply_class; dev->type = &power_supply_dev_type; dev->parent = parent; dev->release = power_supply_dev_release; @@ -1621,12 +1624,12 @@ EXPORT_SYMBOL_GPL(power_supply_get_drvdata); static int __init power_supply_class_init(void) { - power_supply_class = class_create("power_supply"); + int err; - if (IS_ERR(power_supply_class)) - return PTR_ERR(power_supply_class); + err = class_register(&power_supply_class); + if (err) + return err; - power_supply_class->dev_uevent = power_supply_uevent; power_supply_init_attrs(); return 0; @@ -1634,7 +1637,7 @@ static int __init power_supply_class_init(void) static void __exit power_supply_class_exit(void) { - class_destroy(power_supply_class); + class_unregister(&power_supply_class); } subsys_initcall(power_supply_class_init); diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index c0992a77feea..514f652de64d 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -895,7 +895,7 @@ extern int power_supply_powers(struct power_supply *psy, struct device *dev); extern void *power_supply_get_drvdata(struct power_supply *psy); /* For APM emulation, think legacy userspace. */ -extern struct class *power_supply_class; +extern const struct class power_supply_class; static inline bool power_supply_is_amp_property(enum power_supply_property psp) { -- cgit v1.2.3 From bd1ebf2467f9c5d157bec7b025e83f8ffdae1318 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 29 Feb 2024 22:22:26 -0800 Subject: overflow: Allow non-type arg to type_max() and type_min() A common use of type_max() is to find the max for the type of a variable. Using the pattern type_max(typeof(var)) is needlessly verbose. Instead, since typeof(type) == type we can just explicitly call typeof() on the argument to type_max() and type_min(). Add wrappers for readability. We can do some replacements right away: $ git grep '\btype_\(min\|max\)(typeof' | wc -l 11 Link: https://lore.kernel.org/r/20240301062221.work.840-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/overflow.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/overflow.h b/include/linux/overflow.h index bc390f026128..aa691f2119b0 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -31,8 +31,10 @@ * credit to Christian Biere. */ #define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type))) -#define type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T))) -#define type_min(T) ((T)((T)-type_max(T)-(T)1)) +#define __type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T))) +#define type_max(t) __type_max(typeof(t)) +#define __type_min(T) ((T)((T)-type_max(T)-(T)1)) +#define type_min(t) __type_min(typeof(t)) /* * Avoids triggering -Wtype-limits compilation warning, @@ -207,10 +209,10 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define __overflows_type_constexpr(x, T) ( \ is_unsigned_type(typeof(x)) ? \ - (x) > type_max(typeof(T)) : \ + (x) > type_max(T) : \ is_unsigned_type(typeof(T)) ? \ - (x) < 0 || (x) > type_max(typeof(T)) : \ - (x) < type_min(typeof(T)) || (x) > type_max(typeof(T))) + (x) < 0 || (x) > type_max(T) : \ + (x) < type_min(T) || (x) > type_max(T)) #define __overflows_type(x, T) ({ \ typeof(T) v = 0; \ -- cgit v1.2.3 From c3b9a398fb0dae67f91e7ae4bb492e04ac2c80c0 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 29 Feb 2024 20:44:37 -0800 Subject: compiler.h: Explain how __is_constexpr() works The __is_constexpr() macro is dark magic. Shed some light on it with a comment to explain how and why it works. Acked-by: Gustavo A. R. Silva Reviewed-by: Jani Nikula Link: https://lore.kernel.org/r/20240301044428.work.411-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/compiler.h | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'include/linux') diff --git a/include/linux/compiler.h b/include/linux/compiler.h index bb1339c7057b..b688ad992127 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -231,6 +231,45 @@ static inline void *offset_to_ptr(const int *off) * This returns a constant expression while determining if an argument is * a constant expression, most importantly without evaluating the argument. * Glory to Martin Uecker + * + * Details: + * - sizeof() return an integer constant expression, and does not evaluate + * the value of its operand; it only examines the type of its operand. + * - The results of comparing two integer constant expressions is also + * an integer constant expression. + * - The first literal "8" isn't important. It could be any literal value. + * - The second literal "8" is to avoid warnings about unaligned pointers; + * this could otherwise just be "1". + * - (long)(x) is used to avoid warnings about 64-bit types on 32-bit + * architectures. + * - The C Standard defines "null pointer constant", "(void *)0", as + * distinct from other void pointers. + * - If (x) is an integer constant expression, then the "* 0l" resolves + * it into an integer constant expression of value 0. Since it is cast to + * "void *", this makes the second operand a null pointer constant. + * - If (x) is not an integer constant expression, then the second operand + * resolves to a void pointer (but not a null pointer constant: the value + * is not an integer constant 0). + * - The conditional operator's third operand, "(int *)8", is an object + * pointer (to type "int"). + * - The behavior (including the return type) of the conditional operator + * ("operand1 ? operand2 : operand3") depends on the kind of expressions + * given for the second and third operands. This is the central mechanism + * of the macro: + * - When one operand is a null pointer constant (i.e. when x is an integer + * constant expression) and the other is an object pointer (i.e. our + * third operand), the conditional operator returns the type of the + * object pointer operand (i.e. "int *). Here, within the sizeof(), we + * would then get: + * sizeof(*((int *)(...)) == sizeof(int) == 4 + * - When one operand is a void pointer (i.e. when x is not an integer + * constant expression) and the other is an object pointer (i.e. our + * third operand), the conditional operator returns a "void *" type. + * Here, within the sizeof(), we would then get: + * sizeof(*((void *)(...)) == sizeof(void) == 1 + * - The equality comparison to "sizeof(int)" therefore depends on (x): + * sizeof(int) == sizeof(int) (x) was a constant expression + * sizeof(int) != sizeof(void) (x) was not a constant expression */ #define __is_constexpr(x) \ (sizeof(int) == sizeof(*(8 ? ((void *)((long)(x) * 0l)) : (int *)8))) -- cgit v1.2.3 From a74c0c9c3f7fa6fba34196d142bab93509f17dba Mon Sep 17 00:00:00 2001 From: Oliver Neukum Date: Thu, 29 Feb 2024 14:23:57 +0100 Subject: USB: typec: no opencoding FIELD_GET We have a macro. It should be used. Signed-off-by: Oliver Neukum Reviewed-by: Mika Westerberg Link: https://lore.kernel.org/r/20240229132401.3270-1-oneukum@suse.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/typec_tbt.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/typec_tbt.h b/include/linux/usb/typec_tbt.h index c7a2153bd6f5..fa97d7e00f5c 100644 --- a/include/linux/usb/typec_tbt.h +++ b/include/linux/usb/typec_tbt.h @@ -3,6 +3,7 @@ #define __USB_TYPEC_TBT_H #include +#include #define USB_TYPEC_VENDOR_INTEL 0x8087 /* Alias for convenience */ @@ -25,7 +26,7 @@ struct typec_thunderbolt_data { /* TBT3 Device Discover Mode VDO bits */ #define TBT_MODE BIT(0) -#define TBT_ADAPTER(_vdo_) (((_vdo_) & BIT(16)) >> 16) +#define TBT_ADAPTER(_vdo_) FIELD_GET(BIT(16), _vdo_) #define TBT_ADAPTER_LEGACY 0 #define TBT_ADAPTER_TBT3 1 #define TBT_INTEL_SPECIFIC_B0 BIT(26) @@ -35,12 +36,12 @@ struct typec_thunderbolt_data { #define TBT_SET_ADAPTER(a) (((a) & 1) << 16) /* TBT3 Cable Discover Mode VDO bits */ -#define TBT_CABLE_SPEED(_vdo_) (((_vdo_) & GENMASK(18, 16)) >> 16) +#define TBT_CABLE_SPEED(_vdo_) FIELD_GET(GENMASK(18, 16), _vdo_) #define TBT_CABLE_USB3_GEN1 1 #define TBT_CABLE_USB3_PASSIVE 2 #define TBT_CABLE_10_AND_20GBPS 3 -#define TBT_CABLE_ROUNDED_SUPPORT(_vdo_) \ - (((_vdo_) & GENMASK(20, 19)) >> 19) +#define TBT_CABLE_ROUNDED_SUPPORT(_vdo_) FIELD_GET(GENMASK(20, 19), _vdo_) + #define TBT_GEN3_NON_ROUNDED 0 #define TBT_GEN3_GEN4_ROUNDED_NON_ROUNDED 1 #define TBT_CABLE_OPTICAL BIT(21) -- cgit v1.2.3 From caddc92c57451d983c7e31e60b961c5aae4ece63 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Sat, 2 Mar 2024 19:33:29 +0200 Subject: gpio: nomadik: Finish conversion to use firmware node APIs Previously driver got a few updates in order to replace OF APIs by respective firmware node, however it was not finished to the logical end, e.g., some APIs that has been used are still require OF node to be passed. Finish that job by converting leftovers to use firmware node APIs. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240302173401.217830-1-andy.shevchenko@gmail.com Signed-off-by: Linus Walleij --- drivers/gpio/Kconfig | 1 - drivers/gpio/gpio-nomadik.c | 13 ++++++------- drivers/pinctrl/nomadik/pinctrl-nomadik.c | 25 ++++++++++++------------- include/linux/gpio/gpio-nomadik.h | 4 +++- 4 files changed, 21 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index f633be517654..ef20ab921010 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -481,7 +481,6 @@ config GPIO_MXS config GPIO_NOMADIK bool "Nomadik GPIO driver" depends on ARCH_U8500 || ARCH_NOMADIK || MACH_EYEQ5 || COMPILE_TEST - depends on OF_GPIO select GPIOLIB_IRQCHIP help Say yes here to support the Nomadik SoC GPIO block. This block is also diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c index c9fd6631e0aa..483086deb397 100644 --- a/drivers/gpio/gpio-nomadik.c +++ b/drivers/gpio/gpio-nomadik.c @@ -23,10 +23,10 @@ #include #include #include -#include -#include +#include #include #include +#include #include #include #include @@ -504,7 +504,7 @@ static inline void nmk_gpio_dbg_show_one(struct seq_file *s, * it is the pin controller or GPIO driver. However we need to use the right * platform device when looking up resources so pay attention to pdev. */ -struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, +struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode, struct platform_device *pdev) { struct nmk_gpio_chip *nmk_chip; @@ -517,9 +517,9 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, u32 id, ngpio; int ret; - gpio_dev = bus_find_device_by_of_node(&platform_bus_type, np); + gpio_dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode); if (!gpio_dev) { - pr_err("populate \"%pOFn\": device not found\n", np); + dev_err(&pdev->dev, "populate \"%pfwP\": device not found\n", fwnode); return ERR_PTR(-ENODEV); } gpio_pdev = to_platform_device(gpio_dev); @@ -624,7 +624,6 @@ static const struct irq_chip nmk_irq_chip = { static int nmk_gpio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; struct nmk_gpio_chip *nmk_chip; struct gpio_irq_chip *girq; bool supports_sleepmode; @@ -632,7 +631,7 @@ static int nmk_gpio_probe(struct platform_device *pdev) int irq; int ret; - nmk_chip = nmk_gpio_populate_chip(np, pdev); + nmk_chip = nmk_gpio_populate_chip(dev_fwnode(dev), pdev); if (IS_ERR(nmk_chip)) { dev_err(dev, "could not populate nmk chip struct\n"); return PTR_ERR(nmk_chip); diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c index 7849144b3b80..47d5484f6bdf 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c @@ -1190,8 +1190,8 @@ static int nmk_pinctrl_resume(struct device *dev) static int nmk_pinctrl_probe(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; - struct device_node *prcm_np; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); + struct fwnode_handle *prcm_fwnode; struct nmk_pinctrl *npct; uintptr_t version = 0; int i; @@ -1216,28 +1216,27 @@ static int nmk_pinctrl_probe(struct platform_device *pdev) * or after this point: it shouldn't matter as the APIs are orthogonal. */ for (i = 0; i < NMK_MAX_BANKS; i++) { - struct device_node *gpio_np; + struct fwnode_handle *gpio_fwnode; struct nmk_gpio_chip *nmk_chip; - gpio_np = of_parse_phandle(np, "nomadik-gpio-chips", i); - if (!gpio_np) + gpio_fwnode = fwnode_find_reference(fwnode, "nomadik-gpio-chips", i); + if (IS_ERR(gpio_fwnode)) continue; - dev_info(&pdev->dev, "populate NMK GPIO %d \"%pOFn\"\n", - i, gpio_np); - nmk_chip = nmk_gpio_populate_chip(gpio_np, pdev); + dev_info(&pdev->dev, "populate NMK GPIO %d \"%pfwP\"\n", i, gpio_fwnode); + nmk_chip = nmk_gpio_populate_chip(gpio_fwnode, pdev); if (IS_ERR(nmk_chip)) dev_err(&pdev->dev, "could not populate nmk chip struct - continue anyway\n"); - of_node_put(gpio_np); + fwnode_handle_put(gpio_fwnode); /* We are NOT compatible with mobileye,eyeq5-gpio. */ BUG_ON(nmk_chip->is_mobileye_soc); } - prcm_np = of_parse_phandle(np, "prcm", 0); - if (prcm_np) { - npct->prcm_base = of_iomap(prcm_np, 0); - of_node_put(prcm_np); + prcm_fwnode = fwnode_find_reference(fwnode, "prcm", 0); + if (!IS_ERR(prcm_fwnode)) { + npct->prcm_base = fwnode_iomap(prcm_fwnode, 0); + fwnode_handle_put(prcm_fwnode); } if (!npct->prcm_base) { if (version == PINCTRL_NMK_STN8815) { diff --git a/include/linux/gpio/gpio-nomadik.h b/include/linux/gpio/gpio-nomadik.h index 9bdb09fda4c9..4a95ea7935fb 100644 --- a/include/linux/gpio/gpio-nomadik.h +++ b/include/linux/gpio/gpio-nomadik.h @@ -2,6 +2,8 @@ #ifndef __LINUX_GPIO_NOMADIK_H #define __LINUX_GPIO_NOMADIK_H +struct fwnode_handle; + /* Package definitions */ #define PINCTRL_NMK_STN8815 0 #define PINCTRL_NMK_DB8500 1 @@ -263,7 +265,7 @@ void __nmk_gpio_make_output(struct nmk_gpio_chip *nmk_chip, unsigned int offset, int val); void __nmk_gpio_set_slpm(struct nmk_gpio_chip *nmk_chip, unsigned int offset, enum nmk_gpio_slpm mode); -struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, +struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode, struct platform_device *pdev); /* Symbols declared in pinctrl-nomadik used by gpio-nomadik. */ -- cgit v1.2.3 From c029b22f8a98e14988f800d5c0176a9eaec3c8db Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Thu, 14 Dec 2023 21:31:48 +1100 Subject: of: Add of_machine_compatible_match() We have of_machine_is_compatible() to check if a machine is compatible with a single compatible string. However some code is able to support multiple compatible boards, and so wants to check for one of many compatible strings. So add of_machine_compatible_match() which takes a NULL terminated array of compatible strings to check against the root node's compatible property. Compared to an open coded match this is slightly more self documenting, and also avoids the caller needing to juggle the root node either directly or via of_find_node_by_path(). Signed-off-by: Christophe Leroy Reviewed-by: Rob Herring Signed-off-by: Michael Ellerman Link: https://msgid.link/20231214103152.12269-1-mpe@ellerman.id.au --- drivers/of/base.c | 21 +++++++++++++++++++++ include/linux/of.h | 6 ++++++ 2 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/drivers/of/base.c b/drivers/of/base.c index b0ad8fc06e80..3c85afb2697c 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -394,6 +394,27 @@ int of_device_compatible_match(const struct device_node *device, } EXPORT_SYMBOL_GPL(of_device_compatible_match); +/** + * of_machine_compatible_match - Test root of device tree against a compatible array + * @compats: NULL terminated array of compatible strings to look for in root node's compatible property. + * + * Returns true if the root node has any of the given compatible values in its + * compatible property. + */ +bool of_machine_compatible_match(const char *const *compats) +{ + struct device_node *root; + int rc = 0; + + root = of_find_node_by_path("/"); + if (root) { + rc = of_device_compatible_match(root, compats); + of_node_put(root); + } + + return rc != 0; +} + /** * of_machine_is_compatible - Test root of device tree for a given compatible value * @compat: compatible string to look for in root node's compatible property. diff --git a/include/linux/of.h b/include/linux/of.h index 6a9ddf20e79a..e3418babc203 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -403,6 +403,7 @@ extern int of_alias_get_id(struct device_node *np, const char *stem); extern int of_alias_get_highest_id(const char *stem); extern int of_machine_is_compatible(const char *compat); +bool of_machine_compatible_match(const char *const *compats); extern int of_add_property(struct device_node *np, struct property *prop); extern int of_remove_property(struct device_node *np, struct property *prop); @@ -808,6 +809,11 @@ static inline int of_remove_property(struct device_node *np, struct property *pr return 0; } +static inline bool of_machine_compatible_match(const char *const *compats) +{ + return false; +} + static inline bool of_console_check(const struct device_node *dn, const char *name, int index) { return false; -- cgit v1.2.3 From cefdb366dcbe97908b6055595a15bf7689556bf8 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Thu, 14 Dec 2023 21:31:49 +1100 Subject: of: Change of_machine_is_compatible() to return bool of_machine_is_compatible() currently returns a positive integer if it finds a match. However none of the callers ever check the value, they all treat it as a true/false. So change of_machine_is_compatible() to return bool, which will allow the implementation to be changed in a subsequent patch. Suggested-by: Rob Herring Signed-off-by: Michael Ellerman Link: https://msgid.link/20231214103152.12269-2-mpe@ellerman.id.au --- drivers/of/base.c | 5 ++--- include/linux/of.h | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/of/base.c b/drivers/of/base.c index 3c85afb2697c..faa88eb518a9 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -419,10 +419,9 @@ bool of_machine_compatible_match(const char *const *compats) * of_machine_is_compatible - Test root of device tree for a given compatible value * @compat: compatible string to look for in root node's compatible property. * - * Return: A positive integer if the root node has the given value in its - * compatible property. + * Return: true if the root node has the given value in its compatible property. */ -int of_machine_is_compatible(const char *compat) +bool of_machine_is_compatible(const char *compat) { struct device_node *root; int rc = 0; diff --git a/include/linux/of.h b/include/linux/of.h index e3418babc203..c5c663a7fb77 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -402,7 +402,7 @@ extern void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)); extern int of_alias_get_id(struct device_node *np, const char *stem); extern int of_alias_get_highest_id(const char *stem); -extern int of_machine_is_compatible(const char *compat); +extern bool of_machine_is_compatible(const char *compat); bool of_machine_compatible_match(const char *const *compats); extern int of_add_property(struct device_node *np, struct property *prop); -- cgit v1.2.3 From 1ac8205f907517a306b661212496fedce79d7cc5 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 14 Dec 2023 21:31:50 +1100 Subject: of: Reimplement of_machine_is_compatible() using of_machine_compatible_match() of_machine_compatible_match() works with a table of strings. of_machine_is_compatible() is a simplier version with only one string. Re-implement of_machine_is_compatible() by setting a table of strings with a single string then using of_machine_compatible_match(). Suggested-by: Rob Herring Signed-off-by: Christophe Leroy Reviewed-by: Rob Herring Signed-off-by: Michael Ellerman Link: https://msgid.link/20231214103152.12269-3-mpe@ellerman.id.au --- drivers/of/base.c | 21 +-------------------- include/linux/of.h | 14 +++++++++++++- 2 files changed, 14 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/of/base.c b/drivers/of/base.c index faa88eb518a9..df8d9733b054 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -414,26 +414,7 @@ bool of_machine_compatible_match(const char *const *compats) return rc != 0; } - -/** - * of_machine_is_compatible - Test root of device tree for a given compatible value - * @compat: compatible string to look for in root node's compatible property. - * - * Return: true if the root node has the given value in its compatible property. - */ -bool of_machine_is_compatible(const char *compat) -{ - struct device_node *root; - int rc = 0; - - root = of_find_node_by_path("/"); - if (root) { - rc = of_device_is_compatible(root, compat); - of_node_put(root); - } - return rc; -} -EXPORT_SYMBOL(of_machine_is_compatible); +EXPORT_SYMBOL(of_machine_compatible_match); /** * __of_device_is_available - check if a device is available for use diff --git a/include/linux/of.h b/include/linux/of.h index c5c663a7fb77..03ed4e37ca57 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -402,9 +402,21 @@ extern void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)); extern int of_alias_get_id(struct device_node *np, const char *stem); extern int of_alias_get_highest_id(const char *stem); -extern bool of_machine_is_compatible(const char *compat); bool of_machine_compatible_match(const char *const *compats); +/** + * of_machine_is_compatible - Test root of device tree for a given compatible value + * @compat: compatible string to look for in root node's compatible property. + * + * Return: true if the root node has the given value in its compatible property. + */ +static inline bool of_machine_is_compatible(const char *compat) +{ + const char *compats[] = { compat, NULL }; + + return of_machine_compatible_match(compats); +} + extern int of_add_property(struct device_node *np, struct property *prop); extern int of_remove_property(struct device_node *np, struct property *prop); extern int of_update_property(struct device_node *np, struct property *newprop); -- cgit v1.2.3 From 99fea943d9dc2500227bced9acd671e5b39a1471 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 19 Feb 2024 09:14:36 -0300 Subject: soundwire: constify the struct device_type usage Since commit aed65af1cc2f ("drivers: make device_type const"), the driver core can properly handle constant struct device_type. Move the sdw_master_type and sdw_slave_type variables to be constant structures as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Link: https://lore.kernel.org/r/20240219-device_cleanup-soundwire-v1-1-9edd51767611@marliere.net Signed-off-by: Vinod Koul --- drivers/soundwire/master.c | 2 +- drivers/soundwire/slave.c | 2 +- include/linux/soundwire/sdw_type.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/master.c b/drivers/soundwire/master.c index 51abedbbaa66..b2c64512739d 100644 --- a/drivers/soundwire/master.c +++ b/drivers/soundwire/master.c @@ -112,7 +112,7 @@ static const struct dev_pm_ops master_dev_pm = { pm_generic_runtime_resume, NULL) }; -struct device_type sdw_master_type = { +const struct device_type sdw_master_type = { .name = "soundwire_master", .release = sdw_master_device_release, .pm = &master_dev_pm, diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c index 060c2982e26b..9963b92eb505 100644 --- a/drivers/soundwire/slave.c +++ b/drivers/soundwire/slave.c @@ -16,7 +16,7 @@ static void sdw_slave_release(struct device *dev) kfree(slave); } -struct device_type sdw_slave_type = { +const struct device_type sdw_slave_type = { .name = "sdw_slave", .release = sdw_slave_release, .uevent = sdw_slave_uevent, diff --git a/include/linux/soundwire/sdw_type.h b/include/linux/soundwire/sdw_type.h index b445f7200f06..693320b4f5c2 100644 --- a/include/linux/soundwire/sdw_type.h +++ b/include/linux/soundwire/sdw_type.h @@ -5,8 +5,8 @@ #define __SOUNDWIRE_TYPES_H extern const struct bus_type sdw_bus_type; -extern struct device_type sdw_slave_type; -extern struct device_type sdw_master_type; +extern const struct device_type sdw_slave_type; +extern const struct device_type sdw_master_type; static inline int is_sdw_slave(const struct device *dev) { -- cgit v1.2.3 From fbd5f5008fab2203fa21e82579b9b48a7256b8fd Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 3 Mar 2024 15:05:42 -0800 Subject: Input: serio - make serio_bus const Now that the driver core can properly handle constant struct bus_type, move the serio_bus variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Link: https://lore.kernel.org/r/20240210-bus_cleanup-input2-v1-2-0daef7e034e0@marliere.net Signed-off-by: Dmitry Torokhov --- drivers/input/serio/serio.c | 2 +- include/linux/serio.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/input/serio/serio.c b/drivers/input/serio/serio.c index 767fc9efb4a8..a8838b522627 100644 --- a/drivers/input/serio/serio.c +++ b/drivers/input/serio/serio.c @@ -1007,7 +1007,7 @@ irqreturn_t serio_interrupt(struct serio *serio, } EXPORT_SYMBOL(serio_interrupt); -struct bus_type serio_bus = { +const struct bus_type serio_bus = { .name = "serio", .drv_groups = serio_driver_groups, .match = serio_bus_match, diff --git a/include/linux/serio.h b/include/linux/serio.h index 6c27d413da92..7ca41af93b37 100644 --- a/include/linux/serio.h +++ b/include/linux/serio.h @@ -15,7 +15,7 @@ #include #include -extern struct bus_type serio_bus; +extern const struct bus_type serio_bus; struct serio { void *port_data; -- cgit v1.2.3 From cc15bd10e716fcb472d611f24d76c795acb0f8c7 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 29 Feb 2024 09:39:08 +0000 Subject: net: adopt skb_network_header_len() more broadly (skb_transport_header(skb) - skb_network_header(skb)) can be replaced by skb_network_header_len(skb) Add a DEBUG_NET_WARN_ON_ONCE() in skb_network_header_len() to catch cases were the transport_header was not set. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c | 3 +-- drivers/net/ethernet/intel/i40e/i40e_main.c | 2 +- drivers/net/ethernet/intel/iavf/iavf_main.c | 2 +- drivers/net/wireless/intel/iwlwifi/mvm/tx.c | 3 +-- drivers/net/wireless/intel/iwlwifi/pcie/tx.c | 2 +- drivers/net/wireless/intel/iwlwifi/queue/tx.c | 2 +- include/linux/skbuff.h | 1 + 7 files changed, 7 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index adcee8d9d6d9..c9b6acd8c892 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -3621,8 +3621,7 @@ static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb, ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT)); - pbd->ip_hlen_w = (skb_transport_header(skb) - - skb_network_header(skb)) >> 1; + pbd->ip_hlen_w = skb_network_header_len(skb) >> 1; hlen += pbd->ip_hlen_w; diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index a67b13869016..3fada49b8ae2 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -13213,7 +13213,7 @@ static netdev_features_t i40e_features_check(struct sk_buff *skb, goto out_err; /* IPLEN and EIPLEN can support at most 127 dwords */ - len = skb_transport_header(skb) - skb_network_header(skb); + len = skb_network_header_len(skb); if (len & ~(127 * 4)) goto out_err; diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 245c458e38ae..aefec6bd3b67 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -4428,7 +4428,7 @@ static netdev_features_t iavf_features_check(struct sk_buff *skb, goto out_err; /* IPLEN and EIPLEN can support at most 127 dwords */ - len = skb_transport_header(skb) - skb_network_header(skb); + len = skb_network_header_len(skb); if (len & ~(127 * 4)) goto out_err; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c index e502f4ee9e1f..782ddc8c296b 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c @@ -1015,8 +1015,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb, netdev_features_t netdev_flags = NETIF_F_CSUM_MASK | NETIF_F_SG; u8 tid; - snap_ip_tcp = 8 + skb_transport_header(skb) - skb_network_header(skb) + - tcp_hdrlen(skb); + snap_ip_tcp = 8 + skb_network_header_len(skb) + tcp_hdrlen(skb); if (!mvmsta->max_amsdu_len || !ieee80211_is_data_qos(hdr->frame_control) || diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c index 6c2b37e56c78..fa8eba47dc4c 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c @@ -1331,7 +1331,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, trans->txqs.tfd.size, &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, 0); - ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); + ip_hdrlen = skb_network_header_len(skb); snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; amsdu_pad = 0; diff --git a/drivers/net/wireless/intel/iwlwifi/queue/tx.c b/drivers/net/wireless/intel/iwlwifi/queue/tx.c index d3bde2d010b7..33973a60d0bf 100644 --- a/drivers/net/wireless/intel/iwlwifi/queue/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/queue/tx.c @@ -353,7 +353,7 @@ static int iwl_txq_gen2_build_amsdu(struct iwl_trans *trans, trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), &dev_cmd->hdr, start_len, 0); - ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); + ip_hdrlen = skb_network_header_len(skb); snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len; amsdu_pad = 0; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 1470b74fb6d2..d577e0bee18d 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3036,6 +3036,7 @@ static inline int skb_transport_offset(const struct sk_buff *skb) static inline u32 skb_network_header_len(const struct sk_buff *skb) { + DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb)); return skb->transport_header - skb->network_header; } -- cgit v1.2.3 From eb52034436a58e742ceea0dcf2f003f83a3449a5 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 19 Feb 2024 09:38:47 -0300 Subject: i2c: constify the struct device_type usage Since commit aed65af1cc2f ("drivers: make device_type const"), the driver core can properly handle constant struct device_type. Move the i2c_adapter_type and i2c_client_type variables to be constant structures as well, placing it into read-only memory which can not be modified at runtime. Signed-off-by: Ricardo B. Marliere Signed-off-by: Wolfram Sang --- drivers/i2c/i2c-core-base.c | 4 ++-- include/linux/i2c.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 3bd48d4b6318..ff5c486a1dbb 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -701,7 +701,7 @@ const struct bus_type i2c_bus_type = { }; EXPORT_SYMBOL_GPL(i2c_bus_type); -struct device_type i2c_client_type = { +const struct device_type i2c_client_type = { .groups = i2c_dev_groups, .uevent = i2c_device_uevent, .release = i2c_client_dev_release, @@ -1343,7 +1343,7 @@ static struct attribute *i2c_adapter_attrs[] = { }; ATTRIBUTE_GROUPS(i2c_adapter); -struct device_type i2c_adapter_type = { +const struct device_type i2c_adapter_type = { .groups = i2c_adapter_groups, .release = i2c_adapter_dev_release, }; diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 652ecb7abeda..ff93ff8b257c 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -24,8 +24,8 @@ #include extern const struct bus_type i2c_bus_type; -extern struct device_type i2c_adapter_type; -extern struct device_type i2c_client_type; +extern const struct device_type i2c_adapter_type; +extern const struct device_type i2c_client_type; /* --- General options ------------------------------------------------ */ -- cgit v1.2.3 From 469f6acd9a538ea963e2d4d13ba721a7ad1c1813 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 13 Feb 2024 11:46:25 -0300 Subject: tee: make tee_bus_type const Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the tee_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Sumit Garg Reviewed-by: Greg Kroah-Hartman Signed-off-by: Jens Wiklander Signed-off-by: Arnd Bergmann --- drivers/tee/tee_core.c | 2 +- include/linux/tee_drv.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 792d6fae4354..e59c20d74b36 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -1226,7 +1226,7 @@ static int tee_client_device_uevent(const struct device *dev, return add_uevent_var(env, "MODALIAS=tee:%pUb", dev_id); } -struct bus_type tee_bus_type = { +const struct bus_type tee_bus_type = { .name = "tee", .match = tee_client_device_match, .uevent = tee_client_device_uevent, diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h index 911ddf92dcee..71632e3c5f18 100644 --- a/include/linux/tee_drv.h +++ b/include/linux/tee_drv.h @@ -482,7 +482,7 @@ static inline bool tee_param_is_memref(struct tee_param *param) } } -extern struct bus_type tee_bus_type; +extern const struct bus_type tee_bus_type; /** * struct tee_client_device - tee based device -- cgit v1.2.3 From ad8ee969d7e34dd310a618d798cc6d8fe4f04464 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Sun, 3 Mar 2024 11:48:53 +0100 Subject: of: make for_each_property_of_node() available to to !OF for_each_property_of_node() is a macro and so doesn't have a stub inline function for !OF. Move it out of the relevant #ifdef to make it available to all users. Fixes: 611cad720148 ("dt: add of_alias_scan and of_alias_get_id") Signed-off-by: Bartosz Golaszewski Link: https://lore.kernel.org/r/20240303104853.31511-1-brgl@bgdev.pl Signed-off-by: Rob Herring --- include/linux/of.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/of.h b/include/linux/of.h index 6a9ddf20e79a..a3e8e429ad7f 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -362,9 +362,6 @@ extern struct device_node *of_get_cpu_state_node(struct device_node *cpu_node, int index); extern u64 of_get_cpu_hwid(struct device_node *cpun, unsigned int thread); -#define for_each_property_of_node(dn, pp) \ - for (pp = dn->properties; pp != NULL; pp = pp->next) - extern int of_n_addr_cells(struct device_node *np); extern int of_n_size_cells(struct device_node *np); extern const struct of_device_id *of_match_node( @@ -892,6 +889,9 @@ static inline int of_prop_val_eq(struct property *p1, struct property *p2) !memcmp(p1->value, p2->value, (size_t)p1->length); } +#define for_each_property_of_node(dn, pp) \ + for (pp = dn->properties; pp != NULL; pp = pp->next) + #if defined(CONFIG_OF) && defined(CONFIG_NUMA) extern int of_node_to_nid(struct device_node *np); #else -- cgit v1.2.3 From 187e2af05abe6bf80581490239c449456627d17a Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Sat, 24 Feb 2024 14:34:17 -0800 Subject: bpf: struct_ops supports more than one page for trampolines. The BPF struct_ops previously only allowed one page of trampolines. Each function pointer of a struct_ops is implemented by a struct_ops bpf program. Each struct_ops bpf program requires a trampoline. The following selftest patch shows each page can hold a little more than 20 trampolines. While one page is more than enough for the tcp-cc usecase, the sched_ext use case shows that one page is not always enough and hits the one page limit. This patch overcomes the one page limit by allocating another page when needed and it is limited to a total of MAX_IMAGE_PAGES (8) pages which is more than enough for reasonable usages. The variable st_map->image has been changed to st_map->image_pages, and its type has been changed to an array of pointers to pages. Signed-off-by: Kui-Feng Lee Link: https://lore.kernel.org/r/20240224223418.526631-3-thinker.li@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 4 +- kernel/bpf/bpf_struct_ops.c | 130 ++++++++++++++++++++++++++++------------- net/bpf/bpf_dummy_struct_ops.c | 12 ++-- 3 files changed, 96 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 814dc913a968..785660810e6a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1763,7 +1763,9 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, void *stub_func, - void *image, void *image_end); + void **image, u32 *image_off, + bool allow_alloc); +void bpf_struct_ops_image_free(void *image); static inline bool bpf_try_module_get(const void *data, struct module *owner) { if (owner == BPF_MODULE_OWNER) diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 07e554c191d1..43356faaa057 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -18,6 +18,8 @@ struct bpf_struct_ops_value { char data[] ____cacheline_aligned_in_smp; }; +#define MAX_TRAMP_IMAGE_PAGES 8 + struct bpf_struct_ops_map { struct bpf_map map; struct rcu_head rcu; @@ -30,12 +32,11 @@ struct bpf_struct_ops_map { */ struct bpf_link **links; u32 links_cnt; - /* image is a page that has all the trampolines + u32 image_pages_cnt; + /* image_pages is an array of pages that has all the trampolines * that stores the func args before calling the bpf_prog. - * A PAGE_SIZE "image" is enough to store all trampoline for - * "links[]". */ - void *image; + void *image_pages[MAX_TRAMP_IMAGE_PAGES]; /* The owner moduler's btf. */ struct btf *btf; /* uvalue->data stores the kernel struct @@ -116,6 +117,31 @@ static bool is_valid_value_type(struct btf *btf, s32 value_id, return true; } +static void *bpf_struct_ops_image_alloc(void) +{ + void *image; + int err; + + err = bpf_jit_charge_modmem(PAGE_SIZE); + if (err) + return ERR_PTR(err); + image = arch_alloc_bpf_trampoline(PAGE_SIZE); + if (!image) { + bpf_jit_uncharge_modmem(PAGE_SIZE); + return ERR_PTR(-ENOMEM); + } + + return image; +} + +void bpf_struct_ops_image_free(void *image) +{ + if (image) { + arch_free_bpf_trampoline(image, PAGE_SIZE); + bpf_jit_uncharge_modmem(PAGE_SIZE); + } +} + #define MAYBE_NULL_SUFFIX "__nullable" #define MAX_STUB_NAME 128 @@ -461,6 +487,15 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) } } +static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map) +{ + int i; + + for (i = 0; i < st_map->image_pages_cnt; i++) + bpf_struct_ops_image_free(st_map->image_pages[i]); + st_map->image_pages_cnt = 0; +} + static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data) { const struct btf_member *member; @@ -506,9 +541,12 @@ const struct bpf_link_ops bpf_struct_ops_link_lops = { int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, - void *stub_func, void *image, void *image_end) + void *stub_func, + void **_image, u32 *_image_off, + bool allow_alloc) { - u32 flags = BPF_TRAMP_F_INDIRECT; + u32 image_off = *_image_off, flags = BPF_TRAMP_F_INDIRECT; + void *image = *_image; int size; tlinks[BPF_TRAMP_FENTRY].links[0] = link; @@ -518,12 +556,32 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, flags |= BPF_TRAMP_F_RET_FENTRY_RET; size = arch_bpf_trampoline_size(model, flags, tlinks, NULL); - if (size < 0) - return size; - if (size > (unsigned long)image_end - (unsigned long)image) - return -E2BIG; - return arch_prepare_bpf_trampoline(NULL, image, image_end, + if (size <= 0) + return size ? : -EFAULT; + + /* Allocate image buffer if necessary */ + if (!image || size > PAGE_SIZE - image_off) { + if (!allow_alloc) + return -E2BIG; + + image = bpf_struct_ops_image_alloc(); + if (IS_ERR(image)) + return PTR_ERR(image); + image_off = 0; + } + + size = arch_prepare_bpf_trampoline(NULL, image + image_off, + image + PAGE_SIZE, model, flags, tlinks, stub_func); + if (size <= 0) { + if (image != *_image) + bpf_struct_ops_image_free(image); + return size ? : -EFAULT; + } + + *_image = image; + *_image_off = image_off + size; + return 0; } static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, @@ -539,8 +597,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, struct bpf_tramp_links *tlinks; void *udata, *kdata; int prog_fd, err; - void *image, *image_end; - u32 i; + u32 i, trampoline_start, image_off = 0; + void *cur_image = NULL, *image = NULL; if (flags) return -EINVAL; @@ -578,8 +636,6 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, udata = &uvalue->data; kdata = &kvalue->data; - image = st_map->image; - image_end = st_map->image + PAGE_SIZE; module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]); for_each_member(i, t, member) { @@ -658,15 +714,24 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, &bpf_struct_ops_link_lops, prog); st_map->links[i] = &link->link; + trampoline_start = image_off; err = bpf_struct_ops_prepare_trampoline(tlinks, link, - &st_ops->func_models[i], - *(void **)(st_ops->cfi_stubs + moff), - image, image_end); + &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), + &image, &image_off, + st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES); + if (err) + goto reset_unlock; + + if (cur_image != image) { + st_map->image_pages[st_map->image_pages_cnt++] = image; + cur_image = image; + trampoline_start = 0; + } if (err < 0) goto reset_unlock; - *(void **)(kdata + moff) = image + cfi_get_offset(); - image += err; + *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset(); /* put prog_id to udata */ *(unsigned long *)(udata + moff) = prog->aux->id; @@ -677,10 +742,11 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (err) goto reset_unlock; } + for (i = 0; i < st_map->image_pages_cnt; i++) + arch_protect_bpf_trampoline(st_map->image_pages[i], PAGE_SIZE); if (st_map->map.map_flags & BPF_F_LINK) { err = 0; - arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); /* Let bpf_link handle registration & unregistration. * * Pair with smp_load_acquire() during lookup_elem(). @@ -689,7 +755,6 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } - arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); err = st_ops->reg(kdata); if (likely(!err)) { /* This refcnt increment on the map here after @@ -712,9 +777,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, * there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ - arch_unprotect_bpf_trampoline(st_map->image, PAGE_SIZE); reset_unlock: + bpf_struct_ops_map_free_image(st_map); bpf_struct_ops_map_put_progs(st_map); memset(uvalue, 0, map->value_size); memset(kvalue, 0, map->value_size); @@ -781,10 +846,7 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) if (st_map->links) bpf_struct_ops_map_put_progs(st_map); bpf_map_area_free(st_map->links); - if (st_map->image) { - arch_free_bpf_trampoline(st_map->image, PAGE_SIZE); - bpf_jit_uncharge_modmem(PAGE_SIZE); - } + bpf_struct_ops_map_free_image(st_map); bpf_map_area_free(st_map->uvalue); bpf_map_area_free(st_map); } @@ -894,20 +956,6 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) st_map->st_ops_desc = st_ops_desc; map = &st_map->map; - ret = bpf_jit_charge_modmem(PAGE_SIZE); - if (ret) - goto errout_free; - - st_map->image = arch_alloc_bpf_trampoline(PAGE_SIZE); - if (!st_map->image) { - /* __bpf_struct_ops_map_free() uses st_map->image as flag - * for "charged or not". In this case, we need to unchange - * here. - */ - bpf_jit_uncharge_modmem(PAGE_SIZE); - ret = -ENOMEM; - goto errout_free; - } st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); st_map->links_cnt = btf_type_vlen(t); st_map->links = diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 02de71719aed..1b5f812e6972 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -91,6 +91,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, struct bpf_tramp_link *link = NULL; void *image = NULL; unsigned int op_idx; + u32 image_off = 0; int prog_ret; s32 type_id; int err; @@ -114,12 +115,6 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, goto out; } - image = arch_alloc_bpf_trampoline(PAGE_SIZE); - if (!image) { - err = -ENOMEM; - goto out; - } - link = kzalloc(sizeof(*link), GFP_USER); if (!link) { err = -ENOMEM; @@ -133,7 +128,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[op_idx], &dummy_ops_test_ret_function, - image, image + PAGE_SIZE); + &image, &image_off, + true); if (err < 0) goto out; @@ -147,7 +143,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = -EFAULT; out: kfree(args); - arch_free_bpf_trampoline(image, PAGE_SIZE); + bpf_struct_ops_image_free(image); if (link) bpf_link_put(&link->link); kfree(tlinks); -- cgit v1.2.3 From 772dd0342727cc3c3b676b5d97c99708e75730a2 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 23 Feb 2024 17:58:00 -0800 Subject: mm: enumerate all gfp flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce GFP bits enumeration to let compiler track the number of used bits (which depends on the config options) instead of hardcoding them. That simplifies __GFP_BITS_SHIFT calculation. Link: https://lkml.kernel.org/r/20240224015800.2569851-1-surenb@google.com Suggested-by: Petr Tesařík Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Acked-by: Michal Hocko Cc: Kent Overstreet Cc: Petr Tesarik Signed-off-by: Andrew Morton --- include/linux/gfp_types.h | 90 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 1b6053da8754..868c8fb1bbc1 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -21,44 +21,78 @@ typedef unsigned int __bitwise gfp_t; * include/trace/events/mmflags.h and tools/perf/builtin-kmem.c */ +enum { + ___GFP_DMA_BIT, + ___GFP_HIGHMEM_BIT, + ___GFP_DMA32_BIT, + ___GFP_MOVABLE_BIT, + ___GFP_RECLAIMABLE_BIT, + ___GFP_HIGH_BIT, + ___GFP_IO_BIT, + ___GFP_FS_BIT, + ___GFP_ZERO_BIT, + ___GFP_UNUSED_BIT, /* 0x200u unused */ + ___GFP_DIRECT_RECLAIM_BIT, + ___GFP_KSWAPD_RECLAIM_BIT, + ___GFP_WRITE_BIT, + ___GFP_NOWARN_BIT, + ___GFP_RETRY_MAYFAIL_BIT, + ___GFP_NOFAIL_BIT, + ___GFP_NORETRY_BIT, + ___GFP_MEMALLOC_BIT, + ___GFP_COMP_BIT, + ___GFP_NOMEMALLOC_BIT, + ___GFP_HARDWALL_BIT, + ___GFP_THISNODE_BIT, + ___GFP_ACCOUNT_BIT, + ___GFP_ZEROTAGS_BIT, +#ifdef CONFIG_KASAN_HW_TAGS + ___GFP_SKIP_ZERO_BIT, + ___GFP_SKIP_KASAN_BIT, +#endif +#ifdef CONFIG_LOCKDEP + ___GFP_NOLOCKDEP_BIT, +#endif + ___GFP_LAST_BIT +}; + /* Plain integer GFP bitmasks. Do not use this directly. */ -#define ___GFP_DMA 0x01u -#define ___GFP_HIGHMEM 0x02u -#define ___GFP_DMA32 0x04u -#define ___GFP_MOVABLE 0x08u -#define ___GFP_RECLAIMABLE 0x10u -#define ___GFP_HIGH 0x20u -#define ___GFP_IO 0x40u -#define ___GFP_FS 0x80u -#define ___GFP_ZERO 0x100u +#define ___GFP_DMA BIT(___GFP_DMA_BIT) +#define ___GFP_HIGHMEM BIT(___GFP_HIGHMEM_BIT) +#define ___GFP_DMA32 BIT(___GFP_DMA32_BIT) +#define ___GFP_MOVABLE BIT(___GFP_MOVABLE_BIT) +#define ___GFP_RECLAIMABLE BIT(___GFP_RECLAIMABLE_BIT) +#define ___GFP_HIGH BIT(___GFP_HIGH_BIT) +#define ___GFP_IO BIT(___GFP_IO_BIT) +#define ___GFP_FS BIT(___GFP_FS_BIT) +#define ___GFP_ZERO BIT(___GFP_ZERO_BIT) /* 0x200u unused */ -#define ___GFP_DIRECT_RECLAIM 0x400u -#define ___GFP_KSWAPD_RECLAIM 0x800u -#define ___GFP_WRITE 0x1000u -#define ___GFP_NOWARN 0x2000u -#define ___GFP_RETRY_MAYFAIL 0x4000u -#define ___GFP_NOFAIL 0x8000u -#define ___GFP_NORETRY 0x10000u -#define ___GFP_MEMALLOC 0x20000u -#define ___GFP_COMP 0x40000u -#define ___GFP_NOMEMALLOC 0x80000u -#define ___GFP_HARDWALL 0x100000u -#define ___GFP_THISNODE 0x200000u -#define ___GFP_ACCOUNT 0x400000u -#define ___GFP_ZEROTAGS 0x800000u +#define ___GFP_DIRECT_RECLAIM BIT(___GFP_DIRECT_RECLAIM_BIT) +#define ___GFP_KSWAPD_RECLAIM BIT(___GFP_KSWAPD_RECLAIM_BIT) +#define ___GFP_WRITE BIT(___GFP_WRITE_BIT) +#define ___GFP_NOWARN BIT(___GFP_NOWARN_BIT) +#define ___GFP_RETRY_MAYFAIL BIT(___GFP_RETRY_MAYFAIL_BIT) +#define ___GFP_NOFAIL BIT(___GFP_NOFAIL_BIT) +#define ___GFP_NORETRY BIT(___GFP_NORETRY_BIT) +#define ___GFP_MEMALLOC BIT(___GFP_MEMALLOC_BIT) +#define ___GFP_COMP BIT(___GFP_COMP_BIT) +#define ___GFP_NOMEMALLOC BIT(___GFP_NOMEMALLOC_BIT) +#define ___GFP_HARDWALL BIT(___GFP_HARDWALL_BIT) +#define ___GFP_THISNODE BIT(___GFP_THISNODE_BIT) +#define ___GFP_ACCOUNT BIT(___GFP_ACCOUNT_BIT) +#define ___GFP_ZEROTAGS BIT(___GFP_ZEROTAGS_BIT) #ifdef CONFIG_KASAN_HW_TAGS -#define ___GFP_SKIP_ZERO 0x1000000u -#define ___GFP_SKIP_KASAN 0x2000000u +#define ___GFP_SKIP_ZERO BIT(___GFP_SKIP_ZERO_BIT) +#define ___GFP_SKIP_KASAN BIT(___GFP_SKIP_KASAN_BIT) #else #define ___GFP_SKIP_ZERO 0 #define ___GFP_SKIP_KASAN 0 #endif #ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x4000000u +#define ___GFP_NOLOCKDEP BIT(___GFP_NOLOCKDEP_BIT) #else #define ___GFP_NOLOCKDEP 0 #endif -/* If the above are modified, __GFP_BITS_SHIFT may need updating */ /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -249,7 +283,7 @@ typedef unsigned int __bitwise gfp_t; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (26 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT ___GFP_LAST_BIT #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** -- cgit v1.2.3 From 502003bb76b83649cd4ff7f701987ac5cf43bc4b Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Mon, 26 Feb 2024 15:55:29 -0500 Subject: mm/memcg: use order instead of nr in split_page_memcg() We do not have non power of two pages, using nr is error prone if nr is not power-of-two. Use page order instead. Link: https://lkml.kernel.org/r/20240226205534.1603748-4-zi.yan@sent.com Signed-off-by: Zi Yan Acked-by: David Hildenbrand Cc: Hugh Dickins Cc: Kirill A. Shutemov Cc: Luis Chamberlain Cc: "Matthew Wilcox (Oracle)" Cc: Michal Koutny Cc: Roman Gushchin Cc: Ryan Roberts Cc: Yang Shi Cc: Yu Zhao Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 4 ++-- mm/huge_memory.c | 5 +++-- mm/memcontrol.c | 3 ++- mm/page_alloc.c | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 4e4caeaea404..173bbb53c1ec 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1163,7 +1163,7 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm, rcu_read_unlock(); } -void split_page_memcg(struct page *head, unsigned int nr); +void split_page_memcg(struct page *head, int order); unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, gfp_t gfp_mask, @@ -1621,7 +1621,7 @@ void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx) { } -static inline void split_page_memcg(struct page *head, unsigned int nr) +static inline void split_page_memcg(struct page *head, int order) { } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 9840f312c08f..96ac7c62c375 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2889,11 +2889,12 @@ static void __split_huge_page(struct page *page, struct list_head *list, struct lruvec *lruvec; struct address_space *swap_cache = NULL; unsigned long offset = 0; - unsigned int nr = thp_nr_pages(head); int i, nr_dropped = 0; + int order = folio_order(folio); + unsigned int nr = 1 << order; /* complete memcg works before add pages to LRU */ - split_page_memcg(head, nr); + split_page_memcg(head, order); if (folio_test_anon(folio) && folio_test_swapcache(folio)) { offset = swp_offset(folio->swap); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index cb216d30a221..db9a3fb26a99 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3606,11 +3606,12 @@ void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) /* * Because page_memcg(head) is not set on tails, set it now. */ -void split_page_memcg(struct page *head, unsigned int nr) +void split_page_memcg(struct page *head, int order) { struct folio *folio = page_folio(head); struct mem_cgroup *memcg = folio_memcg(folio); int i; + unsigned int nr = 1 << order; if (mem_cgroup_disabled() || !memcg) return; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 5ab921e3d7d1..44ffe1304878 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2617,7 +2617,7 @@ void split_page(struct page *page, unsigned int order) for (i = 1; i < (1 << order); i++) set_page_refcounted(page + i); split_page_owner(page, 1 << order); - split_page_memcg(page, 1 << order); + split_page_memcg(page, order); } EXPORT_SYMBOL_GPL(split_page); @@ -4802,7 +4802,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order, struct page *last = page + nr; split_page_owner(page, 1 << order); - split_page_memcg(page, 1 << order); + split_page_memcg(page, order); while (page < --last) set_page_refcounted(last); -- cgit v1.2.3 From 9a581c12cddb06696fe4811239934fcde57ceb91 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Mon, 26 Feb 2024 15:55:30 -0500 Subject: mm/page_owner: use order instead of nr in split_page_owner() We do not have non power of two pages, using nr is error prone if nr is not power-of-two. Use page order instead. Link: https://lkml.kernel.org/r/20240226205534.1603748-5-zi.yan@sent.com Signed-off-by: Zi Yan Acked-by: David Hildenbrand Cc: Hugh Dickins Cc: Kirill A. Shutemov Cc: Luis Chamberlain Cc: "Matthew Wilcox (Oracle)" Cc: Michal Koutny Cc: Roman Gushchin Cc: Ryan Roberts Cc: Yang Shi Cc: Yu Zhao Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/page_owner.h | 9 ++++----- mm/huge_memory.c | 2 +- mm/page_alloc.c | 4 ++-- mm/page_owner.c | 3 ++- 4 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 119a0c9d2a8b..2b39c8e19d98 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -11,7 +11,7 @@ extern struct page_ext_operations page_owner_ops; extern void __reset_page_owner(struct page *page, unsigned short order); extern void __set_page_owner(struct page *page, unsigned short order, gfp_t gfp_mask); -extern void __split_page_owner(struct page *page, unsigned int nr); +extern void __split_page_owner(struct page *page, int order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); extern void __set_page_owner_migrate_reason(struct page *page, int reason); extern void __dump_page_owner(const struct page *page); @@ -31,10 +31,10 @@ static inline void set_page_owner(struct page *page, __set_page_owner(page, order, gfp_mask); } -static inline void split_page_owner(struct page *page, unsigned int nr) +static inline void split_page_owner(struct page *page, int order) { if (static_branch_unlikely(&page_owner_inited)) - __split_page_owner(page, nr); + __split_page_owner(page, order); } static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) { @@ -59,8 +59,7 @@ static inline void set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask) { } -static inline void split_page_owner(struct page *page, - unsigned short order) +static inline void split_page_owner(struct page *page, int order) { } static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 96ac7c62c375..2b95dbc95fae 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2933,7 +2933,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, unlock_page_lruvec(lruvec); /* Caller disabled irqs, so they are still disabled here */ - split_page_owner(head, nr); + split_page_owner(head, order); /* See comment in __split_huge_page_tail() */ if (PageAnon(head)) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 44ffe1304878..a9d1af34f4d6 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2616,7 +2616,7 @@ void split_page(struct page *page, unsigned int order) for (i = 1; i < (1 << order); i++) set_page_refcounted(page + i); - split_page_owner(page, 1 << order); + split_page_owner(page, order); split_page_memcg(page, order); } EXPORT_SYMBOL_GPL(split_page); @@ -4801,7 +4801,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order, struct page *page = virt_to_page((void *)addr); struct page *last = page + nr; - split_page_owner(page, 1 << order); + split_page_owner(page, order); split_page_memcg(page, order); while (page < --last) set_page_refcounted(last); diff --git a/mm/page_owner.c b/mm/page_owner.c index e56c1e92eccf..b678f7a6e702 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -306,11 +306,12 @@ void __set_page_owner_migrate_reason(struct page *page, int reason) page_ext_put(page_ext); } -void __split_page_owner(struct page *page, unsigned int nr) +void __split_page_owner(struct page *page, int order) { int i; struct page_ext *page_ext = page_ext_get(page); struct page_owner *page_owner; + unsigned int nr = 1 << order; if (unlikely(!page_ext)) return; -- cgit v1.2.3 From b8791381d7edae3706edde207f52d9e483ed400c Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Mon, 26 Feb 2024 15:55:31 -0500 Subject: mm: memcg: make memcg huge page split support any order split It sets memcg information for the pages after the split. A new parameter new_order is added to tell the order of subpages in the new page, always 0 for now. It prepares for upcoming changes to support split huge page to any lower order. Link: https://lkml.kernel.org/r/20240226205534.1603748-6-zi.yan@sent.com Signed-off-by: Zi Yan Acked-by: David Hildenbrand Cc: Hugh Dickins Cc: Kirill A. Shutemov Cc: Luis Chamberlain Cc: "Matthew Wilcox (Oracle)" Cc: Michal Koutny Cc: Roman Gushchin Cc: Ryan Roberts Cc: Yang Shi Cc: Yu Zhao Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 4 ++-- mm/huge_memory.c | 2 +- mm/memcontrol.c | 11 ++++++----- mm/page_alloc.c | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 173bbb53c1ec..9a2dea92be0e 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1163,7 +1163,7 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm, rcu_read_unlock(); } -void split_page_memcg(struct page *head, int order); +void split_page_memcg(struct page *head, int old_order, int new_order); unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, gfp_t gfp_mask, @@ -1621,7 +1621,7 @@ void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx) { } -static inline void split_page_memcg(struct page *head, int order) +static inline void split_page_memcg(struct page *head, int old_order, int new_order) { } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2b95dbc95fae..5d4b7c17b9bc 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2894,7 +2894,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, unsigned int nr = 1 << order; /* complete memcg works before add pages to LRU */ - split_page_memcg(head, order); + split_page_memcg(head, order, 0); if (folio_test_anon(folio) && folio_test_swapcache(folio)) { offset = swp_offset(folio->swap); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index db9a3fb26a99..e4864e2729e6 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3606,23 +3606,24 @@ void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) /* * Because page_memcg(head) is not set on tails, set it now. */ -void split_page_memcg(struct page *head, int order) +void split_page_memcg(struct page *head, int old_order, int new_order) { struct folio *folio = page_folio(head); struct mem_cgroup *memcg = folio_memcg(folio); int i; - unsigned int nr = 1 << order; + unsigned int old_nr = 1 << old_order; + unsigned int new_nr = 1 << new_order; if (mem_cgroup_disabled() || !memcg) return; - for (i = 1; i < nr; i++) + for (i = new_nr; i < old_nr; i += new_nr) folio_page(folio, i)->memcg_data = folio->memcg_data; if (folio_memcg_kmem(folio)) - obj_cgroup_get_many(__folio_objcg(folio), nr - 1); + obj_cgroup_get_many(__folio_objcg(folio), old_nr / new_nr - 1); else - css_get_many(&memcg->css, nr - 1); + css_get_many(&memcg->css, old_nr / new_nr - 1); } #ifdef CONFIG_SWAP diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a9d1af34f4d6..3b3dae66d49d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2617,7 +2617,7 @@ void split_page(struct page *page, unsigned int order) for (i = 1; i < (1 << order); i++) set_page_refcounted(page + i); split_page_owner(page, order); - split_page_memcg(page, order); + split_page_memcg(page, order, 0); } EXPORT_SYMBOL_GPL(split_page); @@ -4802,7 +4802,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order, struct page *last = page + nr; split_page_owner(page, order); - split_page_memcg(page, order); + split_page_memcg(page, order, 0); while (page < --last) set_page_refcounted(last); -- cgit v1.2.3 From 46d44d09d24c5b451d6ab8f0fca5a40f651e3837 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Mon, 26 Feb 2024 15:55:32 -0500 Subject: mm: page_owner: add support for splitting to any order in split page_owner It adds a new_order parameter to set new page order in page owner. It prepares for upcoming changes to support split huge page to any lower order. Link: https://lkml.kernel.org/r/20240226205534.1603748-7-zi.yan@sent.com Signed-off-by: Zi Yan Cc: David Hildenbrand Acked-by: David Hildenbrand Cc: Kirill A. Shutemov Cc: Luis Chamberlain Cc: "Matthew Wilcox (Oracle)" Cc: Michal Koutny Cc: Roman Gushchin Cc: Ryan Roberts Cc: Yang Shi Cc: Yu Zhao Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/page_owner.h | 13 ++++++++----- mm/huge_memory.c | 2 +- mm/page_alloc.c | 4 ++-- mm/page_owner.c | 7 +++---- 4 files changed, 14 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 2b39c8e19d98..debdc25f08b9 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -11,7 +11,8 @@ extern struct page_ext_operations page_owner_ops; extern void __reset_page_owner(struct page *page, unsigned short order); extern void __set_page_owner(struct page *page, unsigned short order, gfp_t gfp_mask); -extern void __split_page_owner(struct page *page, int order); +extern void __split_page_owner(struct page *page, int old_order, + int new_order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); extern void __set_page_owner_migrate_reason(struct page *page, int reason); extern void __dump_page_owner(const struct page *page); @@ -31,10 +32,11 @@ static inline void set_page_owner(struct page *page, __set_page_owner(page, order, gfp_mask); } -static inline void split_page_owner(struct page *page, int order) +static inline void split_page_owner(struct page *page, int old_order, + int new_order) { if (static_branch_unlikely(&page_owner_inited)) - __split_page_owner(page, order); + __split_page_owner(page, old_order, new_order); } static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) { @@ -56,10 +58,11 @@ static inline void reset_page_owner(struct page *page, unsigned short order) { } static inline void set_page_owner(struct page *page, - unsigned int order, gfp_t gfp_mask) + unsigned short order, gfp_t gfp_mask) { } -static inline void split_page_owner(struct page *page, int order) +static inline void split_page_owner(struct page *page, int old_order, + int new_order) { } static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 5d4b7c17b9bc..b2df788c11fa 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2933,7 +2933,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, unlock_page_lruvec(lruvec); /* Caller disabled irqs, so they are still disabled here */ - split_page_owner(head, order); + split_page_owner(head, order, 0); /* See comment in __split_huge_page_tail() */ if (PageAnon(head)) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3b3dae66d49d..ff1f159251df 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2616,7 +2616,7 @@ void split_page(struct page *page, unsigned int order) for (i = 1; i < (1 << order); i++) set_page_refcounted(page + i); - split_page_owner(page, order); + split_page_owner(page, order, 0); split_page_memcg(page, order, 0); } EXPORT_SYMBOL_GPL(split_page); @@ -4801,7 +4801,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order, struct page *page = virt_to_page((void *)addr); struct page *last = page + nr; - split_page_owner(page, order); + split_page_owner(page, order, 0); split_page_memcg(page, order, 0); while (page < --last) set_page_refcounted(last); diff --git a/mm/page_owner.c b/mm/page_owner.c index b678f7a6e702..033e349f6479 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -306,19 +306,18 @@ void __set_page_owner_migrate_reason(struct page *page, int reason) page_ext_put(page_ext); } -void __split_page_owner(struct page *page, int order) +void __split_page_owner(struct page *page, int old_order, int new_order) { int i; struct page_ext *page_ext = page_ext_get(page); struct page_owner *page_owner; - unsigned int nr = 1 << order; if (unlikely(!page_ext)) return; - for (i = 0; i < nr; i++) { + for (i = 0; i < (1 << old_order); i++) { page_owner = get_page_owner(page_ext); - page_owner->order = 0; + page_owner->order = new_order; page_ext = page_ext_next(page_ext); } page_ext_put(page_ext); -- cgit v1.2.3 From c010d47f107f609b9f4d6a103b6dfc53889049e9 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Mon, 26 Feb 2024 15:55:33 -0500 Subject: mm: thp: split huge page to any lower order pages To split a THP to any lower order pages, we need to reform THPs on subpages at given order and add page refcount based on the new page order. Also we need to reinitialize page_deferred_list after removing the page from the split_queue, otherwise a subsequent split will see list corruption when checking the page_deferred_list again. Note: Anonymous order-1 folio is not supported because _deferred_list, which is used by partially mapped folios, is stored in subpage 2 and an order-1 folio only has subpage 0 and 1. File-backed order-1 folios are fine, since they do not use _deferred_list. [ziy@nvidia.com: fixup per discussion with Ryan] Link: https://lkml.kernel.org/r/494F48CD-1F0F-4CAD-884E-6D48F40AF990@nvidia.com Link: https://lkml.kernel.org/r/20240226205534.1603748-8-zi.yan@sent.com Signed-off-by: Zi Yan Cc: David Hildenbrand Cc: Hugh Dickins Cc: Kirill A. Shutemov Cc: Luis Chamberlain Cc: "Matthew Wilcox (Oracle)" Cc: Michal Koutny Cc: Roman Gushchin Cc: Ryan Roberts Cc: Yang Shi Cc: Yu Zhao Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 21 ++++++---- mm/huge_memory.c | 107 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 96 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 5adb86af35fc..de0c89105076 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -265,10 +265,11 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, void folio_prep_large_rmappable(struct folio *folio); bool can_split_folio(struct folio *folio, int *pextra_pins); -int split_huge_page_to_list(struct page *page, struct list_head *list); +int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, + unsigned int new_order); static inline int split_huge_page(struct page *page) { - return split_huge_page_to_list(page, NULL); + return split_huge_page_to_list_to_order(page, NULL, 0); } void deferred_split_folio(struct folio *folio); @@ -422,7 +423,8 @@ can_split_folio(struct folio *folio, int *pextra_pins) return false; } static inline int -split_huge_page_to_list(struct page *page, struct list_head *list) +split_huge_page_to_list_to_order(struct page *page, struct list_head *list, + unsigned int new_order) { return 0; } @@ -519,17 +521,20 @@ static inline bool thp_migration_supported(void) } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline int split_folio_to_list(struct folio *folio, - struct list_head *list) +static inline int split_folio_to_list_to_order(struct folio *folio, + struct list_head *list, int new_order) { - return split_huge_page_to_list(&folio->page, list); + return split_huge_page_to_list_to_order(&folio->page, list, new_order); } -static inline int split_folio(struct folio *folio) +static inline int split_folio_to_order(struct folio *folio, int new_order) { - return split_folio_to_list(folio, NULL); + return split_folio_to_list_to_order(folio, NULL, new_order); } +#define split_folio_to_list(f, l) split_folio_to_list_to_order(f, l, 0) +#define split_folio(f) split_folio_to_order(f, 0) + /* * archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to * limitations in the implementation like arm64 MTE can override this to diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b2df788c11fa..006b192ea6ab 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2770,7 +2770,6 @@ static void lru_add_page_tail(struct page *head, struct page *tail, struct lruvec *lruvec, struct list_head *list) { VM_BUG_ON_PAGE(!PageHead(head), head); - VM_BUG_ON_PAGE(PageCompound(tail), head); VM_BUG_ON_PAGE(PageLRU(tail), head); lockdep_assert_held(&lruvec->lru_lock); @@ -2791,7 +2790,8 @@ static void lru_add_page_tail(struct page *head, struct page *tail, } static void __split_huge_page_tail(struct folio *folio, int tail, - struct lruvec *lruvec, struct list_head *list) + struct lruvec *lruvec, struct list_head *list, + unsigned int new_order) { struct page *head = &folio->page; struct page *page_tail = head + tail; @@ -2861,10 +2861,15 @@ static void __split_huge_page_tail(struct folio *folio, int tail, * which needs correct compound_head(). */ clear_compound_head(page_tail); + if (new_order) { + prep_compound_page(page_tail, new_order); + folio_prep_large_rmappable(new_folio); + } /* Finally unfreeze refcount. Additional reference from page cache. */ - page_ref_unfreeze(page_tail, 1 + (!folio_test_anon(folio) || - folio_test_swapcache(folio))); + page_ref_unfreeze(page_tail, + 1 + ((!folio_test_anon(folio) || folio_test_swapcache(folio)) ? + folio_nr_pages(new_folio) : 0)); if (folio_test_young(folio)) folio_set_young(new_folio); @@ -2882,7 +2887,7 @@ static void __split_huge_page_tail(struct folio *folio, int tail, } static void __split_huge_page(struct page *page, struct list_head *list, - pgoff_t end) + pgoff_t end, unsigned int new_order) { struct folio *folio = page_folio(page); struct page *head = &folio->page; @@ -2890,11 +2895,12 @@ static void __split_huge_page(struct page *page, struct list_head *list, struct address_space *swap_cache = NULL; unsigned long offset = 0; int i, nr_dropped = 0; + unsigned int new_nr = 1 << new_order; int order = folio_order(folio); unsigned int nr = 1 << order; /* complete memcg works before add pages to LRU */ - split_page_memcg(head, order, 0); + split_page_memcg(head, order, new_order); if (folio_test_anon(folio) && folio_test_swapcache(folio)) { offset = swp_offset(folio->swap); @@ -2907,8 +2913,8 @@ static void __split_huge_page(struct page *page, struct list_head *list, ClearPageHasHWPoisoned(head); - for (i = nr - 1; i >= 1; i--) { - __split_huge_page_tail(folio, i, lruvec, list); + for (i = nr - new_nr; i >= new_nr; i -= new_nr) { + __split_huge_page_tail(folio, i, lruvec, list, new_order); /* Some pages can be beyond EOF: drop them from page cache */ if (head[i].index >= end) { struct folio *tail = page_folio(head + i); @@ -2929,24 +2935,30 @@ static void __split_huge_page(struct page *page, struct list_head *list, } } - ClearPageCompound(head); + if (!new_order) + ClearPageCompound(head); + else { + struct folio *new_folio = (struct folio *)head; + + folio_set_order(new_folio, new_order); + } unlock_page_lruvec(lruvec); /* Caller disabled irqs, so they are still disabled here */ - split_page_owner(head, order, 0); + split_page_owner(head, order, new_order); /* See comment in __split_huge_page_tail() */ if (PageAnon(head)) { /* Additional pin to swap cache */ if (PageSwapCache(head)) { - page_ref_add(head, 2); + page_ref_add(head, 1 + new_nr); xa_unlock(&swap_cache->i_pages); } else { page_ref_inc(head); } } else { /* Additional pin to page cache */ - page_ref_add(head, 2); + page_ref_add(head, 1 + new_nr); xa_unlock(&head->mapping->i_pages); } local_irq_enable(); @@ -2958,7 +2970,15 @@ static void __split_huge_page(struct page *page, struct list_head *list, if (folio_test_swapcache(folio)) split_swap_cluster(folio->swap); - for (i = 0; i < nr; i++) { + /* + * set page to its compound_head when split to non order-0 pages, so + * we can skip unlocking it below, since PG_locked is transferred to + * the compound_head of the page and the caller will unlock it. + */ + if (new_order) + page = compound_head(page); + + for (i = 0; i < nr; i += new_nr) { struct page *subpage = head + i; if (subpage == page) continue; @@ -2992,29 +3012,36 @@ bool can_split_folio(struct folio *folio, int *pextra_pins) } /* - * This function splits huge page into normal pages. @page can point to any - * subpage of huge page to split. Split doesn't change the position of @page. + * This function splits huge page into pages in @new_order. @page can point to + * any subpage of huge page to split. Split doesn't change the position of + * @page. + * + * NOTE: order-1 anonymous folio is not supported because _deferred_list, + * which is used by partially mapped folios, is stored in subpage 2 and an + * order-1 folio only has subpage 0 and 1. File-backed order-1 folios are OK, + * since they do not use _deferred_list. * * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. * The huge page must be locked. * * If @list is null, tail pages will be added to LRU list, otherwise, to @list. * - * Both head page and tail pages will inherit mapping, flags, and so on from - * the hugepage. + * Pages in new_order will inherit mapping, flags, and so on from the hugepage. * - * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if - * they are not mapped. + * GUP pin and PG_locked transferred to @page or the compound page @page belongs + * to. Rest subpages can be freed if they are not mapped. * * Returns 0 if the hugepage is split successfully. * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under * us. */ -int split_huge_page_to_list(struct page *page, struct list_head *list) +int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, + unsigned int new_order) { struct folio *folio = page_folio(page); struct deferred_split *ds_queue = get_deferred_split_queue(folio); - XA_STATE(xas, &folio->mapping->i_pages, folio->index); + /* reset xarray order to new order after split */ + XA_STATE_ORDER(xas, &folio->mapping->i_pages, folio->index, new_order); struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; int extra_pins, ret; @@ -3024,6 +3051,31 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); + /* Cannot split anonymous THP to order-1 */ + if (new_order == 1 && folio_test_anon(folio)) { + VM_WARN_ONCE(1, "Cannot split to order-1 folio"); + return -EINVAL; + } + + if (new_order) { + /* Only swapping a whole PMD-mapped folio is supported */ + if (folio_test_swapcache(folio)) + return -EINVAL; + /* Split shmem folio to non-zero order not supported */ + if (shmem_mapping(folio->mapping)) { + VM_WARN_ONCE(1, + "Cannot split shmem folio to non-0 order"); + return -EINVAL; + } + /* No split if the file system does not support large folio */ + if (!mapping_large_folio_support(folio->mapping)) { + VM_WARN_ONCE(1, + "Cannot split file folio to non-0 order"); + return -EINVAL; + } + } + + is_hzp = is_huge_zero_page(&folio->page); if (is_hzp) { pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); @@ -3120,14 +3172,21 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) if (folio_order(folio) > 1 && !list_empty(&folio->_deferred_list)) { ds_queue->split_queue_len--; - list_del(&folio->_deferred_list); + /* + * Reinitialize page_deferred_list after removing the + * page from the split_queue, otherwise a subsequent + * split will see list corruption when checking the + * page_deferred_list. + */ + list_del_init(&folio->_deferred_list); } spin_unlock(&ds_queue->split_queue_lock); if (mapping) { int nr = folio_nr_pages(folio); xas_split(&xas, folio, folio_order(folio)); - if (folio_test_pmd_mappable(folio)) { + if (folio_test_pmd_mappable(folio) && + new_order < HPAGE_PMD_ORDER) { if (folio_test_swapbacked(folio)) { __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); @@ -3139,7 +3198,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) } } - __split_huge_page(page, list, end); + __split_huge_page(page, list, end, new_order); ret = 0; } else { spin_unlock(&ds_queue->split_queue_lock); -- cgit v1.2.3 From 5ce1f4844ba0def4b1b5526d8ccea27a98e840e5 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 26 Feb 2024 15:13:24 +0100 Subject: mm: remove total_mapcount() All users of total_mapcount() are gone, let's remove it. Link: https://lkml.kernel.org/r/20240226141324.278526-3-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 6f4825d82965..49e22a2f6ccc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1183,7 +1183,7 @@ static inline int is_vmalloc_or_module_addr(const void *x) * How many times the entire folio is mapped as a single unit (eg by a * PMD or PUD entry). This is probably not what you want, except for * debugging purposes - it does not include PTE-mapped sub-pages; look - * at folio_mapcount() or page_mapcount() or total_mapcount() instead. + * at folio_mapcount() or page_mapcount() instead. */ static inline int folio_entire_mapcount(struct folio *folio) { @@ -1243,13 +1243,6 @@ static inline int folio_mapcount(struct folio *folio) return folio_total_mapcount(folio); } -static inline int total_mapcount(struct page *page) -{ - if (likely(!PageCompound(page))) - return atomic_read(&page->_mapcount) + 1; - return folio_total_mapcount(page_folio(page)); -} - static inline bool folio_large_is_mapped(struct folio *folio) { /* -- cgit v1.2.3 From 99fbb6bfc16f202adc411ad5d353db214750d121 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 17:42:35 +0000 Subject: mm: make folios_put() the basis of release_pages() Patch series "Rearrange batched folio freeing", v3. Other than the obvious "remove calls to compound_head" changes, the fundamental belief here is that iterating a linked list is much slower than iterating an array (5-15x slower in my testing). There's also an associated belief that since we iterate the batch of folios three times, we do better when the array is small (ie 15 entries) than we do with a batch that is hundreds of entries long, which only gives us the opportunity for the first pages to fall out of cache by the time we get to the end. It is possible we should increase the size of folio_batch. Hopefully the bots let us know if this introduces any performance regressions. This patch (of 3): By making release_pages() call folios_put(), we can get rid of the calls to compound_head() for the callers that already know they have folios. We can also get rid of the lock_batch tracking as we know the size of the batch is limited by folio_batch. This does reduce the maximum number of pages for which the lruvec lock is held, from SWAP_CLUSTER_MAX (32) to PAGEVEC_SIZE (15). I do not expect this to make a significant difference, but if it does, we can increase PAGEVEC_SIZE to 31. Link: https://lkml.kernel.org/r/20240227174254.710559-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240227174254.710559-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Mel Gorman Cc: Ryan Roberts Signed-off-by: Andrew Morton --- include/linux/mm.h | 16 +++++---- mm/mlock.c | 3 +- mm/swap.c | 100 +++++++++++++++++++++++++++++++---------------------- 3 files changed, 70 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 49e22a2f6ccc..5c57fde9b69b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -36,6 +36,7 @@ struct anon_vma; struct anon_vma_chain; struct user_struct; struct pt_regs; +struct folio_batch; extern int sysctl_page_lock_unfairness; @@ -1512,6 +1513,8 @@ static inline void folio_put_refs(struct folio *folio, int refs) __folio_put(folio); } +void folios_put_refs(struct folio_batch *folios, unsigned int *refs); + /* * union release_pages_arg - an array of pages or folios * @@ -1534,18 +1537,19 @@ void release_pages(release_pages_arg, int nr); /** * folios_put - Decrement the reference count on an array of folios. * @folios: The folios. - * @nr: How many folios there are. * - * Like folio_put(), but for an array of folios. This is more efficient - * than writing the loop yourself as it will optimise the locks which - * need to be taken if the folios are freed. + * Like folio_put(), but for a batch of folios. This is more efficient + * than writing the loop yourself as it will optimise the locks which need + * to be taken if the folios are freed. The folios batch is returned + * empty and ready to be reused for another batch; there is no need to + * reinitialise it. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */ -static inline void folios_put(struct folio **folios, unsigned int nr) +static inline void folios_put(struct folio_batch *folios) { - release_pages(folios, nr); + folios_put_refs(folios, NULL); } static inline void put_page(struct page *page) diff --git a/mm/mlock.c b/mm/mlock.c index 086546ac5766..1ed2f2ab37cd 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -206,8 +206,7 @@ static void mlock_folio_batch(struct folio_batch *fbatch) if (lruvec) unlock_page_lruvec_irq(lruvec); - folios_put(fbatch->folios, folio_batch_count(fbatch)); - folio_batch_reinit(fbatch); + folios_put(fbatch); } void mlock_drain_local(void) diff --git a/mm/swap.c b/mm/swap.c index e5380d732c0d..3d51f8c72017 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -89,7 +89,7 @@ static void __page_cache_release(struct folio *folio) __folio_clear_lru_flags(folio); unlock_page_lruvec_irqrestore(lruvec, flags); } - /* See comment on folio_test_mlocked in release_pages() */ + /* See comment on folio_test_mlocked in folios_put() */ if (unlikely(folio_test_mlocked(folio))) { long nr_pages = folio_nr_pages(folio); @@ -175,7 +175,7 @@ static void lru_add_fn(struct lruvec *lruvec, struct folio *folio) * while the LRU lock is held. * * (That is not true of __page_cache_release(), and not necessarily - * true of release_pages(): but those only clear the mlocked flag after + * true of folios_put(): but those only clear the mlocked flag after * folio_put_testzero() has excluded any other users of the folio.) */ if (folio_evictable(folio)) { @@ -221,8 +221,7 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); - folios_put(fbatch->folios, folio_batch_count(fbatch)); - folio_batch_reinit(fbatch); + folios_put(fbatch); } static void folio_batch_add_and_move(struct folio_batch *fbatch, @@ -946,47 +945,30 @@ void lru_cache_disable(void) } /** - * release_pages - batched put_page() - * @arg: array of pages to release - * @nr: number of pages + * folios_put_refs - Reduce the reference count on a batch of folios. + * @folios: The folios. + * @refs: The number of refs to subtract from each folio. * - * Decrement the reference count on all the pages in @arg. If it - * fell to zero, remove the page from the LRU and free it. + * Like folio_put(), but for a batch of folios. This is more efficient + * than writing the loop yourself as it will optimise the locks which need + * to be taken if the folios are freed. The folios batch is returned + * empty and ready to be reused for another batch; there is no need + * to reinitialise it. If @refs is NULL, we subtract one from each + * folio refcount. * - * Note that the argument can be an array of pages, encoded pages, - * or folio pointers. We ignore any encoded bits, and turn any of - * them into just a folio that gets free'd. + * Context: May be called in process or interrupt context, but not in NMI + * context. May be called while holding a spinlock. */ -void release_pages(release_pages_arg arg, int nr) +void folios_put_refs(struct folio_batch *folios, unsigned int *refs) { int i; - struct encoded_page **encoded = arg.encoded_pages; LIST_HEAD(pages_to_free); struct lruvec *lruvec = NULL; unsigned long flags = 0; - unsigned int lock_batch; - for (i = 0; i < nr; i++) { - unsigned int nr_refs = 1; - struct folio *folio; - - /* Turn any of the argument types into a folio */ - folio = page_folio(encoded_page_ptr(encoded[i])); - - /* Is our next entry actually "nr_pages" -> "nr_refs" ? */ - if (unlikely(encoded_page_flags(encoded[i]) & - ENCODED_PAGE_BIT_NR_PAGES_NEXT)) - nr_refs = encoded_nr_pages(encoded[++i]); - - /* - * Make sure the IRQ-safe lock-holding time does not get - * excessive with a continuous string of pages from the - * same lruvec. The lock is held only if lruvec != NULL. - */ - if (lruvec && ++lock_batch == SWAP_CLUSTER_MAX) { - unlock_page_lruvec_irqrestore(lruvec, flags); - lruvec = NULL; - } + for (i = 0; i < folios->nr; i++) { + struct folio *folio = folios->folios[i]; + unsigned int nr_refs = refs ? refs[i] : 1; if (is_huge_zero_page(&folio->page)) continue; @@ -1016,13 +998,8 @@ void release_pages(release_pages_arg arg, int nr) } if (folio_test_lru(folio)) { - struct lruvec *prev_lruvec = lruvec; - lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); - if (prev_lruvec != lruvec) - lock_batch = 0; - lruvec_del_folio(lruvec, folio); __folio_clear_lru_flags(folio); } @@ -1046,6 +1023,47 @@ void release_pages(release_pages_arg arg, int nr) mem_cgroup_uncharge_list(&pages_to_free); free_unref_page_list(&pages_to_free); + folio_batch_reinit(folios); +} +EXPORT_SYMBOL(folios_put_refs); + +/** + * release_pages - batched put_page() + * @arg: array of pages to release + * @nr: number of pages + * + * Decrement the reference count on all the pages in @arg. If it + * fell to zero, remove the page from the LRU and free it. + * + * Note that the argument can be an array of pages, encoded pages, + * or folio pointers. We ignore any encoded bits, and turn any of + * them into just a folio that gets free'd. + */ +void release_pages(release_pages_arg arg, int nr) +{ + struct folio_batch fbatch; + int refs[PAGEVEC_SIZE]; + struct encoded_page **encoded = arg.encoded_pages; + int i; + + folio_batch_init(&fbatch); + for (i = 0; i < nr; i++) { + /* Turn any of the argument types into a folio */ + struct folio *folio = page_folio(encoded_page_ptr(encoded[i])); + + /* Is our next entry actually "nr_pages" -> "nr_refs" ? */ + refs[fbatch.nr] = 1; + if (unlikely(encoded_page_flags(encoded[i]) & + ENCODED_PAGE_BIT_NR_PAGES_NEXT)) + refs[fbatch.nr] = encoded_nr_pages(encoded[++i]); + + if (folio_batch_add(&fbatch, folio) > 0) + continue; + folios_put_refs(&fbatch, refs); + } + + if (fbatch.nr) + folios_put_refs(&fbatch, refs); } EXPORT_SYMBOL(release_pages); -- cgit v1.2.3 From 4882c80975e2bf7241a5b043eb1dbe8df2726a29 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 17:42:39 +0000 Subject: memcg: add mem_cgroup_uncharge_folios() Almost identical to mem_cgroup_uncharge_list(), except it takes a folio_batch instead of a list_head. Link: https://lkml.kernel.org/r/20240227174254.710559-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Ryan Roberts Cc: David Hildenbrand Cc: Mel Gorman Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 14 ++++++++++++-- mm/memcontrol.c | 13 +++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 9a2dea92be0e..35a0d7a851f0 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -721,10 +721,16 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list) __mem_cgroup_uncharge_list(page_list); } -void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages); +void __mem_cgroup_uncharge_folios(struct folio_batch *folios); +static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) +{ + if (mem_cgroup_disabled()) + return; + __mem_cgroup_uncharge_folios(folios); +} +void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages); void mem_cgroup_replace_folio(struct folio *old, struct folio *new); - void mem_cgroup_migrate(struct folio *old, struct folio *new); /** @@ -1299,6 +1305,10 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list) { } +static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) +{ +} + static inline void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e4864e2729e6..2edb250d131b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -7531,6 +7532,18 @@ void __mem_cgroup_uncharge_list(struct list_head *page_list) uncharge_batch(&ug); } +void __mem_cgroup_uncharge_folios(struct folio_batch *folios) +{ + struct uncharge_gather ug; + unsigned int i; + + uncharge_gather_clear(&ug); + for (i = 0; i < folios->nr; i++) + uncharge_folio(folios->folios[i], &ug); + if (ug.memcg) + uncharge_batch(&ug); +} + /** * mem_cgroup_replace_folio - Charge a folio's replacement. * @old: Currently circulating folio. -- cgit v1.2.3 From f1ee018baee9f4e724e08859c2559323be768be3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 17:42:42 +0000 Subject: mm: use __page_cache_release() in folios_put() Pass a pointer to the lruvec so we can take advantage of the folio_lruvec_relock_irqsave(). Adjust the calling convention of folio_lruvec_relock_irqsave() to suit and add a page_cache_release() wrapper. Link: https://lkml.kernel.org/r/20240227174254.710559-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Mel Gorman Cc: Ryan Roberts Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 16 ++++++------ mm/swap.c | 62 ++++++++++++++++++++++------------------------ 2 files changed, 37 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 35a0d7a851f0..b7f5e0c17de7 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1705,18 +1705,18 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, return folio_lruvec_lock_irq(folio); } -/* Don't lock again iff page's lruvec locked */ -static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio, - struct lruvec *locked_lruvec, unsigned long *flags) +/* Don't lock again iff folio's lruvec locked */ +static inline void folio_lruvec_relock_irqsave(struct folio *folio, + struct lruvec **lruvecp, unsigned long *flags) { - if (locked_lruvec) { - if (folio_matches_lruvec(folio, locked_lruvec)) - return locked_lruvec; + if (*lruvecp) { + if (folio_matches_lruvec(folio, *lruvecp)) + return; - unlock_page_lruvec_irqrestore(locked_lruvec, *flags); + unlock_page_lruvec_irqrestore(*lruvecp, *flags); } - return folio_lruvec_lock_irqsave(folio, flags); + *lruvecp = folio_lruvec_lock_irqsave(folio, flags); } #ifdef CONFIG_CGROUP_WRITEBACK diff --git a/mm/swap.c b/mm/swap.c index ad3f2e9448a4..dce5ea67ae05 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -74,22 +74,21 @@ static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = { .lock = INIT_LOCAL_LOCK(lock), }; -/* - * This path almost never happens for VM activity - pages are normally freed - * in batches. But it gets used by networking - and for compound pages. - */ -static void __page_cache_release(struct folio *folio) +static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp, + unsigned long *flagsp) { if (folio_test_lru(folio)) { - struct lruvec *lruvec; - unsigned long flags; - - lruvec = folio_lruvec_lock_irqsave(folio, &flags); - lruvec_del_folio(lruvec, folio); + folio_lruvec_relock_irqsave(folio, lruvecp, flagsp); + lruvec_del_folio(*lruvecp, folio); __folio_clear_lru_flags(folio); - unlock_page_lruvec_irqrestore(lruvec, flags); } - /* See comment on folio_test_mlocked in folios_put() */ + + /* + * In rare cases, when truncation or holepunching raced with + * munlock after VM_LOCKED was cleared, Mlocked may still be + * found set here. This does not indicate a problem, unless + * "unevictable_pgs_cleared" appears worryingly large. + */ if (unlikely(folio_test_mlocked(folio))) { long nr_pages = folio_nr_pages(folio); @@ -99,9 +98,23 @@ static void __page_cache_release(struct folio *folio) } } +/* + * This path almost never happens for VM activity - pages are normally freed + * in batches. But it gets used by networking - and for compound pages. + */ +static void page_cache_release(struct folio *folio) +{ + struct lruvec *lruvec = NULL; + unsigned long flags; + + __page_cache_release(folio, &lruvec, &flags); + if (lruvec) + unlock_page_lruvec_irqrestore(lruvec, flags); +} + static void __folio_put_small(struct folio *folio) { - __page_cache_release(folio); + page_cache_release(folio); mem_cgroup_uncharge(folio); free_unref_page(&folio->page, 0); } @@ -115,7 +128,7 @@ static void __folio_put_large(struct folio *folio) * be called for hugetlb (it has a separate hugetlb_cgroup.) */ if (!folio_test_hugetlb(folio)) - __page_cache_release(folio); + page_cache_release(folio); destroy_large_folio(folio); } @@ -216,7 +229,7 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) if (move_fn != lru_add_fn && !folio_test_clear_lru(folio)) continue; - lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags); + folio_lruvec_relock_irqsave(folio, &lruvec, &flags); move_fn(lruvec, folio); folio_set_lru(folio); @@ -999,24 +1012,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs) continue; } - if (folio_test_lru(folio)) { - lruvec = folio_lruvec_relock_irqsave(folio, lruvec, - &flags); - lruvec_del_folio(lruvec, folio); - __folio_clear_lru_flags(folio); - } - - /* - * In rare cases, when truncation or holepunching raced with - * munlock after VM_LOCKED was cleared, Mlocked may still be - * found set here. This does not indicate a problem, unless - * "unevictable_pgs_cleared" appears worryingly large. - */ - if (unlikely(folio_test_mlocked(folio))) { - __folio_clear_mlocked(folio); - zone_stat_sub_folio(folio, NR_MLOCK); - count_vm_event(UNEVICTABLE_PGCLEARED); - } + __page_cache_release(folio, &lruvec, &flags); if (j != i) folios->folios[j] = folio; -- cgit v1.2.3 From be5a9e17a2ccbecfb7020aa1938e2c62d8a9189c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 17:42:47 +0000 Subject: memcg: remove mem_cgroup_uncharge_list() All users have been converted to mem_cgroup_uncharge_folios() so we can remove this API. Link: https://lkml.kernel.org/r/20240227174254.710559-14-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Ryan Roberts Cc: David Hildenbrand Cc: Mel Gorman Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 12 ------------ mm/memcontrol.c | 19 ------------------- 2 files changed, 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index b7f5e0c17de7..394fd0a887ae 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -713,14 +713,6 @@ static inline void mem_cgroup_uncharge(struct folio *folio) __mem_cgroup_uncharge(folio); } -void __mem_cgroup_uncharge_list(struct list_head *page_list); -static inline void mem_cgroup_uncharge_list(struct list_head *page_list) -{ - if (mem_cgroup_disabled()) - return; - __mem_cgroup_uncharge_list(page_list); -} - void __mem_cgroup_uncharge_folios(struct folio_batch *folios); static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { @@ -1301,10 +1293,6 @@ static inline void mem_cgroup_uncharge(struct folio *folio) { } -static inline void mem_cgroup_uncharge_list(struct list_head *page_list) -{ -} - static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2edb250d131b..fabce2b50c69 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -7513,25 +7513,6 @@ void __mem_cgroup_uncharge(struct folio *folio) uncharge_batch(&ug); } -/** - * __mem_cgroup_uncharge_list - uncharge a list of page - * @page_list: list of pages to uncharge - * - * Uncharge a list of pages previously charged with - * __mem_cgroup_charge(). - */ -void __mem_cgroup_uncharge_list(struct list_head *page_list) -{ - struct uncharge_gather ug; - struct folio *folio; - - uncharge_gather_clear(&ug); - list_for_each_entry(folio, page_list, lru) - uncharge_folio(folio, &ug); - if (ug.memcg) - uncharge_batch(&ug); -} - void __mem_cgroup_uncharge_folios(struct folio_batch *folios) { struct uncharge_gather ug; -- cgit v1.2.3 From f39ec4dcb9e9b03b2a280829b8c15e3ae607398c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 17:42:49 +0000 Subject: mm: remove lru_to_page() The last user was removed over a year ago; remove the definition. Link: https://lkml.kernel.org/r/20240227174254.710559-16-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Mel Gorman Cc: Ryan Roberts Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 5c57fde9b69b..d45eadc440f5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -227,7 +227,6 @@ int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *, /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */ #define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE) -#define lru_to_page(head) (list_entry((head)->prev, struct page, lru)) static inline struct folio *lru_to_folio(struct list_head *head) { return list_entry((head)->prev, struct folio, lru); -- cgit v1.2.3 From 63b774993dd02b17127cb404b7362fc436632995 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 17:42:52 +0000 Subject: mm: convert free_swap_cache() to take a folio All but one caller already has a folio, so convert free_page_and_swap_cache() to have a folio and remove the call to page_folio(). Link: https://lkml.kernel.org/r/20240227174254.710559-19-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Ryan Roberts Reviewed-by: David Hildenbrand Cc: Mel Gorman Signed-off-by: Andrew Morton --- include/linux/swap.h | 8 ++++---- mm/khugepaged.c | 2 +- mm/memory.c | 2 +- mm/swap_state.c | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 8d28f6091a32..1ad6f63d1a52 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -447,9 +447,9 @@ static inline unsigned long total_swapcache_pages(void) return global_node_page_state(NR_SWAPCACHE); } -extern void free_swap_cache(struct page *page); -extern void free_page_and_swap_cache(struct page *); -extern void free_pages_and_swap_cache(struct encoded_page **, int); +void free_swap_cache(struct folio *folio); +void free_page_and_swap_cache(struct page *); +void free_pages_and_swap_cache(struct encoded_page **, int); /* linux/mm/swapfile.c */ extern atomic_long_t nr_swap_pages; extern long total_swap_pages; @@ -531,7 +531,7 @@ static inline void put_swap_device(struct swap_info_struct *si) /* used to sanity check ptes in zap_pte_range when CONFIG_SWAP=0 */ #define free_swap_and_cache(e) is_pfn_swap_entry(e) -static inline void free_swap_cache(struct page *page) +static inline void free_swap_cache(struct folio *folio) { } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index b9223e803262..38830174608f 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -731,7 +731,7 @@ static void __collapse_huge_page_copy_succeeded(pte_t *pte, node_stat_sub_folio(src, NR_ISOLATED_ANON + folio_is_file_lru(src)); folio_unlock(src); - free_swap_cache(&src->page); + free_swap_cache(src); folio_putback_lru(src); } } diff --git a/mm/memory.c b/mm/memory.c index 1c45b6a42a1b..75e0abda8642 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3452,7 +3452,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) folio_put(new_folio); if (old_folio) { if (page_copied) - free_swap_cache(&old_folio->page); + free_swap_cache(old_folio); folio_put(old_folio); } diff --git a/mm/swap_state.c b/mm/swap_state.c index 96b5c585f047..bfc7e8c58a6d 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -283,10 +283,8 @@ void clear_shadow_from_swap_cache(int type, unsigned long begin, * folio_free_swap() _with_ the lock. * - Marcelo */ -void free_swap_cache(struct page *page) +void free_swap_cache(struct folio *folio) { - struct folio *folio = page_folio(page); - if (folio_test_swapcache(folio) && !folio_mapped(folio) && folio_trylock(folio)) { folio_free_swap(folio); @@ -300,9 +298,11 @@ void free_swap_cache(struct page *page) */ void free_page_and_swap_cache(struct page *page) { - free_swap_cache(page); + struct folio *folio = page_folio(page); + + free_swap_cache(folio); if (!is_huge_zero_page(page)) - put_page(page); + folio_put(folio); } /* @@ -319,7 +319,7 @@ void free_pages_and_swap_cache(struct encoded_page **pages, int nr) for (int i = 0; i < nr; i++) { struct folio *folio = page_folio(encoded_page_ptr(pages[i])); - free_swap_cache(&folio->page); + free_swap_cache(folio); refs[folios.nr] = 1; if (unlikely(encoded_page_flags(pages[i]) & ENCODED_PAGE_BIT_NR_PAGES_NEXT)) -- cgit v1.2.3 From 8f8cd6c0a43ed637e620bbe45a8d0e0c2f4d5130 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Tue, 27 Feb 2024 10:35:46 +0800 Subject: modules: wait do_free_init correctly The synchronization here is to ensure the ordering of freeing of a module init so that it happens before W+X checking. It is worth noting it is not that the freeing was not happening, it is just that our sanity checkers raced against the permission checkers which assume init memory is already gone. Commit 1a7b7d922081 ("modules: Use vmalloc special flag") moved calling do_free_init() into a global workqueue instead of relying on it being called through call_rcu(..., do_free_init), which used to allowed us call do_free_init() asynchronously after the end of a subsequent grace period. The move to a global workqueue broke the gaurantees for code which needed to be sure the do_free_init() would complete with rcu_barrier(). To fix this callers which used to rely on rcu_barrier() must now instead use flush_work(&init_free_wq). Without this fix, we still could encounter false positive reports in W+X checking since the rcu_barrier() here can not ensure the ordering now. Even worse, the rcu_barrier() can introduce significant delay. Eric Chanudet reported that the rcu_barrier introduces ~0.1s delay on a PREEMPT_RT kernel. [ 0.291444] Freeing unused kernel memory: 5568K [ 0.402442] Run /sbin/init as init process With this fix, the above delay can be eliminated. Link: https://lkml.kernel.org/r/20240227023546.2490667-1-changbin.du@huawei.com Fixes: 1a7b7d922081 ("modules: Use vmalloc special flag") Signed-off-by: Changbin Du Tested-by: Eric Chanudet Acked-by: Luis Chamberlain Cc: Xiaoyi Su Signed-off-by: Andrew Morton --- include/linux/moduleloader.h | 8 ++++++++ init/main.c | 5 +++-- kernel/module/main.c | 9 +++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index 001b2ce83832..89b1e0ed9811 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -115,6 +115,14 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod); +#ifdef CONFIG_MODULES +void flush_module_init_free_work(void); +#else +static inline void flush_module_init_free_work(void) +{ +} +#endif + /* Any cleanup needed when module leaves. */ void module_arch_cleanup(struct module *mod); diff --git a/init/main.c b/init/main.c index 749a9f8d2c9b..5ecd4e8cf6d3 100644 --- a/init/main.c +++ b/init/main.c @@ -88,6 +88,7 @@ #include #include #include +#include #include #include #include @@ -1403,11 +1404,11 @@ static void mark_readonly(void) if (rodata_enabled) { /* * load_module() results in W+X mappings, which are cleaned - * up with call_rcu(). Let's make sure that queued work is + * up with init_free_wq. Let's make sure that queued work is * flushed so that we don't hit false positives looking for * insecure pages which are W+X. */ - rcu_barrier(); + flush_module_init_free_work(); mark_rodata_ro(); debug_checkwx(); rodata_test(); diff --git a/kernel/module/main.c b/kernel/module/main.c index 36681911c05a..b0b99348e1a8 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2489,6 +2489,11 @@ static void do_free_init(struct work_struct *w) } } +void flush_module_init_free_work(void) +{ + flush_work(&init_free_wq); +} + #undef MODULE_PARAM_PREFIX #define MODULE_PARAM_PREFIX "module." /* Default value for module->async_probe_requested */ @@ -2593,8 +2598,8 @@ static noinline int do_init_module(struct module *mod) * Note that module_alloc() on most architectures creates W+X page * mappings which won't be cleaned up until do_free_init() runs. Any * code such as mark_rodata_ro() which depends on those mappings to - * be cleaned up needs to sync with the queued work - ie - * rcu_barrier() + * be cleaned up needs to sync with the queued work by invoking + * flush_module_init_free_work(). */ if (llist_add(&freeinit->node, &init_free_list)) schedule_work(&init_free_wq); -- cgit v1.2.3 From 345a6e2631c1267221b684e110bba03e4c26ece0 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 1 Mar 2024 17:19:45 +0000 Subject: tcp: align tcp_sock_write_rx group Stephen Rothwell and kernel test robot reported that some arches (parisc, hexagon) and/or compilers would not like blamed commit. Lets make sure tcp_sock_write_rx group does not start with a hole. While we are at it, correct tcp_sock_write_tx CACHELINE_ASSERT_GROUP_SIZE() since after the blamed commit, we went to 105 bytes. Fixes: 99123622050f ("tcp: remove some holes in struct tcp_sock") Reported-by: Stephen Rothwell Reported-by: kernel test robot Link: https://lore.kernel.org/netdev/20240301121108.5d39e4f9@canb.auug.org.au/ Closes: https://lore.kernel.org/oe-kbuild-all/202403011451.csPYOS3C-lkp@intel.com/ Signed-off-by: Eric Dumazet Reviewed-by: Simon Horman Tested-by: Simon Horman # build-tested Link: https://lore.kernel.org/r/20240301171945.2958176-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/tcp.h | 2 +- net/ipv4/tcp.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 988a30ef6bfe..55399ee2a57e 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -304,7 +304,7 @@ struct tcp_sock { __cacheline_group_end(tcp_sock_write_txrx); /* RX read-write hotpath cache lines */ - __cacheline_group_begin(tcp_sock_write_rx); + __cacheline_group_begin(tcp_sock_write_rx) __aligned(8); u64 bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived * sum(delta(rcv_nxt)), or how many bytes diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index c82dc42f57c6..7e1b848398d0 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4651,7 +4651,7 @@ static void __init tcp_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags); - CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 113); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 105); /* TXRX read-write hotpath cache lines */ CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags); -- cgit v1.2.3 From 411c5f36805c02c7c412f1ad6bfa4459a1148011 Mon Sep 17 00:00:00 2001 From: Yunsheng Lin Date: Wed, 28 Feb 2024 17:30:08 +0800 Subject: mm/page_alloc: modify page_frag_alloc_align() to accept align as an argument napi_alloc_frag_align() and netdev_alloc_frag_align() accept align as an argument, and they are thin wrappers around the __napi_alloc_frag_align() and __netdev_alloc_frag_align() APIs doing the alignment checking and align mask conversion, in order to call page_frag_alloc_align() directly. The intention here is to keep the alignment checking and the alignmask conversion in in-line wrapper to avoid those kind of operations during execution time since it can usually be handled during compile time. We are going to use page_frag_alloc_align() in vhost_net.c, it need the same kind of alignment checking and alignmask conversion, so split up page_frag_alloc_align into an inline wrapper doing the above operation, and add __page_frag_alloc_align() which is passed with the align mask the original function expected as suggested by Alexander. Signed-off-by: Yunsheng Lin CC: Alexander Duyck Acked-by: Michael S. Tsirkin Signed-off-by: Paolo Abeni --- include/linux/gfp.h | 15 +++++++++++---- mm/page_alloc.c | 8 ++++---- net/core/skbuff.c | 9 ++++++--- 3 files changed, 21 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index de292a007138..28aea17fa59b 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -312,14 +312,21 @@ extern void free_pages(unsigned long addr, unsigned int order); struct page_frag_cache; extern void __page_frag_cache_drain(struct page *page, unsigned int count); -extern void *page_frag_alloc_align(struct page_frag_cache *nc, - unsigned int fragsz, gfp_t gfp_mask, - unsigned int align_mask); +void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz, + gfp_t gfp_mask, unsigned int align_mask); + +static inline void *page_frag_alloc_align(struct page_frag_cache *nc, + unsigned int fragsz, gfp_t gfp_mask, + unsigned int align) +{ + WARN_ON_ONCE(!is_power_of_2(align)); + return __page_frag_alloc_align(nc, fragsz, gfp_mask, -align); +} static inline void *page_frag_alloc(struct page_frag_cache *nc, unsigned int fragsz, gfp_t gfp_mask) { - return page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u); + return __page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u); } extern void page_frag_free(void *addr); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 150d4f23b010..c0f7e67c4250 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4708,9 +4708,9 @@ void __page_frag_cache_drain(struct page *page, unsigned int count) } EXPORT_SYMBOL(__page_frag_cache_drain); -void *page_frag_alloc_align(struct page_frag_cache *nc, - unsigned int fragsz, gfp_t gfp_mask, - unsigned int align_mask) +void *__page_frag_alloc_align(struct page_frag_cache *nc, + unsigned int fragsz, gfp_t gfp_mask, + unsigned int align_mask) { unsigned int size = PAGE_SIZE; struct page *page; @@ -4779,7 +4779,7 @@ refill: return nc->va + offset; } -EXPORT_SYMBOL(page_frag_alloc_align); +EXPORT_SYMBOL(__page_frag_alloc_align); /* * Frees a page fragment allocated out of either a compound or order 0 page. diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1f918e602bc4..43d7fc150acc 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -315,7 +315,8 @@ void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) fragsz = SKB_DATA_ALIGN(fragsz); - return page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); + return __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, + align_mask); } EXPORT_SYMBOL(__napi_alloc_frag_align); @@ -327,13 +328,15 @@ void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) if (in_hardirq() || irqs_disabled()) { struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); - data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); + data = __page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, + align_mask); } else { struct napi_alloc_cache *nc; local_bh_disable(); nc = this_cpu_ptr(&napi_alloc_cache); - data = page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, align_mask); + data = __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC, + align_mask); local_bh_enable(); } return data; -- cgit v1.2.3 From a0727489ac22d6fbd2e390d38a51193bba61da83 Mon Sep 17 00:00:00 2001 From: Yunsheng Lin Date: Wed, 28 Feb 2024 17:30:10 +0800 Subject: net: introduce page_frag_cache_drain() When draining a page_frag_cache, most user are doing the similar steps, so introduce an API to avoid code duplication. Signed-off-by: Yunsheng Lin Acked-by: Jason Wang Reviewed-by: Alexander Duyck Acked-by: Michael S. Tsirkin Signed-off-by: Paolo Abeni --- drivers/net/ethernet/google/gve/gve_main.c | 11 ++--------- drivers/net/ethernet/mediatek/mtk_wed_wo.c | 17 ++--------------- drivers/nvme/host/tcp.c | 7 +------ drivers/nvme/target/tcp.c | 4 +--- include/linux/gfp.h | 1 + mm/page_alloc.c | 10 ++++++++++ 6 files changed, 17 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 7b89b66adb53..166bd827a6d7 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -1276,17 +1276,10 @@ static void gve_unreg_xdp_info(struct gve_priv *priv) static void gve_drain_page_cache(struct gve_priv *priv) { - struct page_frag_cache *nc; int i; - for (i = 0; i < priv->rx_cfg.num_queues; i++) { - nc = &priv->rx[i].page_cache; - if (nc->va) { - __page_frag_cache_drain(virt_to_page(nc->va), - nc->pagecnt_bias); - nc->va = NULL; - } - } + for (i = 0; i < priv->rx_cfg.num_queues; i++) + page_frag_cache_drain(&priv->rx[i].page_cache); } static void gve_qpls_get_curr_alloc_cfg(struct gve_priv *priv, diff --git a/drivers/net/ethernet/mediatek/mtk_wed_wo.c b/drivers/net/ethernet/mediatek/mtk_wed_wo.c index d58b07e7e123..7063c78bd35f 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_wo.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_wo.c @@ -286,7 +286,6 @@ mtk_wed_wo_queue_free(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q) static void mtk_wed_wo_queue_tx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q) { - struct page *page; int i; for (i = 0; i < q->n_desc; i++) { @@ -301,19 +300,12 @@ mtk_wed_wo_queue_tx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q) entry->buf = NULL; } - if (!q->cache.va) - return; - - page = virt_to_page(q->cache.va); - __page_frag_cache_drain(page, q->cache.pagecnt_bias); - memset(&q->cache, 0, sizeof(q->cache)); + page_frag_cache_drain(&q->cache); } static void mtk_wed_wo_queue_rx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q) { - struct page *page; - for (;;) { void *buf = mtk_wed_wo_dequeue(wo, q, NULL, true); @@ -323,12 +315,7 @@ mtk_wed_wo_queue_rx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q) skb_free_frag(buf); } - if (!q->cache.va) - return; - - page = virt_to_page(q->cache.va); - __page_frag_cache_drain(page, q->cache.pagecnt_bias); - memset(&q->cache, 0, sizeof(q->cache)); + page_frag_cache_drain(&q->cache); } static void diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index a6d596e05602..3692b56cb58d 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1344,7 +1344,6 @@ static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) { - struct page *page; struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); struct nvme_tcp_queue *queue = &ctrl->queues[qid]; unsigned int noreclaim_flag; @@ -1355,11 +1354,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) if (queue->hdr_digest || queue->data_digest) nvme_tcp_free_crypto(queue); - if (queue->pf_cache.va) { - page = virt_to_head_page(queue->pf_cache.va); - __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); - queue->pf_cache.va = NULL; - } + page_frag_cache_drain(&queue->pf_cache); noreclaim_flag = memalloc_noreclaim_save(); /* ->sock will be released by fput() */ diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index c8655fc5aa5b..2aa5762e9f50 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1591,7 +1591,6 @@ static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue) static void nvmet_tcp_release_queue_work(struct work_struct *w) { - struct page *page; struct nvmet_tcp_queue *queue = container_of(w, struct nvmet_tcp_queue, release_work); @@ -1615,8 +1614,7 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w) if (queue->hdr_digest || queue->data_digest) nvmet_tcp_free_crypto(queue); ida_free(&nvmet_tcp_queue_ida, queue->idx); - page = virt_to_head_page(queue->pf_cache.va); - __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); + page_frag_cache_drain(&queue->pf_cache); kfree(queue); } diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 28aea17fa59b..6cef1c241180 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -311,6 +311,7 @@ extern void __free_pages(struct page *page, unsigned int order); extern void free_pages(unsigned long addr, unsigned int order); struct page_frag_cache; +void page_frag_cache_drain(struct page_frag_cache *nc); extern void __page_frag_cache_drain(struct page *page, unsigned int count); void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz, gfp_t gfp_mask, unsigned int align_mask); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 636145c29f70..06aa1ebbd21c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4699,6 +4699,16 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, return page; } +void page_frag_cache_drain(struct page_frag_cache *nc) +{ + if (!nc->va) + return; + + __page_frag_cache_drain(virt_to_head_page(nc->va), nc->pagecnt_bias); + nc->va = NULL; +} +EXPORT_SYMBOL(page_frag_cache_drain); + void __page_frag_cache_drain(struct page *page, unsigned int count) { VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); -- cgit v1.2.3 From 88803989ff6d73155969df94cc84f1ecd28c4d9c Mon Sep 17 00:00:00 2001 From: Yang Xiwen Date: Thu, 29 Feb 2024 09:36:19 +0800 Subject: mmc: core: Use a struct device* as in-param to mmc_of_parse_clk_phase() Parsing dt usually happens very early, sometimes even before the struct mmc_host has been allocated (e.g. dw_mci_probe() and dw_mci_parse_dt() in dw_mmc.c). Looking at the source of mmc_of_parse_clk_phase(), it's actually not needed to have an initialized mmc_host, let's therefore pass a struct device* to it instead. Also update the only current user, sdhci-of-aspeed. Reviewed-by: Paul Menzel Acked-by: Andrew Jeffery Signed-off-by: Yang Xiwen Link: https://lore.kernel.org/r/20240229-b4-mmc-hi3798mv200-v7-1-10c03f316285@outlook.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/host.c | 4 +--- drivers/mmc/host/sdhci-of-aspeed.c | 2 +- include/linux/mmc/host.h | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 7cc9a33d28ca..3aa9dda6fd4f 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -234,10 +234,8 @@ static void mmc_of_parse_timing_phase(struct device *dev, const char *prop, } void -mmc_of_parse_clk_phase(struct mmc_host *host, struct mmc_clk_phase_map *map) +mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map) { - struct device *dev = host->parent; - mmc_of_parse_timing_phase(dev, "clk-phase-legacy", &map->phase[MMC_TIMING_LEGACY]); mmc_of_parse_timing_phase(dev, "clk-phase-mmc-hs", diff --git a/drivers/mmc/host/sdhci-of-aspeed.c b/drivers/mmc/host/sdhci-of-aspeed.c index 42d54532cabe..430c1f90037b 100644 --- a/drivers/mmc/host/sdhci-of-aspeed.c +++ b/drivers/mmc/host/sdhci-of-aspeed.c @@ -435,7 +435,7 @@ static int aspeed_sdhci_probe(struct platform_device *pdev) goto err_sdhci_add; if (dev->phase_desc) - mmc_of_parse_clk_phase(host->mmc, &dev->phase_map); + mmc_of_parse_clk_phase(&pdev->dev, &dev->phase_map); ret = sdhci_add_host(host); if (ret) diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 2f445c651742..5894bf912f7b 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -539,7 +539,7 @@ struct mmc_host *devm_mmc_alloc_host(struct device *dev, int extra); int mmc_add_host(struct mmc_host *); void mmc_remove_host(struct mmc_host *); void mmc_free_host(struct mmc_host *); -void mmc_of_parse_clk_phase(struct mmc_host *host, +void mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map); int mmc_of_parse(struct mmc_host *host); int mmc_of_parse_voltage(struct mmc_host *host, u32 *mask); -- cgit v1.2.3 From 885c36e59f46375c138de18ff1692f18eff67b7f Mon Sep 17 00:00:00 2001 From: Abhishek Chauhan Date: Fri, 1 Mar 2024 12:13:48 -0800 Subject: net: Re-use and set mono_delivery_time bit for userspace tstamp packets Bridge driver today has no support to forward the userspace timestamp packets and ends up resetting the timestamp. ETF qdisc checks the packet coming from userspace and encounters to be 0 thereby dropping time sensitive packets. These changes will allow userspace timestamps packets to be forwarded from the bridge to NIC drivers. Setting the same bit (mono_delivery_time) to avoid dropping of userspace tstamp packets in the forwarding path. Existing functionality of mono_delivery_time remains unaltered here, instead just extended with userspace tstamp support for bridge forwarding path. Signed-off-by: Abhishek Chauhan Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20240301201348.2815102-1-quic_abchauha@quicinc.com Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 6 +++--- net/ipv4/ip_output.c | 1 + net/ipv4/raw.c | 1 + net/ipv6/ip6_output.c | 2 +- net/ipv6/raw.c | 2 +- net/packet/af_packet.c | 4 +++- 6 files changed, 10 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d577e0bee18d..3013355b63f5 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -822,9 +822,9 @@ typedef unsigned char *sk_buff_data_t; * @decrypted: Decrypted SKB * @slow_gro: state present at GRO time, slower prepare step required * @mono_delivery_time: When set, skb->tstamp has the - * delivery_time in mono clock base (i.e. EDT). Otherwise, the - * skb->tstamp has the (rcv) timestamp at ingress and - * delivery_time at egress. + * delivery_time in mono clock base (i.e., EDT) or a clock base chosen + * by SO_TXTIME. If zero, skb->tstamp has the (rcv) timestamp at + * ingress. * @napi_id: id of the NAPI struct this skb came from * @sender_cpu: (aka @napi_id) source CPU in XPS * @alloc_cpu: CPU which did the skb allocation. diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 1fe794967211..33f93dc730a3 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1458,6 +1458,7 @@ struct sk_buff *__ip_make_skb(struct sock *sk, skb->priority = (cork->tos != -1) ? cork->priority: READ_ONCE(sk->sk_priority); skb->mark = cork->mark; skb->tstamp = cork->transmit_time; + skb->mono_delivery_time = !!skb->tstamp; /* * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec * on dst refcount diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index a82fd102df05..494a6284bd7e 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -353,6 +353,7 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4, skb->priority = READ_ONCE(sk->sk_priority); skb->mark = sockc->mark; skb->tstamp = sockc->transmit_time; + skb->mono_delivery_time = !!skb->tstamp; skb_dst_set(skb, &rt->dst); *rtp = NULL; diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index b9dd3a66e423..02eeca5492cd 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1925,7 +1925,7 @@ struct sk_buff *__ip6_make_skb(struct sock *sk, skb->priority = READ_ONCE(sk->sk_priority); skb->mark = cork->base.mark; skb->tstamp = cork->base.transmit_time; - + skb->mono_delivery_time = !!skb->tstamp; ip6_cork_steal_dst(skb, cork); IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS); if (proto == IPPROTO_ICMPV6) { diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 76e6eb3b643d..779274055abf 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -615,7 +615,7 @@ static int rawv6_send_hdrinc(struct sock *sk, struct msghdr *msg, int length, skb->priority = READ_ONCE(sk->sk_priority); skb->mark = sockc->mark; skb->tstamp = sockc->transmit_time; - + skb->mono_delivery_time = !!skb->tstamp; skb_put(skb, length); skb_reset_network_header(skb); iph = ipv6_hdr(skb); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index c9bbc2686690..0db31ca4982d 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2057,7 +2057,7 @@ retry: skb->priority = READ_ONCE(sk->sk_priority); skb->mark = READ_ONCE(sk->sk_mark); skb->tstamp = sockc.transmit_time; - + skb->mono_delivery_time = !!skb->tstamp; skb_setup_tx_timestamp(skb, sockc.tsflags); if (unlikely(extra_len == 4)) @@ -2586,6 +2586,7 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, skb->priority = READ_ONCE(po->sk.sk_priority); skb->mark = READ_ONCE(po->sk.sk_mark); skb->tstamp = sockc->transmit_time; + skb->mono_delivery_time = !!skb->tstamp; skb_setup_tx_timestamp(skb, sockc->tsflags); skb_zcopy_set_nouarg(skb, ph.raw); @@ -3064,6 +3065,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) skb->priority = READ_ONCE(sk->sk_priority); skb->mark = sockc.mark; skb->tstamp = sockc.transmit_time; + skb->mono_delivery_time = !!skb->tstamp; if (unlikely(extra_len == 4)) skb->no_fcs = 1; -- cgit v1.2.3 From d28240785e00ffb889d90368882bf382e22e9555 Mon Sep 17 00:00:00 2001 From: Oliver Neukum Date: Thu, 29 Feb 2024 14:17:33 +0100 Subject: usb: typec: pd: no opencoding of FIELD_GET If we have a neat macro, at least new code should use it. Signed-off-by: Oliver Neukum Link: https://lore.kernel.org/r/20240229131851.16148-2-oneukum@suse.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/pd_vdo.h | 5 +++-- include/linux/usb/typec_dp.h | 11 ++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/pd_vdo.h b/include/linux/usb/pd_vdo.h index c09c5a12e273..5c48e8a81403 100644 --- a/include/linux/usb/pd_vdo.h +++ b/include/linux/usb/pd_vdo.h @@ -7,6 +7,7 @@ #define __LINUX_USB_PD_VDO_H #include "pd.h" +#include /* * VDO : Vendor Defined Message Object @@ -188,7 +189,7 @@ * <5:3> :: Alternate modes * <2:0> :: USB highest speed */ -#define PD_VDO_UFP_DEVCAP(vdo) (((vdo) & GENMASK(27, 24)) >> 24) +#define PD_VDO_UFP_DEVCAP(vdo) FIELD_GET(GENMASK(27, 24), vdo) /* UFP VDO Version */ #define UFP_VDO_VER1_2 2 @@ -247,7 +248,7 @@ * <21:5> :: Reserved * <4:0> :: Port number */ -#define PD_VDO_DFP_HOSTCAP(vdo) (((vdo) & GENMASK(26, 24)) >> 24) +#define PD_VDO_DFP_HOSTCAP(vdo) FIELD_GET(GENMASK(26, 24), vdo) #define DFP_VDO_VER1_1 1 #define HOST_USB2_CAPABLE BIT(0) diff --git a/include/linux/usb/typec_dp.h b/include/linux/usb/typec_dp.h index 1f358098522d..f2da264d9c14 100644 --- a/include/linux/usb/typec_dp.h +++ b/include/linux/usb/typec_dp.h @@ -3,6 +3,7 @@ #define __USB_TYPEC_DP_H #include +#include #define USB_TYPEC_DP_SID 0xff01 /* USB IF has not assigned a Standard ID (SID) for VirtualLink, @@ -67,21 +68,21 @@ enum { #define DP_CAP_UFP_D 1 #define DP_CAP_DFP_D 2 #define DP_CAP_DFP_D_AND_UFP_D 3 -#define DP_CAP_DP_SIGNALLING(_cap_) (((_cap_) & GENMASK(5, 2)) >> 2) +#define DP_CAP_DP_SIGNALLING(_cap_) FIELD_GET(GENMASK(5, 2), _cap_) #define DP_CAP_SIGNALLING_HBR3 1 #define DP_CAP_SIGNALLING_UHBR10 2 #define DP_CAP_SIGNALLING_UHBR20 3 #define DP_CAP_RECEPTACLE BIT(6) #define DP_CAP_USB BIT(7) -#define DP_CAP_DFP_D_PIN_ASSIGN(_cap_) (((_cap_) & GENMASK(15, 8)) >> 8) -#define DP_CAP_UFP_D_PIN_ASSIGN(_cap_) (((_cap_) & GENMASK(23, 16)) >> 16) +#define DP_CAP_DFP_D_PIN_ASSIGN(_cap_) FIELD_GET(GENMASK(15, 8), _cap_) +#define DP_CAP_UFP_D_PIN_ASSIGN(_cap_) FIELD_GET(GENMASK(23, 16), _cap_) /* Get pin assignment taking plug & receptacle into consideration */ #define DP_CAP_PIN_ASSIGN_UFP_D(_cap_) ((_cap_ & DP_CAP_RECEPTACLE) ? \ DP_CAP_UFP_D_PIN_ASSIGN(_cap_) : DP_CAP_DFP_D_PIN_ASSIGN(_cap_)) #define DP_CAP_PIN_ASSIGN_DFP_D(_cap_) ((_cap_ & DP_CAP_RECEPTACLE) ? \ DP_CAP_DFP_D_PIN_ASSIGN(_cap_) : DP_CAP_UFP_D_PIN_ASSIGN(_cap_)) #define DP_CAP_UHBR_13_5_SUPPORT BIT(26) -#define DP_CAP_CABLE_TYPE(_cap_) (((_cap_) & GENMASK(29, 28)) >> 28) +#define DP_CAP_CABLE_TYPE(_cap_) FIELD_GET(GENMASK(29, 28), _cap_) #define DP_CAP_CABLE_TYPE_PASSIVE 0 #define DP_CAP_CABLE_TYPE_RE_TIMER 1 #define DP_CAP_CABLE_TYPE_RE_DRIVER 2 @@ -116,7 +117,7 @@ enum { /* Helper for setting/getting the pin assignment value to the configuration */ #define DP_CONF_SET_PIN_ASSIGN(_a_) ((_a_) << 8) -#define DP_CONF_GET_PIN_ASSIGN(_conf_) (((_conf_) & GENMASK(15, 8)) >> 8) +#define DP_CONF_GET_PIN_ASSIGN(_conf_) FIELD_GET(GENMASK(15, 8), _conf_) #define DP_CONF_UHBR13_5_SUPPORT BIT(26) #define DP_CONF_CABLE_TYPE_MASK GENMASK(29, 28) #define DP_CONF_CABLE_TYPE_SHIFT 28 -- cgit v1.2.3 From 82e82130a78b75a9ce5225df24d5a0b1b3290eb0 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Thu, 22 Feb 2024 16:58:21 -0800 Subject: usb: core: Set connect_type of ports based on DT node When a USB hub is described in DT, such as any device that matches the onboard-hub driver, the connect_type is set to "unknown" or USB_PORT_CONNECT_TYPE_UNKNOWN. This makes any device plugged into that USB port report their 'removable' device attribute as "unknown". ChromeOS userspace would like to know if the USB device is actually removable or not so that security policies can be applied. Improve the connect_type attribute for ports, and in turn the removable attribute for USB devices, by looking for child devices with a reg property or an OF graph when the device is described in DT. If the graph exists, endpoints that are connected to a remote node must be something like a usb-{a,b,c}-connector compatible node, or an intermediate node like a redriver, and not a hardwired USB device on the board. Set the connect_type to USB_PORT_CONNECT_TYPE_HOT_PLUG in this case because the device is going to be plugged in. Set the connect_type to USB_PORT_CONNECT_TYPE_HARD_WIRED if there's a child node for the port like 'device@2' for port2. Set the connect_type to USB_PORT_NOT_USED if there isn't an endpoint or child node corresponding to the port number. To make sure things don't change, only set the port to not used if there are child nodes. This way an onboard hub connect_type doesn't change until ports are added or child nodes are added to describe hardwired devices. It's assumed that all ports or no ports will be described for a device. Cc: Matthias Kaehlcke Cc: linux-usb@vger.kernel.org Cc: devicetree@vger.kernel.org Cc: Pin-yen Lin Cc: maciek swiech Signed-off-by: Stephen Boyd Link: https://lore.kernel.org/r/20240223005823.3074029-3-swboyd@chromium.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/of.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ drivers/usb/core/port.c | 2 ++ include/linux/usb/of.h | 7 +++++ 3 files changed, 80 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/core/of.c b/drivers/usb/core/of.c index db4ccf9ce3d9..f1a499ee482c 100644 --- a/drivers/usb/core/of.c +++ b/drivers/usb/core/of.c @@ -8,6 +8,7 @@ */ #include +#include #include /** @@ -75,6 +76,76 @@ bool usb_of_has_combined_node(struct usb_device *udev) } EXPORT_SYMBOL_GPL(usb_of_has_combined_node); +static bool usb_of_has_devices_or_graph(const struct usb_device *hub) +{ + const struct device_node *np = hub->dev.of_node; + struct device_node *child; + + if (of_graph_is_present(np)) + return true; + + for_each_child_of_node(np, child) + if (of_property_present(child, "reg")) + return true; + + return false; +} + +/** + * usb_of_get_connect_type() - get a USB hub's port connect_type + * @hub: hub to which port is for @port1 + * @port1: one-based index of port + * + * Get the connect_type of @port1 based on the device node for @hub. If the + * port is described in the OF graph, the connect_type is "hotplug". If the + * @hub has a child device has with a 'reg' property equal to @port1 the + * connect_type is "hard-wired". If there isn't an OF graph or child node at + * all then the connect_type is "unknown". Otherwise, the port is considered + * "unused" because it isn't described at all. + * + * Return: A connect_type for @port1 based on the device node for @hub. + */ +enum usb_port_connect_type usb_of_get_connect_type(struct usb_device *hub, int port1) +{ + struct device_node *np, *child, *ep, *remote_np; + enum usb_port_connect_type connect_type; + + /* Only set connect_type if binding has ports/hardwired devices. */ + if (!usb_of_has_devices_or_graph(hub)) + return USB_PORT_CONNECT_TYPE_UNKNOWN; + + /* Assume port is unused if there's a graph or a child node. */ + connect_type = USB_PORT_NOT_USED; + + np = hub->dev.of_node; + /* + * Hotplug ports are connected to an available remote node, e.g. + * usb-a-connector compatible node, in the OF graph. + */ + if (of_graph_is_present(np)) { + ep = of_graph_get_endpoint_by_regs(np, port1, -1); + if (ep) { + remote_np = of_graph_get_remote_port_parent(ep); + of_node_put(ep); + if (of_device_is_available(remote_np)) + connect_type = USB_PORT_CONNECT_TYPE_HOT_PLUG; + of_node_put(remote_np); + } + } + + /* + * Hard-wired ports are child nodes with a reg property corresponding + * to the port number, i.e. a usb device. + */ + child = usb_of_get_device_node(hub, port1); + if (of_device_is_available(child)) + connect_type = USB_PORT_CONNECT_TYPE_HARD_WIRED; + of_node_put(child); + + return connect_type; +} +EXPORT_SYMBOL_GPL(usb_of_get_connect_type); + /** * usb_of_get_interface_node() - get a USB interface node * @udev: USB device of interface diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 84d36172b040..e1613b02c19f 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "hub.h" @@ -708,6 +709,7 @@ int usb_hub_create_port_device(struct usb_hub *hub, int port1) return -ENOMEM; } + port_dev->connect_type = usb_of_get_connect_type(hdev, port1); hub->ports[port1 - 1] = port_dev; port_dev->portnum = port1; set_bit(port1, hub->power_bits); diff --git a/include/linux/usb/of.h b/include/linux/usb/of.h index 98487fd7ab11..de42f14bd280 100644 --- a/include/linux/usb/of.h +++ b/include/linux/usb/of.h @@ -6,6 +6,7 @@ #ifndef __LINUX_USB_OF_H #define __LINUX_USB_OF_H +#include #include #include #include @@ -17,6 +18,7 @@ enum usb_dr_mode of_usb_get_dr_mode_by_phy(struct device_node *np, int arg0); bool of_usb_host_tpl_support(struct device_node *np); int of_usb_update_otg_caps(struct device_node *np, struct usb_otg_caps *otg_caps); +enum usb_port_connect_type usb_of_get_connect_type(struct usb_device *hub, int port1); struct device_node *usb_of_get_device_node(struct usb_device *hub, int port1); bool usb_of_has_combined_node(struct usb_device *udev); struct device_node *usb_of_get_interface_node(struct usb_device *udev, @@ -37,6 +39,11 @@ static inline int of_usb_update_otg_caps(struct device_node *np, { return 0; } +static inline enum usb_port_connect_type +usb_of_get_connect_type(const struct usb_device *hub, int port1) +{ + return USB_PORT_CONNECT_TYPE_UNKNOWN; +} static inline struct device_node * usb_of_get_device_node(struct usb_device *hub, int port1) { -- cgit v1.2.3 From ac92ea6b656374abab230f9f38fd3f0ab6cd0d61 Mon Sep 17 00:00:00 2001 From: Marco Felsch Date: Thu, 22 Feb 2024 22:09:02 +0100 Subject: usb: typec: tcpm: add support to set tcpc connector orientatition This adds the support to set the connector orientation value accordingly. This is part of the optional CONFIG_STANDARD_OUTPUT register 0x18, specified within the USB port controller spsicification rev. 2.0 [1]. [1] https://www.usb.org/sites/default/files/documents/usb-port_controller_specification_rev2.0_v1.0_0.pdf Signed-off-by: Marco Felsch Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240222210903.208901-4-m.felsch@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 6 ++++++ include/linux/usb/tcpm.h | 2 ++ 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index b6d7131228f9..3d505614bff1 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1167,6 +1167,12 @@ static int tcpm_set_roles(struct tcpm_port *port, bool attached, if (ret < 0) return ret; + if (port->tcpc->set_orientation) { + ret = port->tcpc->set_orientation(port->tcpc, orientation); + if (ret < 0) + return ret; + } + port->pwr_role = role; port->data_role = data; typec_set_data_role(port->typec_port, data); diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index 6671427f7eeb..061da9546a81 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -144,6 +144,8 @@ struct tcpc_dev { enum typec_cc_status *cc2); int (*set_polarity)(struct tcpc_dev *dev, enum typec_cc_polarity polarity); + int (*set_orientation)(struct tcpc_dev *dev, + enum typec_orientation orientation); int (*set_vconn)(struct tcpc_dev *dev, bool on); int (*set_vbus)(struct tcpc_dev *dev, bool on, bool charge); int (*set_current_limit)(struct tcpc_dev *dev, u32 max_ma, u32 mv); -- cgit v1.2.3 From 7bfb915a597a301abb892f620fe5c283a9fdbd77 Mon Sep 17 00:00:00 2001 From: Jonas Gorski Date: Sun, 3 Mar 2024 16:08:07 +0100 Subject: serial: core: only stop transmit when HW fifo is empty If the circular buffer is empty, it just means we fit all characters to send into the HW fifo, but not that the hardware finished transmitting them. So if we immediately call stop_tx() after that, this may abort any pending characters in the HW fifo, and cause dropped characters on the console. Fix this by only stopping tx when the tx HW fifo is actually empty. Fixes: 8275b48b2780 ("tty: serial: introduce transmit helpers") Cc: stable@vger.kernel.org Signed-off-by: Jonas Gorski Link: https://lore.kernel.org/r/20240303150807.68117-1-jonas.gorski@gmail.com Signed-off-by: Greg Kroah-Hartman --- include/linux/serial_core.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 55b1f3ba48ac..bb0f2d4ac62f 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -786,7 +786,8 @@ enum UART_TX_FLAGS { if (pending < WAKEUP_CHARS) { \ uart_write_wakeup(__port); \ \ - if (!((flags) & UART_TX_NOSTOP) && pending == 0) \ + if (!((flags) & UART_TX_NOSTOP) && pending == 0 && \ + __port->ops->tx_empty(__port)) \ __port->ops->stop_tx(__port); \ } \ \ -- cgit v1.2.3 From 35c822a34b2293aedf475238c395e75858d1e8c8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 4 Mar 2024 14:27:02 +0200 Subject: serial: core: Move struct uart_port::quirks closer to possible values Currently it's not crystal clear what UPIO_* and UPQ_* definitions belong to. Reindent the code, so it will be easy to read and understand. No functional changes intended. Reviewed-by: Andi Shyti Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240304123035.758700-2-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/serial_core.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index bb0f2d4ac62f..f9d7f0a625fd 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -467,8 +467,8 @@ struct uart_port { unsigned int fifosize; /* tx fifo size */ unsigned char x_char; /* xon/xoff char */ unsigned char regshift; /* reg offset shift */ + unsigned char iotype; /* io access style */ - unsigned char quirks; /* internal quirks */ #define UPIO_PORT (SERIAL_IO_PORT) /* 8b I/O port access */ #define UPIO_HUB6 (SERIAL_IO_HUB6) /* Hub6 ISA card */ @@ -479,7 +479,9 @@ struct uart_port { #define UPIO_MEM32BE (SERIAL_IO_MEM32BE) /* 32b big endian */ #define UPIO_MEM16 (SERIAL_IO_MEM16) /* 16b little endian */ - /* quirks must be updated while holding port mutex */ + unsigned char quirks; /* internal quirks */ + + /* internal quirks must be updated while holding port mutex */ #define UPQ_NO_TXEN_TEST BIT(0) unsigned int read_status_mask; /* driver specific */ -- cgit v1.2.3 From 79d713baf63c8f23cc58b304c40be33d64a12aaf Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 4 Mar 2024 14:27:03 +0200 Subject: serial: core: Add UPIO_UNKNOWN constant for unknown port type In some APIs we would like to assign the special value to iotype and compare against it in another places. Introduce UPIO_UNKNOWN for this purpose. Note, we can't use 0, because it's a valid value for IO port access. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240304123035.758700-3-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/serial_core.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index f9d7f0a625fd..3b64c9a26945 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -470,6 +470,7 @@ struct uart_port { unsigned char iotype; /* io access style */ +#define UPIO_UNKNOWN ((unsigned char)~0U) /* UCHAR_MAX */ #define UPIO_PORT (SERIAL_IO_PORT) /* 8b I/O port access */ #define UPIO_HUB6 (SERIAL_IO_HUB6) /* Hub6 ISA card */ #define UPIO_MEM (SERIAL_IO_MEM) /* driver-specific */ -- cgit v1.2.3 From e894b6005dce0ed621b2788d6a249708fb6f95f9 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 4 Mar 2024 14:27:04 +0200 Subject: serial: port: Introduce a common helper to read properties Several serial drivers want to read the same or similar set of the port properties. Make a common helper for them. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240304123035.758700-4-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/serial_port.c | 145 +++++++++++++++++++++++++++++++++++++++ include/linux/serial_core.h | 2 + 2 files changed, 147 insertions(+) (limited to 'include/linux') diff --git a/drivers/tty/serial/serial_port.c b/drivers/tty/serial/serial_port.c index 88975a4df306..6779a008e67a 100644 --- a/drivers/tty/serial/serial_port.c +++ b/drivers/tty/serial/serial_port.c @@ -8,7 +8,10 @@ #include #include +#include +#include #include +#include #include #include @@ -82,6 +85,148 @@ void uart_remove_one_port(struct uart_driver *drv, struct uart_port *port) } EXPORT_SYMBOL(uart_remove_one_port); +/** + * __uart_read_properties - read firmware properties of the given UART port + * @port: corresponding port + * @use_defaults: apply defaults (when %true) or validate the values (when %false) + * + * The following device properties are supported: + * - clock-frequency (optional) + * - fifo-size (optional) + * - no-loopback-test (optional) + * - reg-shift (defaults may apply) + * - reg-offset (value may be validated) + * - reg-io-width (defaults may apply or value may be validated) + * - interrupts (OF only) + * - serial [alias ID] (OF only) + * + * If the port->dev is of struct platform_device type the interrupt line + * will be retrieved via platform_get_irq() call against that device. + * Otherwise it will be assigned by fwnode_irq_get() call. In both cases + * the index 0 of the resource is used. + * + * The caller is responsible to initialize the following fields of the @port + * ->dev (must be valid) + * ->flags + * ->mapbase + * ->mapsize + * ->regshift (if @use_defaults is false) + * before calling this function. Alternatively the above mentioned fields + * may be zeroed, in such case the only ones, that have associated properties + * found, will be set to the respective values. + * + * If no error happened, the ->irq, ->mapbase, ->mapsize will be altered. + * The ->iotype is always altered. + * + * When @use_defaults is true and the respective property is not found + * the following values will be applied: + * ->regshift = 0 + * In this case IRQ must be provided, otherwise an error will be returned. + * + * When @use_defaults is false and the respective property is found + * the following values will be validated: + * - reg-io-width (->iotype) + * - reg-offset (->mapsize against ->mapbase) + * + * Returns: 0 on success or negative errno on failure + */ +static int __uart_read_properties(struct uart_port *port, bool use_defaults) +{ + struct device *dev = port->dev; + u32 value; + int ret; + + /* Read optional UART functional clock frequency */ + device_property_read_u32(dev, "clock-frequency", &port->uartclk); + + /* Read the registers alignment (default: 8-bit) */ + ret = device_property_read_u32(dev, "reg-shift", &value); + if (ret) + port->regshift = use_defaults ? 0 : port->regshift; + else + port->regshift = value; + + /* Read the registers I/O access type (default: MMIO 8-bit) */ + ret = device_property_read_u32(dev, "reg-io-width", &value); + if (ret) { + port->iotype = UPIO_MEM; + } else { + switch (value) { + case 1: + port->iotype = UPIO_MEM; + break; + case 2: + port->iotype = UPIO_MEM16; + break; + case 4: + port->iotype = device_is_big_endian(dev) ? UPIO_MEM32BE : UPIO_MEM32; + break; + default: + if (!use_defaults) { + dev_err(dev, "Unsupported reg-io-width (%u)\n", value); + return -EINVAL; + } + port->iotype = UPIO_UNKNOWN; + break; + } + } + + /* Read the address mapping base offset (default: no offset) */ + ret = device_property_read_u32(dev, "reg-offset", &value); + if (ret) + value = 0; + + /* Check for shifted address mapping overflow */ + if (!use_defaults && port->mapsize < value) { + dev_err(dev, "reg-offset %u exceeds region size %pa\n", value, &port->mapsize); + return -EINVAL; + } + + port->mapbase += value; + port->mapsize -= value; + + /* Read optional FIFO size */ + device_property_read_u32(dev, "fifo-size", &port->fifosize); + + if (device_property_read_bool(dev, "no-loopback-test")) + port->flags |= UPF_SKIP_TEST; + + /* Get index of serial line, if found in DT aliases */ + ret = of_alias_get_id(dev_of_node(dev), "serial"); + if (ret >= 0) + port->line = ret; + + if (dev_is_platform(dev)) + ret = platform_get_irq(to_platform_device(dev), 0); + else + ret = fwnode_irq_get(dev_fwnode(dev), 0); + if (ret == -EPROBE_DEFER) + return ret; + if (ret > 0) + port->irq = ret; + else if (use_defaults) + /* By default IRQ support is mandatory */ + return ret; + else + port->irq = 0; + + port->flags |= UPF_SHARE_IRQ; + + return 0; +} + +int uart_read_port_properties(struct uart_port *port) +{ + return __uart_read_properties(port, true); +} +EXPORT_SYMBOL_GPL(uart_read_port_properties); + +int uart_read_and_validate_port_properties(struct uart_port *port) +{ + return __uart_read_properties(port, false); +} +EXPORT_SYMBOL_GPL(uart_read_and_validate_port_properties); + static struct device_driver serial_port_driver = { .name = "port", .suppress_bind_attrs = true, diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 3b64c9a26945..0a0f6e21d40e 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -963,6 +963,8 @@ int uart_register_driver(struct uart_driver *uart); void uart_unregister_driver(struct uart_driver *uart); int uart_add_one_port(struct uart_driver *reg, struct uart_port *port); void uart_remove_one_port(struct uart_driver *reg, struct uart_port *port); +int uart_read_port_properties(struct uart_port *port); +int uart_read_and_validate_port_properties(struct uart_port *port); bool uart_match_port(const struct uart_port *port1, const struct uart_port *port2); -- cgit v1.2.3 From 7ba59ac7da2aae2fbcfe90352ee40d30cecca10d Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 4 Mar 2024 13:19:45 -0800 Subject: greybus: Avoid fake flexible array for response data FORTIFY_SOURCE has been ignoring 0-sized destinations while the kernel code base has been converted to flexible arrays. In order to enforce the 0-sized destinations (e.g. with __counted_by), the remaining 0-sized destinations need to be handled. Instead of converting an empty struct into using a flexible array, just directly use a pointer without any additional indirection. Remove struct gb_bootrom_get_firmware_response and struct gb_fw_download_fetch_firmware_response. Signed-off-by: Kees Cook Reviewed-by: Alex Elder Link: https://lore.kernel.org/r/20240304211940.it.083-kees@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/greybus/bootrom.c | 8 ++++---- drivers/staging/greybus/fw-download.c | 8 ++++---- include/linux/greybus/greybus_protocols.h | 8 ++------ 3 files changed, 10 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/greybus/bootrom.c b/drivers/staging/greybus/bootrom.c index 79581457c4af..c0d338db6b52 100644 --- a/drivers/staging/greybus/bootrom.c +++ b/drivers/staging/greybus/bootrom.c @@ -243,10 +243,10 @@ static int gb_bootrom_get_firmware(struct gb_operation *op) struct gb_bootrom *bootrom = gb_connection_get_data(op->connection); const struct firmware *fw; struct gb_bootrom_get_firmware_request *firmware_request; - struct gb_bootrom_get_firmware_response *firmware_response; struct device *dev = &op->connection->bundle->dev; unsigned int offset, size; enum next_request_type next_request; + u8 *firmware_response; int ret = 0; /* Disable timeouts */ @@ -280,15 +280,15 @@ static int gb_bootrom_get_firmware(struct gb_operation *op) goto unlock; } - if (!gb_operation_response_alloc(op, sizeof(*firmware_response) + size, - GFP_KERNEL)) { + /* gb_bootrom_get_firmware_response contains only a byte array */ + if (!gb_operation_response_alloc(op, size, GFP_KERNEL)) { dev_err(dev, "%s: error allocating response\n", __func__); ret = -ENOMEM; goto unlock; } firmware_response = op->response->payload; - memcpy(firmware_response->data, fw->data + offset, size); + memcpy(firmware_response, fw->data + offset, size); dev_dbg(dev, "responding with firmware (offs = %u, size = %u)\n", offset, size); diff --git a/drivers/staging/greybus/fw-download.c b/drivers/staging/greybus/fw-download.c index 2a5c6d1b049c..9a09bd3af79b 100644 --- a/drivers/staging/greybus/fw-download.c +++ b/drivers/staging/greybus/fw-download.c @@ -270,11 +270,11 @@ static int fw_download_fetch_firmware(struct gb_operation *op) struct gb_connection *connection = op->connection; struct fw_download *fw_download = gb_connection_get_data(connection); struct gb_fw_download_fetch_firmware_request *request; - struct gb_fw_download_fetch_firmware_response *response; struct fw_request *fw_req; const struct firmware *fw; unsigned int offset, size; u8 firmware_id; + u8 *response; int ret = 0; if (op->request->payload_size != sizeof(*request)) { @@ -324,8 +324,8 @@ static int fw_download_fetch_firmware(struct gb_operation *op) goto put_fw; } - if (!gb_operation_response_alloc(op, sizeof(*response) + size, - GFP_KERNEL)) { + /* gb_fw_download_fetch_firmware_response contains only a byte array */ + if (!gb_operation_response_alloc(op, size, GFP_KERNEL)) { dev_err(fw_download->parent, "error allocating fetch firmware response\n"); ret = -ENOMEM; @@ -333,7 +333,7 @@ static int fw_download_fetch_firmware(struct gb_operation *op) } response = op->response->payload; - memcpy(response->data, fw->data + offset, size); + memcpy(response, fw->data + offset, size); dev_dbg(fw_download->parent, "responding with firmware (offs = %u, size = %u)\n", offset, diff --git a/include/linux/greybus/greybus_protocols.h b/include/linux/greybus/greybus_protocols.h index aeb8f9243545..820134b0105c 100644 --- a/include/linux/greybus/greybus_protocols.h +++ b/include/linux/greybus/greybus_protocols.h @@ -232,9 +232,7 @@ struct gb_fw_download_fetch_firmware_request { __le32 size; } __packed; -struct gb_fw_download_fetch_firmware_response { - __u8 data[0]; -} __packed; +/* gb_fw_download_fetch_firmware_response contains no other data */ /* firmware download release firmware request */ struct gb_fw_download_release_firmware_request { @@ -414,9 +412,7 @@ struct gb_bootrom_get_firmware_request { __le32 size; } __packed; -struct gb_bootrom_get_firmware_response { - __u8 data[0]; -} __packed; +/* gb_bootrom_get_firmware_response contains no other data */ /* Bootrom protocol Ready to boot request */ struct gb_bootrom_ready_to_boot_request { -- cgit v1.2.3 From 68ade0976df7979eac5f1d46320ff798f5043af6 Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Fri, 1 Mar 2024 23:58:26 +0100 Subject: power: supply: core: add power_supply_for_each_device() Introduce power_supply_for_each_device(), which is a wrapper for class_for_each_device() using the power_supply_class and going through all devices. This allows making the power_supply_class itself a local variable, so that drivers cannot mess with it and simplifies the code slightly. Reviewed-by: Ricardo B. Marliere Link: https://lore.kernel.org/r/20240301-psy-class-cleanup-v1-1-aebe8c4b6b08@collabora.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/ab8500_btemp.c | 3 +-- drivers/power/supply/ab8500_chargalg.c | 3 +-- drivers/power/supply/ab8500_charger.c | 3 +-- drivers/power/supply/ab8500_fg.c | 3 +-- drivers/power/supply/apm_power.c | 3 +-- drivers/power/supply/power_supply_core.c | 34 +++++++++++++++----------------- include/linux/power_supply.h | 3 +-- 7 files changed, 22 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/ab8500_btemp.c b/drivers/power/supply/ab8500_btemp.c index 41dba40fffdf..56f136b2d071 100644 --- a/drivers/power/supply/ab8500_btemp.c +++ b/drivers/power/supply/ab8500_btemp.c @@ -617,8 +617,7 @@ static int ab8500_btemp_get_ext_psy_data(struct device *dev, void *data) */ static void ab8500_btemp_external_power_changed(struct power_supply *psy) { - class_for_each_device(&power_supply_class, NULL, psy, - ab8500_btemp_get_ext_psy_data); + power_supply_for_each_device(psy, ab8500_btemp_get_ext_psy_data); } /* ab8500 btemp driver interrupts and their respective isr */ diff --git a/drivers/power/supply/ab8500_chargalg.c b/drivers/power/supply/ab8500_chargalg.c index 329ae784a72d..55ab7a28056e 100644 --- a/drivers/power/supply/ab8500_chargalg.c +++ b/drivers/power/supply/ab8500_chargalg.c @@ -1231,8 +1231,7 @@ static void ab8500_chargalg_algorithm(struct ab8500_chargalg *di) int ret; /* Collect data from all power_supply class devices */ - class_for_each_device(&power_supply_class, NULL, - di->chargalg_psy, ab8500_chargalg_get_ext_psy_data); + power_supply_for_each_device(di->chargalg_psy, ab8500_chargalg_get_ext_psy_data); ab8500_chargalg_end_of_charge(di); ab8500_chargalg_check_temp(di); diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c index 1c2b69bbed17..9b34d1a60f66 100644 --- a/drivers/power/supply/ab8500_charger.c +++ b/drivers/power/supply/ab8500_charger.c @@ -1949,8 +1949,7 @@ static void ab8500_charger_check_vbat_work(struct work_struct *work) struct ab8500_charger *di = container_of(work, struct ab8500_charger, check_vbat_work.work); - class_for_each_device(&power_supply_class, NULL, - &di->usb_chg, ab8500_charger_get_ext_psy_data); + power_supply_for_each_device(&di->usb_chg, ab8500_charger_get_ext_psy_data); /* First run old_vbat is 0. */ if (di->old_vbat == 0) diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index e49e704023e1..2ccaf6116c09 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -2407,8 +2407,7 @@ out: */ static void ab8500_fg_external_power_changed(struct power_supply *psy) { - class_for_each_device(&power_supply_class, NULL, psy, - ab8500_fg_get_ext_psy_data); + power_supply_for_each_device(psy, ab8500_fg_get_ext_psy_data); } /** diff --git a/drivers/power/supply/apm_power.c b/drivers/power/supply/apm_power.c index 034f28699977..8ef1b6f1f787 100644 --- a/drivers/power/supply/apm_power.c +++ b/drivers/power/supply/apm_power.c @@ -79,8 +79,7 @@ static void find_main_battery(void) main_battery = NULL; bp.main = main_battery; - error = class_for_each_device(&power_supply_class, NULL, &bp, - __find_main_battery); + error = power_supply_for_each_device(&bp, __find_main_battery); if (error) { main_battery = bp.main; return; diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index 4f27f17f8741..0eb8a57dda70 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -25,12 +25,10 @@ #include "power_supply.h" #include "samsung-sdi-battery.h" -/* exported for the APM Power driver, APM emulation */ -const struct class power_supply_class = { +static const struct class power_supply_class = { .name = "power_supply", .dev_uevent = power_supply_uevent, }; -EXPORT_SYMBOL_GPL(power_supply_class); static BLOCKING_NOTIFIER_HEAD(power_supply_notifier); @@ -100,8 +98,7 @@ static void power_supply_changed_work(struct work_struct *work) if (likely(psy->changed)) { psy->changed = false; spin_unlock_irqrestore(&psy->changed_lock, flags); - class_for_each_device(&power_supply_class, NULL, psy, - __power_supply_changed_work); + power_supply_for_each_device(psy, __power_supply_changed_work); power_supply_update_leds(psy); blocking_notifier_call_chain(&power_supply_notifier, PSY_EVENT_PROP_CHANGED, psy); @@ -119,6 +116,12 @@ static void power_supply_changed_work(struct work_struct *work) spin_unlock_irqrestore(&psy->changed_lock, flags); } +int power_supply_for_each_device(void *data, int (*fn)(struct device *dev, void *data)) +{ + return class_for_each_device(&power_supply_class, NULL, data, fn); +} +EXPORT_SYMBOL_GPL(power_supply_for_each_device); + void power_supply_changed(struct power_supply *psy) { unsigned long flags; @@ -194,8 +197,7 @@ static int power_supply_populate_supplied_from(struct power_supply *psy) { int error; - error = class_for_each_device(&power_supply_class, NULL, psy, - __power_supply_populate_supplied_from); + error = power_supply_for_each_device(psy, __power_supply_populate_supplied_from); dev_dbg(&psy->dev, "%s %d\n", __func__, error); @@ -208,7 +210,7 @@ static int __power_supply_find_supply_from_node(struct device *dev, struct device_node *np = data; struct power_supply *epsy = dev_get_drvdata(dev); - /* returning non-zero breaks out of class_for_each_device loop */ + /* returning non-zero breaks out of power_supply_for_each_device loop */ if (epsy->of_node == np) return 1; @@ -220,17 +222,16 @@ static int power_supply_find_supply_from_node(struct device_node *supply_node) int error; /* - * class_for_each_device() either returns its own errors or values + * power_supply_for_each_device() either returns its own errors or values * returned by __power_supply_find_supply_from_node(). * * __power_supply_find_supply_from_node() will return 0 (no match) * or 1 (match). * - * We return 0 if class_for_each_device() returned 1, -EPROBE_DEFER if + * We return 0 if power_supply_for_each_device() returned 1, -EPROBE_DEFER if * it returned 0, or error as returned by it. */ - error = class_for_each_device(&power_supply_class, NULL, supply_node, - __power_supply_find_supply_from_node); + error = power_supply_for_each_device(supply_node, __power_supply_find_supply_from_node); return error ? (error == 1 ? 0 : error) : -EPROBE_DEFER; } @@ -336,8 +337,7 @@ int power_supply_am_i_supplied(struct power_supply *psy) struct psy_am_i_supplied_data data = { psy, 0 }; int error; - error = class_for_each_device(&power_supply_class, NULL, &data, - __power_supply_am_i_supplied); + error = power_supply_for_each_device(&data, __power_supply_am_i_supplied); dev_dbg(&psy->dev, "%s count %u err %d\n", __func__, data.count, error); @@ -372,8 +372,7 @@ int power_supply_is_system_supplied(void) int error; unsigned int count = 0; - error = class_for_each_device(&power_supply_class, NULL, &count, - __power_supply_is_system_supplied); + error = power_supply_for_each_device(&count, __power_supply_is_system_supplied); /* * If no system scope power class device was found at all, most probably we @@ -419,8 +418,7 @@ int power_supply_get_property_from_supplier(struct power_supply *psy, * This function is not intended for use with a supply with multiple * suppliers, we simply pick the first supply to report the psp. */ - ret = class_for_each_device(&power_supply_class, NULL, &data, - __power_supply_get_supplier_property); + ret = power_supply_for_each_device(&data, __power_supply_get_supplier_property); if (ret < 0) return ret; if (ret == 0) diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 514f652de64d..92dd205774ec 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -894,8 +894,7 @@ extern int power_supply_powers(struct power_supply *psy, struct device *dev); #define to_power_supply(device) container_of(device, struct power_supply, dev) extern void *power_supply_get_drvdata(struct power_supply *psy); -/* For APM emulation, think legacy userspace. */ -extern const struct class power_supply_class; +extern int power_supply_for_each_device(void *data, int (*fn)(struct device *dev, void *data)); static inline bool power_supply_is_amp_property(enum power_supply_property psp) { -- cgit v1.2.3 From 4e61f1e9d58fb0765f59f47d4d1f318b36c14d95 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Sun, 3 Mar 2024 16:31:15 +0100 Subject: power: supply: core: fix charge_behaviour formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This property is documented to have a special format which exposes all available behaviours and the currently active one at the same time. For this special format some helpers are provided. When the charge_behaviour property was added in 1b0b6cc8030d ("power: supply: add charge_behaviour attributes"), it did not update the default logic in in power_supply_sysfs.c to use the format helpers. Thus by default only the currently active behaviour is printed. This fixes the default logic to follow the documented format. There is currently only one in-tree drivers exposing charge behaviours - thinkpad_acpi, which is not affected by the change, as it directly uses the helpers and does not use the power_supply_sysfs.c logic. Signed-off-by: Thomas Weißschuh Link: https://lore.kernel.org/r/20240303-power_supply-charge_behaviour_prop-v2-3-8ebb0a7c2409@weissschuh.net Signed-off-by: Sebastian Reichel --- drivers/power/supply/power_supply_sysfs.c | 20 ++++++++++++++++++++ include/linux/power_supply.h | 1 + 2 files changed, 21 insertions(+) (limited to 'include/linux') diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index 9b461e997fcb..0d2c3724d0bc 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -271,6 +271,23 @@ static ssize_t power_supply_show_usb_type(struct device *dev, return count; } +static ssize_t power_supply_show_charge_behaviour(struct device *dev, + struct power_supply *psy, + union power_supply_propval *value, + char *buf) +{ + int ret; + + ret = power_supply_get_property(psy, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + value); + if (ret < 0) + return ret; + + return power_supply_charge_behaviour_show(dev, psy->desc->charge_behaviours, + value->intval, buf); +} + static ssize_t power_supply_show_property(struct device *dev, struct device_attribute *attr, char *buf) { @@ -303,6 +320,9 @@ static ssize_t power_supply_show_property(struct device *dev, ret = power_supply_show_usb_type(dev, psy->desc, &value, buf); break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + ret = power_supply_show_charge_behaviour(dev, psy, &value, buf); + break; case POWER_SUPPLY_PROP_MODEL_NAME ... POWER_SUPPLY_PROP_SERIAL_NUMBER: ret = sysfs_emit(buf, "%s\n", value.strval); break; diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 92dd205774ec..8e5705a56b85 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -242,6 +242,7 @@ struct power_supply_config { struct power_supply_desc { const char *name; enum power_supply_type type; + u8 charge_behaviours; const enum power_supply_usb_type *usb_types; size_t num_usb_types; const enum power_supply_property *properties; -- cgit v1.2.3 From e3b6876ab85061e7de198f023a0c2bfc7478b420 Mon Sep 17 00:00:00 2001 From: Andrew Lunn Date: Sat, 2 Mar 2024 20:53:01 +0100 Subject: net: phy: Add phydev->enable_tx_lpi to simplify adjust link callbacks MAC drivers which support EEE need to know the results of the EEE auto-neg in order to program the hardware to perform EEE or not. The oddly named phy_init_eee() can be used to determine this, it returns 0 if EEE should be used, or a negative error code, e.g. -EOPPROTONOTSUPPORT if the PHY does not support EEE or negotiate resulted in it not being used. However, many MAC drivers get this wrong. Add phydev->enable_tx_lpi which indicates the result of the autoneg for EEE, including if EEE is administratively disabled with ethtool. The MAC driver can then access this in the same way as link speed and duplex in the adjust link callback. If enable_tx_lpi is true, the MAC should send low power indications and does not need to consider anything else with respect to EEE. Reviewed-by: Florian Fainelli Signed-off-by: Andrew Lunn Signed-off-by: Oleksij Rempel Link: https://lore.kernel.org/r/20240302195306.3207716-3-o.rempel@pengutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy.c | 7 +++++++ include/linux/phy.h | 2 ++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 14224e06d69f..2bc0a7d51c63 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -983,9 +983,16 @@ static int phy_check_link_status(struct phy_device *phydev) if (phydev->link && phydev->state != PHY_RUNNING) { phy_check_downshift(phydev); phydev->state = PHY_RUNNING; + err = genphy_c45_eee_is_active(phydev, + NULL, NULL, NULL); + if (err < 0) + phydev->enable_tx_lpi = false; + else + phydev->enable_tx_lpi = !!err; phy_link_up(phydev); } else if (!phydev->link && phydev->state != PHY_NOLINK) { phydev->state = PHY_NOLINK; + phydev->enable_tx_lpi = false; phy_link_down(phydev); } diff --git a/include/linux/phy.h b/include/linux/phy.h index e3ab2c347a59..a880f6d7170e 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -594,6 +594,7 @@ struct macsec_ops; * @supported_eee: supported PHY EEE linkmodes * @advertising_eee: Currently advertised EEE linkmodes * @eee_enabled: Flag indicating whether the EEE feature is enabled + * @enable_tx_lpi: When True, MAC should transmit LPI to PHY * @lp_advertising: Current link partner advertised linkmodes * @host_interfaces: PHY interface modes supported by host * @eee_broken_modes: Energy efficient ethernet modes which should be prohibited @@ -713,6 +714,7 @@ struct phy_device { /* Energy efficient ethernet modes which should be prohibited */ u32 eee_broken_modes; + bool enable_tx_lpi; #ifdef CONFIG_LED_TRIGGER_PHY struct phy_led_trigger *phy_led_triggers; -- cgit v1.2.3 From fe0d4fd9285e5013b4bafbd3338847235b805a1c Mon Sep 17 00:00:00 2001 From: Andrew Lunn Date: Sat, 2 Mar 2024 20:53:02 +0100 Subject: net: phy: Keep track of EEE configuration Have phylib keep track of the EEE configuration. This simplifies the MAC drivers, in that they don't need to store it. Future patches to phylib will also make use of this information to further simplify the MAC drivers. Reviewed-by: Russell King (Oracle) Signed-off-by: Andrew Lunn Reviewed-by: Florian Fainelli Signed-off-by: Oleksij Rempel Link: https://lore.kernel.org/r/20240302195306.3207716-4-o.rempel@pengutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy.c | 7 +++++-- include/linux/phy.h | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 2bc0a7d51c63..95c4ef5d4e97 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -1640,8 +1640,8 @@ EXPORT_SYMBOL(phy_get_eee_err); * @phydev: target phy_device struct * @data: ethtool_keee data * - * Description: it reportes the Supported/Advertisement/LP Advertisement - * capabilities. + * Description: reports the Supported/Advertisement/LP Advertisement + * capabilities, etc. */ int phy_ethtool_get_eee(struct phy_device *phydev, struct ethtool_keee *data) { @@ -1652,6 +1652,7 @@ int phy_ethtool_get_eee(struct phy_device *phydev, struct ethtool_keee *data) mutex_lock(&phydev->lock); ret = genphy_c45_ethtool_get_eee(phydev, data); + eeecfg_to_eee(data, &phydev->eee_cfg); mutex_unlock(&phydev->lock); return ret; @@ -1674,6 +1675,8 @@ int phy_ethtool_set_eee(struct phy_device *phydev, struct ethtool_keee *data) mutex_lock(&phydev->lock); ret = genphy_c45_ethtool_set_eee(phydev, data); + if (!ret) + eee_to_eeecfg(&phydev->eee_cfg, data); mutex_unlock(&phydev->lock); return ret; diff --git a/include/linux/phy.h b/include/linux/phy.h index a880f6d7170e..695e366bd75c 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -30,6 +30,7 @@ #include #include +#include #define PHY_DEFAULT_FEATURES (SUPPORTED_Autoneg | \ SUPPORTED_TP | \ @@ -595,6 +596,7 @@ struct macsec_ops; * @advertising_eee: Currently advertised EEE linkmodes * @eee_enabled: Flag indicating whether the EEE feature is enabled * @enable_tx_lpi: When True, MAC should transmit LPI to PHY + * @eee_cfg: User configuration of EEE * @lp_advertising: Current link partner advertised linkmodes * @host_interfaces: PHY interface modes supported by host * @eee_broken_modes: Energy efficient ethernet modes which should be prohibited @@ -715,6 +717,7 @@ struct phy_device { /* Energy efficient ethernet modes which should be prohibited */ u32 eee_broken_modes; bool enable_tx_lpi; + struct eee_config eee_cfg; #ifdef CONFIG_LED_TRIGGER_PHY struct phy_led_trigger *phy_led_triggers; -- cgit v1.2.3 From 49168d1980e220cf3d1b761e1eafac62041cb94d Mon Sep 17 00:00:00 2001 From: Andrew Lunn Date: Sat, 2 Mar 2024 20:53:04 +0100 Subject: net: phy: Add phy_support_eee() indicating MAC support EEE In order for EEE to operate, both the MAC and the PHY need to support it, similar to how pause works. With some exception - a number of PHYs have SmartEEE or AutoGrEEEn support in order to provide some EEE-like power savings with non-EEE capable MACs. Copy the pause concept and add the call phy_support_eee() which the MAC makes after connecting the PHY to indicate it supports EEE. phylib will then advertise EEE when auto-neg is performed. Signed-off-by: Andrew Lunn Signed-off-by: Oleksij Rempel Link: https://lore.kernel.org/r/20240302195306.3207716-6-o.rempel@pengutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy_device.c | 28 ++++++++++++++++++++++++++++ include/linux/phy.h | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 2eefee970851..72452e6a478c 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -2910,6 +2910,34 @@ void phy_advertise_eee_all(struct phy_device *phydev) } EXPORT_SYMBOL_GPL(phy_advertise_eee_all); +/** + * phy_support_eee - Set initial EEE policy configuration + * @phydev: Target phy_device struct + * + * This function configures the initial policy for Energy Efficient Ethernet + * (EEE) on the specified PHY device, influencing that EEE capabilities are + * advertised before the link is established. It should be called during PHY + * registration by the MAC driver and/or the PHY driver (for SmartEEE PHYs) + * if MAC supports LPI or PHY is capable to compensate missing LPI functionality + * of the MAC. + * + * The function sets default EEE policy parameters, including preparing the PHY + * to advertise EEE capabilities based on hardware support. + * + * It also sets the expected configuration for Low Power Idle (LPI) in the MAC + * driver. If the PHY framework determines that both local and remote + * advertisements support EEE, and the negotiated link mode is compatible with + * EEE, it will set enable_tx_lpi = true. The MAC driver is expected to act on + * this setting by enabling the LPI timer if enable_tx_lpi is set. + */ +void phy_support_eee(struct phy_device *phydev) +{ + linkmode_copy(phydev->advertising_eee, phydev->supported_eee); + phydev->eee_cfg.tx_lpi_enabled = true; + phydev->eee_cfg.eee_enabled = true; +} +EXPORT_SYMBOL(phy_support_eee); + /** * phy_support_sym_pause - Enable support of symmetrical pause * @phydev: target phy_device struct diff --git a/include/linux/phy.h b/include/linux/phy.h index 695e366bd75c..3f68b8239bb1 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -706,7 +706,7 @@ struct phy_device { __ETHTOOL_DECLARE_LINK_MODE_MASK(lp_advertising); /* used with phy_speed_down */ __ETHTOOL_DECLARE_LINK_MODE_MASK(adv_old); - /* used for eee validation */ + /* used for eee validation and configuration*/ __ETHTOOL_DECLARE_LINK_MODE_MASK(supported_eee); __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising_eee); bool eee_enabled; @@ -1973,6 +1973,7 @@ void phy_advertise_supported(struct phy_device *phydev); void phy_advertise_eee_all(struct phy_device *phydev); void phy_support_sym_pause(struct phy_device *phydev); void phy_support_asym_pause(struct phy_device *phydev); +void phy_support_eee(struct phy_device *phydev); void phy_set_sym_pause(struct phy_device *phydev, bool rx, bool tx, bool autoneg); void phy_set_asym_pause(struct phy_device *phydev, bool rx, bool tx); -- cgit v1.2.3 From ad86f7e959dc1814c3b2bcbeb08c3c02214110a7 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 14:56:59 +0100 Subject: firmware: arm_scmi: Populate perf commands rate_limit Arm SCMI spec. v3.2, s4.5.3.4 PERFORMANCE_DOMAIN_ATTRIBUTES defines a per-domain rate_limit for performance requests: """ Rate Limit in microseconds, indicating the minimum time required between successive requests. A value of 0 indicates that this field is not supported by the platform. This field does not apply to FastChannels. """" The field is first defined in SCMI v1.0. Add support to fetch this value and advertise it through a rate_limit_get() callback. Signed-off-by: Pierre Gondois Reviewed-by: Cristian Marussi Acked-by: Sudeep Holla Signed-off-by: Viresh Kumar --- drivers/firmware/arm_scmi/perf.c | 21 +++++++++++++++++++++ include/linux/scmi_protocol.h | 4 ++++ 2 files changed, 25 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 8ea2a7b3d35d..3269415ff77d 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -153,6 +153,7 @@ struct perf_dom_info { bool perf_fastchannels; bool level_indexing_mode; u32 opp_count; + u32 rate_limit_us; u32 sustained_freq_khz; u32 sustained_perf_level; unsigned long mult_factor; @@ -266,6 +267,8 @@ scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph, if (PROTOCOL_REV_MAJOR(version) >= 0x4) dom_info->level_indexing_mode = SUPPORTS_LEVEL_INDEXING(flags); + dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) & + GENMASK(19, 0); dom_info->sustained_freq_khz = le32_to_cpu(attr->sustained_freq_khz); dom_info->sustained_perf_level = @@ -842,6 +845,23 @@ scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph, return dom->opp[dom->opp_count - 1].trans_latency_us * 1000; } +static int +scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit) +{ + struct perf_dom_info *dom; + + if (!rate_limit) + return -EINVAL; + + dom = scmi_perf_domain_lookup(ph, domain); + if (IS_ERR(dom)) + return PTR_ERR(dom); + + *rate_limit = dom->rate_limit_us; + return 0; +} + static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain, unsigned long freq, bool poll) { @@ -957,6 +977,7 @@ static const struct scmi_perf_proto_ops perf_proto_ops = { .level_set = scmi_perf_level_set, .level_get = scmi_perf_level_get, .transition_latency_get = scmi_dvfs_transition_latency_get, + .rate_limit_get = scmi_dvfs_rate_limit_get, .device_opps_add = scmi_dvfs_device_opps_add, .freq_set = scmi_dvfs_freq_set, .freq_get = scmi_dvfs_freq_get, diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index f2f05fb42d28..acd956ffcb84 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -128,6 +128,8 @@ struct scmi_perf_domain_info { * @level_set: sets the performance level of a domain * @level_get: gets the performance level of a domain * @transition_latency_get: gets the DVFS transition latency for a given device + * @rate_limit_get: gets the minimum time (us) required between successive + * requests * @device_opps_add: adds all the OPPs for a given device * @freq_set: sets the frequency for a given device using sustained frequency * to sustained performance level mapping @@ -154,6 +156,8 @@ struct scmi_perf_proto_ops { u32 *level, bool poll); int (*transition_latency_get)(const struct scmi_protocol_handle *ph, u32 domain); + int (*rate_limit_get)(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit); int (*device_opps_add)(const struct scmi_protocol_handle *ph, struct device *dev, u32 domain); int (*freq_set)(const struct scmi_protocol_handle *ph, u32 domain, -- cgit v1.2.3 From 2441caa84aac8abf1be9e20db3e6bb921e74c8a2 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 14:57:00 +0100 Subject: firmware: arm_scmi: Populate fast channel rate_limit Arm SCMI spec. v3.2, s4.5.3.12 PERFORMANCE_DESCRIBE_FASTCHANNEL defines a per-domain rate_limit for performance requests: """ Rate Limit in microseconds, indicating the minimum time required between successive requests. A value of 0 indicates that this field is not applicable or supported on the platform. """" The field is first defined in SCMI v2.0. Add support to fetch this value and advertise it through a fast_switch_rate_limit() callback. Signed-off-by: Pierre Gondois Reviewed-by: Cristian Marussi Acked-by: Sudeep Holla Signed-off-by: Viresh Kumar --- drivers/firmware/arm_scmi/driver.c | 5 ++++- drivers/firmware/arm_scmi/perf.c | 32 ++++++++++++++++++++++++++++---- drivers/firmware/arm_scmi/powercap.c | 12 ++++++++---- drivers/firmware/arm_scmi/protocols.h | 4 +++- include/linux/scmi_protocol.h | 4 ++++ 5 files changed, 47 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 3ea64b22cf0d..1d38ecfafc59 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -1617,7 +1617,7 @@ static void scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, u8 describe_id, u32 message_id, u32 valid_size, u32 domain, void __iomem **p_addr, - struct scmi_fc_db_info **p_db) + struct scmi_fc_db_info **p_db, u32 *rate_limit) { int ret; u32 flags; @@ -1661,6 +1661,9 @@ scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, goto err_xfer; } + if (rate_limit) + *rate_limit = le32_to_cpu(resp->rate_limit) & GENMASK(19, 0); + phys_addr = le32_to_cpu(resp->chan_addr_low); phys_addr |= (u64)le32_to_cpu(resp->chan_addr_high) << 32; addr = devm_ioremap(ph->dev, phys_addr, size); diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 3269415ff77d..be1801fb848d 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -776,23 +776,27 @@ static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph, ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LEVEL_GET, 4, dom->id, - &fc[PERF_FC_LEVEL].get_addr, NULL); + &fc[PERF_FC_LEVEL].get_addr, NULL, + &fc[PERF_FC_LEVEL].rate_limit); ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LIMITS_GET, 8, dom->id, - &fc[PERF_FC_LIMIT].get_addr, NULL); + &fc[PERF_FC_LIMIT].get_addr, NULL, + &fc[PERF_FC_LIMIT].rate_limit); if (dom->info.set_perf) ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LEVEL_SET, 4, dom->id, &fc[PERF_FC_LEVEL].set_addr, - &fc[PERF_FC_LEVEL].set_db); + &fc[PERF_FC_LEVEL].set_db, + &fc[PERF_FC_LEVEL].rate_limit); if (dom->set_limits) ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LIMITS_SET, 8, dom->id, &fc[PERF_FC_LIMIT].set_addr, - &fc[PERF_FC_LIMIT].set_db); + &fc[PERF_FC_LIMIT].set_db, + &fc[PERF_FC_LIMIT].rate_limit); dom->fc_info = fc; } @@ -961,6 +965,25 @@ static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph, return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr; } +static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit) +{ + struct perf_dom_info *dom; + + if (!rate_limit) + return -EINVAL; + + dom = scmi_perf_domain_lookup(ph, domain); + if (IS_ERR(dom)) + return PTR_ERR(dom); + + if (!dom->fc_info) + return -EINVAL; + + *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit; + return 0; +} + static enum scmi_power_scale scmi_power_scale_get(const struct scmi_protocol_handle *ph) { @@ -983,6 +1006,7 @@ static const struct scmi_perf_proto_ops perf_proto_ops = { .freq_get = scmi_dvfs_freq_get, .est_power_get = scmi_dvfs_est_power_get, .fast_switch_possible = scmi_fast_switch_possible, + .fast_switch_rate_limit = scmi_fast_switch_rate_limit, .power_scale_get = scmi_power_scale_get, }; diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c index a4c6cd4716fe..604184c044ff 100644 --- a/drivers/firmware/arm_scmi/powercap.c +++ b/drivers/firmware/arm_scmi/powercap.c @@ -703,20 +703,24 @@ static void scmi_powercap_domain_init_fc(const struct scmi_protocol_handle *ph, ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_CAP_SET, 4, domain, &fc[POWERCAP_FC_CAP].set_addr, - &fc[POWERCAP_FC_CAP].set_db); + &fc[POWERCAP_FC_CAP].set_db, + &fc[POWERCAP_FC_CAP].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_CAP_GET, 4, domain, - &fc[POWERCAP_FC_CAP].get_addr, NULL); + &fc[POWERCAP_FC_CAP].get_addr, NULL, + &fc[POWERCAP_FC_CAP].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_PAI_SET, 4, domain, &fc[POWERCAP_FC_PAI].set_addr, - &fc[POWERCAP_FC_PAI].set_db); + &fc[POWERCAP_FC_PAI].set_db, + &fc[POWERCAP_FC_PAI].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_PAI_GET, 4, domain, - &fc[POWERCAP_FC_PAI].get_addr, NULL); + &fc[POWERCAP_FC_PAI].get_addr, NULL, + &fc[POWERCAP_PAI_GET].rate_limit); *p_fc = fc; } diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h index e683c26f24eb..8b5d9ce4a33a 100644 --- a/drivers/firmware/arm_scmi/protocols.h +++ b/drivers/firmware/arm_scmi/protocols.h @@ -234,6 +234,7 @@ struct scmi_fc_info { void __iomem *set_addr; void __iomem *get_addr; struct scmi_fc_db_info *set_db; + u32 rate_limit; }; /** @@ -268,7 +269,8 @@ struct scmi_proto_helpers_ops { u8 describe_id, u32 message_id, u32 valid_size, u32 domain, void __iomem **p_addr, - struct scmi_fc_db_info **p_db); + struct scmi_fc_db_info **p_db, + u32 *rate_limit); void (*fastchannel_db_ring)(struct scmi_fc_db_info *db); }; diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index acd956ffcb84..fafedb3b6604 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -139,6 +139,8 @@ struct scmi_perf_domain_info { * at a given frequency * @fast_switch_possible: indicates if fast DVFS switching is possible or not * for a given device + * @fast_switch_rate_limit: gets the minimum time (us) required between + * successive fast_switching requests * @power_scale_mw_get: indicates if the power values provided are in milliWatts * or in some other (abstract) scale */ @@ -168,6 +170,8 @@ struct scmi_perf_proto_ops { unsigned long *rate, unsigned long *power); bool (*fast_switch_possible)(const struct scmi_protocol_handle *ph, u32 domain); + int (*fast_switch_rate_limit)(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit); enum scmi_power_scale (*power_scale_get)(const struct scmi_protocol_handle *ph); }; -- cgit v1.2.3 From 9fe0c03f0bfc5f74dad6e818090ab967d8603095 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Tue, 5 Mar 2024 18:08:27 -0800 Subject: fsnotify: Fix misspelling of "writable" Several file system notification system headers have "writable" misspelled as "writtable" in the comments. This patch fixes it in the fsnotify header. Signed-off-by: Vicki Pfau Signed-off-by: Jan Kara Message-Id: <20240306020831.1404033-2-vi@endrift.com> --- include/linux/fsnotify_backend.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 7f63be5ca0f1..8f40c349b228 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -31,8 +31,8 @@ #define FS_ACCESS 0x00000001 /* File was accessed */ #define FS_MODIFY 0x00000002 /* File was modified */ #define FS_ATTRIB 0x00000004 /* Metadata changed */ -#define FS_CLOSE_WRITE 0x00000008 /* Writtable file was closed */ -#define FS_CLOSE_NOWRITE 0x00000010 /* Unwrittable file closed */ +#define FS_CLOSE_WRITE 0x00000008 /* Writable file was closed */ +#define FS_CLOSE_NOWRITE 0x00000010 /* Unwritable file closed */ #define FS_OPEN 0x00000020 /* File was opened */ #define FS_MOVED_FROM 0x00000040 /* File was moved from X */ #define FS_MOVED_TO 0x00000080 /* File was moved to Y */ -- cgit v1.2.3 From 00b9850e7307ff690639dd0584a50dd8a72d3548 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 5 Jan 2024 11:16:17 +0100 Subject: greybus: make greybus_bus_type const Now that the driver core can properly handle constant struct bus_type, move the greybus_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Alex Elder Cc: greybus-dev@lists.linaro.org Reviewed-by: Johan Hovold Link: https://lore.kernel.org/r/2024010517-handgun-scoreless-05e7@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/greybus/core.c | 2 +- include/linux/greybus.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/greybus/core.c b/drivers/greybus/core.c index 5714be740470..b0aadc1e7bbe 100644 --- a/drivers/greybus/core.c +++ b/drivers/greybus/core.c @@ -155,7 +155,7 @@ static void greybus_shutdown(struct device *dev) } } -struct bus_type greybus_bus_type = { +const struct bus_type greybus_bus_type = { .name = "greybus", .match = greybus_match_device, .uevent = greybus_uevent, diff --git a/include/linux/greybus.h b/include/linux/greybus.h index 18c0fb958b74..92da9ec4f5f0 100644 --- a/include/linux/greybus.h +++ b/include/linux/greybus.h @@ -104,7 +104,7 @@ void gb_debugfs_init(void); void gb_debugfs_cleanup(void); struct dentry *gb_debugfs_get(void); -extern struct bus_type greybus_bus_type; +extern const struct bus_type greybus_bus_type; extern struct device_type greybus_hd_type; extern struct device_type greybus_module_type; -- cgit v1.2.3 From e869b72b33731063b50433eb6146d51a479995a1 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 19 Feb 2024 09:40:50 -0300 Subject: greybus: constify the struct device_type usage Since commit aed65af1cc2f ("drivers: make device_type const"), the driver core can properly handle constant struct device_type. Move the greybus_hd_type, greybus_module_type, greybus_interface_type, greybus_control_type, greybus_bundle_type and greybus_svc_type variables to be constant structures as well, placing it into read-only memory which can not be modified at runtime. Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Alex Elder Link: https://lore.kernel.org/r/20240219-device_cleanup-greybus-v1-1-babb3f65e8cc@marliere.net Signed-off-by: Greg Kroah-Hartman --- drivers/greybus/bundle.c | 2 +- drivers/greybus/control.c | 2 +- drivers/greybus/hd.c | 2 +- drivers/greybus/interface.c | 2 +- drivers/greybus/module.c | 2 +- drivers/greybus/svc.c | 2 +- include/linux/greybus.h | 12 ++++++------ 7 files changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/greybus/bundle.c b/drivers/greybus/bundle.c index 84660729538b..a6e1cca06172 100644 --- a/drivers/greybus/bundle.c +++ b/drivers/greybus/bundle.c @@ -166,7 +166,7 @@ static const struct dev_pm_ops gb_bundle_pm_ops = { SET_RUNTIME_PM_OPS(gb_bundle_suspend, gb_bundle_resume, gb_bundle_idle) }; -struct device_type greybus_bundle_type = { +const struct device_type greybus_bundle_type = { .name = "greybus_bundle", .release = gb_bundle_release, .pm = &gb_bundle_pm_ops, diff --git a/drivers/greybus/control.c b/drivers/greybus/control.c index 359a25841973..b5cf49d09df2 100644 --- a/drivers/greybus/control.c +++ b/drivers/greybus/control.c @@ -436,7 +436,7 @@ static void gb_control_release(struct device *dev) kfree(control); } -struct device_type greybus_control_type = { +const struct device_type greybus_control_type = { .name = "greybus_control", .release = gb_control_release, }; diff --git a/drivers/greybus/hd.c b/drivers/greybus/hd.c index 72b21bf2d7d3..e2f3496bddc3 100644 --- a/drivers/greybus/hd.c +++ b/drivers/greybus/hd.c @@ -116,7 +116,7 @@ static void gb_hd_release(struct device *dev) kfree(hd); } -struct device_type greybus_hd_type = { +const struct device_type greybus_hd_type = { .name = "greybus_host_device", .release = gb_hd_release, }; diff --git a/drivers/greybus/interface.c b/drivers/greybus/interface.c index 9ec949a438ef..a88dc701289c 100644 --- a/drivers/greybus/interface.c +++ b/drivers/greybus/interface.c @@ -765,7 +765,7 @@ static const struct dev_pm_ops gb_interface_pm_ops = { gb_interface_runtime_idle) }; -struct device_type greybus_interface_type = { +const struct device_type greybus_interface_type = { .name = "greybus_interface", .release = gb_interface_release, .pm = &gb_interface_pm_ops, diff --git a/drivers/greybus/module.c b/drivers/greybus/module.c index 36f77f9e1d74..7f7153a1dd60 100644 --- a/drivers/greybus/module.c +++ b/drivers/greybus/module.c @@ -81,7 +81,7 @@ static void gb_module_release(struct device *dev) kfree(module); } -struct device_type greybus_module_type = { +const struct device_type greybus_module_type = { .name = "greybus_module", .release = gb_module_release, }; diff --git a/drivers/greybus/svc.c b/drivers/greybus/svc.c index 0d7e749174a4..4256467fcd35 100644 --- a/drivers/greybus/svc.c +++ b/drivers/greybus/svc.c @@ -1305,7 +1305,7 @@ static void gb_svc_release(struct device *dev) kfree(svc); } -struct device_type greybus_svc_type = { +const struct device_type greybus_svc_type = { .name = "greybus_svc", .release = gb_svc_release, }; diff --git a/include/linux/greybus.h b/include/linux/greybus.h index 92da9ec4f5f0..2cc570ea63bf 100644 --- a/include/linux/greybus.h +++ b/include/linux/greybus.h @@ -106,12 +106,12 @@ struct dentry *gb_debugfs_get(void); extern const struct bus_type greybus_bus_type; -extern struct device_type greybus_hd_type; -extern struct device_type greybus_module_type; -extern struct device_type greybus_interface_type; -extern struct device_type greybus_control_type; -extern struct device_type greybus_bundle_type; -extern struct device_type greybus_svc_type; +extern const struct device_type greybus_hd_type; +extern const struct device_type greybus_module_type; +extern const struct device_type greybus_interface_type; +extern const struct device_type greybus_control_type; +extern const struct device_type greybus_bundle_type; +extern const struct device_type greybus_svc_type; static inline int is_gb_host_device(const struct device *dev) { -- cgit v1.2.3 From 39714fd73c6b60a8d27bcc5b431afb0828bf4434 Mon Sep 17 00:00:00 2001 From: Ethan Zhao Date: Tue, 5 Mar 2024 20:21:14 +0800 Subject: PCI: Make pci_dev_is_disconnected() helper public for other drivers Make pci_dev_is_disconnected() public so that it can be called from Intel VT-d driver to quickly fix/workaround the surprise removal unplug hang issue for those ATS capable devices on PCIe switch downstream hotplug capable ports. Beside pci_device_is_present() function, this one has no config space space access, so is light enough to optimize the normal pure surprise removal and safe removal flow. Acked-by: Bjorn Helgaas Reviewed-by: Dan Carpenter Tested-by: Haorong Ye Signed-off-by: Ethan Zhao Link: https://lore.kernel.org/r/20240301080727.3529832-2-haifeng.zhao@linux.intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/pci/pci.h | 5 ----- include/linux/pci.h | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index e9750b1b19ba..bfc56f7bee1c 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -368,11 +368,6 @@ static inline int pci_dev_set_disconnected(struct pci_dev *dev, void *unused) return 0; } -static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) -{ - return dev->error_state == pci_channel_io_perm_failure; -} - /* pci_dev priv_flags */ #define PCI_DEV_ADDED 0 #define PCI_DPC_RECOVERED 1 diff --git a/include/linux/pci.h b/include/linux/pci.h index 7ab0d13672da..213109d3c601 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2517,6 +2517,11 @@ static inline struct pci_dev *pcie_find_root_port(struct pci_dev *dev) return NULL; } +static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) +{ + return dev->error_state == pci_channel_io_perm_failure; +} + void pci_request_acs(void); bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags); bool pci_acs_path_enabled(struct pci_dev *start, -- cgit v1.2.3 From 0061ffe289e19caabeea8103e69cb0f1896e34d8 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 5 Mar 2024 20:21:17 +0800 Subject: iommu: Add static iommu_ops->release_domain The current device_release callback for individual iommu drivers does the following: 1) Silent IOMMU DMA translation: It detaches any existing domain from the device and puts it into a blocking state (some drivers might use the identity state). 2) Resource release: It releases resources allocated during the device_probe callback and restores the device to its pre-probe state. Step 1 is challenging for individual iommu drivers because each must check if a domain is already attached to the device. Additionally, if a deferred attach never occurred, the device_release should avoid modifying hardware configuration regardless of the reason for its call. To simplify this process, introduce a static release_domain within the iommu_ops structure. It can be either a blocking or identity domain depending on the iommu hardware. The iommu core will decide whether to attach this domain before the device_release callback, eliminating the need for repetitive code in various drivers. Consequently, the device_release callback can focus solely on the opposite operations of device_probe, including releasing all resources allocated during that callback. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240305013305.204605-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 19 +++++++++++++++---- include/linux/iommu.h | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..cd1210026ac5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -463,13 +463,24 @@ static void iommu_deinit_device(struct device *dev) /* * release_device() must stop using any attached domain on the device. - * If there are still other devices in the group they are not effected + * If there are still other devices in the group, they are not affected * by this callback. * - * The IOMMU driver must set the device to either an identity or - * blocking translation and stop using any domain pointer, as it is - * going to be freed. + * If the iommu driver provides release_domain, the core code ensures + * that domain is attached prior to calling release_device. Drivers can + * use this to enforce a translation on the idle iommu. Typically, the + * global static blocked_domain is a good choice. + * + * Otherwise, the iommu driver must set the device to either an identity + * or a blocking translation in release_device() and stop using any + * domain pointer, as it is going to be freed. + * + * Regardless, if a delayed attach never occurred, then the release + * should still avoid touching any hardware configuration either. */ + if (!dev->iommu->attach_deferred && ops->release_domain) + ops->release_domain->ops->attach_dev(ops->release_domain, dev); + if (ops->release_device) ops->release_device(dev); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1ea2a820e1eb..a0a07e1680a2 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -487,6 +487,7 @@ struct iommu_ops { struct module *owner; struct iommu_domain *identity_domain; struct iommu_domain *blocked_domain; + struct iommu_domain *release_domain; struct iommu_domain *default_domain; }; -- cgit v1.2.3 From eb52286634f042432ec775077a73334603a1c6e4 Mon Sep 17 00:00:00 2001 From: "Gang Li Subject: padata: dispatch works on" Date: Wed, 6 Mar 2024 13:04:17 -0800 Subject: Author: Gang Li padata: dispatch works on different nodes Date: Thu, 22 Feb 2024 22:04:17 +0800 When a group of tasks that access different nodes are scheduled on the same node, they may encounter bandwidth bottlenecks and access latency. Thus, numa_aware flag is introduced here, allowing tasks to be distributed across different nodes to fully utilize the advantage of multi-node systems. Link: https://lkml.kernel.org/r/20240222140422.393911-5-gang.li@linux.dev Signed-off-by: Gang Li Tested-by: David Rientjes Reviewed-by: Muchun Song Reviewed-by: Tim Chen Cc: Alexey Dobriyan Cc: Daniel Jordan Cc: David Hildenbrand Cc: Jane Chu Cc: Mike Kravetz Cc: Paul E. McKenney Cc: Randy Dunlap Cc: Steffen Klassert Signed-off-by: Andrew Morton --- include/linux/padata.h | 2 ++ kernel/padata.c | 14 ++++++++++++-- mm/mm_init.c | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/padata.h b/include/linux/padata.h index 495b16b6b4d7..8f418711351b 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -137,6 +137,7 @@ struct padata_shell { * appropriate for one worker thread to do at once. * @max_threads: Max threads to use for the job, actual number may be less * depending on task size and minimum chunk size. + * @numa_aware: Distribute jobs to different nodes with CPU in a round robin fashion. */ struct padata_mt_job { void (*thread_fn)(unsigned long start, unsigned long end, void *arg); @@ -146,6 +147,7 @@ struct padata_mt_job { unsigned long align; unsigned long min_chunk; int max_threads; + bool numa_aware; }; /** diff --git a/kernel/padata.c b/kernel/padata.c index 179fb1518070..e3f639ff1670 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -485,7 +485,8 @@ void __init padata_do_multithreaded(struct padata_mt_job *job) struct padata_work my_work, *pw; struct padata_mt_job_state ps; LIST_HEAD(works); - int nworks; + int nworks, nid; + static atomic_t last_used_nid __initdata; if (job->size == 0) return; @@ -517,7 +518,16 @@ void __init padata_do_multithreaded(struct padata_mt_job *job) ps.chunk_size = roundup(ps.chunk_size, job->align); list_for_each_entry(pw, &works, pw_list) - queue_work(system_unbound_wq, &pw->pw_work); + if (job->numa_aware) { + int old_node = atomic_read(&last_used_nid); + + do { + nid = next_node_in(old_node, node_states[N_CPU]); + } while (!atomic_try_cmpxchg(&last_used_nid, &old_node, nid)); + queue_work_node(nid, system_unbound_wq, &pw->pw_work); + } else { + queue_work(system_unbound_wq, &pw->pw_work); + } /* Use the current thread, which saves starting a workqueue worker. */ padata_work_init(&my_work, padata_mt_helper, &ps, PADATA_WORK_ONSTACK); diff --git a/mm/mm_init.c b/mm/mm_init.c index 2c19f5515e36..549e76af8f82 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2231,6 +2231,7 @@ static int __init deferred_init_memmap(void *data) .align = PAGES_PER_SECTION, .min_chunk = PAGES_PER_SECTION, .max_threads = max_threads, + .numa_aware = false, }; padata_do_multithreaded(&job); -- cgit v1.2.3 From bd5ed02e23958cb56d0f5a90ebe620c8b47dab47 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 22 Feb 2024 22:04:18 +0800 Subject: padata: downgrade padata_do_multithreaded to serial execution for non-SMP hugetlb parallelization depends on PADATA, and PADATA depends on SMP. PADATA consists of two distinct functionality: One part is padata_do_multithreaded which disregards order and simply divides tasks into several groups for parallel execution. Hugetlb init parallelization depends on padata_do_multithreaded. The other part is composed of a set of APIs that, while handling data in an out-of-order parallel manner, can eventually return the data with ordered sequence. Currently Only `crypto/pcrypt.c` use them. All users of PADATA of non-SMP case currently only use padata_do_multithreaded. It is easy to implement a serial one in include/linux/padata.h. And it is not necessary to implement another functionality unless the only user of crypto/pcrypt.c does not depend on SMP in the future. Link: https://lkml.kernel.org/r/20240222140422.393911-6-gang.li@linux.dev Signed-off-by: Gang Li Tested-by: Paul E. McKenney Acked-by: Daniel Jordan Cc: David Hildenbrand Cc: David Rientjes Cc: Jane Chu Cc: Muchun Song Cc: Randy Dunlap Cc: Steffen Klassert Cc: Tim Chen Cc: Alexey Dobriyan Cc: Mike Kravetz Signed-off-by: Andrew Morton --- include/linux/padata.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/padata.h b/include/linux/padata.h index 8f418711351b..0146daf34430 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -180,10 +180,6 @@ struct padata_instance { #ifdef CONFIG_PADATA extern void __init padata_init(void); -#else -static inline void __init padata_init(void) {} -#endif - extern struct padata_instance *padata_alloc(const char *name); extern void padata_free(struct padata_instance *pinst); extern struct padata_shell *padata_alloc_shell(struct padata_instance *pinst); @@ -194,4 +190,12 @@ extern void padata_do_serial(struct padata_priv *padata); extern void __init padata_do_multithreaded(struct padata_mt_job *job); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); +#else +static inline void __init padata_init(void) {} +static inline void __init padata_do_multithreaded(struct padata_mt_job *job) +{ + job->thread_fn(job->start, job->start + job->size, job->fn_arg); +} +#endif + #endif -- cgit v1.2.3 From b78b27d02930f6f0262353080d0f784ce7aa377e Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 22 Feb 2024 22:04:21 +0800 Subject: hugetlb: parallelize 1G hugetlb initialization Optimizing the initialization speed of 1G huge pages through parallelization. 1G hugetlbs are allocated from bootmem, a process that is already very fast and does not currently require optimization. Therefore, we focus on parallelizing only the initialization phase in `gather_bootmem_prealloc`. Here are some test results: test case no patch(ms) patched(ms) saved ------------------- -------------- ------------- -------- 256c2T(4 node) 1G 4745 2024 57.34% 128c1T(2 node) 1G 3358 1712 49.02% 12T 1G 77000 18300 76.23% [akpm@linux-foundation.org: s/initialied/initialized/, per Alexey] Link: https://lkml.kernel.org/r/20240222140422.393911-9-gang.li@linux.dev Signed-off-by: Gang Li Tested-by: David Rientjes Reviewed-by: Muchun Song Cc: Alexey Dobriyan Cc: Daniel Jordan Cc: David Hildenbrand Cc: Jane Chu Cc: Mike Kravetz Cc: Paul E. McKenney Cc: Randy Dunlap Cc: Steffen Klassert Cc: Tim Chen Signed-off-by: Andrew Morton --- arch/powerpc/mm/hugetlbpage.c | 2 +- include/linux/hugetlb.h | 2 +- mm/hugetlb.c | 51 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 45 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 16557d008eef..594a4b7b2ca2 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -226,7 +226,7 @@ static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) return 0; m = phys_to_virt(gpage_freearray[--nr_gpages]); gpage_freearray[nr_gpages] = 0; - list_add(&m->list, &huge_boot_pages); + list_add(&m->list, &huge_boot_pages[0]); m->hstate = hstate; return 1; } diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index c1ee640d87b1..77b30a8c6076 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -178,7 +178,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage); extern int sysctl_hugetlb_shm_group; -extern struct list_head huge_boot_pages; +extern struct list_head huge_boot_pages[MAX_NUMNODES]; /* arch callbacks */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 9934ed8aad68..418d66953224 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -69,7 +69,7 @@ static bool hugetlb_cma_folio(struct folio *folio, unsigned int order) #endif static unsigned long hugetlb_cma_size __initdata; -__initdata LIST_HEAD(huge_boot_pages); +__initdata struct list_head huge_boot_pages[MAX_NUMNODES]; /* for command line parsing */ static struct hstate * __initdata parsed_hstate; @@ -3301,7 +3301,7 @@ int alloc_bootmem_huge_page(struct hstate *h, int nid) int __alloc_bootmem_huge_page(struct hstate *h, int nid) { struct huge_bootmem_page *m = NULL; /* initialize for clang */ - int nr_nodes, node; + int nr_nodes, node = nid; /* do node specific alloc */ if (nid != NUMA_NO_NODE) { @@ -3339,7 +3339,7 @@ found: huge_page_size(h) - PAGE_SIZE); /* Put them into a private list first because mem_map is not up yet */ INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages); + list_add(&m->list, &huge_boot_pages[node]); m->hstate = h; return 1; } @@ -3390,8 +3390,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, /* Send list for bulk vmemmap optimization processing */ hugetlb_vmemmap_optimize_folios(h, folio_list); - /* Add all new pool pages to free lists in one lock cycle */ - spin_lock_irqsave(&hugetlb_lock, flags); list_for_each_entry_safe(folio, tmp_f, folio_list, lru) { if (!folio_test_hugetlb_vmemmap_optimized(folio)) { /* @@ -3404,23 +3402,25 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, HUGETLB_VMEMMAP_RESERVE_PAGES, pages_per_huge_page(h)); } + /* Subdivide locks to achieve better parallel performance */ + spin_lock_irqsave(&hugetlb_lock, flags); __prep_account_new_huge_page(h, folio_nid(folio)); enqueue_hugetlb_folio(h, folio); + spin_unlock_irqrestore(&hugetlb_lock, flags); } - spin_unlock_irqrestore(&hugetlb_lock, flags); } /* * Put bootmem huge pages into the standard lists after mem_map is up. * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages. */ -static void __init gather_bootmem_prealloc(void) +static void __init gather_bootmem_prealloc_node(unsigned long nid) { LIST_HEAD(folio_list); struct huge_bootmem_page *m; struct hstate *h = NULL, *prev_h = NULL; - list_for_each_entry(m, &huge_boot_pages, list) { + list_for_each_entry(m, &huge_boot_pages[nid], list) { struct page *page = virt_to_page(m); struct folio *folio = (void *)page; @@ -3453,6 +3453,31 @@ static void __init gather_bootmem_prealloc(void) prep_and_add_bootmem_folios(h, &folio_list); } +static void __init gather_bootmem_prealloc_parallel(unsigned long start, + unsigned long end, void *arg) +{ + int nid; + + for (nid = start; nid < end; nid++) + gather_bootmem_prealloc_node(nid); +} + +static void __init gather_bootmem_prealloc(void) +{ + struct padata_mt_job job = { + .thread_fn = gather_bootmem_prealloc_parallel, + .fn_arg = NULL, + .start = 0, + .size = num_node_state(N_MEMORY), + .align = 1, + .min_chunk = 1, + .max_threads = num_node_state(N_MEMORY), + .numa_aware = true, + }; + + padata_do_multithreaded(&job); +} + static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) { unsigned long i; @@ -3600,6 +3625,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) static void __init hugetlb_hstate_alloc_pages(struct hstate *h) { unsigned long allocated; + static bool initialized __initdata; /* skip gigantic hugepages allocation if hugetlb_cma enabled */ if (hstate_is_gigantic(h) && hugetlb_cma_size) { @@ -3607,6 +3633,15 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h) return; } + /* hugetlb_hstate_alloc_pages will be called many times, initialize huge_boot_pages once */ + if (!initialized) { + int i = 0; + + for (i = 0; i < MAX_NUMNODES; i++) + INIT_LIST_HEAD(&huge_boot_pages[i]); + initialized = true; + } + /* do node specific alloc */ if (hugetlb_hstate_alloc_pages_specific_nodes(h)) return; -- cgit v1.2.3 From dfbac6dc68bae989bd68a56947dcca16c5574fda Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:28 +0000 Subject: mm: separate out FOLIO_FLAGS from PAGEFLAGS Patch series "PageFlags cleanups". We have now successfully removed all of the uses of some of the PageFlags from the kernel, but there's nothing to stop somebody reintroducing them. By splitting out FOLIO_FLAGS from PAGEFLAGS, we can stop defining the old flags; and we do that in some of the later patches. After doing this, I realised that dump_page() was living dangerously; we could end up calling folio_test_foo() on a pointer which no longer pointed to a folio (as dump_page() is not necessarily called when the caller has a reference to the page). So I fixed that up. And then I realised that this was the key to making dump_page() take a const argument, which means we can constify the page flags testing, which means we can remove more cast-away-the-const bad code. And here's where I ended up. This patch (of 8): We've progressed far enough with the folio transition that some flags are now no longer checked on pages, but only on folios. To prevent new users appearing, prepare to only define the folio versions of the flag test/set/clear. Link: https://lkml.kernel.org/r/20240227192337.757313-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240227192337.757313-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 63 +++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 735cddc13d20..95ab75d0b39c 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -367,54 +367,77 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) #define FOLIO_PF_NO_COMPOUND 0 #define FOLIO_PF_SECOND 1 +#define FOLIO_HEAD_PAGE 0 +#define FOLIO_SECOND_PAGE 1 + /* * Macros to create function definitions for page flags */ +#define FOLIO_TEST_FLAG(name, page) \ +static __always_inline bool folio_test_##name(struct folio *folio) \ +{ return test_bit(PG_##name, folio_flags(folio, page)); } + +#define FOLIO_SET_FLAG(name, page) \ +static __always_inline void folio_set_##name(struct folio *folio) \ +{ set_bit(PG_##name, folio_flags(folio, page)); } + +#define FOLIO_CLEAR_FLAG(name, page) \ +static __always_inline void folio_clear_##name(struct folio *folio) \ +{ clear_bit(PG_##name, folio_flags(folio, page)); } + +#define __FOLIO_SET_FLAG(name, page) \ +static __always_inline void __folio_set_##name(struct folio *folio) \ +{ __set_bit(PG_##name, folio_flags(folio, page)); } + +#define __FOLIO_CLEAR_FLAG(name, page) \ +static __always_inline void __folio_clear_##name(struct folio *folio) \ +{ __clear_bit(PG_##name, folio_flags(folio, page)); } + +#define FOLIO_TEST_SET_FLAG(name, page) \ +static __always_inline bool folio_test_set_##name(struct folio *folio) \ +{ return test_and_set_bit(PG_##name, folio_flags(folio, page)); } + +#define FOLIO_TEST_CLEAR_FLAG(name, page) \ +static __always_inline bool folio_test_clear_##name(struct folio *folio) \ +{ return test_and_clear_bit(PG_##name, folio_flags(folio, page)); } + +#define FOLIO_FLAG(name, page) \ +FOLIO_TEST_FLAG(name, page) \ +FOLIO_SET_FLAG(name, page) \ +FOLIO_CLEAR_FLAG(name, page) + #define TESTPAGEFLAG(uname, lname, policy) \ -static __always_inline bool folio_test_##lname(struct folio *folio) \ -{ return test_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +FOLIO_TEST_FLAG(lname, FOLIO_##policy) \ static __always_inline int Page##uname(struct page *page) \ { return test_bit(PG_##lname, &policy(page, 0)->flags); } #define SETPAGEFLAG(uname, lname, policy) \ -static __always_inline \ -void folio_set_##lname(struct folio *folio) \ -{ set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +FOLIO_SET_FLAG(lname, FOLIO_##policy) \ static __always_inline void SetPage##uname(struct page *page) \ { set_bit(PG_##lname, &policy(page, 1)->flags); } #define CLEARPAGEFLAG(uname, lname, policy) \ -static __always_inline \ -void folio_clear_##lname(struct folio *folio) \ -{ clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \ static __always_inline void ClearPage##uname(struct page *page) \ { clear_bit(PG_##lname, &policy(page, 1)->flags); } #define __SETPAGEFLAG(uname, lname, policy) \ -static __always_inline \ -void __folio_set_##lname(struct folio *folio) \ -{ __set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +__FOLIO_SET_FLAG(lname, FOLIO_##policy) \ static __always_inline void __SetPage##uname(struct page *page) \ { __set_bit(PG_##lname, &policy(page, 1)->flags); } #define __CLEARPAGEFLAG(uname, lname, policy) \ -static __always_inline \ -void __folio_clear_##lname(struct folio *folio) \ -{ __clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +__FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \ static __always_inline void __ClearPage##uname(struct page *page) \ { __clear_bit(PG_##lname, &policy(page, 1)->flags); } #define TESTSETFLAG(uname, lname, policy) \ -static __always_inline \ -bool folio_test_set_##lname(struct folio *folio) \ -{ return test_and_set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +FOLIO_TEST_SET_FLAG(lname, FOLIO_##policy) \ static __always_inline int TestSetPage##uname(struct page *page) \ { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); } #define TESTCLEARFLAG(uname, lname, policy) \ -static __always_inline \ -bool folio_test_clear_##lname(struct folio *folio) \ -{ return test_and_clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \ +FOLIO_TEST_CLEAR_FLAG(lname, FOLIO_##policy) \ static __always_inline int TestClearPage##uname(struct page *page) \ { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); } -- cgit v1.2.3 From 0d846469fd216d37a91845945e9baad11dfa107b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:29 +0000 Subject: mm: remove PageWaiters, PageSetWaiters and PageClearWaiters All callers have been converted to use folios. This was the only user of PF_ONLY_HEAD, so remove that too. Link: https://lkml.kernel.org/r/20240227192337.757313-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 95ab75d0b39c..d8f5127ae72e 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -328,9 +328,6 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) * for compound page all operations related to the page flag applied to * head page. * - * PF_ONLY_HEAD: - * for compound page, callers only ever operate on the head page. - * * PF_NO_TAIL: * modifications of the page flag must be done on small or head pages, * checks can be done on tail pages too. @@ -346,9 +343,6 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) page; }) #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) -#define PF_ONLY_HEAD(page, enforce) ({ \ - VM_BUG_ON_PGFLAGS(PageTail(page), page); \ - PF_POISONED_CHECK(page); }) #define PF_NO_TAIL(page, enforce) ({ \ VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \ PF_POISONED_CHECK(compound_head(page)); }) @@ -362,7 +356,6 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) /* Which page is the flag stored in */ #define FOLIO_PF_ANY 0 #define FOLIO_PF_HEAD 0 -#define FOLIO_PF_ONLY_HEAD 0 #define FOLIO_PF_NO_TAIL 0 #define FOLIO_PF_NO_COMPOUND 0 #define FOLIO_PF_SECOND 1 @@ -488,7 +481,7 @@ static inline int TestClearPage##uname(struct page *page) { return 0; } TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname) __PAGEFLAG(Locked, locked, PF_NO_TAIL) -PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) +FOLIO_FLAG(waiters, FOLIO_HEAD_PAGE) PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL) PAGEFLAG(Referenced, referenced, PF_HEAD) TESTCLEARFLAG(Referenced, referenced, PF_HEAD) @@ -1138,7 +1131,6 @@ static inline bool folio_has_private(struct folio *folio) #undef PF_ANY #undef PF_HEAD -#undef PF_ONLY_HEAD #undef PF_NO_TAIL #undef PF_NO_COMPOUND #undef PF_SECOND -- cgit v1.2.3 From 7da8988c7c0e28dad8d0e9a697d6e7baa66f4534 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:30 +0000 Subject: mm: remove PageYoung and PageIdle definitions All callers have been converted to use folios, so remove the various set/clear/test functions defined on pages. Link: https://lkml.kernel.org/r/20240227192337.757313-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index d8f5127ae72e..582ca7400eca 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -599,10 +599,10 @@ PAGEFLAG_FALSE(HWPoison, hwpoison) #endif #if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) -TESTPAGEFLAG(Young, young, PF_ANY) -SETPAGEFLAG(Young, young, PF_ANY) -TESTCLEARFLAG(Young, young, PF_ANY) -PAGEFLAG(Idle, idle, PF_ANY) +FOLIO_TEST_FLAG(young, FOLIO_HEAD_PAGE) +FOLIO_SET_FLAG(young, FOLIO_HEAD_PAGE) +FOLIO_TEST_CLEAR_FLAG(young, FOLIO_HEAD_PAGE) +FOLIO_FLAG(idle, FOLIO_HEAD_PAGE) #endif /* -- cgit v1.2.3 From fae7d834c43ccdb9fcecaf4d0f33145d884b3e5c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:31 +0000 Subject: mm: add __dump_folio() Turn __dump_page() into a wrapper around __dump_folio(). Snapshot the page & folio into a stack variable so we don't hit BUG_ON() if an allocation is freed under us and what was a folio pointer becomes a pointer to a tail page. [willy@infradead.org: fix build issue] Link: https://lkml.kernel.org/r/ZeAKCyTn_xS3O9cE@casper.infradead.org [willy@infradead.org: fix __dump_folio] Link: https://lkml.kernel.org/r/ZeJJegP8zM7S9GTy@casper.infradead.org [willy@infradead.org: fix pointer confusion] Link: https://lkml.kernel.org/r/ZeYa00ixxC4k1ot-@casper.infradead.org [akpm@linux-foundation.org: s/printk/pr_warn/] Link: https://lkml.kernel.org/r/20240227192337.757313-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 7 +++ include/linux/mmzone.h | 3 ++ mm/debug.c | 128 ++++++++++++++++++++++++++++--------------------- 3 files changed, 83 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index d45eadc440f5..02547c8adda0 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2066,6 +2066,13 @@ static inline long folio_nr_pages(struct folio *folio) #endif } +/* Only hugetlbfs can allocate folios larger than MAX_ORDER */ +#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE +#define MAX_FOLIO_NR_PAGES (1UL << PUD_ORDER) +#else +#define MAX_FOLIO_NR_PAGES MAX_ORDER_NR_PAGES +#endif + /* * compound_nr() returns the number of pages in this potentially compound * page. compound_nr() can be called on a tail page, and is defined to diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 633812a1d220..c11b7cde81ef 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -76,9 +76,12 @@ extern const char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define is_migrate_cma_folio(folio, pfn) (MIGRATE_CMA == \ + get_pfnblock_flags_mask(&folio->page, pfn, MIGRATETYPE_MASK)) #else # define is_migrate_cma(migratetype) false # define is_migrate_cma_page(_page) false +# define is_migrate_cma_folio(folio, pfn) false #endif static inline bool is_migrate_movable(int mt) diff --git a/mm/debug.c b/mm/debug.c index ee533a5ceb79..12fed592c955 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -51,84 +51,102 @@ const struct trace_print_flags vmaflag_names[] = { {0, NULL} }; -static void __dump_page(struct page *page) +static void __dump_folio(struct folio *folio, struct page *page, + unsigned long pfn, unsigned long idx) { - struct folio *folio = page_folio(page); - struct page *head = &folio->page; - struct address_space *mapping; - bool compound = PageCompound(page); - /* - * Accessing the pageblock without the zone lock. It could change to - * "isolate" again in the meantime, but since we are just dumping the - * state for debugging, it should be fine to accept a bit of - * inaccuracy here due to racing. - */ - bool page_cma = is_migrate_cma_page(page); - int mapcount; + struct address_space *mapping = folio_mapping(folio); + int mapcount = 0; char *type = ""; - if (page < head || (page >= head + MAX_ORDER_NR_PAGES)) { - /* - * Corrupt page, so we cannot call page_mapping. Instead, do a - * safe subset of the steps that page_mapping() does. Caution: - * this will be misleading for tail pages, PageSwapCache pages, - * and potentially other situations. (See the page_mapping() - * implementation for what's missing here.) - */ - unsigned long tmp = (unsigned long)page->mapping; - - if (tmp & PAGE_MAPPING_ANON) - mapping = NULL; - else - mapping = (void *)(tmp & ~PAGE_MAPPING_FLAGS); - head = page; - folio = (struct folio *)page; - compound = false; - } else { - mapping = page_mapping(page); - } - /* - * Avoid VM_BUG_ON() in page_mapcount(). - * page->_mapcount space in struct page is used by sl[aou]b pages to - * encode own info. + * page->_mapcount space in struct page is used by slab pages to + * encode own info, and we must avoid calling page_folio() again. */ - mapcount = PageSlab(head) ? 0 : page_mapcount(page); - - pr_warn("page:%p refcount:%d mapcount:%d mapping:%p index:%#lx pfn:%#lx\n", - page, page_ref_count(head), mapcount, mapping, - page_to_pgoff(page), page_to_pfn(page)); - if (compound) { - pr_warn("head:%p order:%u entire_mapcount:%d nr_pages_mapped:%d pincount:%d\n", - head, compound_order(head), + if (!folio_test_slab(folio)) { + mapcount = atomic_read(&page->_mapcount) + 1; + if (folio_test_large(folio)) + mapcount += folio_entire_mapcount(folio); + } + + pr_warn("page: refcount:%d mapcount:%d mapping:%p index:%#lx pfn:%#lx\n", + folio_ref_count(folio), mapcount, mapping, + folio->index + idx, pfn); + if (folio_test_large(folio)) { + pr_warn("head: order:%u entire_mapcount:%d nr_pages_mapped:%d pincount:%d\n", + folio_order(folio), folio_entire_mapcount(folio), folio_nr_pages_mapped(folio), atomic_read(&folio->_pincount)); } #ifdef CONFIG_MEMCG - if (head->memcg_data) - pr_warn("memcg:%lx\n", head->memcg_data); + if (folio->memcg_data) + pr_warn("memcg:%lx\n", folio->memcg_data); #endif - if (PageKsm(page)) + if (folio_test_ksm(folio)) type = "ksm "; - else if (PageAnon(page)) + else if (folio_test_anon(folio)) type = "anon "; else if (mapping) dump_mapping(mapping); BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS + 1); - pr_warn("%sflags: %pGp%s\n", type, &head->flags, - page_cma ? " CMA" : ""); - pr_warn("page_type: %pGt\n", &head->page_type); + /* + * Accessing the pageblock without the zone lock. It could change to + * "isolate" again in the meantime, but since we are just dumping the + * state for debugging, it should be fine to accept a bit of + * inaccuracy here due to racing. + */ + pr_warn("%sflags: %pGp%s\n", type, &folio->flags, + is_migrate_cma_folio(folio, pfn) ? " CMA" : ""); + pr_warn("page_type: %pGt\n", &folio->page.page_type); print_hex_dump(KERN_WARNING, "raw: ", DUMP_PREFIX_NONE, 32, sizeof(unsigned long), page, sizeof(struct page), false); - if (head != page) + if (folio_test_large(folio)) print_hex_dump(KERN_WARNING, "head: ", DUMP_PREFIX_NONE, 32, - sizeof(unsigned long), head, - sizeof(struct page), false); + sizeof(unsigned long), folio, + 2 * sizeof(struct page), false); +} + +static void __dump_page(const struct page *page) +{ + struct folio *foliop, folio; + struct page precise; + unsigned long pfn = page_to_pfn(page); + unsigned long idx, nr_pages = 1; + int loops = 5; + +again: + memcpy(&precise, page, sizeof(*page)); + foliop = page_folio(&precise); + if (foliop == (struct folio *)&precise) { + idx = 0; + if (!folio_test_large(foliop)) + goto dump; + foliop = (struct folio *)page; + } else { + idx = folio_page_idx(foliop, page); + } + + if (idx < MAX_FOLIO_NR_PAGES) { + memcpy(&folio, foliop, 2 * sizeof(struct page)); + nr_pages = folio_nr_pages(&folio); + foliop = &folio; + } + + if (idx > nr_pages) { + if (loops-- > 0) + goto again; + pr_warn("page does not match folio\n"); + precise.compound_head &= ~1UL; + foliop = (struct folio *)&precise; + idx = 0; + } + +dump: + __dump_folio(foliop, &precise, pfn, idx); } void dump_page(struct page *page, const char *reason) -- cgit v1.2.3 From b3a3203309c89061452250f7384507787b7badcb Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:32 +0000 Subject: mm: make dump_page() take a const argument Now that __dump_page() takes a const argument, we can make dump_page() take a const struct page too. Link: https://lkml.kernel.org/r/20240227192337.757313-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/mmdebug.h | 2 +- mm/debug.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmdebug.h b/include/linux/mmdebug.h index 7c3e7b0b0e8f..39a7714605a7 100644 --- a/include/linux/mmdebug.h +++ b/include/linux/mmdebug.h @@ -10,7 +10,7 @@ struct vm_area_struct; struct mm_struct; struct vma_iterator; -void dump_page(struct page *page, const char *reason); +void dump_page(const struct page *page, const char *reason); void dump_vma(const struct vm_area_struct *vma); void dump_mm(const struct mm_struct *mm); void vma_iter_dump_tree(const struct vma_iterator *vmi); diff --git a/mm/debug.c b/mm/debug.c index 12fed592c955..c1c1a6a484e4 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -149,7 +149,7 @@ dump: __dump_folio(foliop, &precise, pfn, idx); } -void dump_page(struct page *page, const char *reason) +void dump_page(const struct page *page, const char *reason) { if (PagePoisoned(page)) pr_warn("page:%p is uninitialized and poisoned", page); -- cgit v1.2.3 From ce3467af6bded1c0018ca67ea1599f45fbb8100b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:33 +0000 Subject: mm: constify testing page/folio flags Now that dump_page() takes a const argument, we can constify all the page flag tests. Link: https://lkml.kernel.org/r/20240227192337.757313-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 582ca7400eca..3463cd1baebf 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -237,7 +237,7 @@ static inline const struct page *page_fixed_fake_head(const struct page *page) } #endif -static __always_inline int page_is_fake_head(struct page *page) +static __always_inline int page_is_fake_head(const struct page *page) { return page_fixed_fake_head(page) != page; } @@ -281,12 +281,12 @@ static inline unsigned long _compound_head(const struct page *page) */ #define folio_page(folio, n) nth_page(&(folio)->page, n) -static __always_inline int PageTail(struct page *page) +static __always_inline int PageTail(const struct page *page) { return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page); } -static __always_inline int PageCompound(struct page *page) +static __always_inline int PageCompound(const struct page *page) { return test_bit(PG_head, &page->flags) || READ_ONCE(page->compound_head) & 1; @@ -306,6 +306,16 @@ static inline void page_init_poison(struct page *page, size_t size) } #endif +static const unsigned long *const_folio_flags(const struct folio *folio, + unsigned n) +{ + const struct page *page = &folio->page; + + VM_BUG_ON_PGFLAGS(PageTail(page), page); + VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page); + return &page[n].flags; +} + static unsigned long *folio_flags(struct folio *folio, unsigned n) { struct page *page = &folio->page; @@ -367,8 +377,8 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) * Macros to create function definitions for page flags */ #define FOLIO_TEST_FLAG(name, page) \ -static __always_inline bool folio_test_##name(struct folio *folio) \ -{ return test_bit(PG_##name, folio_flags(folio, page)); } +static __always_inline bool folio_test_##name(const struct folio *folio) \ +{ return test_bit(PG_##name, const_folio_flags(folio, page)); } #define FOLIO_SET_FLAG(name, page) \ static __always_inline void folio_set_##name(struct folio *folio) \ @@ -401,7 +411,7 @@ FOLIO_CLEAR_FLAG(name, page) #define TESTPAGEFLAG(uname, lname, policy) \ FOLIO_TEST_FLAG(lname, FOLIO_##policy) \ -static __always_inline int Page##uname(struct page *page) \ +static __always_inline int Page##uname(const struct page *page) \ { return test_bit(PG_##lname, &policy(page, 0)->flags); } #define SETPAGEFLAG(uname, lname, policy) \ @@ -801,7 +811,7 @@ static __always_inline bool folio_test_head(struct folio *folio) return test_bit(PG_head, folio_flags(folio, FOLIO_PF_ANY)); } -static __always_inline int PageHead(struct page *page) +static __always_inline int PageHead(const struct page *page) { PF_POISONED_CHECK(page); return test_bit(PG_head, &page->flags) && !page_is_fake_head(page); -- cgit v1.2.3 From 29cfe7556bfd6be043b6eb602a29c89d43565d71 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:34 +0000 Subject: mm: constify more page/folio tests Constify the flag tests that aren't automatically generated and the tests that look like flag tests but are more complicated. Link: https://lkml.kernel.org/r/20240227192337.757313-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 52 +++++++++++++++++++++++----------------------- mm/hugetlb.c | 4 ++-- 2 files changed, 28 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 3463cd1baebf..652d77805e99 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -558,13 +558,13 @@ PAGEFLAG_FALSE(HighMem, highmem) #endif #ifdef CONFIG_SWAP -static __always_inline bool folio_test_swapcache(struct folio *folio) +static __always_inline bool folio_test_swapcache(const struct folio *folio) { return folio_test_swapbacked(folio) && - test_bit(PG_swapcache, folio_flags(folio, 0)); + test_bit(PG_swapcache, const_folio_flags(folio, 0)); } -static __always_inline bool PageSwapCache(struct page *page) +static __always_inline bool PageSwapCache(const struct page *page) { return folio_test_swapcache(page_folio(page)); } @@ -663,22 +663,22 @@ PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted) */ #define PAGE_MAPPING_DAX_SHARED ((void *)0x1) -static __always_inline bool folio_mapping_flags(struct folio *folio) +static __always_inline bool folio_mapping_flags(const struct folio *folio) { return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0; } -static __always_inline int PageMappingFlags(struct page *page) +static __always_inline int PageMappingFlags(const struct page *page) { return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; } -static __always_inline bool folio_test_anon(struct folio *folio) +static __always_inline bool folio_test_anon(const struct folio *folio) { return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0; } -static __always_inline bool PageAnon(struct page *page) +static __always_inline bool PageAnon(const struct page *page) { return folio_test_anon(page_folio(page)); } @@ -689,7 +689,7 @@ static __always_inline bool __folio_test_movable(const struct folio *folio) PAGE_MAPPING_MOVABLE; } -static __always_inline int __PageMovable(struct page *page) +static __always_inline int __PageMovable(const struct page *page) { return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == PAGE_MAPPING_MOVABLE; @@ -702,13 +702,13 @@ static __always_inline int __PageMovable(struct page *page) * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any * anon_vma, but to that page's node of the stable tree. */ -static __always_inline bool folio_test_ksm(struct folio *folio) +static __always_inline bool folio_test_ksm(const struct folio *folio) { return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) == PAGE_MAPPING_KSM; } -static __always_inline bool PageKsm(struct page *page) +static __always_inline bool PageKsm(const struct page *page) { return folio_test_ksm(page_folio(page)); } @@ -747,9 +747,9 @@ static inline bool folio_xor_flags_has_waiters(struct folio *folio, * some of the bytes in it may be; see the is_partially_uptodate() * address_space operation. */ -static inline bool folio_test_uptodate(struct folio *folio) +static inline bool folio_test_uptodate(const struct folio *folio) { - bool ret = test_bit(PG_uptodate, folio_flags(folio, 0)); + bool ret = test_bit(PG_uptodate, const_folio_flags(folio, 0)); /* * Must ensure that the data we read out of the folio is loaded * _after_ we've loaded folio->flags to check the uptodate bit. @@ -764,7 +764,7 @@ static inline bool folio_test_uptodate(struct folio *folio) return ret; } -static inline int PageUptodate(struct page *page) +static inline int PageUptodate(const struct page *page) { return folio_test_uptodate(page_folio(page)); } @@ -806,9 +806,9 @@ void set_page_writeback(struct page *page); #define folio_start_writeback_keepwrite(folio) \ __folio_start_writeback(folio, true) -static __always_inline bool folio_test_head(struct folio *folio) +static __always_inline bool folio_test_head(const struct folio *folio) { - return test_bit(PG_head, folio_flags(folio, FOLIO_PF_ANY)); + return test_bit(PG_head, const_folio_flags(folio, FOLIO_PF_ANY)); } static __always_inline int PageHead(const struct page *page) @@ -827,7 +827,7 @@ CLEARPAGEFLAG(Head, head, PF_ANY) * * Return: True if the folio is larger than one page. */ -static inline bool folio_test_large(struct folio *folio) +static inline bool folio_test_large(const struct folio *folio) { return folio_test_head(folio); } @@ -856,7 +856,7 @@ TESTPAGEFLAG_FALSE(LargeRmappable, large_rmappable) #define PG_head_mask ((1UL << PG_head)) #ifdef CONFIG_HUGETLB_PAGE -int PageHuge(struct page *page); +int PageHuge(const struct page *page); SETPAGEFLAG(HugeTLB, hugetlb, PF_SECOND) CLEARPAGEFLAG(HugeTLB, hugetlb, PF_SECOND) @@ -869,10 +869,10 @@ CLEARPAGEFLAG(HugeTLB, hugetlb, PF_SECOND) * Return: True for hugetlbfs folios, false for anon folios or folios * belonging to other filesystems. */ -static inline bool folio_test_hugetlb(struct folio *folio) +static inline bool folio_test_hugetlb(const struct folio *folio) { return folio_test_large(folio) && - test_bit(PG_hugetlb, folio_flags(folio, 1)); + test_bit(PG_hugetlb, const_folio_flags(folio, 1)); } #else TESTPAGEFLAG_FALSE(Huge, hugetlb) @@ -887,7 +887,7 @@ TESTPAGEFLAG_FALSE(Huge, hugetlb) * hugetlbfs pages, but not normal pages. PageTransHuge() can only be * called only in the core VM paths where hugetlbfs pages can't exist. */ -static inline int PageTransHuge(struct page *page) +static inline int PageTransHuge(const struct page *page) { VM_BUG_ON_PAGE(PageTail(page), page); return PageHead(page); @@ -898,7 +898,7 @@ static inline int PageTransHuge(struct page *page) * and hugetlbfs pages, so it should only be called when it's known * that hugetlbfs pages aren't involved. */ -static inline int PageTransCompound(struct page *page) +static inline int PageTransCompound(const struct page *page) { return PageCompound(page); } @@ -908,7 +908,7 @@ static inline int PageTransCompound(struct page *page) * and hugetlbfs pages, so it should only be called when it's known * that hugetlbfs pages aren't involved. */ -static inline int PageTransTail(struct page *page) +static inline int PageTransTail(const struct page *page) { return PageTail(page); } @@ -972,7 +972,7 @@ static inline int page_type_has_type(unsigned int page_type) return (int)page_type < PAGE_MAPCOUNT_RESERVE; } -static inline int page_has_type(struct page *page) +static inline int page_has_type(const struct page *page) { return page_type_has_type(page->page_type); } @@ -1056,7 +1056,7 @@ extern bool is_free_buddy_page(struct page *page); PAGEFLAG(Isolated, isolated, PF_ANY); -static __always_inline int PageAnonExclusive(struct page *page) +static __always_inline int PageAnonExclusive(const struct page *page) { VM_BUG_ON_PGFLAGS(!PageAnon(page), page); VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); @@ -1129,12 +1129,12 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) * Determine if a page has private stuff, indicating that release routines * should be invoked upon it. */ -static inline int page_has_private(struct page *page) +static inline int page_has_private(const struct page *page) { return !!(page->flags & PAGE_FLAGS_PRIVATE); } -static inline bool folio_has_private(struct folio *folio) +static inline bool folio_has_private(const struct folio *folio) { return page_has_private(&folio->page); } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 418d66953224..bb17e5c22759 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2164,9 +2164,9 @@ static bool prep_compound_gigantic_folio_for_demote(struct folio *folio, * transparent huge pages. See the PageTransHuge() documentation for more * details. */ -int PageHuge(struct page *page) +int PageHuge(const struct page *page) { - struct folio *folio; + const struct folio *folio; if (!PageCompound(page)) return 0; -- cgit v1.2.3 From 9164448d3100d5118bda5e9d38b69a9f32cea509 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 27 Feb 2024 19:23:35 +0000 Subject: mm: remove cast from page_to_nid() Now that PF_POISONED_CHECK() can take a const argument, we can drop the cast. Link: https://lkml.kernel.org/r/20240227192337.757313-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/mm.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 02547c8adda0..699e850d143c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1641,13 +1641,11 @@ static inline int page_zone_id(struct page *page) } #ifdef NODE_NOT_IN_PAGE_FLAGS -extern int page_to_nid(const struct page *page); +int page_to_nid(const struct page *page); #else static inline int page_to_nid(const struct page *page) { - struct page *p = (struct page *)page; - - return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; + return (PF_POISONED_CHECK(page)->flags >> NODES_PGSHIFT) & NODES_MASK; } #endif -- cgit v1.2.3 From 22beb471b46a1a408720498f7895232edab559d1 Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 4 Mar 2024 19:07:18 +0800 Subject: mm: pgtable: correct the wrong comment about ptdesc->__page_flags Patch series "minor fixes and supplement for ptdesc". In this series, the [PATCH 1/3] and [PATCH 2/3] are fixes for some issues discovered during code inspection. The [PATCH 3/3] is a supplement to ptdesc conversion in s390, I don't know why this is not done in the commit 6326c26c1514 ("s390: convert various pgalloc functions to use ptdescs"), maybe I missed something. And since I don't have an s390 environment, I hope kernel test robot can help compile and test, and this is why I did not fold [PATCH 2/3] and [PATCH 3/3] into one patch. This patch (of 3): The commit 32cc0b7c9d50 ("powerpc: add pte_free_defer() for pgtables sharing page") introduced the use of PageActive flag to page table fragments tracking, so the ptdesc->__page_flags is not unused, so correct the wrong comment. Link: https://lkml.kernel.org/r/cover.1709541697.git.zhengqi.arch@bytedance.com Link: https://lkml.kernel.org/r/cc42d5915fd98fd802f920de243f535efcfe01db.1709541697.git.zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Cc: David Hildenbrand Cc: Hugh Dickins Cc: Matthew Wilcox (Oracle) Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Vishal Moola (Oracle) Cc: Christian Borntraeger Cc: Claudio Imbrenda Cc: Janosch Frank Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a7223ba3ea1e..5ea77969daae 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -419,7 +419,7 @@ FOLIO_MATCH(compound_head, _head_2a); /** * struct ptdesc - Memory descriptor for page tables. - * @__page_flags: Same as page flags. Unused for page tables. + * @__page_flags: Same as page flags. Powerpc only. * @pt_rcu_head: For freeing page table pages. * @pt_list: List of used page tables. Used for s390 and x86. * @_pt_pad_1: Padding that aliases with page's compound head. -- cgit v1.2.3 From ea919671517a46b75f975fcf126e08ccf7e9c09f Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 4 Mar 2024 19:07:19 +0800 Subject: mm: pgtable: add missing pt_index to struct ptdesc In s390, the page->index field is used for gmap (see gmap_shadow_pgt()), so add the corresponding pt_index to struct ptdesc and add a comment to clarify this. Link: https://lkml.kernel.org/r/283624c2af45fb2090b41a6b1b5481bb0a45bad7.1709541697.git.zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Cc: Christian Borntraeger Cc: Claudio Imbrenda Cc: David Hildenbrand Cc: Hugh Dickins Cc: Janosch Frank Cc: Matthew Wilcox (Oracle) Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 5ea77969daae..5240bd7bca33 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -425,6 +425,7 @@ FOLIO_MATCH(compound_head, _head_2a); * @_pt_pad_1: Padding that aliases with page's compound head. * @pmd_huge_pte: Protected by ptdesc->ptl, used for THPs. * @__page_mapping: Aliases with page->mapping. Unused for page tables. + * @pt_index: Used for s390 gmap. * @pt_mm: Used for x86 pgds. * @pt_frag_refcount: For fragmented page table tracking. Powerpc only. * @_pt_pad_2: Padding to ensure proper alignment. @@ -450,6 +451,7 @@ struct ptdesc { unsigned long __page_mapping; union { + pgoff_t pt_index; struct mm_struct *pt_mm; atomic_t pt_frag_refcount; }; @@ -475,6 +477,7 @@ TABLE_MATCH(flags, __page_flags); TABLE_MATCH(compound_head, pt_list); TABLE_MATCH(compound_head, _pt_pad_1); TABLE_MATCH(mapping, __page_mapping); +TABLE_MATCH(index, pt_index); TABLE_MATCH(rcu_head, pt_rcu_head); TABLE_MATCH(page_type, __page_type); TABLE_MATCH(_refcount, __page_refcount); -- cgit v1.2.3 From c05995b7ec2a73bf813a8944978e175f8e4ec3ac Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Tue, 5 Mar 2024 12:37:50 +0800 Subject: mm/treewide: align up pXd_leaf() retval across archs Even if pXd_leaf() API is defined globally, it's not clear on the retval, and there are three types used (bool, int, unsigned log). Always return a boolean for pXd_leaf() APIs. Link: https://lkml.kernel.org/r/20240305043750.93762-11-peterx@redhat.com Signed-off-by: Peter Xu Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: Mike Rapoport (IBM) Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: "Aneesh Kumar K.V" Cc: Borislav Petkov Cc: Christophe Leroy Cc: Dave Hansen Cc: Dmitry Vyukov Cc: Ingo Molnar Cc: Kirill A. Shutemov Cc: Michael Ellerman Cc: Muchun Song Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Thomas Gleixner Cc: Vincenzo Frascino Cc: Yang Shi Signed-off-by: Andrew Morton --- arch/riscv/include/asm/pgtable-64.h | 2 +- arch/riscv/include/asm/pgtable.h | 2 +- arch/s390/include/asm/pgtable.h | 4 ++-- arch/sparc/include/asm/pgtable_64.h | 4 ++-- arch/x86/include/asm/pgtable.h | 8 ++++---- include/linux/pgtable.h | 8 ++++---- 6 files changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index b42017d76924..2c7e1661db01 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -190,7 +190,7 @@ static inline int pud_bad(pud_t pud) } #define pud_leaf pud_leaf -static inline int pud_leaf(pud_t pud) +static inline bool pud_leaf(pud_t pud) { return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); } diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index add5cd30ab34..6839520dbcb1 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -241,7 +241,7 @@ static inline int pmd_bad(pmd_t pmd) } #define pmd_leaf pmd_leaf -static inline int pmd_leaf(pmd_t pmd) +static inline bool pmd_leaf(pmd_t pmd) { return pmd_present(pmd) && (pmd_val(pmd) & _PAGE_LEAF); } diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 9e08af5b9247..60950e7a25f5 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -706,7 +706,7 @@ static inline int pud_none(pud_t pud) } #define pud_leaf pud_leaf -static inline int pud_leaf(pud_t pud) +static inline bool pud_leaf(pud_t pud) { if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3) return 0; @@ -714,7 +714,7 @@ static inline int pud_leaf(pud_t pud) } #define pmd_leaf pmd_leaf -static inline int pmd_leaf(pmd_t pmd) +static inline bool pmd_leaf(pmd_t pmd) { return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0; } diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 6ff0a28d5fd1..4d1bafaba942 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -681,7 +681,7 @@ static inline unsigned long pte_special(pte_t pte) } #define pmd_leaf pmd_leaf -static inline unsigned long pmd_leaf(pmd_t pmd) +static inline bool pmd_leaf(pmd_t pmd) { pte_t pte = __pte(pmd_val(pmd)); @@ -868,7 +868,7 @@ static inline pmd_t *pud_pgtable(pud_t pud) #define p4d_page(p4d) NULL #define pud_leaf pud_leaf -static inline unsigned long pud_leaf(pud_t pud) +static inline bool pud_leaf(pud_t pud) { pte_t pte = __pte(pud_val(pud)); diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index cfc84c55d0e6..7621a5acb13e 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -252,7 +252,7 @@ static inline unsigned long pgd_pfn(pgd_t pgd) } #define p4d_leaf p4d_leaf -static inline int p4d_leaf(p4d_t p4d) +static inline bool p4d_leaf(p4d_t p4d) { /* No 512 GiB pages yet */ return 0; @@ -261,7 +261,7 @@ static inline int p4d_leaf(p4d_t p4d) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) #define pmd_leaf pmd_leaf -static inline int pmd_leaf(pmd_t pte) +static inline bool pmd_leaf(pmd_t pte) { return pmd_flags(pte) & _PAGE_PSE; } @@ -1086,7 +1086,7 @@ static inline pmd_t *pud_pgtable(pud_t pud) #define pud_page(pud) pfn_to_page(pud_pfn(pud)) #define pud_leaf pud_leaf -static inline int pud_leaf(pud_t pud) +static inline bool pud_leaf(pud_t pud) { return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) == (_PAGE_PSE | _PAGE_PRESENT); @@ -1413,7 +1413,7 @@ static inline bool pgdp_maps_userspace(void *__ptr) } #define pgd_leaf pgd_leaf -static inline int pgd_leaf(pgd_t pgd) { return 0; } +static inline bool pgd_leaf(pgd_t pgd) { return false; } #ifdef CONFIG_PAGE_TABLE_ISOLATION /* diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index a36cf4e124b0..85fc7554cd52 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1777,16 +1777,16 @@ typedef unsigned int pgtbl_mod_mask; * Only meaningful when called on a valid entry. */ #ifndef pgd_leaf -#define pgd_leaf(x) 0 +#define pgd_leaf(x) false #endif #ifndef p4d_leaf -#define p4d_leaf(x) 0 +#define p4d_leaf(x) false #endif #ifndef pud_leaf -#define pud_leaf(x) 0 +#define pud_leaf(x) false #endif #ifndef pmd_leaf -#define pmd_leaf(x) 0 +#define pmd_leaf(x) false #endif #ifndef pgd_leaf_size -- cgit v1.2.3 From e6f798225a31485e47a6e4f6aa07ee9fdf80c2cb Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 4 Mar 2024 19:05:16 -0800 Subject: mm: Introduce VM_SPARSE kind and vm_area_[un]map_pages(). vmap/vmalloc APIs are used to map a set of pages into contiguous kernel virtual space. get_vm_area() with appropriate flag is used to request an area of kernel address range. It's used for vmalloc, vmap, ioremap, xen use cases. - vmalloc use case dominates the usage. Such vm areas have VM_ALLOC flag. - the areas created by vmap() function should be tagged with VM_MAP. - ioremap areas are tagged with VM_IOREMAP. BPF would like to extend the vmap API to implement a lazily-populated sparse, yet contiguous kernel virtual space. Introduce VM_SPARSE flag and vm_area_map_pages(area, start_addr, count, pages) API to map a set of pages within a given area. It has the same sanity checks as vmap() does. It also checks that get_vm_area() was created with VM_SPARSE flag which identifies such areas in /proc/vmallocinfo and returns zero pages on read through /proc/kcore. The next commits will introduce bpf_arena which is a sparsely populated shared memory region between bpf program and user space process. It will map privately-managed pages into a sparse vm area with the following steps: // request virtual memory region during bpf prog verification area = get_vm_area(area_size, VM_SPARSE); // on demand vm_area_map_pages(area, kaddr, kend, pages); vm_area_unmap_pages(area, kaddr, kend); // after bpf program is detached and unloaded free_vm_area(area); Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Reviewed-by: Christoph Hellwig Reviewed-by: Pasha Tatashin Link: https://lore.kernel.org/bpf/20240305030516.41519-3-alexei.starovoitov@gmail.com --- include/linux/vmalloc.h | 5 +++++ mm/vmalloc.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index c720be70c8dd..0f72c85a377b 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -35,6 +35,7 @@ struct iov_iter; /* in uio.h */ #else #define VM_DEFER_KMEMLEAK 0 #endif +#define VM_SPARSE 0x00001000 /* sparse vm_area. not all pages are present. */ /* bits [20..32] reserved for arch specific ioremap internals */ @@ -232,6 +233,10 @@ static inline bool is_vm_area_hugepages(const void *addr) } #ifdef CONFIG_MMU +int vm_area_map_pages(struct vm_struct *area, unsigned long start, + unsigned long end, struct page **pages); +void vm_area_unmap_pages(struct vm_struct *area, unsigned long start, + unsigned long end); void vunmap_range(unsigned long addr, unsigned long end); static inline void set_vm_flush_reset_perms(void *addr) { diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f42f98a127d5..e5b8c70950bc 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -648,6 +648,58 @@ static int vmap_pages_range(unsigned long addr, unsigned long end, return err; } +static int check_sparse_vm_area(struct vm_struct *area, unsigned long start, + unsigned long end) +{ + might_sleep(); + if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS)) + return -EINVAL; + if (WARN_ON_ONCE(area->flags & VM_NO_GUARD)) + return -EINVAL; + if (WARN_ON_ONCE(!(area->flags & VM_SPARSE))) + return -EINVAL; + if ((end - start) >> PAGE_SHIFT > totalram_pages()) + return -E2BIG; + if (start < (unsigned long)area->addr || + (void *)end > area->addr + get_vm_area_size(area)) + return -ERANGE; + return 0; +} + +/** + * vm_area_map_pages - map pages inside given sparse vm_area + * @area: vm_area + * @start: start address inside vm_area + * @end: end address inside vm_area + * @pages: pages to map (always PAGE_SIZE pages) + */ +int vm_area_map_pages(struct vm_struct *area, unsigned long start, + unsigned long end, struct page **pages) +{ + int err; + + err = check_sparse_vm_area(area, start, end); + if (err) + return err; + + return vmap_pages_range(start, end, PAGE_KERNEL, pages, PAGE_SHIFT); +} + +/** + * vm_area_unmap_pages - unmap pages inside given sparse vm_area + * @area: vm_area + * @start: start address inside vm_area + * @end: end address inside vm_area + */ +void vm_area_unmap_pages(struct vm_struct *area, unsigned long start, + unsigned long end) +{ + if (check_sparse_vm_area(area, start, end)) + return; + + vunmap_range(start, end); +} + int is_vmalloc_or_module_addr(const void *x) { /* @@ -3822,9 +3874,9 @@ long vread_iter(struct iov_iter *iter, const char *addr, size_t count) if (flags & VMAP_RAM) copied = vmap_ram_vread_iter(iter, addr, n, flags); - else if (!(vm && (vm->flags & VM_IOREMAP))) + else if (!(vm && (vm->flags & (VM_IOREMAP | VM_SPARSE)))) copied = aligned_vread_iter(iter, addr, n); - else /* IOREMAP area is treated as memory hole */ + else /* IOREMAP | SPARSE area is treated as memory hole */ copied = zero_iter(iter, n); addr += copied; @@ -4415,6 +4467,9 @@ static int s_show(struct seq_file *m, void *p) if (v->flags & VM_IOREMAP) seq_puts(m, " ioremap"); + if (v->flags & VM_SPARSE) + seq_puts(m, " sparse"); + if (v->flags & VM_ALLOC) seq_puts(m, " vmalloc"); -- cgit v1.2.3 From 011832b97b311bb9e3c27945bc0d1089a14209c9 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 5 Mar 2024 19:19:26 -0800 Subject: bpf: Introduce may_goto instruction Introduce may_goto instruction that from the verifier pov is similar to open coded iterators bpf_for()/bpf_repeat() and bpf_loop() helper, but it doesn't iterate any objects. In assembly 'may_goto' is a nop most of the time until bpf runtime has to terminate the program for whatever reason. In the current implementation may_goto has a hidden counter, but other mechanisms can be used. For programs written in C the later patch introduces 'cond_break' macro that combines 'may_goto' with 'break' statement and has similar semantics: cond_break is a nop until bpf runtime has to break out of this loop. It can be used in any normal "for" or "while" loop, like for (i = zero; i < cnt; cond_break, i++) { The verifier recognizes that may_goto is used in the program, reserves additional 8 bytes of stack, initializes them in subprog prologue, and replaces may_goto instruction with: aux_reg = *(u64 *)(fp - 40) if aux_reg == 0 goto pc+off aux_reg -= 1 *(u64 *)(fp - 40) = aux_reg may_goto instruction can be used by LLVM to implement __builtin_memcpy, __builtin_strcmp. may_goto is not a full substitute for bpf_for() macro. bpf_for() doesn't have induction variable that verifiers sees, so 'i' in bpf_for(i, 0, 100) is seen as imprecise and bounded. But when the code is written as: for (i = 0; i < 100; cond_break, i++) the verifier see 'i' as precise constant zero, hence cond_break (aka may_goto) doesn't help to converge the loop. A static or global variable can be used as a workaround: static int zero = 0; for (i = zero; i < 100; cond_break, i++) // works! may_goto works well with arena pointers that don't need to be bounds checked on access. Load/store from arena returns imprecise unbounded scalar and loops with may_goto pass the verifier. Reserve new opcode BPF_JMP | BPF_JCOND for may_goto insn. JCOND stands for conditional pseudo jump. Since goto_or_nop insn was proposed, it may use the same opcode. may_goto vs goto_or_nop can be distinguished by src_reg: code = BPF_JMP | BPF_JCOND src_reg = 0 - may_goto src_reg = 1 - goto_or_nop Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Acked-by: Eduard Zingerman Acked-by: John Fastabend Tested-by: John Fastabend Link: https://lore.kernel.org/bpf/20240306031929.42666-2-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 2 + include/uapi/linux/bpf.h | 5 ++ kernel/bpf/core.c | 1 + kernel/bpf/disasm.c | 4 + kernel/bpf/verifier.c | 163 +++++++++++++++++++++++++++++++++-------- tools/include/uapi/linux/bpf.h | 5 ++ 6 files changed, 150 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 84365e6dd85d..4b0f6600e499 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -449,6 +449,7 @@ struct bpf_verifier_state { u32 jmp_history_cnt; u32 dfs_depth; u32 callback_unroll_depth; + u32 may_goto_depth; }; #define bpf_get_spilled_reg(slot, frame, mask) \ @@ -619,6 +620,7 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u16 stack_depth; /* max. stack depth used by this function */ + u16 stack_extra; bool has_tail_call: 1; bool tail_call_reachable: 1; bool has_ld_abs: 1; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a241f407c234..85ec7fc799d7 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -42,6 +42,7 @@ #define BPF_JSGE 0x70 /* SGE is signed '>=', GE in x86 */ #define BPF_JSLT 0xc0 /* SLT is signed, '<' */ #define BPF_JSLE 0xd0 /* SLE is signed, '<=' */ +#define BPF_JCOND 0xe0 /* conditional pseudo jumps: may_goto, goto_or_nop */ #define BPF_CALL 0x80 /* function call */ #define BPF_EXIT 0x90 /* function return */ @@ -50,6 +51,10 @@ #define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */ #define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */ +enum bpf_cond_pseudo_jmp { + BPF_MAY_GOTO = 0, +}; + /* Register numbers */ enum { BPF_REG_0 = 0, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 71c459a51d9e..9ee4536d0a09 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1675,6 +1675,7 @@ bool bpf_opcode_in_insntable(u8 code) [BPF_LD | BPF_IND | BPF_B] = true, [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, + [BPF_JMP | BPF_JCOND] = true, }; #undef BPF_INSN_3_TBL #undef BPF_INSN_2_TBL diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 49940c26a227..82b2dbdd048f 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -322,6 +322,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, } else if (insn->code == (BPF_JMP | BPF_JA)) { verbose(cbs->private_data, "(%02x) goto pc%+d\n", insn->code, insn->off); + } else if (insn->code == (BPF_JMP | BPF_JCOND) && + insn->src_reg == BPF_MAY_GOTO) { + verbose(cbs->private_data, "(%02x) may_goto pc%+d\n", + insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { verbose(cbs->private_data, "(%02x) gotol pc%+d\n", insn->code, insn->imm); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4dd84e13bbfe..8030b50d3b45 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -533,6 +533,16 @@ static bool is_async_callback_calling_insn(struct bpf_insn *insn) return bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm); } +static bool is_may_goto_insn(struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; +} + +static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx) +{ + return is_may_goto_insn(&env->prog->insnsi[insn_idx]); +} + static bool is_storage_get_function(enum bpf_func_id func_id) { return func_id == BPF_FUNC_sk_storage_get || @@ -1429,6 +1439,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, dst_state->dfs_depth = src->dfs_depth; dst_state->callback_unroll_depth = src->callback_unroll_depth; dst_state->used_as_loop_entry = src->used_as_loop_entry; + dst_state->may_goto_depth = src->may_goto_depth; for (i = 0; i <= src->curframe; i++) { dst = dst_state->frame[i]; if (!dst) { @@ -14871,11 +14882,36 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, int err; /* Only conditional jumps are expected to reach here. */ - if (opcode == BPF_JA || opcode > BPF_JSLE) { + if (opcode == BPF_JA || opcode > BPF_JCOND) { verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); return -EINVAL; } + if (opcode == BPF_JCOND) { + struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; + int idx = *insn_idx; + + if (insn->code != (BPF_JMP | BPF_JCOND) || + insn->src_reg != BPF_MAY_GOTO || + insn->dst_reg || insn->imm || insn->off == 0) { + verbose(env, "invalid may_goto off %d imm %d\n", + insn->off, insn->imm); + return -EINVAL; + } + prev_st = find_prev_entry(env, cur_st->parent, idx); + + /* branch out 'fallthrough' insn as a new state to explore */ + queued_st = push_stack(env, idx + 1, idx, false); + if (!queued_st) + return -ENOMEM; + + queued_st->may_goto_depth++; + if (prev_st) + widen_imprecise_scalars(env, prev_st, queued_st); + *insn_idx += insn->off; + return 0; + } + /* check src2 operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) @@ -15659,6 +15695,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) default: /* conditional jump with two edges */ mark_prune_point(env, t); + if (is_may_goto_insn(insn)) + mark_force_checkpoint(env, t); ret = push_insn(t, t + 1, FALLTHROUGH, env); if (ret) @@ -17135,6 +17173,13 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) } goto skip_inf_loop_check; } + if (is_may_goto_insn_at(env, insn_idx)) { + if (states_equal(env, &sl->state, cur, true)) { + update_loop_entry(cur, &sl->state); + goto hit; + } + goto skip_inf_loop_check; + } if (calls_callback(env, insn_idx)) { if (states_equal(env, &sl->state, cur, true)) goto hit; @@ -17144,6 +17189,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur, true) && !iter_active_depths_differ(&sl->state, cur) && + sl->state.may_goto_depth == cur->may_goto_depth && sl->state.callback_unroll_depth == cur->callback_unroll_depth) { verbose_linfo(env, insn_idx, "; "); verbose(env, "infinite loop detected at insn %d\n", insn_idx); @@ -19408,7 +19454,10 @@ static int do_misc_fixups(struct bpf_verifier_env *env) struct bpf_insn insn_buf[16]; struct bpf_prog *new_prog; struct bpf_map *map_ptr; - int i, ret, cnt, delta = 0; + int i, ret, cnt, delta = 0, cur_subprog = 0; + struct bpf_subprog_info *subprogs = env->subprog_info; + u16 stack_depth = subprogs[cur_subprog].stack_depth; + u16 stack_depth_extra = 0; if (env->seen_exception && !env->exception_callback_subprog) { struct bpf_insn patch[] = { @@ -19428,7 +19477,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) mark_subprog_exc_cb(env, env->exception_callback_subprog); } - for (i = 0; i < insn_cnt; i++, insn++) { + for (i = 0; i < insn_cnt;) { /* Make divide-by-zero exceptions impossible. */ if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || @@ -19467,7 +19516,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ @@ -19487,7 +19536,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Rewrite pointer arithmetic to mitigate speculation attacks. */ @@ -19502,7 +19551,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) aux = &env->insn_aux_data[i + delta]; if (!aux->alu_state || aux->alu_state == BPF_ALU_NON_POINTER) - continue; + goto next_insn; isneg = aux->alu_state & BPF_ALU_NEG_VALUE; issrc = (aux->alu_state & BPF_ALU_SANITIZE) == @@ -19540,19 +19589,39 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; + } + + if (is_may_goto_insn(insn)) { + int stack_off = -stack_depth - 8; + + stack_depth_extra = 8; + insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); + insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); + insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); + insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); + cnt = 4; + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + goto next_insn; } if (insn->code != (BPF_JMP | BPF_CALL)) - continue; + goto next_insn; if (insn->src_reg == BPF_PSEUDO_CALL) - continue; + goto next_insn; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); if (ret) return ret; if (cnt == 0) - continue; + goto next_insn; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -19561,7 +19630,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } if (insn->imm == BPF_FUNC_get_route_realm) @@ -19609,11 +19678,11 @@ static int do_misc_fixups(struct bpf_verifier_env *env) } insn->imm = ret + 1; - continue; + goto next_insn; } if (!bpf_map_ptr_unpriv(aux)) - continue; + goto next_insn; /* instead of changing every JIT dealing with tail_call * emit two extra insns: @@ -19642,7 +19711,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } if (insn->imm == BPF_FUNC_timer_set_callback) { @@ -19754,7 +19823,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, @@ -19785,31 +19854,31 @@ patch_map_ops_generic: switch (insn->imm) { case BPF_FUNC_map_lookup_elem: insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); - continue; + goto next_insn; case BPF_FUNC_map_update_elem: insn->imm = BPF_CALL_IMM(ops->map_update_elem); - continue; + goto next_insn; case BPF_FUNC_map_delete_elem: insn->imm = BPF_CALL_IMM(ops->map_delete_elem); - continue; + goto next_insn; case BPF_FUNC_map_push_elem: insn->imm = BPF_CALL_IMM(ops->map_push_elem); - continue; + goto next_insn; case BPF_FUNC_map_pop_elem: insn->imm = BPF_CALL_IMM(ops->map_pop_elem); - continue; + goto next_insn; case BPF_FUNC_map_peek_elem: insn->imm = BPF_CALL_IMM(ops->map_peek_elem); - continue; + goto next_insn; case BPF_FUNC_redirect_map: insn->imm = BPF_CALL_IMM(ops->map_redirect); - continue; + goto next_insn; case BPF_FUNC_for_each_map_elem: insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); - continue; + goto next_insn; case BPF_FUNC_map_lookup_percpu_elem: insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); - continue; + goto next_insn; } goto patch_call_imm; @@ -19837,7 +19906,7 @@ patch_map_ops_generic: delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Implement bpf_get_func_arg inline. */ @@ -19862,7 +19931,7 @@ patch_map_ops_generic: delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Implement bpf_get_func_ret inline. */ @@ -19890,7 +19959,7 @@ patch_map_ops_generic: delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Implement get_func_arg_cnt inline. */ @@ -19905,7 +19974,7 @@ patch_map_ops_generic: env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Implement bpf_get_func_ip inline. */ @@ -19920,7 +19989,7 @@ patch_map_ops_generic: env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } /* Implement bpf_kptr_xchg inline */ @@ -19938,7 +20007,7 @@ patch_map_ops_generic: delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; - continue; + goto next_insn; } patch_call_imm: fn = env->ops->get_func_proto(insn->imm, env->prog); @@ -19952,6 +20021,40 @@ patch_call_imm: return -EFAULT; } insn->imm = fn->func - __bpf_call_base; +next_insn: + if (subprogs[cur_subprog + 1].start == i + delta + 1) { + subprogs[cur_subprog].stack_depth += stack_depth_extra; + subprogs[cur_subprog].stack_extra = stack_depth_extra; + cur_subprog++; + stack_depth = subprogs[cur_subprog].stack_depth; + stack_depth_extra = 0; + } + i++; + insn++; + } + + env->prog->aux->stack_depth = subprogs[0].stack_depth; + for (i = 0; i < env->subprog_cnt; i++) { + int subprog_start = subprogs[i].start; + int stack_slots = subprogs[i].stack_extra / 8; + + if (!stack_slots) + continue; + if (stack_slots > 1) { + verbose(env, "verifier bug: stack_slots supports may_goto only\n"); + return -EFAULT; + } + + /* Add ST insn to subprog prologue to init extra stack */ + insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, + -subprogs[i].stack_depth, BPF_MAX_LOOPS); + /* Copy first actual insn to preserve it */ + insn_buf[1] = env->prog->insnsi[subprog_start]; + + new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2); + if (!new_prog) + return -ENOMEM; + env->prog = prog = new_prog; } /* Since poke tab is now finalized, publish aux to tracker. */ diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index a241f407c234..85ec7fc799d7 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -42,6 +42,7 @@ #define BPF_JSGE 0x70 /* SGE is signed '>=', GE in x86 */ #define BPF_JSLT 0xc0 /* SLT is signed, '<' */ #define BPF_JSLE 0xd0 /* SLE is signed, '<=' */ +#define BPF_JCOND 0xe0 /* conditional pseudo jumps: may_goto, goto_or_nop */ #define BPF_CALL 0x80 /* function call */ #define BPF_EXIT 0x90 /* function return */ @@ -50,6 +51,10 @@ #define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */ #define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */ +enum bpf_cond_pseudo_jmp { + BPF_MAY_GOTO = 0, +}; + /* Register numbers */ enum { BPF_REG_0 = 0, -- cgit v1.2.3 From f311507c5336ad6b9ca7687c35c3bd7a4fe2868c Mon Sep 17 00:00:00 2001 From: "Hsin-Yu.Chen" Date: Wed, 6 Mar 2024 12:19:00 +0800 Subject: i2c: remove redundant condition I2C_M_RD is defined as and guaranteed to be 1 and 'flag & I2C_M_RD' is one or zero. No need for an additional condition to obtain the value. Signed-off-by: Hsin-Yu.Chen Reviewed-by: Andi Shyti [wsa: slightly updated commit message] Signed-off-by: Wolfram Sang --- include/linux/i2c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/i2c.h b/include/linux/i2c.h index ff93ff8b257c..5e6cd43a6dbd 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -931,7 +931,7 @@ static inline int i2c_adapter_id(struct i2c_adapter *adap) static inline u8 i2c_8bit_addr_from_msg(const struct i2c_msg *msg) { - return (msg->addr << 1) | (msg->flags & I2C_M_RD ? 1 : 0); + return (msg->addr << 1) | (msg->flags & I2C_M_RD); } u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold); -- cgit v1.2.3 From a0873a5d542559698edfd4c8fc6e6636d338eea2 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 14 Feb 2024 19:08:00 -0800 Subject: net/mlx5: Add MPIR bit in mcam_access_reg Add a cap bit in mcam_access_reg to check for MPIR support. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 628a3aa7a7e0..2756bdb654b4 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -10253,7 +10253,9 @@ struct mlx5_ifc_mcam_access_reg_bits { u8 mcqi[0x1]; u8 mcqs[0x1]; - u8 regs_95_to_87[0x9]; + u8 regs_95_to_90[0x6]; + u8 mpir[0x1]; + u8 regs_88_to_87[0x2]; u8 mpegc[0x1]; u8 mtutc[0x1]; u8 regs_84_to_68[0x11]; -- cgit v1.2.3 From ed29705e4ed1d5c1b2184fecc4684bd56c5d24ee Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 14 Feb 2024 19:08:13 -0800 Subject: net/mlx5: Enable SD feature Have an actual mlx5_sd instance in the core device, and fix the getter accordingly. This allows SD stuff to flow, the feature becomes supported only here. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h | 3 ++- include/linux/mlx5/driver.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 0810b92b48d0..37d5f445598c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -59,10 +59,11 @@ struct mlx5_sd; static inline struct mlx5_sd *mlx5_get_sd(struct mlx5_core_dev *dev) { - return NULL; + return dev->sd; } static inline void mlx5_set_sd(struct mlx5_core_dev *dev, struct mlx5_sd *sd) { + dev->sd = sd; } #endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 41f03b352401..bf9324a31ae9 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -823,6 +823,7 @@ struct mlx5_core_dev { struct blocking_notifier_head macsec_nh; #endif u64 num_ipsec_offloads; + struct mlx5_sd *sd; }; struct mlx5_db { -- cgit v1.2.3 From 1368d06dd2c99186174290c03d79c132db16efe2 Mon Sep 17 00:00:00 2001 From: Duje Mihanović Date: Thu, 25 Jan 2024 16:30:53 +0100 Subject: leds: Introduce ExpressWire library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ExpressWire protocol is shared between at least KTD2692 and KTD2801 with slight differences such as timings and the former not having a defined set of pulses for enabling the protocol (possibly because it does not support PWM unlike KTD2801). Despite these differences the ExpressWire handling code can be shared between the two, so in preparation for adding KTD2801 support introduce a library implementing this protocol. Suggested-by: Daniel Thompson Reviewed-by: Linus Walleij Reviewed-by: Daniel Thompson Signed-off-by: Duje Mihanović Link: https://lore.kernel.org/r/20240125-ktd2801-v5-1-e22da232a825@skole.hr Signed-off-by: Lee Jones --- MAINTAINERS | 7 +++++ drivers/leds/Kconfig | 4 +++ drivers/leds/Makefile | 3 ++ drivers/leds/leds-expresswire.c | 68 ++++++++++++++++++++++++++++++++++++++++ include/linux/leds-expresswire.h | 36 +++++++++++++++++++++ 5 files changed, 118 insertions(+) create mode 100644 drivers/leds/leds-expresswire.c create mode 100644 include/linux/leds-expresswire.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 8d1052fa6a69..e1c83e0e837a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7979,6 +7979,13 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git F: fs/exfat/ +EXPRESSWIRE PROTOCOL LIBRARY +M: Duje Mihanović +L: linux-leds@vger.kernel.org +S: Maintained +F: drivers/leds/leds-expresswire.c +F: include/linux/leds-expresswire.h + EXT2 FILE SYSTEM M: Jan Kara L: linux-ext4@vger.kernel.org diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index d721b254e1e4..64bb2de237e9 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -186,6 +186,10 @@ config LEDS_EL15203000 To compile this driver as a module, choose M here: the module will be called leds-el15203000. +config LEDS_EXPRESSWIRE + bool + depends on GPIOLIB + config LEDS_TURRIS_OMNIA tristate "LED support for CZ.NIC's Turris Omnia" depends on LEDS_CLASS_MULTICOLOR diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile index ce07dc295ff0..effdfc6f1e95 100644 --- a/drivers/leds/Makefile +++ b/drivers/leds/Makefile @@ -91,6 +91,9 @@ obj-$(CONFIG_LEDS_WM831X_STATUS) += leds-wm831x-status.o obj-$(CONFIG_LEDS_WM8350) += leds-wm8350.o obj-$(CONFIG_LEDS_WRAP) += leds-wrap.o +# Kinetic ExpressWire Protocol +obj-$(CONFIG_LEDS_EXPRESSWIRE) += leds-expresswire.o + # LED SPI Drivers obj-$(CONFIG_LEDS_CR0014114) += leds-cr0014114.o obj-$(CONFIG_LEDS_DAC124S085) += leds-dac124s085.o diff --git a/drivers/leds/leds-expresswire.c b/drivers/leds/leds-expresswire.c new file mode 100644 index 000000000000..89e147b0e019 --- /dev/null +++ b/drivers/leds/leds-expresswire.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Shared library for Kinetic's ExpressWire protocol. + * This protocol works by pulsing the ExpressWire IC's control GPIO. + * ktd2692 and ktd2801 are known to use this protocol. + */ + +#include +#include +#include + +void expresswire_power_off(struct expresswire_common_props *props) +{ + gpiod_set_value_cansleep(props->ctrl_gpio, 0); + usleep_range(props->timing.poweroff_us, props->timing.poweroff_us * 2); +} +EXPORT_SYMBOL_NS_GPL(expresswire_power_off, EXPRESSWIRE); + +void expresswire_enable(struct expresswire_common_props *props) +{ + gpiod_set_value(props->ctrl_gpio, 1); + udelay(props->timing.detect_delay_us); + gpiod_set_value(props->ctrl_gpio, 0); + udelay(props->timing.detect_us); + gpiod_set_value(props->ctrl_gpio, 1); +} +EXPORT_SYMBOL_NS_GPL(expresswire_enable, EXPRESSWIRE); + +void expresswire_start(struct expresswire_common_props *props) +{ + gpiod_set_value(props->ctrl_gpio, 1); + udelay(props->timing.data_start_us); +} +EXPORT_SYMBOL_NS_GPL(expresswire_start, EXPRESSWIRE); + +void expresswire_end(struct expresswire_common_props *props) +{ + gpiod_set_value(props->ctrl_gpio, 0); + udelay(props->timing.end_of_data_low_us); + gpiod_set_value(props->ctrl_gpio, 1); + udelay(props->timing.end_of_data_high_us); +} +EXPORT_SYMBOL_NS_GPL(expresswire_end, EXPRESSWIRE); + +void expresswire_set_bit(struct expresswire_common_props *props, bool bit) +{ + if (bit) { + gpiod_set_value(props->ctrl_gpio, 0); + udelay(props->timing.short_bitset_us); + gpiod_set_value(props->ctrl_gpio, 1); + udelay(props->timing.long_bitset_us); + } else { + gpiod_set_value(props->ctrl_gpio, 0); + udelay(props->timing.long_bitset_us); + gpiod_set_value(props->ctrl_gpio, 1); + udelay(props->timing.short_bitset_us); + } +} +EXPORT_SYMBOL_NS_GPL(expresswire_set_bit, EXPRESSWIRE); + +void expresswire_write_u8(struct expresswire_common_props *props, u8 val) +{ + expresswire_start(props); + for (int i = 7; i >= 0; i--) + expresswire_set_bit(props, val & BIT(i)); + expresswire_end(props); +} +EXPORT_SYMBOL_NS_GPL(expresswire_write_u8, EXPRESSWIRE); diff --git a/include/linux/leds-expresswire.h b/include/linux/leds-expresswire.h new file mode 100644 index 000000000000..3c61902ccac8 --- /dev/null +++ b/include/linux/leds-expresswire.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Shared library for Kinetic's ExpressWire protocol. + * This protocol works by pulsing the ExpressWire IC's control GPIO. + * ktd2692 and ktd2801 are known to use this protocol. + */ + +#ifndef _LEDS_EXPRESSWIRE_H +#define _LEDS_EXPRESSWIRE_H + +#include + +struct expresswire_timing { + unsigned long poweroff_us; + unsigned long detect_delay_us; + unsigned long detect_us; + unsigned long data_start_us; + unsigned long end_of_data_low_us; + unsigned long end_of_data_high_us; + unsigned long short_bitset_us; + unsigned long long_bitset_us; +}; + +struct expresswire_common_props { + struct gpio_desc *ctrl_gpio; + struct expresswire_timing timing; +}; + +void expresswire_power_off(struct expresswire_common_props *props); +void expresswire_enable(struct expresswire_common_props *props); +void expresswire_start(struct expresswire_common_props *props); +void expresswire_end(struct expresswire_common_props *props); +void expresswire_set_bit(struct expresswire_common_props *props, bool bit); +void expresswire_write_u8(struct expresswire_common_props *props, u8 val); + +#endif /* _LEDS_EXPRESSWIRE_H */ -- cgit v1.2.3 From 7774f3d1dd3822e938e236df67766436c0debd11 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 23 Feb 2024 22:30:00 +0200 Subject: leds: expresswire: Don't use "proxy" headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update header inclusions to follow IWYU (Include What You Use) principle. Signed-off-by: Andy Shevchenko Reviewed-by: Duje Mihanović Link: https://lore.kernel.org/r/20240223203010.881065-1-andriy.shevchenko@linux.intel.com Signed-off-by: Lee Jones --- drivers/leds/leds-expresswire.c | 4 ++++ include/linux/leds-expresswire.h | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/leds/leds-expresswire.c b/drivers/leds/leds-expresswire.c index 89e147b0e019..e4937a8e0f44 100644 --- a/drivers/leds/leds-expresswire.c +++ b/drivers/leds/leds-expresswire.c @@ -5,8 +5,12 @@ * ktd2692 and ktd2801 are known to use this protocol. */ +#include #include +#include #include +#include + #include void expresswire_power_off(struct expresswire_common_props *props) diff --git a/include/linux/leds-expresswire.h b/include/linux/leds-expresswire.h index 3c61902ccac8..a422921f4159 100644 --- a/include/linux/leds-expresswire.h +++ b/include/linux/leds-expresswire.h @@ -8,7 +8,9 @@ #ifndef _LEDS_EXPRESSWIRE_H #define _LEDS_EXPRESSWIRE_H -#include +#include + +struct gpio_desc; struct expresswire_timing { unsigned long poweroff_us; -- cgit v1.2.3 From 211f8ec9400b58fb97cf4b6bd7033781e889bf53 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 9 Jan 2024 10:06:39 +0100 Subject: leds: Remove led_init_default_state_get() and devm_led_classdev_register_ext() stubs These two functions have stub implementations that are called when NEW_LEDS and/or LEDS_CLASS are disabled, theorerically allowing drivers to optionally use the LED subsystem. However, this has never really worked because a built-in driver is unable to link against these functions if the LED class is in a loadable module. Heiner ran into this problem with a driver that newly gained a LEDS_CLASS dependency and suggested using an IS_REACHABLE() check. This is the reverse approach, removing the stub entirely to acknowledge that it is pointless in its current form, and that not having it avoids misleading developers into thinking that they can rely on it. This survived around 1000 randconfig builds to validate that any callers of the interface already have the correct Kconfig dependency already, with the exception of the one that Heiner just added. Cc: Heiner Kallweit Link: https://lore.kernel.org/linux-leds/0f6f432b-c650-4bb8-a1b5-fe3372804d52@gmail.com/T/#u Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20240109090715.982332-1-arnd@kernel.org Signed-off-by: Lee Jones --- include/linux/leds.h | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index 4754b02d3a2c..7598d472903a 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -82,15 +82,7 @@ struct led_init_data { bool devname_mandatory; }; -#if IS_ENABLED(CONFIG_NEW_LEDS) enum led_default_state led_init_default_state_get(struct fwnode_handle *fwnode); -#else -static inline enum led_default_state -led_init_default_state_get(struct fwnode_handle *fwnode) -{ - return LEDS_DEFSTATE_OFF; -} -#endif struct led_hw_trigger_type { int dummy; @@ -279,20 +271,9 @@ static inline int led_classdev_register(struct device *parent, return led_classdev_register_ext(parent, led_cdev, NULL); } -#if IS_ENABLED(CONFIG_LEDS_CLASS) int devm_led_classdev_register_ext(struct device *parent, struct led_classdev *led_cdev, struct led_init_data *init_data); -#else -static inline int -devm_led_classdev_register_ext(struct device *parent, - struct led_classdev *led_cdev, - struct led_init_data *init_data) -{ - return 0; -} -#endif - static inline int devm_led_classdev_register(struct device *parent, struct led_classdev *led_cdev) { -- cgit v1.2.3 From 09e3f3244e8480d53873bb86a3808edaa3f4e314 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 9 Jan 2024 10:06:40 +0100 Subject: leds: Make flash and multicolor dependencies unconditional Along the same lines as making devm_led_classdev_register() declared extern unconditional, do the same thing for the two sub-classes that have similar stubs. The users of these interfaces go to great lengths to allow building with both the generic leds API and the extended version, but realistically there is not much use in this, so just simplify it to always rely on it and remove the confusing fallback logic. Signed-off-by: Arnd Bergmann Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240109090715.982332-2-arnd@kernel.org Signed-off-by: Lee Jones --- drivers/leds/Kconfig | 4 ++-- drivers/leds/flash/Kconfig | 4 ++-- drivers/staging/greybus/Kconfig | 2 +- drivers/staging/greybus/light.c | 21 --------------------- include/linux/led-class-flash.h | 24 ------------------------ include/linux/led-class-multicolor.h | 29 ----------------------------- 6 files changed, 5 insertions(+), 79 deletions(-) (limited to 'include/linux') diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index 64bb2de237e9..52328d295b4e 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -399,7 +399,7 @@ config LEDS_LP3952 config LEDS_LP50XX tristate "LED Support for TI LP5036/30/24/18/12/09 LED driver chip" depends on LEDS_CLASS && REGMAP_I2C - depends on LEDS_CLASS_MULTICOLOR || !LEDS_CLASS_MULTICOLOR + depends on LEDS_CLASS_MULTICOLOR help If you say yes here you get support for the Texas Instruments LP5036, LP5030, LP5024, LP5018, LP5012 and LP5009 LED driver. @@ -410,7 +410,7 @@ config LEDS_LP50XX config LEDS_LP55XX_COMMON tristate "Common Driver for TI/National LP5521/5523/55231/5562/8501" depends on LEDS_CLASS - depends on LEDS_CLASS_MULTICOLOR || !LEDS_CLASS_MULTICOLOR + depends on LEDS_CLASS_MULTICOLOR depends on OF depends on I2C select FW_LOADER diff --git a/drivers/leds/flash/Kconfig b/drivers/leds/flash/Kconfig index 24722d581369..809b6d98bb3e 100644 --- a/drivers/leds/flash/Kconfig +++ b/drivers/leds/flash/Kconfig @@ -52,8 +52,8 @@ config LEDS_MAX77693 config LEDS_MT6360 tristate "LED Support for Mediatek MT6360 PMIC" depends on LEDS_CLASS && OF - depends on LEDS_CLASS_FLASH || !LEDS_CLASS_FLASH - depends on LEDS_CLASS_MULTICOLOR || !LEDS_CLASS_MULTICOLOR + depends on LEDS_CLASS_FLASH + depends on LEDS_CLASS_MULTICOLOR depends on V4L2_FLASH_LED_CLASS || !V4L2_FLASH_LED_CLASS depends on MFD_MT6360 help diff --git a/drivers/staging/greybus/Kconfig b/drivers/staging/greybus/Kconfig index 927cfa4bc989..1e745a8d439c 100644 --- a/drivers/staging/greybus/Kconfig +++ b/drivers/staging/greybus/Kconfig @@ -64,7 +64,7 @@ config GREYBUS_HID config GREYBUS_LIGHT tristate "Greybus LED Class driver" - depends on LEDS_CLASS + depends on LEDS_CLASS_FLASH help Select this option if you have a device that follows the Greybus LED Class specification. diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c index 87d36948c610..d62f97249aca 100644 --- a/drivers/staging/greybus/light.c +++ b/drivers/staging/greybus/light.c @@ -29,13 +29,9 @@ struct gb_channel { struct attribute_group *attr_group; const struct attribute_group **attr_groups; struct led_classdev *led; -#if IS_REACHABLE(CONFIG_LEDS_CLASS_FLASH) struct led_classdev_flash fled; struct led_flash_setting intensity_uA; struct led_flash_setting timeout_us; -#else - struct led_classdev cled; -#endif struct gb_light *light; bool is_registered; bool releasing; @@ -84,7 +80,6 @@ static bool is_channel_flash(struct gb_channel *channel) | GB_CHANNEL_MODE_INDICATOR)); } -#if IS_REACHABLE(CONFIG_LEDS_CLASS_FLASH) static struct gb_channel *get_channel_from_cdev(struct led_classdev *cdev) { struct led_classdev_flash *fled_cdev = lcdev_to_flcdev(cdev); @@ -153,22 +148,6 @@ static int __gb_lights_flash_brightness_set(struct gb_channel *channel) return __gb_lights_flash_intensity_set(channel, intensity); } -#else -static struct gb_channel *get_channel_from_cdev(struct led_classdev *cdev) -{ - return container_of(cdev, struct gb_channel, cled); -} - -static struct led_classdev *get_channel_cdev(struct gb_channel *channel) -{ - return &channel->cled; -} - -static int __gb_lights_flash_brightness_set(struct gb_channel *channel) -{ - return 0; -} -#endif static int gb_lights_color_set(struct gb_channel *channel, u32 color); static int gb_lights_fade_set(struct gb_channel *channel); diff --git a/include/linux/led-class-flash.h b/include/linux/led-class-flash.h index 612b4cab3819..36df927ec4b7 100644 --- a/include/linux/led-class-flash.h +++ b/include/linux/led-class-flash.h @@ -85,7 +85,6 @@ static inline struct led_classdev_flash *lcdev_to_flcdev( return container_of(lcdev, struct led_classdev_flash, led_cdev); } -#if IS_ENABLED(CONFIG_LEDS_CLASS_FLASH) /** * led_classdev_flash_register_ext - register a new object of LED class with * init data and with support for flash LEDs @@ -116,29 +115,6 @@ int devm_led_classdev_flash_register_ext(struct device *parent, void devm_led_classdev_flash_unregister(struct device *parent, struct led_classdev_flash *fled_cdev); -#else - -static inline int led_classdev_flash_register_ext(struct device *parent, - struct led_classdev_flash *fled_cdev, - struct led_init_data *init_data) -{ - return 0; -} - -static inline void led_classdev_flash_unregister(struct led_classdev_flash *fled_cdev) {}; -static inline int devm_led_classdev_flash_register_ext(struct device *parent, - struct led_classdev_flash *fled_cdev, - struct led_init_data *init_data) -{ - return 0; -} - -static inline void devm_led_classdev_flash_unregister(struct device *parent, - struct led_classdev_flash *fled_cdev) -{}; - -#endif /* IS_ENABLED(CONFIG_LEDS_CLASS_FLASH) */ - static inline int led_classdev_flash_register(struct device *parent, struct led_classdev_flash *fled_cdev) { diff --git a/include/linux/led-class-multicolor.h b/include/linux/led-class-multicolor.h index 210d57bcd767..db9f34c6736e 100644 --- a/include/linux/led-class-multicolor.h +++ b/include/linux/led-class-multicolor.h @@ -30,7 +30,6 @@ static inline struct led_classdev_mc *lcdev_to_mccdev( return container_of(led_cdev, struct led_classdev_mc, led_cdev); } -#if IS_ENABLED(CONFIG_LEDS_CLASS_MULTICOLOR) /** * led_classdev_multicolor_register_ext - register a new object of led_classdev * class with support for multicolor LEDs @@ -64,34 +63,6 @@ int devm_led_classdev_multicolor_register_ext(struct device *parent, void devm_led_classdev_multicolor_unregister(struct device *parent, struct led_classdev_mc *mcled_cdev); -#else - -static inline int led_classdev_multicolor_register_ext(struct device *parent, - struct led_classdev_mc *mcled_cdev, - struct led_init_data *init_data) -{ - return 0; -} - -static inline void led_classdev_multicolor_unregister(struct led_classdev_mc *mcled_cdev) {}; -static inline int led_mc_calc_color_components(struct led_classdev_mc *mcled_cdev, - enum led_brightness brightness) -{ - return 0; -} - -static inline int devm_led_classdev_multicolor_register_ext(struct device *parent, - struct led_classdev_mc *mcled_cdev, - struct led_init_data *init_data) -{ - return 0; -} - -static inline void devm_led_classdev_multicolor_unregister(struct device *parent, - struct led_classdev_mc *mcled_cdev) -{}; - -#endif /* IS_ENABLED(CONFIG_LEDS_CLASS_MULTICOLOR) */ static inline int led_classdev_multicolor_register(struct device *parent, struct led_classdev_mc *mcled_cdev) -- cgit v1.2.3 From 08b7dab9f025e20dc02bb4ad19b8e519c3948d20 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 28 Feb 2024 10:38:26 +0100 Subject: leds: Fix ifdef check for gpio_led_register_device() gpio_led_register_device() is built whenever CONFIG_LEDS_GPIO_REGISTER is enabled, and this may be used even when CONFIG_NEW_LEDS is turned off. However, the stub declaration in the header is provided for all configs without CONFIG_NEW_LEDS, resulting in a build failure: drivers/leds/leds-gpio-register.c:24:1: error: redefinition of 'gpio_led_register_device' 24 | gpio_led_register_device(int id, const struct gpio_led_platform_data *pdata) | ^ include/linux/leds.h:646:39: note: previous definition is here Change the #ifdef check to match the definition. Note: this apparently took years of randconfig builds to hit, since a number of other drivers just 'select NEW_LEDS' anyway. Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20240228093834.2230004-1-arnd@kernel.org Signed-off-by: Lee Jones --- include/linux/leds.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index 7598d472903a..db6b114bb3d9 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -639,7 +639,7 @@ struct gpio_led_platform_data { gpio_blink_set_t gpio_blink_set; }; -#ifdef CONFIG_NEW_LEDS +#ifdef CONFIG_LEDS_GPIO_REGISTER struct platform_device *gpio_led_register_device( int id, const struct gpio_led_platform_data *pdata); #else -- cgit v1.2.3 From 5d51a794414359dd387c3da7a2ea7602c67f84a6 Mon Sep 17 00:00:00 2001 From: Richard Fitzgerald Date: Thu, 7 Mar 2024 10:55:16 +0000 Subject: firmware: cirrus: cs_dsp: Remove non-existent member from kerneldoc The kerneldoc for struct cs_dsp refers to a fw_file_name member but there's no such member. Signed-off-by: Richard Fitzgerald Link: https://msgid.link/r/20240307105516.40250-1-rf@opensource.cirrus.com Signed-off-by: Mark Brown --- include/linux/firmware/cirrus/cs_dsp.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 29cd11d5a3cf..23384a54d575 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -123,7 +123,6 @@ struct cs_dsp_client_ops; * @sysclk_mask: Mask of frequency bits within sysclk register (ADSP1 only) * @sysclk_shift: Shift of frequency bits within sysclk register (ADSP1 only) * @alg_regions: List of currently loaded algorithm regions - * @fw_file_name: Filename of the current firmware * @fw_name: Name of the current firmware * @fw_id: ID of the current firmware, obtained from the wmfw * @fw_id_version: Version of the firmware, obtained from the wmfw -- cgit v1.2.3 From 68ac1e46425c54653ddb5f559bc37abe19071024 Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Fri, 1 Mar 2024 17:43:08 +0100 Subject: net: phylink: clean the pcs_get_state documentation commit 4d72c3bb60dd ("net: phylink: strip out pre-March 2020 legacy code") dropped the mac_pcs_get_state ops in phylink_mac_ops in favor of dedicated PCS operation pcs_get_state. However, the documentation for the pcs_get_state ops was incorrectly converted and now self-references. Drop the extra comment. Signed-off-by: Maxime Chevallier Signed-off-by: Paolo Abeni --- include/linux/phylink.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 6ba411732a0d..9a57deefcb07 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -480,9 +480,6 @@ void pcs_disable(struct phylink_pcs *pcs); * negotiation completion state in @state->an_complete, and link up state * in @state->link. If possible, @state->lp_advertising should also be * populated. - * - * When present, this overrides pcs_get_state() in &struct - * phylink_pcs_ops. */ void pcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state); -- cgit v1.2.3 From 14fe5a98fb24192f73639590d9d3cdb5640d48db Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 7 Mar 2024 17:01:00 +0200 Subject: spi: Fix types of the last chip select storage variables First of all, last_cs_index_mask should be aligned with the original cs_index_mask, which is 16-bit (for now) wide. Use the same pattern for the last_cs_index_mask. Second, last_cs can be negative and since 'char' is equal to 'unsigned char' in the kernel, it's incorrect, strictly speaking, to assign signed number to it. Use s8 type as it's done for *_native_cs ones. With this change, regroup a bit the ordering to avoid too much memory space to be wasted due to paddings. Shuffle kernel documentation accordignly. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240307150256.3789138-3-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index ddfb66dd4caf..b05d5a87c313 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -450,9 +450,11 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * the @cur_msg_completion. This flag is used to signal the context that * is running spi_finalize_current_message() that it needs to complete() * @cur_msg_mapped: message has been mapped for DMA + * @fallback: fallback to PIO if DMA transfer return failure with + * SPI_TRANS_FAIL_NO_START. + * @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs. * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip * selected - * @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs. * @xfer_completion: used by core transfer_one_message() * @busy: message pump is busy * @running: message pump is running @@ -529,8 +531,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * If the driver does not set this, the SPI core takes the snapshot as * close to the driver hand-over as possible. * @irq_flags: Interrupt enable state during PTP system timestamping - * @fallback: fallback to PIO if DMA transfer return failure with - * SPI_TRANS_FAIL_NO_START. * @queue_empty: signal green light for opportunistically skipping the queue * for spi_sync transfers. * @must_async: disable all fast paths in the core @@ -710,10 +710,10 @@ struct spi_controller { bool rt; bool auto_runtime_pm; bool cur_msg_mapped; - char last_cs[SPI_CS_CNT_MAX]; - char last_cs_index_mask; - bool last_cs_mode_high; bool fallback; + bool last_cs_mode_high; + s8 last_cs[SPI_CS_CNT_MAX]; + u32 last_cs_index_mask : SPI_CS_CNT_MAX; struct completion xfer_completion; size_t max_dma_len; -- cgit v1.2.3 From ab23f1bffcf690ffae2b0c4bb8b09420299be05c Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 24 Feb 2024 11:44:03 +0000 Subject: slimbus: core: make slimbus_bus const Since commit d492cc2573a0 ("driver core: device.h: make struct bus_type a const *"), the driver core can properly handle constant struct bus_type, move the slimbus_bus variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20240224114403.86230-3-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/slimbus/core.c | 2 +- include/linux/slimbus.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/slimbus/core.c b/drivers/slimbus/core.c index 01cbd4621981..41e62de1f91f 100644 --- a/drivers/slimbus/core.c +++ b/drivers/slimbus/core.c @@ -100,7 +100,7 @@ static int slim_device_uevent(const struct device *dev, struct kobj_uevent_env * return add_uevent_var(env, "MODALIAS=slim:%s", dev_name(&sbdev->dev)); } -struct bus_type slimbus_bus = { +const struct bus_type slimbus_bus = { .name = "slimbus", .match = slim_device_match, .probe = slim_device_probe, diff --git a/include/linux/slimbus.h b/include/linux/slimbus.h index 12c9719b2a55..3042385b7b40 100644 --- a/include/linux/slimbus.h +++ b/include/linux/slimbus.h @@ -10,7 +10,7 @@ #include #include -extern struct bus_type slimbus_bus; +extern const struct bus_type slimbus_bus; /** * struct slim_eaddr - Enumeration address for a SLIMbus device -- cgit v1.2.3 From e34b943068d30f20db31f28100affdaaedc7efab Mon Sep 17 00:00:00 2001 From: Praveen Teja Kundanala Date: Sat, 24 Feb 2024 11:45:10 +0000 Subject: firmware: xilinx: Add ZynqMP efuse access API Add zynqmp_pm_efuse_access API in the ZynqMP firmware for read/write access of efuse memory. Signed-off-by: Praveen Teja Kundanala Acked-by: Michal Simek Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20240224114516.86365-6-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/xilinx/zynqmp.c | 25 +++++++++++++++++++++++++ include/linux/firmware/xlnx-zynqmp.h | 8 ++++++++ 2 files changed, 33 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 79789f0563f6..9bc45357e1a8 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -3,6 +3,7 @@ * Xilinx Zynq MPSoC Firmware layer * * Copyright (C) 2014-2022 Xilinx, Inc. + * Copyright (C) 2022 - 2023, Advanced Micro Devices, Inc. * * Michal Simek * Davorin Mista @@ -1384,6 +1385,30 @@ int zynqmp_pm_aes_engine(const u64 address, u32 *out) } EXPORT_SYMBOL_GPL(zynqmp_pm_aes_engine); +/** + * zynqmp_pm_efuse_access - Provides access to efuse memory. + * @address: Address of the efuse params structure + * @out: Returned output value + * + * Return: Returns status, either success or error code. + */ +int zynqmp_pm_efuse_access(const u64 address, u32 *out) +{ + u32 ret_payload[PAYLOAD_ARG_CNT]; + int ret; + + if (!out) + return -EINVAL; + + ret = zynqmp_pm_invoke_fn(PM_EFUSE_ACCESS, ret_payload, 2, + upper_32_bits(address), + lower_32_bits(address)); + *out = ret_payload[1]; + + return ret; +} +EXPORT_SYMBOL_GPL(zynqmp_pm_efuse_access); + /** * zynqmp_pm_sha_hash - Access the SHA engine to calculate the hash * @address: Address of the data/ Address of output buffer where diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 9a7e52739251..1a069a56c961 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -3,6 +3,7 @@ * Xilinx Zynq MPSoC Firmware layer * * Copyright (C) 2014-2021 Xilinx + * Copyright (C) 2022 - 2023, Advanced Micro Devices, Inc. * * Michal Simek * Davorin Mista @@ -171,6 +172,7 @@ enum pm_api_id { PM_CLOCK_GETPARENT = 44, PM_FPGA_READ = 46, PM_SECURE_AES = 47, + PM_EFUSE_ACCESS = 53, PM_FEATURE_CHECK = 63, }; @@ -562,6 +564,7 @@ int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities, const u32 qos, const enum zynqmp_pm_request_ack ack); int zynqmp_pm_aes_engine(const u64 address, u32 *out); +int zynqmp_pm_efuse_access(const u64 address, u32 *out); int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags); int zynqmp_pm_fpga_load(const u64 address, const u32 size, const u32 flags); int zynqmp_pm_fpga_get_status(u32 *value); @@ -749,6 +752,11 @@ static inline int zynqmp_pm_aes_engine(const u64 address, u32 *out) return -ENODEV; } +static inline int zynqmp_pm_efuse_access(const u64 address, u32 *out) +{ + return -ENODEV; +} + static inline int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags) { -- cgit v1.2.3 From cb1c1224193e648b4108dd06ebb7cc86b5c514ad Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 12 Feb 2024 08:41:01 -0300 Subject: dio: make dio_bus_type const Now that the driver core can properly handle constant struct bus_type, move the dio_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240212-bus_cleanup-dio-v2-1-3b1ba4c0547d@marliere.net Signed-off-by: Greg Kroah-Hartman --- drivers/dio/dio-driver.c | 2 +- include/linux/dio.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/dio/dio-driver.c b/drivers/dio/dio-driver.c index 69c46935ffc7..2d9fa6011945 100644 --- a/drivers/dio/dio-driver.c +++ b/drivers/dio/dio-driver.c @@ -123,7 +123,7 @@ static int dio_bus_match(struct device *dev, struct device_driver *drv) } -struct bus_type dio_bus_type = { +const struct bus_type dio_bus_type = { .name = "dio", .match = dio_bus_match, .probe = dio_device_probe, diff --git a/include/linux/dio.h b/include/linux/dio.h index 5abd07361eb5..2b5923909f96 100644 --- a/include/linux/dio.h +++ b/include/linux/dio.h @@ -68,7 +68,7 @@ struct dio_bus { }; extern struct dio_bus dio_bus; /* Single DIO bus */ -extern struct bus_type dio_bus_type; +extern const struct bus_type dio_bus_type; /* * DIO device IDs -- cgit v1.2.3 From d843f031d9e90462253015bc0bd9e3852d206bf2 Mon Sep 17 00:00:00 2001 From: Wayne Chang Date: Thu, 7 Mar 2024 11:03:27 +0800 Subject: phy: tegra: xusb: Add API to retrieve the port number of phy This patch introduces a new API, tegra_xusb_padctl_get_port_number, to the Tegra XUSB Pad Controller driver. This API is used to identify the USB port that is associated with a given PHY. The function takes a PHY pointer for either a USB2 PHY or USB3 PHY as input and returns the corresponding port number. If the PHY pointer is invalid, it returns -ENODEV. Cc: stable@vger.kernel.org Signed-off-by: Wayne Chang Reviewed-by: Jon Hunter Tested-by: Jon Hunter Link: https://lore.kernel.org/r/20240307030328.1487748-2-waynec@nvidia.com Signed-off-by: Greg Kroah-Hartman --- drivers/phy/tegra/xusb.c | 13 +++++++++++++ include/linux/phy/tegra/xusb.h | 1 + 2 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c index 142ebe0247cc..983a6e6173bd 100644 --- a/drivers/phy/tegra/xusb.c +++ b/drivers/phy/tegra/xusb.c @@ -1531,6 +1531,19 @@ int tegra_xusb_padctl_get_usb3_companion(struct tegra_xusb_padctl *padctl, } EXPORT_SYMBOL_GPL(tegra_xusb_padctl_get_usb3_companion); +int tegra_xusb_padctl_get_port_number(struct phy *phy) +{ + struct tegra_xusb_lane *lane; + + if (!phy) + return -ENODEV; + + lane = phy_get_drvdata(phy); + + return lane->index; +} +EXPORT_SYMBOL_GPL(tegra_xusb_padctl_get_port_number); + MODULE_AUTHOR("Thierry Reding "); MODULE_DESCRIPTION("Tegra XUSB Pad Controller driver"); MODULE_LICENSE("GPL v2"); diff --git a/include/linux/phy/tegra/xusb.h b/include/linux/phy/tegra/xusb.h index 70998e6dd6fd..6ca51e0080ec 100644 --- a/include/linux/phy/tegra/xusb.h +++ b/include/linux/phy/tegra/xusb.h @@ -26,6 +26,7 @@ void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy); int tegra_phy_xusb_utmi_port_reset(struct phy *phy); int tegra_xusb_padctl_get_usb3_companion(struct tegra_xusb_padctl *padctl, unsigned int port); +int tegra_xusb_padctl_get_port_number(struct phy *phy); int tegra_xusb_padctl_enable_phy_sleepwalk(struct tegra_xusb_padctl *padctl, struct phy *phy, enum usb_device_speed speed); int tegra_xusb_padctl_disable_phy_sleepwalk(struct tegra_xusb_padctl *padctl, struct phy *phy); -- cgit v1.2.3 From a13bd6f3c936edff957f2d02cf65c44046cb1243 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 26 Feb 2024 18:05:19 -0300 Subject: greybus: move is_gb_* functions out of greybus.h The functions below are only used within the context of drivers/greybus/core.c, so move them all into core and drop their 'inline' specifiers: is_gb_host_device(), is_gb_module(), is_gb_interface(), is_gb_control(), is_gb_bundle() and is_gb_svc(). Suggested-by: Alex Elder Cc: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Alex Elder Link: https://lore.kernel.org/r/20240226-device_cleanup-greybus2-v1-1-5f7d1161e684@marliere.net Signed-off-by: Greg Kroah-Hartman --- drivers/greybus/core.c | 30 ++++++++++++++++++++++++++++++ include/linux/greybus.h | 30 ------------------------------ 2 files changed, 30 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/greybus/core.c b/drivers/greybus/core.c index b0aadc1e7bbe..95c09d4f3a86 100644 --- a/drivers/greybus/core.c +++ b/drivers/greybus/core.c @@ -27,6 +27,36 @@ int greybus_disabled(void) } EXPORT_SYMBOL_GPL(greybus_disabled); +static int is_gb_host_device(const struct device *dev) +{ + return dev->type == &greybus_hd_type; +} + +static int is_gb_module(const struct device *dev) +{ + return dev->type == &greybus_module_type; +} + +static int is_gb_interface(const struct device *dev) +{ + return dev->type == &greybus_interface_type; +} + +static int is_gb_control(const struct device *dev) +{ + return dev->type == &greybus_control_type; +} + +static int is_gb_bundle(const struct device *dev) +{ + return dev->type == &greybus_bundle_type; +} + +static int is_gb_svc(const struct device *dev) +{ + return dev->type == &greybus_svc_type; +} + static bool greybus_match_one_id(struct gb_bundle *bundle, const struct greybus_bundle_id *id) { diff --git a/include/linux/greybus.h b/include/linux/greybus.h index 2cc570ea63bf..634c9511cf78 100644 --- a/include/linux/greybus.h +++ b/include/linux/greybus.h @@ -113,36 +113,6 @@ extern const struct device_type greybus_control_type; extern const struct device_type greybus_bundle_type; extern const struct device_type greybus_svc_type; -static inline int is_gb_host_device(const struct device *dev) -{ - return dev->type == &greybus_hd_type; -} - -static inline int is_gb_module(const struct device *dev) -{ - return dev->type == &greybus_module_type; -} - -static inline int is_gb_interface(const struct device *dev) -{ - return dev->type == &greybus_interface_type; -} - -static inline int is_gb_control(const struct device *dev) -{ - return dev->type == &greybus_control_type; -} - -static inline int is_gb_bundle(const struct device *dev) -{ - return dev->type == &greybus_bundle_type; -} - -static inline int is_gb_svc(const struct device *dev) -{ - return dev->type == &greybus_svc_type; -} - static inline bool cport_id_valid(struct gb_host_device *hd, u16 cport_id) { return cport_id != CPORT_ID_BAD && cport_id < hd->num_cports; -- cgit v1.2.3 From 0e439ba38e615e505404b3935585f1898bafaea9 Mon Sep 17 00:00:00 2001 From: Nipun Gupta Date: Mon, 26 Feb 2024 13:58:16 +0530 Subject: cdx: add MSI support for CDX bus Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain. In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device. Co-developed-by: Nikhil Agarwal Signed-off-by: Nikhil Agarwal Co-developed-by: Abhijit Gangurde Signed-off-by: Abhijit Gangurde Signed-off-by: Nipun Gupta Reviewed-by: Pieter Jansen van Vuuren Reviewed-by: Thomas Gleixner Tested-by: Nikhil Agarwal Link: https://lore.kernel.org/r/20240226082816.100872-1-nipun.gupta@amd.com Signed-off-by: Greg Kroah-Hartman --- drivers/cdx/Makefile | 4 + drivers/cdx/cdx.c | 20 ++++ drivers/cdx/cdx.h | 12 ++ drivers/cdx/cdx_msi.c | 192 ++++++++++++++++++++++++++++++++ drivers/cdx/controller/Kconfig | 1 + drivers/cdx/controller/cdx_controller.c | 25 +++++ drivers/cdx/controller/mc_cdx_pcol.h | 64 +++++++++++ drivers/cdx/controller/mcdi_functions.c | 33 +++++- drivers/cdx/controller/mcdi_functions.h | 33 ++++++ include/linux/cdx/cdx_bus.h | 53 ++++++++- 10 files changed, 434 insertions(+), 3 deletions(-) create mode 100644 drivers/cdx/cdx_msi.c (limited to 'include/linux') diff --git a/drivers/cdx/Makefile b/drivers/cdx/Makefile index 5d1ea482419f..749a3295c2bd 100644 --- a/drivers/cdx/Makefile +++ b/drivers/cdx/Makefile @@ -8,3 +8,7 @@ ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CDX_BUS obj-$(CONFIG_CDX_BUS) += cdx.o controller/ + +ifdef CONFIG_GENERIC_MSI_IRQ +obj-$(CONFIG_CDX_BUS) += cdx_msi.o +endif diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c index b74d76afccb6..236d381dc5f7 100644 --- a/drivers/cdx/cdx.c +++ b/drivers/cdx/cdx.c @@ -56,6 +56,7 @@ */ #include +#include #include #include #include @@ -302,8 +303,19 @@ static int cdx_probe(struct device *dev) { struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); struct cdx_device *cdx_dev = to_cdx_device(dev); + struct cdx_controller *cdx = cdx_dev->cdx; int error; + /* + * Setup MSI device data so that generic MSI alloc/free can + * be used by the device driver. + */ + if (cdx->msi_domain) { + error = msi_setup_device_data(&cdx_dev->dev); + if (error) + return error; + } + error = cdx_drv->probe(cdx_dev); if (error) { dev_err_probe(dev, error, "%s failed\n", __func__); @@ -787,6 +799,7 @@ int cdx_device_add(struct cdx_dev_params *dev_params) /* Populate CDX dev params */ cdx_dev->req_id = dev_params->req_id; + cdx_dev->msi_dev_id = dev_params->msi_dev_id; cdx_dev->vendor = dev_params->vendor; cdx_dev->device = dev_params->device; cdx_dev->subsystem_vendor = dev_params->subsys_vendor; @@ -804,12 +817,19 @@ int cdx_device_add(struct cdx_dev_params *dev_params) cdx_dev->dev.bus = &cdx_bus_type; cdx_dev->dev.dma_mask = &cdx_dev->dma_mask; cdx_dev->dev.release = cdx_device_release; + cdx_dev->msi_write_pending = false; + mutex_init(&cdx_dev->irqchip_lock); /* Set Name */ dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)), cdx_dev->dev_num); + if (cdx->msi_domain) { + cdx_dev->num_msi = dev_params->num_msi; + dev_set_msi_domain(&cdx_dev->dev, cdx->msi_domain); + } + ret = device_add(&cdx_dev->dev); if (ret) { dev_err(&cdx_dev->dev, diff --git a/drivers/cdx/cdx.h b/drivers/cdx/cdx.h index 300ad8be7a34..9c60c04dcf87 100644 --- a/drivers/cdx/cdx.h +++ b/drivers/cdx/cdx.h @@ -25,6 +25,8 @@ * @req_id: Requestor ID associated with CDX device * @class: Class of the CDX Device * @revision: Revision of the CDX device + * @msi_dev_id: MSI device ID associated with CDX device + * @num_msi: Number of MSI's supported by the device */ struct cdx_dev_params { struct cdx_controller *cdx; @@ -40,6 +42,8 @@ struct cdx_dev_params { u32 req_id; u32 class; u8 revision; + u32 msi_dev_id; + u32 num_msi; }; /** @@ -79,4 +83,12 @@ int cdx_device_add(struct cdx_dev_params *dev_params); */ struct device *cdx_bus_add(struct cdx_controller *cdx, u8 bus_num); +/** + * cdx_msi_domain_init - Init the CDX bus MSI domain. + * @dev: Device of the CDX bus controller + * + * Return: CDX MSI domain, NULL on failure + */ +struct irq_domain *cdx_msi_domain_init(struct device *dev); + #endif /* _CDX_H_ */ diff --git a/drivers/cdx/cdx_msi.c b/drivers/cdx/cdx_msi.c new file mode 100644 index 000000000000..e55f1716cfcb --- /dev/null +++ b/drivers/cdx/cdx_msi.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD CDX bus driver MSI support + * + * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cdx.h" + +static void cdx_msi_write_msg(struct irq_data *irq_data, struct msi_msg *msg) +{ + struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data); + struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev); + + /* We would not operate on msg here rather we wait for irq_bus_sync_unlock() + * to be called from preemptible task context. + */ + msi_desc->msg = *msg; + cdx_dev->msi_write_pending = true; +} + +static void cdx_msi_write_irq_lock(struct irq_data *irq_data) +{ + struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data); + struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev); + + mutex_lock(&cdx_dev->irqchip_lock); +} + +static void cdx_msi_write_irq_unlock(struct irq_data *irq_data) +{ + struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data); + struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev); + struct cdx_controller *cdx = cdx_dev->cdx; + struct cdx_device_config dev_config; + + if (!cdx_dev->msi_write_pending) { + mutex_unlock(&cdx_dev->irqchip_lock); + return; + } + + cdx_dev->msi_write_pending = false; + mutex_unlock(&cdx_dev->irqchip_lock); + + dev_config.msi.msi_index = msi_desc->msi_index; + dev_config.msi.data = msi_desc->msg.data; + dev_config.msi.addr = ((u64)(msi_desc->msg.address_hi) << 32) | msi_desc->msg.address_lo; + + /* + * dev_configure() is a controller callback which can interact with + * Firmware or other entities, and can sleep, so invoke this function + * outside of the mutex held region. + */ + dev_config.type = CDX_DEV_MSI_CONF; + if (cdx->ops->dev_configure) + cdx->ops->dev_configure(cdx, cdx_dev->bus_num, cdx_dev->dev_num, &dev_config); +} + +int cdx_enable_msi(struct cdx_device *cdx_dev) +{ + struct cdx_controller *cdx = cdx_dev->cdx; + struct cdx_device_config dev_config; + + dev_config.type = CDX_DEV_MSI_ENABLE; + dev_config.msi_enable = true; + if (cdx->ops->dev_configure) { + return cdx->ops->dev_configure(cdx, cdx_dev->bus_num, cdx_dev->dev_num, + &dev_config); + } + + return -EOPNOTSUPP; +} +EXPORT_SYMBOL_GPL(cdx_enable_msi); + +void cdx_disable_msi(struct cdx_device *cdx_dev) +{ + struct cdx_controller *cdx = cdx_dev->cdx; + struct cdx_device_config dev_config; + + dev_config.type = CDX_DEV_MSI_ENABLE; + dev_config.msi_enable = false; + if (cdx->ops->dev_configure) + cdx->ops->dev_configure(cdx, cdx_dev->bus_num, cdx_dev->dev_num, &dev_config); +} +EXPORT_SYMBOL_GPL(cdx_disable_msi); + +static struct irq_chip cdx_msi_irq_chip = { + .name = "CDX-MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = msi_domain_set_affinity, + .irq_write_msi_msg = cdx_msi_write_msg, + .irq_bus_lock = cdx_msi_write_irq_lock, + .irq_bus_sync_unlock = cdx_msi_write_irq_unlock +}; + +/* Convert an msi_desc to a unique identifier within the domain. */ +static irq_hw_number_t cdx_domain_calc_hwirq(struct cdx_device *dev, + struct msi_desc *desc) +{ + return ((irq_hw_number_t)dev->msi_dev_id << 10) | desc->msi_index; +} + +static void cdx_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = cdx_domain_calc_hwirq(to_cdx_device(desc->dev), desc); +} + +static int cdx_msi_prepare(struct irq_domain *msi_domain, + struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + struct device *parent = cdx_dev->cdx->dev; + struct msi_domain_info *msi_info; + u32 dev_id; + int ret; + + /* Retrieve device ID from requestor ID using parent device */ + ret = of_map_id(parent->of_node, cdx_dev->msi_dev_id, "msi-map", "msi-map-mask", + NULL, &dev_id); + if (ret) { + dev_err(dev, "of_map_id failed for MSI: %d\n", ret); + return ret; + } + +#ifdef GENERIC_MSI_DOMAIN_OPS + /* Set the device Id to be passed to the GIC-ITS */ + info->scratchpad[0].ul = dev_id; +#endif + + msi_info = msi_get_domain_info(msi_domain->parent); + + return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info); +} + +static struct msi_domain_ops cdx_msi_ops = { + .msi_prepare = cdx_msi_prepare, + .set_desc = cdx_msi_set_desc +}; + +static struct msi_domain_info cdx_msi_domain_info = { + .ops = &cdx_msi_ops, + .chip = &cdx_msi_irq_chip, + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS +}; + +struct irq_domain *cdx_msi_domain_init(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct fwnode_handle *fwnode_handle; + struct irq_domain *cdx_msi_domain; + struct device_node *parent_node; + struct irq_domain *parent; + + fwnode_handle = of_node_to_fwnode(np); + + parent_node = of_parse_phandle(np, "msi-map", 1); + if (!parent_node) { + dev_err(dev, "msi-map not present on cdx controller\n"); + return NULL; + } + + parent = irq_find_matching_fwnode(of_node_to_fwnode(parent_node), DOMAIN_BUS_NEXUS); + if (!parent || !msi_get_domain_info(parent)) { + dev_err(dev, "unable to locate ITS domain\n"); + return NULL; + } + + cdx_msi_domain = msi_create_irq_domain(fwnode_handle, &cdx_msi_domain_info, parent); + if (!cdx_msi_domain) { + dev_err(dev, "unable to create CDX-MSI domain\n"); + return NULL; + } + + dev_dbg(dev, "CDX-MSI domain created\n"); + + return cdx_msi_domain; +} +EXPORT_SYMBOL_NS_GPL(cdx_msi_domain_init, CDX_BUS_CONTROLLER); diff --git a/drivers/cdx/controller/Kconfig b/drivers/cdx/controller/Kconfig index 61bf17fbe433..f8e729761aee 100644 --- a/drivers/cdx/controller/Kconfig +++ b/drivers/cdx/controller/Kconfig @@ -9,6 +9,7 @@ if CDX_BUS config CDX_CONTROLLER tristate "CDX bus controller" + select GENERIC_MSI_IRQ select REMOTEPROC select RPMSG help diff --git a/drivers/cdx/controller/cdx_controller.c b/drivers/cdx/controller/cdx_controller.c index 85fe4b1c4e5e..112a1541de6d 100644 --- a/drivers/cdx/controller/cdx_controller.c +++ b/drivers/cdx/controller/cdx_controller.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "cdx_controller.h" #include "../cdx.h" @@ -60,9 +61,19 @@ static int cdx_configure_device(struct cdx_controller *cdx, u8 bus_num, u8 dev_num, struct cdx_device_config *dev_config) { + u16 msi_index; int ret = 0; + u32 data; + u64 addr; switch (dev_config->type) { + case CDX_DEV_MSI_CONF: + msi_index = dev_config->msi.msi_index; + data = dev_config->msi.data; + addr = dev_config->msi.addr; + + ret = cdx_mcdi_write_msi(cdx->priv, bus_num, dev_num, msi_index, addr, data); + break; case CDX_DEV_RESET_CONF: ret = cdx_mcdi_reset_device(cdx->priv, bus_num, dev_num); break; @@ -70,6 +81,9 @@ static int cdx_configure_device(struct cdx_controller *cdx, ret = cdx_mcdi_bus_master_enable(cdx->priv, bus_num, dev_num, dev_config->bus_master_enable); break; + case CDX_DEV_MSI_ENABLE: + ret = cdx_mcdi_msi_enable(cdx->priv, bus_num, dev_num, dev_config->msi_enable); + break; default: ret = -EINVAL; } @@ -178,6 +192,14 @@ static int xlnx_cdx_probe(struct platform_device *pdev) cdx->priv = cdx_mcdi; cdx->ops = &cdx_ops; + /* Create MSI domain */ + cdx->msi_domain = cdx_msi_domain_init(&pdev->dev); + if (!cdx->msi_domain) { + dev_err(&pdev->dev, "cdx_msi_domain_init() failed"); + ret = -ENODEV; + goto cdx_msi_fail; + } + ret = cdx_setup_rpmsg(pdev); if (ret) { if (ret != -EPROBE_DEFER) @@ -189,6 +211,8 @@ static int xlnx_cdx_probe(struct platform_device *pdev) return 0; cdx_rpmsg_fail: + irq_domain_remove(cdx->msi_domain); +cdx_msi_fail: kfree(cdx); cdx_alloc_fail: cdx_mcdi_finish(cdx_mcdi); @@ -205,6 +229,7 @@ static int xlnx_cdx_remove(struct platform_device *pdev) cdx_destroy_rpmsg(pdev); + irq_domain_remove(cdx->msi_domain); kfree(cdx); cdx_mcdi_finish(cdx_mcdi); diff --git a/drivers/cdx/controller/mc_cdx_pcol.h b/drivers/cdx/controller/mc_cdx_pcol.h index 2de019406b57..832a44af963e 100644 --- a/drivers/cdx/controller/mc_cdx_pcol.h +++ b/drivers/cdx/controller/mc_cdx_pcol.h @@ -455,6 +455,12 @@ #define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_OFST 84 #define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_LEN 4 +/* MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_V2 msgresponse */ +#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_V2_LEN 92 +/* Requester ID used by device for GIC ITS DeviceID */ +#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_V2_REQUESTER_DEVICE_ID_OFST 88 +#define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_V2_REQUESTER_DEVICE_ID_LEN 4 + /***********************************/ /* * MC_CMD_CDX_BUS_DOWN @@ -616,6 +622,64 @@ #define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MMIO_REGIONS_ENABLE_LBN 2 #define MC_CMD_CDX_DEVICE_CONTROL_GET_OUT_MMIO_REGIONS_ENABLE_WIDTH 1 +/***********************************/ +/* + * MC_CMD_CDX_DEVICE_WRITE_MSI_MSG + * Populates the MSI message to be used by the hardware to raise the specified + * interrupt vector. Versal-net implementation specific limitations are that + * only 4 CDX devices with MSI interrupt capability are supported and all + * vectors within a device must use the same write address. The command will + * return EINVAL if any of these limitations is violated. + */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG 0x9 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_MSGSET 0x9 +#undef MC_CMD_0x9_PRIVILEGE_CTG + +#define MC_CMD_0x9_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN msgrequest */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_LEN 28 +/* Device bus number, in range 0 to BUS_COUNT-1 */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_BUS_OFST 0 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_BUS_LEN 4 +/* Device number relative to the bus, in range 0 to DEVICE_COUNT-1 for that bus */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_DEVICE_OFST 4 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_DEVICE_LEN 4 +/* + * Device-relative MSI vector number. Must be < MSI_COUNT reported for the + * device. + */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_VECTOR_OFST 8 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_VECTOR_LEN 4 +/* Reserved (alignment) */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_RESERVED_OFST 12 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_RESERVED_LEN 4 +/* + * MSI address to be used by the hardware. Typically, on ARM systems this + * address is translated by the IOMMU (if enabled) and it is the responsibility + * of the entity managing the IOMMU (APU kernel) to supply the correct IOVA + * here. + */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_OFST 16 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_LEN 8 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_LO_OFST 16 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_LO_LEN 4 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_LO_LBN 128 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_LO_WIDTH 32 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_HI_OFST 20 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_HI_LEN 4 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_HI_LBN 160 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS_HI_WIDTH 32 +/* + * MSI data to be used by the hardware. On versal-net, only the lower 16-bits + * are used, the remaining bits are ignored and should be set to zero. + */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_DATA_OFST 24 +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_DATA_LEN 4 + +/* MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_OUT msgresponse */ +#define MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_OUT_LEN 0 + /***********************************/ /* MC_CMD_V2_EXTN - Encapsulation for a v2 extended command */ #define MC_CMD_V2_EXTN 0x7f diff --git a/drivers/cdx/controller/mcdi_functions.c b/drivers/cdx/controller/mcdi_functions.c index b1f530946389..885c69e6ebe5 100644 --- a/drivers/cdx/controller/mcdi_functions.c +++ b/drivers/cdx/controller/mcdi_functions.c @@ -49,7 +49,7 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num, struct cdx_dev_params *dev_params) { - MCDI_DECLARE_BUF(outbuf, MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_LEN); + MCDI_DECLARE_BUF(outbuf, MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_V2_LEN); MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_IN_LEN); struct resource *res = &dev_params->res[0]; size_t outlen; @@ -64,7 +64,7 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, if (ret) return ret; - if (outlen != MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_LEN) + if (outlen != MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_V2_LEN) return -EIO; dev_params->bus_num = bus_num; @@ -73,6 +73,9 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, req_id = MCDI_DWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID); dev_params->req_id = req_id; + dev_params->msi_dev_id = MCDI_DWORD(outbuf, + CDX_BUS_GET_DEVICE_CONFIG_OUT_V2_REQUESTER_DEVICE_ID); + dev_params->res_count = 0; if (MCDI_QWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MMIO_REGION0_SIZE) != 0) { res[dev_params->res_count].start = @@ -127,6 +130,7 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, dev_params->class = MCDI_DWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_CLASS) & 0xFFFFFF; dev_params->revision = MCDI_BYTE(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_DEVICE_REVISION); + dev_params->num_msi = MCDI_DWORD(outbuf, CDX_BUS_GET_DEVICE_CONFIG_OUT_MSI_COUNT); return 0; } @@ -155,6 +159,24 @@ int cdx_mcdi_bus_disable(struct cdx_mcdi *cdx, u8 bus_num) return ret; } +int cdx_mcdi_write_msi(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num, + u32 msi_vector, u64 msi_address, u32 msi_data) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_WRITE_MSI_MSG_IN_LEN); + int ret; + + MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_BUS, bus_num); + MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_DEVICE, dev_num); + MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_VECTOR, msi_vector); + MCDI_SET_QWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_ADDRESS, msi_address); + MCDI_SET_DWORD(inbuf, CDX_DEVICE_WRITE_MSI_MSG_IN_MSI_DATA, msi_data); + + ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_DEVICE_WRITE_MSI_MSG, inbuf, sizeof(inbuf), + NULL, 0, NULL); + + return ret; +} + int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num) { MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_RESET_IN_LEN); @@ -226,3 +248,10 @@ int cdx_mcdi_bus_master_enable(struct cdx_mcdi *cdx, u8 bus_num, return cdx_mcdi_ctrl_flag_set(cdx, bus_num, dev_num, enable, MC_CMD_CDX_DEVICE_CONTROL_SET_IN_BUS_MASTER_ENABLE_LBN); } + +int cdx_mcdi_msi_enable(struct cdx_mcdi *cdx, u8 bus_num, + u8 dev_num, bool enable) +{ + return cdx_mcdi_ctrl_flag_set(cdx, bus_num, dev_num, enable, + MC_CMD_CDX_DEVICE_CONTROL_SET_IN_MSI_ENABLE_LBN); +} diff --git a/drivers/cdx/controller/mcdi_functions.h b/drivers/cdx/controller/mcdi_functions.h index 258a5462fbe3..b9942affdc6b 100644 --- a/drivers/cdx/controller/mcdi_functions.h +++ b/drivers/cdx/controller/mcdi_functions.h @@ -65,6 +65,26 @@ int cdx_mcdi_bus_enable(struct cdx_mcdi *cdx, u8 bus_num); */ int cdx_mcdi_bus_disable(struct cdx_mcdi *cdx, u8 bus_num); +/** + * cdx_mcdi_write_msi - Write MSI configuration for CDX device + * @cdx: pointer to MCDI interface. + * @bus_num: Bus number. + * @dev_num: Device number. + * @msi_vector: Device-relative MSI vector number. + * Must be < MSI_COUNT reported for the device. + * @msi_address: MSI address to be used by the hardware. Typically, on ARM + * systems this address is translated by the IOMMU (if enabled) and + * it is the responsibility of the entity managing the IOMMU (APU kernel) + * to supply the correct IOVA here. + * @msi_data: MSI data to be used by the hardware. On versal-net, only the + * lower 16-bits are used, the remaining bits are ignored and should be + * set to zero. + * + * Return: 0 on success, <0 on failure + */ +int cdx_mcdi_write_msi(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num, + u32 msi_vector, u64 msi_address, u32 msi_data); + /** * cdx_mcdi_reset_device - Reset cdx device represented by bus_num:dev_num * @cdx: pointer to MCDI interface. @@ -89,4 +109,17 @@ int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, int cdx_mcdi_bus_master_enable(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num, bool enable); +/** + * cdx_mcdi_msi_enable - Enable/Disable MSIs for cdx device represented + * by bus_num:dev_num + * @cdx: pointer to MCDI interface. + * @bus_num: Bus number. + * @dev_num: Device number. + * @enable: Enable msi's if set, disable otherwise. + * + * Return: 0 on success, <0 on failure + */ +int cdx_mcdi_msi_enable(struct cdx_mcdi *cdx, u8 bus_num, + u8 dev_num, bool enable); + #endif /* CDX_MCDI_FUNCTIONS_H */ diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h index 6355a36a3f81..b57118aaa679 100644 --- a/include/linux/cdx/cdx_bus.h +++ b/include/linux/cdx/cdx_bus.h @@ -12,6 +12,7 @@ #include #include #include +#include #define MAX_CDX_DEV_RESOURCES 4 #define CDX_CONTROLLER_ID_SHIFT 4 @@ -21,13 +22,25 @@ struct cdx_controller; enum { + CDX_DEV_MSI_CONF, CDX_DEV_BUS_MASTER_CONF, CDX_DEV_RESET_CONF, + CDX_DEV_MSI_ENABLE, +}; + +struct cdx_msi_config { + u64 addr; + u32 data; + u16 msi_index; }; struct cdx_device_config { u8 type; - bool bus_master_enable; + union { + struct cdx_msi_config msi; + bool bus_master_enable; + bool msi_enable; + }; }; typedef int (*cdx_bus_enable_cb)(struct cdx_controller *cdx, u8 bus_num); @@ -87,6 +100,7 @@ struct cdx_ops { * struct cdx_controller: CDX controller object * @dev: Linux device associated with the CDX controller. * @priv: private data + * @msi_domain: MSI domain * @id: Controller ID * @controller_registered: controller registered with bus * @ops: CDX controller ops @@ -94,6 +108,7 @@ struct cdx_ops { struct cdx_controller { struct device *dev; void *priv; + struct irq_domain *msi_domain; u32 id; bool controller_registered; struct cdx_ops *ops; @@ -120,9 +135,13 @@ struct cdx_controller { * @req_id: Requestor ID associated with CDX device * @is_bus: Is this bus device * @enabled: is this bus enabled + * @msi_dev_id: MSI Device ID associated with CDX device + * @num_msi: Number of MSI's supported by the device * @driver_override: driver name to force a match; do not set directly, * because core frees it; use driver_set_override() to * set or clear it. + * @irqchip_lock: lock to synchronize irq/msi configuration + * @msi_write_pending: MSI write pending for this device */ struct cdx_device { struct device dev; @@ -144,7 +163,11 @@ struct cdx_device { u32 req_id; bool is_bus; bool enabled; + u32 msi_dev_id; + u32 num_msi; const char *driver_override; + struct mutex irqchip_lock; + bool msi_write_pending; }; #define to_cdx_device(_dev) \ @@ -237,4 +260,32 @@ int cdx_set_master(struct cdx_device *cdx_dev); */ int cdx_clear_master(struct cdx_device *cdx_dev); +#ifdef CONFIG_GENERIC_MSI_IRQ +/** + * cdx_enable_msi - Enable MSI for the CDX device. + * @cdx_dev: device pointer + * + * Return: 0 for success, -errno on failure + */ +int cdx_enable_msi(struct cdx_device *cdx_dev); + +/** + * cdx_disable_msi - Disable MSI for the CDX device. + * @cdx_dev: device pointer + */ +void cdx_disable_msi(struct cdx_device *cdx_dev); + +#else /* CONFIG_GENERIC_MSI_IRQ */ + +static inline int cdx_enable_msi(struct cdx_device *cdx_dev) +{ + return -ENODEV; +} + +static inline void cdx_disable_msi(struct cdx_device *cdx_dev) +{ +} + +#endif /* CONFIG_GENERIC_MSI_IRQ */ + #endif /* _CDX_BUS_H_ */ -- cgit v1.2.3 From 576882ef5e7fce030b65c92b508a0f84ea5a81c2 Mon Sep 17 00:00:00 2001 From: Chris Leech Date: Mon, 5 Feb 2024 12:01:37 -0800 Subject: uio: introduce UIO_MEM_DMA_COHERENT type Add a UIO memtype specifically for sharing dma_alloc_coherent memory with userspace, backed by dma_mmap_coherent. This is mainly for the bnx2/bnx2x/bnx2i "cnic" interface, although there are a few other uio drivers which map dma_alloc_coherent memory and will be converted to use dma_mmap_coherent as well. Signed-off-by: Nilesh Javali Signed-off-by: Chris Leech Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20240205200137.138302-1-cleech@redhat.com Signed-off-by: Greg Kroah-Hartman --- drivers/uio/uio.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/uio_driver.h | 13 +++++++++++++ 2 files changed, 60 insertions(+) (limited to 'include/linux') diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c index 2d572f6c8ec8..bb77de6fa067 100644 --- a/drivers/uio/uio.c +++ b/drivers/uio/uio.c @@ -24,6 +24,7 @@ #include #include #include +#include #define UIO_MAX_DEVICES (1U << MINORBITS) @@ -759,6 +760,49 @@ static int uio_mmap_physical(struct vm_area_struct *vma) vma->vm_page_prot); } +static int uio_mmap_dma_coherent(struct vm_area_struct *vma) +{ + struct uio_device *idev = vma->vm_private_data; + struct uio_mem *mem; + void *addr; + int ret = 0; + int mi; + + mi = uio_find_mem_index(vma); + if (mi < 0) + return -EINVAL; + + mem = idev->info->mem + mi; + + if (mem->addr & ~PAGE_MASK) + return -ENODEV; + if (mem->dma_addr & ~PAGE_MASK) + return -ENODEV; + if (!mem->dma_device) + return -ENODEV; + if (vma->vm_end - vma->vm_start > mem->size) + return -EINVAL; + + dev_warn(mem->dma_device, + "use of UIO_MEM_DMA_COHERENT is highly discouraged"); + + /* + * UIO uses offset to index into the maps for a device. + * We need to clear vm_pgoff for dma_mmap_coherent. + */ + vma->vm_pgoff = 0; + + addr = (void *)mem->addr; + ret = dma_mmap_coherent(mem->dma_device, + vma, + addr, + mem->dma_addr, + vma->vm_end - vma->vm_start); + vma->vm_pgoff = mi; + + return ret; +} + static int uio_mmap(struct file *filep, struct vm_area_struct *vma) { struct uio_listener *listener = filep->private_data; @@ -806,6 +850,9 @@ static int uio_mmap(struct file *filep, struct vm_area_struct *vma) case UIO_MEM_VIRTUAL: ret = uio_mmap_logical(vma); break; + case UIO_MEM_DMA_COHERENT: + ret = uio_mmap_dma_coherent(vma); + break; default: ret = -EINVAL; } diff --git a/include/linux/uio_driver.h b/include/linux/uio_driver.h index 47c5962b876b..18238dc8bfd3 100644 --- a/include/linux/uio_driver.h +++ b/include/linux/uio_driver.h @@ -28,19 +28,26 @@ struct uio_map; * logical, virtual, or physical & phys_addr_t * should always be large enough to handle any of * the address types) + * @dma_addr: DMA handle set by dma_alloc_coherent, used with + * UIO_MEM_DMA_COHERENT only (@addr should be the + * void * returned from the same dma_alloc_coherent call) * @offs: offset of device memory within the page * @size: size of IO (multiple of page size) * @memtype: type of memory addr points to * @internal_addr: ioremap-ped version of addr, for driver internal use + * @dma_device: device struct that was passed to dma_alloc_coherent, + * used with UIO_MEM_DMA_COHERENT only * @map: for use by the UIO core only. */ struct uio_mem { const char *name; phys_addr_t addr; + dma_addr_t dma_addr; unsigned long offs; resource_size_t size; int memtype; void __iomem *internal_addr; + struct device *dma_device; struct uio_map *map; }; @@ -158,6 +165,12 @@ extern int __must_check #define UIO_MEM_LOGICAL 2 #define UIO_MEM_VIRTUAL 3 #define UIO_MEM_IOVA 4 +/* + * UIO_MEM_DMA_COHERENT exists for legacy drivers that had been getting by with + * improperly mapping DMA coherent allocations through the other modes. + * Do not use in new drivers. + */ +#define UIO_MEM_DMA_COHERENT 5 /* defines for uio_port->porttype */ #define UIO_PORT_NONE 0 -- cgit v1.2.3 From 8dde8fa0cc3edce73c050b9882d06c1a575f6402 Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Wed, 17 Jan 2024 00:33:07 -0800 Subject: firmware_loader: introduce __free() cleanup hanler Define cleanup handler using facilities from linux/cleanup.h to simplify error handling in code using firmware loader. This will allow writing code like this: int driver_update_firmware(...) { const struct firmware *fw_entry __free(firmware) = NULL; int error; ... error = request_firmware(&fw_entry, fw_name, dev); if (error) { dev_err(dev, "failed to request firmware %s: %d", fw_name, error); return error; } error = check_firmware_valid(fw_entry); if (error) return error; guard(mutex)(&instance->lock); error = use_firmware(instance, fw); if (error) return error; return 0; } Signed-off-by: Dmitry Torokhov Acked-by: Luis Chamberalin Link: https://lore.kernel.org/r/ZaeQw7VXhnirX4pQ@google.com Signed-off-by: Greg Kroah-Hartman --- include/linux/firmware.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/firmware.h b/include/linux/firmware.h index 0311858b46ce..f026f8926d79 100644 --- a/include/linux/firmware.h +++ b/include/linux/firmware.h @@ -4,6 +4,7 @@ #include #include +#include #include #define FW_ACTION_NOUEVENT 0 @@ -198,4 +199,6 @@ static inline void firmware_upload_unregister(struct fw_upload *fw_upload) int firmware_request_cache(struct device *device, const char *name); +DEFINE_FREE(firmware, struct firmware *, release_firmware(_T)) + #endif -- cgit v1.2.3 From bbf6cfba49a117c502ec5df66d3ab3b485c113f8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Mar 2024 20:00:05 +0200 Subject: driver core: Drop unneeded 'extern' keyword in fwnode.h We do not use 'extern' keyword with functions. Remove the last one mistakenly added to fwnode.h. Reviewed-by: Sakari Ailus Acked-by: Saravana Kannan Acked-by: "Rafael J. Wysocki" Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240301180138.271590-2-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/fwnode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 2a72f55d26eb..2d23a14857c7 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -209,9 +209,9 @@ static inline void fwnode_dev_initialized(struct fwnode_handle *fwnode, fwnode->flags &= ~FWNODE_FLAG_INITIALIZED; } -extern bool fw_devlink_is_strict(void); int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup); void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); +bool fw_devlink_is_strict(void); #endif -- cgit v1.2.3 From 1c4002aeab3c81afa8a00ae76b1ea38d066e9978 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Mar 2024 20:00:06 +0200 Subject: driver core: Move fw_devlink stuff to where it belongs A few APIs, i.e. fwnode_is_ancestor_of(), fwnode_get_next_parent_dev(), and get_dev_from_fwnode(), that belong specifically to the fw_devlink APIs, may be static, but they are not. Resolve this mess by moving them to the driver/base/core where the all users are being resided and make static. No functional changes intended. Reviewed-by: Sakari Ailus Acked-by: "Rafael J. Wysocki" Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240301180138.271590-3-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ drivers/base/property.c | 56 ---------------------------------------------- include/linux/fwnode.h | 1 - include/linux/property.h | 2 -- 4 files changed, 58 insertions(+), 59 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 9828da9b933c..35ccd8bb2c9b 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1871,6 +1871,7 @@ static void fw_devlink_unblock_consumers(struct device *dev) device_links_write_unlock(); } +#define get_dev_from_fwnode(fwnode) get_device((fwnode)->dev) static bool fwnode_init_without_drv(struct fwnode_handle *fwnode) { @@ -1901,6 +1902,63 @@ static bool fwnode_ancestor_init_without_drv(struct fwnode_handle *fwnode) return false; } +/** + * fwnode_is_ancestor_of - Test if @ancestor is ancestor of @child + * @ancestor: Firmware which is tested for being an ancestor + * @child: Firmware which is tested for being the child + * + * A node is considered an ancestor of itself too. + * + * Return: true if @ancestor is an ancestor of @child. Otherwise, returns false. + */ +static bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, + const struct fwnode_handle *child) +{ + struct fwnode_handle *parent; + + if (IS_ERR_OR_NULL(ancestor)) + return false; + + if (child == ancestor) + return true; + + fwnode_for_each_parent_node(child, parent) { + if (parent == ancestor) { + fwnode_handle_put(parent); + return true; + } + } + return false; +} + +/** + * fwnode_get_next_parent_dev - Find device of closest ancestor fwnode + * @fwnode: firmware node + * + * Given a firmware node (@fwnode), this function finds its closest ancestor + * firmware node that has a corresponding struct device and returns that struct + * device. + * + * The caller is responsible for calling put_device() on the returned device + * pointer. + * + * Return: a pointer to the device of the @fwnode's closest ancestor. + */ +static struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode) +{ + struct fwnode_handle *parent; + struct device *dev; + + fwnode_for_each_parent_node(fwnode, parent) { + dev = get_dev_from_fwnode(parent); + if (dev) { + fwnode_handle_put(parent); + return dev; + } + } + return NULL; +} + /** * __fw_devlink_relax_cycles - Relax and mark dependency cycles. * @con: Potential consumer device. diff --git a/drivers/base/property.c b/drivers/base/property.c index a1b01ab42052..afa1bf2b3c5a 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -699,34 +699,6 @@ struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(fwnode_get_next_parent); -/** - * fwnode_get_next_parent_dev - Find device of closest ancestor fwnode - * @fwnode: firmware node - * - * Given a firmware node (@fwnode), this function finds its closest ancestor - * firmware node that has a corresponding struct device and returns that struct - * device. - * - * The caller is responsible for calling put_device() on the returned device - * pointer. - * - * Return: a pointer to the device of the @fwnode's closest ancestor. - */ -struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode) -{ - struct fwnode_handle *parent; - struct device *dev; - - fwnode_for_each_parent_node(fwnode, parent) { - dev = get_dev_from_fwnode(parent); - if (dev) { - fwnode_handle_put(parent); - return dev; - } - } - return NULL; -} - /** * fwnode_count_parents - Return the number of parents a node has * @fwnode: The node the parents of which are to be counted @@ -773,34 +745,6 @@ struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwnode, } EXPORT_SYMBOL_GPL(fwnode_get_nth_parent); -/** - * fwnode_is_ancestor_of - Test if @ancestor is ancestor of @child - * @ancestor: Firmware which is tested for being an ancestor - * @child: Firmware which is tested for being the child - * - * A node is considered an ancestor of itself too. - * - * Return: true if @ancestor is an ancestor of @child. Otherwise, returns false. - */ -bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, const struct fwnode_handle *child) -{ - struct fwnode_handle *parent; - - if (IS_ERR_OR_NULL(ancestor)) - return false; - - if (child == ancestor) - return true; - - fwnode_for_each_parent_node(child, parent) { - if (parent == ancestor) { - fwnode_handle_put(parent); - return true; - } - } - return false; -} - /** * fwnode_get_next_child_node - Return the next child node handle for a node * @fwnode: Firmware node to find the next child node for. diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 2d23a14857c7..416cbe72f0c7 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -187,7 +187,6 @@ struct fwnode_operations { if (fwnode_has_op(fwnode, op)) \ (fwnode)->ops->op(fwnode, ## __VA_ARGS__); \ } while (false) -#define get_dev_from_fwnode(fwnode) get_device((fwnode)->dev) static inline void fwnode_init(struct fwnode_handle *fwnode, const struct fwnode_operations *ops) diff --git a/include/linux/property.h b/include/linux/property.h index e6516d0b7d52..284ff79ebf03 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -156,11 +156,9 @@ struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode); for (parent = fwnode_get_parent(fwnode); parent; \ parent = fwnode_get_next_parent(parent)) -struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode); unsigned int fwnode_count_parents(const struct fwnode_handle *fwn); struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwn, unsigned int depth); -bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, const struct fwnode_handle *child); struct fwnode_handle *fwnode_get_next_child_node( const struct fwnode_handle *fwnode, struct fwnode_handle *child); struct fwnode_handle *fwnode_get_next_available_child_node( -- cgit v1.2.3 From 420b104dd116cddd1615588a400b557bf4e436b4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Mar 2024 20:00:07 +0200 Subject: device property: Move enum dev_dma_attr to fwnode.h The struct fwnode_operations defines one of the callback to return enum dev_dma_attr. But this currently is defined in property.h. Move it to the correct location. Reviewed-by: Sakari Ailus Acked-by: "Rafael J. Wysocki" Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240301180138.271590-4-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/fwnode.h | 6 ++++++ include/linux/property.h | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 416cbe72f0c7..4228c45d5ccc 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -14,6 +14,12 @@ #include #include +enum dev_dma_attr { + DEV_DMA_NOT_SUPPORTED, + DEV_DMA_NON_COHERENT, + DEV_DMA_COHERENT, +}; + struct fwnode_operations; struct device; diff --git a/include/linux/property.h b/include/linux/property.h index 284ff79ebf03..1f0135e24d00 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -27,12 +27,6 @@ enum dev_prop_type { DEV_PROP_REF, }; -enum dev_dma_attr { - DEV_DMA_NOT_SUPPORTED, - DEV_DMA_NON_COHERENT, - DEV_DMA_COHERENT, -}; - const struct fwnode_handle *__dev_fwnode_const(const struct device *dev); struct fwnode_handle *__dev_fwnode(struct device *dev); #define dev_fwnode(dev) \ -- cgit v1.2.3 From 4dc3d612ee5c3be2a4d1a73ab31bcfaaa850aa19 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Mar 2024 20:00:08 +0200 Subject: device property: Don't use "proxy" headers Update header inclusions to follow IWYU (Include What You Use) principle. Reviewed-by: Sakari Ailus Acked-by: "Rafael J. Wysocki" Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240301180138.271590-5-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/property.c | 11 ++++++----- drivers/base/swnode.c | 13 ++++++++++++- include/linux/fwnode.h | 4 ++-- include/linux/property.h | 1 + 4 files changed, 21 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index afa1bf2b3c5a..7324a704a9a1 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -7,15 +7,16 @@ * Mika Westerberg */ -#include +#include +#include #include -#include +#include #include -#include -#include -#include #include #include +#include +#include +#include struct fwnode_handle *__dev_fwnode(struct device *dev) { diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 36512fb75a20..eb6eb25b343b 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -6,10 +6,21 @@ * Author: Heikki Krogerus */ +#include #include -#include +#include +#include +#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include #include "base.h" diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 4228c45d5ccc..80f3cd91b471 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -9,10 +9,10 @@ #ifndef _LINUX_FWNODE_H_ #define _LINUX_FWNODE_H_ -#include -#include #include #include +#include +#include enum dev_dma_attr { DEV_DMA_NOT_SUPPORTED, diff --git a/include/linux/property.h b/include/linux/property.h index 1f0135e24d00..3a1045eb786c 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -11,6 +11,7 @@ #define _LINUX_PROPERTY_H_ #include +#include #include #include #include -- cgit v1.2.3 From 75cde56a5b504d07a64ce0e3f8c7410df70308a3 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Mon, 4 Mar 2024 21:04:54 -0800 Subject: driver core: Adds flags param to fwnode_link_add() Allow the callers to set fwnode link flags when adding fwnode links. Signed-off-by: Saravana Kannan Acked-by: "Rafael J. Wysocki" Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240305050458.1400667-2-saravanak@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 5 +++-- drivers/firmware/efi/sysfb_efi.c | 2 +- drivers/of/property.c | 2 +- include/linux/fwnode.h | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 35ccd8bb2c9b..83a6f429bddb 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -92,12 +92,13 @@ static int __fwnode_link_add(struct fwnode_handle *con, return 0; } -int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup) +int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, + u8 flags) { int ret; mutex_lock(&fwnode_link_lock); - ret = __fwnode_link_add(con, sup, 0); + ret = __fwnode_link_add(con, sup, flags); mutex_unlock(&fwnode_link_lock); return ret; } diff --git a/drivers/firmware/efi/sysfb_efi.c b/drivers/firmware/efi/sysfb_efi.c index 456d0e5eaf78..cc807ed35aed 100644 --- a/drivers/firmware/efi/sysfb_efi.c +++ b/drivers/firmware/efi/sysfb_efi.c @@ -336,7 +336,7 @@ static int efifb_add_links(struct fwnode_handle *fwnode) if (!sup_np) return 0; - fwnode_link_add(fwnode, of_fwnode_handle(sup_np)); + fwnode_link_add(fwnode, of_fwnode_handle(sup_np), 0); of_node_put(sup_np); return 0; diff --git a/drivers/of/property.c b/drivers/of/property.c index b71267c6667c..bce849f21ae2 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -1085,7 +1085,7 @@ static void of_link_to_phandle(struct device_node *con_np, tmp_np = of_get_next_parent(tmp_np); } - fwnode_link_add(of_fwnode_handle(con_np), of_fwnode_handle(sup_np)); + fwnode_link_add(of_fwnode_handle(con_np), of_fwnode_handle(sup_np), 0); } /** diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 80f3cd91b471..70d9c40269b9 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -214,7 +214,8 @@ static inline void fwnode_dev_initialized(struct fwnode_handle *fwnode, fwnode->flags &= ~FWNODE_FLAG_INITIALIZED; } -int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup); +int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, + u8 flags); void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); bool fw_devlink_is_strict(void); -- cgit v1.2.3 From b7e1241d8f77ed64404a5e4450f43a319310fc91 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Mon, 4 Mar 2024 21:04:55 -0800 Subject: driver core: Add FWLINK_FLAG_IGNORE to completely ignore a fwnode link A fwnode link between specific supplier-consumer fwnodes can be added multiple times for multiple reasons. If that dependency doesn't exist, deleting the fwnode link once doesn't guarantee that it won't get created again. So, add FWLINK_FLAG_IGNORE flag to mark a fwnode link as one that needs to be completely ignored. Since a fwnode link's flags is an OR of all the flags passed to all the fwnode_link_add() calls to create that specific fwnode link, the FWLINK_FLAG_IGNORE flag is preserved and can be used to mark a fwnode link as on that need to be completely ignored until it is deleted. Signed-off-by: Saravana Kannan Acked-by: "Rafael J. Wysocki" Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240305050458.1400667-3-saravanak@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 9 ++++++++- include/linux/fwnode.h | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 83a6f429bddb..b93f3c5716ae 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1012,7 +1012,8 @@ static struct fwnode_handle *fwnode_links_check_suppliers( return NULL; list_for_each_entry(link, &fwnode->suppliers, c_hook) - if (!(link->flags & FWLINK_FLAG_CYCLE)) + if (!(link->flags & + (FWLINK_FLAG_CYCLE | FWLINK_FLAG_IGNORE))) return link->supplier; return NULL; @@ -2021,6 +2022,9 @@ static bool __fw_devlink_relax_cycles(struct device *con, } list_for_each_entry(link, &sup_handle->suppliers, c_hook) { + if (link->flags & FWLINK_FLAG_IGNORE) + continue; + if (__fw_devlink_relax_cycles(con, link->supplier)) { __fwnode_link_cycle(link); ret = true; @@ -2099,6 +2103,9 @@ static int fw_devlink_create_devlink(struct device *con, int ret = 0; u32 flags; + if (link->flags & FWLINK_FLAG_IGNORE) + return 0; + if (con->fwnode == link->consumer) flags = fw_devlink_get_flags(link->flags); else diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 70d9c40269b9..0d79070c5a70 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -59,8 +59,10 @@ struct fwnode_handle { * fwnode link flags * * CYCLE: The fwnode link is part of a cycle. Don't defer probe. + * IGNORE: Completely ignore this link, even during cycle detection. */ #define FWLINK_FLAG_CYCLE BIT(0) +#define FWLINK_FLAG_IGNORE BIT(1) struct fwnode_link { struct fwnode_handle *supplier; -- cgit v1.2.3 From cf2c2e4a3d910270903d50462aaa75140cdb2c96 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 6 Mar 2024 19:12:25 -0800 Subject: bpf: Plumb get_unmapped_area() callback into bpf_map_ops Subsequent patches introduce bpf_arena that imposes special alignment requirements on address selection. Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/r/20240307031228.42896-4-alexei.starovoitov@gmail.com Signed-off-by: Martin KaFai Lau --- include/linux/bpf.h | 3 +++ kernel/bpf/syscall.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 785660810e6a..95e07673cdc1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -139,6 +139,9 @@ struct bpf_map_ops { int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); + unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags); /* Functions called by bpf_local_storage maps */ int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index b2750b79ac80..f63f4da4db5e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -937,6 +937,21 @@ static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts) return EPOLLERR; } +static unsigned long bpf_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + struct bpf_map *map = filp->private_data; + + if (map->ops->map_get_unmapped_area) + return map->ops->map_get_unmapped_area(filp, addr, len, pgoff, flags); +#ifdef CONFIG_MMU + return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); +#else + return addr; +#endif +} + const struct file_operations bpf_map_fops = { #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_map_show_fdinfo, @@ -946,6 +961,7 @@ const struct file_operations bpf_map_fops = { .write = bpf_dummy_write, .mmap = bpf_map_mmap, .poll = bpf_map_poll, + .get_unmapped_area = bpf_get_unmapped_area, }; int bpf_map_new_fd(struct bpf_map *map, int flags) -- cgit v1.2.3 From 2658b5a8a4eee5fad378d0bde2f221deacbc58f1 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:14 +0000 Subject: net: introduce struct net_hotdata Instead of spreading networking critical fields all over the places, add a custom net_hotdata structure so that we can precisely control its layout. In this first patch, move : - gro_normal_batch used in rx (GRO stack) - offload_base used in rx and tx (GRO and TSO stacks) Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-2-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 1 - include/net/gro.h | 5 ++--- include/net/hotdata.h | 15 +++++++++++++++ net/core/Makefile | 1 + net/core/gro.c | 15 ++++++--------- net/core/gso.c | 4 ++-- net/core/hotdata.c | 9 +++++++++ net/core/sysctl_net_core.c | 3 ++- 8 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 include/net/hotdata.h create mode 100644 net/core/hotdata.c (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 2767467138a0..6643452af543 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4796,7 +4796,6 @@ void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); extern int netdev_max_backlog; extern int dev_rx_weight; extern int dev_tx_weight; -extern int gro_normal_batch; enum { NESTED_SYNC_IMM_BIT, diff --git a/include/net/gro.h b/include/net/gro.h index 2b58671a6549..d6fc8fbd3730 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -9,6 +9,7 @@ #include #include #include +#include struct napi_gro_cb { union { @@ -446,7 +447,7 @@ static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, { list_add_tail(&skb->list, &napi->rx_list); napi->rx_count += segs; - if (napi->rx_count >= READ_ONCE(gro_normal_batch)) + if (napi->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch)) gro_normal_list(napi); } @@ -493,6 +494,4 @@ static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int * #endif } -extern struct list_head offload_base; - #endif /* _NET_IPV6_GRO_H */ diff --git a/include/net/hotdata.h b/include/net/hotdata.h new file mode 100644 index 000000000000..6ed32e4e34aa --- /dev/null +++ b/include/net/hotdata.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _NET_HOTDATA_H +#define _NET_HOTDATA_H + +#include + +/* Read mostly data used in network fast paths. */ +struct net_hotdata { + struct list_head offload_base; + int gro_normal_batch; +}; + +extern struct net_hotdata net_hotdata; + +#endif /* _NET_HOTDATA_H */ diff --git a/net/core/Makefile b/net/core/Makefile index 821aec06abf1..6e6548011fae 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -18,6 +18,7 @@ obj-y += dev.o dev_addr_lists.o dst.o netevent.o \ obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o obj-y += net-sysfs.o +obj-y += hotdata.o obj-$(CONFIG_PAGE_POOL) += page_pool.o page_pool_user.o obj-$(CONFIG_PROC_FS) += net-procfs.o obj-$(CONFIG_NET_PKTGEN) += pktgen.o diff --git a/net/core/gro.c b/net/core/gro.c index 6a0edbd826a1..ee30d4f0c038 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -10,9 +10,6 @@ #define GRO_MAX_HEAD (MAX_HEADER + 128) static DEFINE_SPINLOCK(offload_lock); -struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base); -/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ -int gro_normal_batch __read_mostly = 8; /** * dev_add_offload - register offload handlers @@ -31,7 +28,7 @@ void dev_add_offload(struct packet_offload *po) struct packet_offload *elem; spin_lock(&offload_lock); - list_for_each_entry(elem, &offload_base, list) { + list_for_each_entry(elem, &net_hotdata.offload_base, list) { if (po->priority < elem->priority) break; } @@ -55,7 +52,7 @@ EXPORT_SYMBOL(dev_add_offload); */ static void __dev_remove_offload(struct packet_offload *po) { - struct list_head *head = &offload_base; + struct list_head *head = &net_hotdata.offload_base; struct packet_offload *po1; spin_lock(&offload_lock); @@ -235,9 +232,9 @@ done: static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) { + struct list_head *head = &net_hotdata.offload_base; struct packet_offload *ptype; __be16 type = skb->protocol; - struct list_head *head = &offload_base; int err = -ENOENT; BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); @@ -444,7 +441,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff { u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); struct gro_list *gro_list = &napi->gro_hash[bucket]; - struct list_head *head = &offload_base; + struct list_head *head = &net_hotdata.offload_base; struct packet_offload *ptype; __be16 type = skb->protocol; struct sk_buff *pp = NULL; @@ -550,7 +547,7 @@ normal: struct packet_offload *gro_find_receive_by_type(__be16 type) { - struct list_head *offload_head = &offload_base; + struct list_head *offload_head = &net_hotdata.offload_base; struct packet_offload *ptype; list_for_each_entry_rcu(ptype, offload_head, list) { @@ -564,7 +561,7 @@ EXPORT_SYMBOL(gro_find_receive_by_type); struct packet_offload *gro_find_complete_by_type(__be16 type) { - struct list_head *offload_head = &offload_base; + struct list_head *offload_head = &net_hotdata.offload_base; struct packet_offload *ptype; list_for_each_entry_rcu(ptype, offload_head, list) { diff --git a/net/core/gso.c b/net/core/gso.c index 9e1803bfc9c6..bcd156372f4d 100644 --- a/net/core/gso.c +++ b/net/core/gso.c @@ -17,7 +17,7 @@ struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb, struct packet_offload *ptype; rcu_read_lock(); - list_for_each_entry_rcu(ptype, &offload_base, list) { + list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) { if (ptype->type == type && ptype->callbacks.gso_segment) { segs = ptype->callbacks.gso_segment(skb, features); break; @@ -48,7 +48,7 @@ struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, __skb_pull(skb, vlan_depth); rcu_read_lock(); - list_for_each_entry_rcu(ptype, &offload_base, list) { + list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) { if (ptype->type == type && ptype->callbacks.gso_segment) { segs = ptype->callbacks.gso_segment(skb, features); break; diff --git a/net/core/hotdata.c b/net/core/hotdata.c new file mode 100644 index 000000000000..abb8ad19d59a --- /dev/null +++ b/net/core/hotdata.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include +#include + +struct net_hotdata net_hotdata __cacheline_aligned = { + .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base), + .gro_normal_batch = 8, +}; diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 986f15e5d6c4..0eb1242eabbe 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "dev.h" @@ -632,7 +633,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "gro_normal_batch", - .data = &gro_normal_batch, + .data = &net_hotdata.gro_normal_batch, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_dointvec_minmax, -- cgit v1.2.3 From 0b91fa4bfb1caedd01cb6eb3b733cbc77c9edb0e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:17 +0000 Subject: net: move ptype_all into net_hotdata ptype_all is used in rx/tx fast paths. Move it to net_hotdata for better cache locality. Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-5-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 1 - include/net/hotdata.h | 1 + net/core/dev.c | 16 +++++++--------- net/core/hotdata.c | 1 + net/core/net-procfs.c | 7 ++++--- 5 files changed, 13 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 6643452af543..b18ac8072f18 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -5306,7 +5306,6 @@ static inline const char *netdev_reg_state(const struct net_device *dev) #define PTYPE_HASH_SIZE (16) #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1) -extern struct list_head ptype_all __read_mostly; extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; extern struct net_device *blackhole_netdev; diff --git a/include/net/hotdata.h b/include/net/hotdata.h index 149e56528537..d462cb8f16ba 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -7,6 +7,7 @@ /* Read mostly data used in network fast paths. */ struct net_hotdata { struct list_head offload_base; + struct list_head ptype_all; int gro_normal_batch; int netdev_budget; int netdev_budget_usecs; diff --git a/net/core/dev.c b/net/core/dev.c index f2f44303c035..e1493e558407 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -161,7 +161,6 @@ static DEFINE_SPINLOCK(ptype_lock); struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; -struct list_head ptype_all __read_mostly; /* Taps */ static int netif_rx_internal(struct sk_buff *skb); static int call_netdevice_notifiers_extack(unsigned long val, @@ -540,7 +539,7 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev) static inline struct list_head *ptype_head(const struct packet_type *pt) { if (pt->type == htons(ETH_P_ALL)) - return pt->dev ? &pt->dev->ptype_all : &ptype_all; + return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all; else return pt->dev ? &pt->dev->ptype_specific : &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; @@ -2226,7 +2225,8 @@ static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) */ bool dev_nit_active(struct net_device *dev) { - return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all); + return !list_empty(&net_hotdata.ptype_all) || + !list_empty(&dev->ptype_all); } EXPORT_SYMBOL_GPL(dev_nit_active); @@ -2237,10 +2237,9 @@ EXPORT_SYMBOL_GPL(dev_nit_active); void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) { - struct packet_type *ptype; + struct list_head *ptype_list = &net_hotdata.ptype_all; + struct packet_type *ptype, *pt_prev = NULL; struct sk_buff *skb2 = NULL; - struct packet_type *pt_prev = NULL; - struct list_head *ptype_list = &ptype_all; rcu_read_lock(); again: @@ -2286,7 +2285,7 @@ again: pt_prev = ptype; } - if (ptype_list == &ptype_all) { + if (ptype_list == &net_hotdata.ptype_all) { ptype_list = &dev->ptype_all; goto again; } @@ -5387,7 +5386,7 @@ another_round: if (pfmemalloc) goto skip_taps; - list_for_each_entry_rcu(ptype, &ptype_all, list) { + list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) { if (pt_prev) ret = deliver_skb(skb, pt_prev, orig_dev); pt_prev = ptype; @@ -11749,7 +11748,6 @@ static int __init net_dev_init(void) if (netdev_kobject_init()) goto out; - INIT_LIST_HEAD(&ptype_all); for (i = 0; i < PTYPE_HASH_SIZE; i++) INIT_LIST_HEAD(&ptype_base[i]); diff --git a/net/core/hotdata.c b/net/core/hotdata.c index 087c4c84987d..29fcfe89fd9a 100644 --- a/net/core/hotdata.c +++ b/net/core/hotdata.c @@ -7,6 +7,7 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base), + .ptype_all = LIST_HEAD_INIT(net_hotdata.ptype_all), .gro_normal_batch = 8, .netdev_budget = 300, diff --git a/net/core/net-procfs.c b/net/core/net-procfs.c index 2e4e96d30ee1..a97eceb84e61 100644 --- a/net/core/net-procfs.c +++ b/net/core/net-procfs.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "dev.h" @@ -183,7 +184,7 @@ static void *ptype_get_idx(struct seq_file *seq, loff_t pos) } } - list_for_each_entry_rcu(pt, &ptype_all, list) { + list_for_each_entry_rcu(pt, &net_hotdata.ptype_all, list) { if (i == pos) return pt; ++i; @@ -231,13 +232,13 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) } } - nxt = ptype_all.next; + nxt = net_hotdata.ptype_all.next; goto ptype_all; } if (pt->type == htons(ETH_P_ALL)) { ptype_all: - if (nxt != &ptype_all) + if (nxt != &net_hotdata.ptype_all) goto found; hash = 0; nxt = ptype_base[0].next; -- cgit v1.2.3 From edbc666cdcbf4a80ada4311c272a2078af87b880 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:18 +0000 Subject: net: move netdev_max_backlog to net_hotdata netdev_max_backlog is used in rx fat path. Move it to net_hodata for better cache locality. Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-6-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 1 - include/net/hotdata.h | 1 + net/core/dev.c | 8 +++----- net/core/gro_cells.c | 3 ++- net/core/hotdata.c | 2 ++ net/core/sysctl_net_core.c | 2 +- net/xfrm/espintcp.c | 4 +++- net/xfrm/xfrm_input.c | 3 ++- 8 files changed, 14 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index b18ac8072f18..c9a671b7bb37 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4793,7 +4793,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, const struct pcpu_sw_netstats __percpu *netstats); void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); -extern int netdev_max_backlog; extern int dev_rx_weight; extern int dev_tx_weight; diff --git a/include/net/hotdata.h b/include/net/hotdata.h index d462cb8f16ba..dc50b200a94b 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -12,6 +12,7 @@ struct net_hotdata { int netdev_budget; int netdev_budget_usecs; int tstamp_prequeue; + int max_backlog; }; extern struct net_hotdata net_hotdata; diff --git a/net/core/dev.c b/net/core/dev.c index e1493e558407..e23edbaff392 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4404,9 +4404,6 @@ EXPORT_SYMBOL(__dev_direct_xmit); * Receiver routines *************************************************************************/ -int netdev_max_backlog __read_mostly = 1000; -EXPORT_SYMBOL(netdev_max_backlog); - unsigned int sysctl_skb_defer_max __read_mostly = 64; int weight_p __read_mostly = 64; /* old backlog weight */ int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ @@ -4713,7 +4710,7 @@ static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) struct softnet_data *sd; unsigned int old_flow, new_flow; - if (qlen < (READ_ONCE(netdev_max_backlog) >> 1)) + if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1)) return false; sd = this_cpu_ptr(&softnet_data); @@ -4761,7 +4758,8 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, if (!netif_running(skb->dev)) goto drop; qlen = skb_queue_len(&sd->input_pkt_queue); - if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) { + if (qlen <= READ_ONCE(net_hotdata.max_backlog) && + !skb_flow_limit(skb, qlen)) { if (qlen) { enqueue: __skb_queue_tail(&sd->input_pkt_queue, skb); diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c index ed5ec5de47f6..ff8e5b64bf6b 100644 --- a/net/core/gro_cells.c +++ b/net/core/gro_cells.c @@ -3,6 +3,7 @@ #include #include #include +#include struct gro_cell { struct sk_buff_head napi_skbs; @@ -26,7 +27,7 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) cell = this_cpu_ptr(gcells->cells); - if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(netdev_max_backlog)) { + if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) { drop: dev_core_stats_rx_dropped_inc(dev); kfree_skb(skb); diff --git a/net/core/hotdata.c b/net/core/hotdata.c index 29fcfe89fd9a..35ed5a83ecc7 100644 --- a/net/core/hotdata.c +++ b/net/core/hotdata.c @@ -15,4 +15,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .netdev_budget_usecs = 2 * USEC_PER_SEC / HZ, .tstamp_prequeue = 1, + .max_backlog = 1000, }; +EXPORT_SYMBOL(net_hotdata); diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index bddd07da0998..8eaeeb289914 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -440,7 +440,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "netdev_max_backlog", - .data = &netdev_max_backlog, + .data = &net_hotdata.max_backlog, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index d3b3f9e720b3..fe82e2d07300 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -10,6 +10,7 @@ #if IS_ENABLED(CONFIG_IPV6) #include #endif +#include static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb, struct sock *sk) @@ -169,7 +170,8 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) { struct espintcp_ctx *ctx = espintcp_getctx(sk); - if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog)) + if (skb_queue_len(&ctx->out_queue) >= + READ_ONCE(net_hotdata.max_backlog)) return -ENOBUFS; __skb_queue_tail(&ctx->out_queue, skb); diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index bd4ce21d76d7..161f535c8b94 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "xfrm_inout.h" @@ -764,7 +765,7 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, trans = this_cpu_ptr(&xfrm_trans_tasklet); - if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog)) + if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog)) return -ENOBUFS; BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb)); -- cgit v1.2.3 From 26722dc74bf08fd79564cbcad1e5f3e2aa3bf9cc Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:21 +0000 Subject: net: move dev_tx_weight to net_hotdata dev_tx_weight is used in tx fast path. Move it to net_hotdata for better cache locality. Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-9-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 1 - include/net/hotdata.h | 1 + net/core/dev.c | 1 - net/core/hotdata.c | 1 + net/core/sysctl_net_core.c | 2 +- net/sched/sch_generic.c | 3 ++- 6 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c9a671b7bb37..ad4b031098ff 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4794,7 +4794,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); extern int dev_rx_weight; -extern int dev_tx_weight; enum { NESTED_SYNC_IMM_BIT, diff --git a/include/net/hotdata.h b/include/net/hotdata.h index d86d02f156fc..ffea9cc263e5 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -21,6 +21,7 @@ struct net_hotdata { int netdev_budget_usecs; int tstamp_prequeue; int max_backlog; + int dev_tx_weight; }; extern struct net_hotdata net_hotdata; diff --git a/net/core/dev.c b/net/core/dev.c index e23edbaff392..5d6bd481103f 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4409,7 +4409,6 @@ int weight_p __read_mostly = 64; /* old backlog weight */ int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ int dev_rx_weight __read_mostly = 64; -int dev_tx_weight __read_mostly = 64; /* Called with irq disabled */ static inline void ____napi_schedule(struct softnet_data *sd, diff --git a/net/core/hotdata.c b/net/core/hotdata.c index 35ed5a83ecc7..ec8c3b48e8fe 100644 --- a/net/core/hotdata.c +++ b/net/core/hotdata.c @@ -16,5 +16,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .tstamp_prequeue = 1, .max_backlog = 1000, + .dev_tx_weight = 64, }; EXPORT_SYMBOL(net_hotdata); diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 8eaeeb289914..a30016a8660e 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -302,7 +302,7 @@ static int proc_do_dev_weight(struct ctl_table *table, int write, if (!ret && write) { weight = READ_ONCE(weight_p); WRITE_ONCE(dev_rx_weight, weight * dev_weight_rx_bias); - WRITE_ONCE(dev_tx_weight, weight * dev_weight_tx_bias); + WRITE_ONCE(net_hotdata.dev_tx_weight, weight * dev_weight_tx_bias); } mutex_unlock(&dev_weight_mutex); diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 9b3e9262040b..ff5336493777 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -409,7 +410,7 @@ static inline bool qdisc_restart(struct Qdisc *q, int *packets) void __qdisc_run(struct Qdisc *q) { - int quota = READ_ONCE(dev_tx_weight); + int quota = READ_ONCE(net_hotdata.dev_tx_weight); int packets; while (qdisc_restart(q, &packets)) { -- cgit v1.2.3 From 71c0de9bac9c1dda503322c86be4924f055dc6c9 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:22 +0000 Subject: net: move dev_rx_weight to net_hotdata dev_rx_weight is read from process_backlog(). Move it to net_hotdata for better cache locality. Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-10-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 -- include/net/hotdata.h | 1 + net/core/dev.c | 3 +-- net/core/hotdata.c | 1 + net/core/sysctl_net_core.c | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index ad4b031098ff..dd641297e807 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4793,8 +4793,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, const struct pcpu_sw_netstats __percpu *netstats); void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); -extern int dev_rx_weight; - enum { NESTED_SYNC_IMM_BIT, NESTED_SYNC_TODO_BIT, diff --git a/include/net/hotdata.h b/include/net/hotdata.h index ffea9cc263e5..e6595ed2c3be 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -22,6 +22,7 @@ struct net_hotdata { int tstamp_prequeue; int max_backlog; int dev_tx_weight; + int dev_rx_weight; }; extern struct net_hotdata net_hotdata; diff --git a/net/core/dev.c b/net/core/dev.c index 5d6bd481103f..40ba02e04bcb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4408,7 +4408,6 @@ unsigned int sysctl_skb_defer_max __read_mostly = 64; int weight_p __read_mostly = 64; /* old backlog weight */ int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ -int dev_rx_weight __read_mostly = 64; /* Called with irq disabled */ static inline void ____napi_schedule(struct softnet_data *sd, @@ -5978,7 +5977,7 @@ static int process_backlog(struct napi_struct *napi, int quota) net_rps_action_and_irq_enable(sd); } - napi->weight = READ_ONCE(dev_rx_weight); + napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); while (again) { struct sk_buff *skb; diff --git a/net/core/hotdata.c b/net/core/hotdata.c index ec8c3b48e8fe..c8a7a451c18a 100644 --- a/net/core/hotdata.c +++ b/net/core/hotdata.c @@ -17,5 +17,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .tstamp_prequeue = 1, .max_backlog = 1000, .dev_tx_weight = 64, + .dev_rx_weight = 64, }; EXPORT_SYMBOL(net_hotdata); diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index a30016a8660e..8a4c698dad9c 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -301,7 +301,7 @@ static int proc_do_dev_weight(struct ctl_table *table, int write, ret = proc_dointvec(table, write, buffer, lenp, ppos); if (!ret && write) { weight = READ_ONCE(weight_p); - WRITE_ONCE(dev_rx_weight, weight * dev_weight_rx_bias); + WRITE_ONCE(net_hotdata.dev_rx_weight, weight * dev_weight_rx_bias); WRITE_ONCE(net_hotdata.dev_tx_weight, weight * dev_weight_tx_bias); } mutex_unlock(&dev_weight_mutex); -- cgit v1.2.3 From aa70d2d16f280efe8aa52afc25a33b2ec8d346b6 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:23 +0000 Subject: net: move skbuff_cache(s) to net_hotdata skbuff_cache, skbuff_fclone_cache and skb_small_head_cache are used in rx/tx fast paths. Move them to net_hotdata for better cache locality. Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-11-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 1 - include/net/hotdata.h | 3 +++ kernel/bpf/cpumap.c | 4 +++- net/bpf/test_run.c | 4 +++- net/core/skbuff.c | 44 ++++++++++++++++++++------------------------ net/core/xdp.c | 5 +++-- 6 files changed, 32 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 3013355b63f5..d0508f90bed5 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1271,7 +1271,6 @@ static inline void consume_skb(struct sk_buff *skb) void __consume_stateless_skb(struct sk_buff *skb); void __kfree_skb(struct sk_buff *skb); -extern struct kmem_cache *skbuff_cache; void kfree_skb_partial(struct sk_buff *skb, bool head_stolen); bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, diff --git a/include/net/hotdata.h b/include/net/hotdata.h index e6595ed2c3be..a8f7e5e826fb 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -16,6 +16,9 @@ struct net_hotdata { #endif struct list_head offload_base; struct list_head ptype_all; + struct kmem_cache *skbuff_cache; + struct kmem_cache *skbuff_fclone_cache; + struct kmem_cache *skb_small_head_cache; int gro_normal_batch; int netdev_budget; int netdev_budget_usecs; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index ef82ffc90cbe..9ee8da477465 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -326,7 +327,8 @@ static int cpu_map_kthread_run(void *data) /* Support running another XDP prog on this CPU */ nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list); if (nframes) { - m = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, skbs); + m = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + gfp, nframes, skbs); if (unlikely(m == 0)) { for (i = 0; i < nframes; i++) skbs[i] = NULL; /* effect: xdp_return_frame */ diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 5535f9adc658..61efeadaff8d 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -254,7 +255,8 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes, int i, n; LIST_HEAD(list); - n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs); + n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes, + (void **)skbs); if (unlikely(n == 0)) { for (i = 0; i < nframes; i++) xdp_return_frame(frames[i]); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 43d7fc150acc..766219011aea 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -69,6 +69,7 @@ #include #include #include +#include #include #include #include @@ -88,15 +89,10 @@ #include "dev.h" #include "sock_destructor.h" -struct kmem_cache *skbuff_cache __ro_after_init; -static struct kmem_cache *skbuff_fclone_cache __ro_after_init; #ifdef CONFIG_SKB_EXTENSIONS static struct kmem_cache *skbuff_ext_cache __ro_after_init; #endif - -static struct kmem_cache *skb_small_head_cache __ro_after_init; - #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER) /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two. @@ -349,7 +345,7 @@ static struct sk_buff *napi_skb_cache_get(void) struct sk_buff *skb; if (unlikely(!nc->skb_count)) { - nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache, + nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, GFP_ATOMIC, NAPI_SKB_CACHE_BULK, nc->skb_cache); @@ -358,7 +354,7 @@ static struct sk_buff *napi_skb_cache_get(void) } skb = nc->skb_cache[--nc->skb_count]; - kasan_mempool_unpoison_object(skb, kmem_cache_size(skbuff_cache)); + kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache)); return skb; } @@ -416,7 +412,7 @@ struct sk_buff *slab_build_skb(void *data) struct sk_buff *skb; unsigned int size; - skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); + skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC); if (unlikely(!skb)) return NULL; @@ -467,7 +463,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size) { struct sk_buff *skb; - skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); + skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC); if (unlikely(!skb)) return NULL; @@ -578,7 +574,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, obj_size = SKB_HEAD_ALIGN(*size); if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && !(flags & KMALLOC_NOT_NORMAL_BITS)) { - obj = kmem_cache_alloc_node(skb_small_head_cache, + obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags | __GFP_NOMEMALLOC | __GFP_NOWARN, node); *size = SKB_SMALL_HEAD_CACHE_SIZE; @@ -586,7 +582,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, goto out; /* Try again but now we are using pfmemalloc reserves */ ret_pfmemalloc = true; - obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node); + obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node); goto out; } @@ -649,7 +645,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, u8 *data; cache = (flags & SKB_ALLOC_FCLONE) - ? skbuff_fclone_cache : skbuff_cache; + ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache; if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) gfp_mask |= __GFP_MEMALLOC; @@ -1095,7 +1091,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb) static void skb_kfree_head(void *head, unsigned int end_offset) { if (end_offset == SKB_SMALL_HEAD_HEADROOM) - kmem_cache_free(skb_small_head_cache, head); + kmem_cache_free(net_hotdata.skb_small_head_cache, head); else kfree(head); } @@ -1162,7 +1158,7 @@ static void kfree_skbmem(struct sk_buff *skb) switch (skb->fclone) { case SKB_FCLONE_UNAVAILABLE: - kmem_cache_free(skbuff_cache, skb); + kmem_cache_free(net_hotdata.skbuff_cache, skb); return; case SKB_FCLONE_ORIG: @@ -1183,7 +1179,7 @@ static void kfree_skbmem(struct sk_buff *skb) if (!refcount_dec_and_test(&fclones->fclone_ref)) return; fastpath: - kmem_cache_free(skbuff_fclone_cache, fclones); + kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones); } void skb_release_head_state(struct sk_buff *skb) @@ -1280,7 +1276,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb, sa->skb_array[sa->skb_count++] = skb; if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { - kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE, + kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE, sa->skb_array); sa->skb_count = 0; } @@ -1305,7 +1301,7 @@ kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) } if (sa.skb_count) - kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array); + kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array); } EXPORT_SYMBOL(kfree_skb_list_reason); @@ -1467,9 +1463,9 @@ static void napi_skb_cache_put(struct sk_buff *skb) if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) kasan_mempool_unpoison_object(nc->skb_cache[i], - kmem_cache_size(skbuff_cache)); + kmem_cache_size(net_hotdata.skbuff_cache)); - kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF, + kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF, nc->skb_cache + NAPI_SKB_CACHE_HALF); nc->skb_count = NAPI_SKB_CACHE_HALF; } @@ -2066,7 +2062,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) if (skb_pfmemalloc(skb)) gfp_mask |= __GFP_MEMALLOC; - n = kmem_cache_alloc(skbuff_cache, gfp_mask); + n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask); if (!n) return NULL; @@ -5005,7 +5001,7 @@ static void skb_extensions_init(void) {} void __init skb_init(void) { - skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", + net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", sizeof(struct sk_buff), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC| @@ -5013,7 +5009,7 @@ void __init skb_init(void) offsetof(struct sk_buff, cb), sizeof_field(struct sk_buff, cb), NULL); - skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", + net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", sizeof(struct sk_buff_fclones), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, @@ -5022,7 +5018,7 @@ void __init skb_init(void) * struct skb_shared_info is located at the end of skb->head, * and should not be copied to/from user. */ - skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", + net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", SKB_SMALL_HEAD_CACHE_SIZE, 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, @@ -5895,7 +5891,7 @@ void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) { if (head_stolen) { skb_release_head_state(skb); - kmem_cache_free(skbuff_cache, skb); + kmem_cache_free(net_hotdata.skbuff_cache, skb); } else { __kfree_skb(skb); } diff --git a/net/core/xdp.c b/net/core/xdp.c index 0e3709a29175..41693154e426 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -16,6 +16,7 @@ #include #include +#include #include #include /* struct xdp_mem_allocator */ #include @@ -589,7 +590,7 @@ EXPORT_SYMBOL_GPL(xdp_warn); int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp) { - n_skb = kmem_cache_alloc_bulk(skbuff_cache, gfp, n_skb, skbs); + n_skb = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, n_skb, skbs); if (unlikely(!n_skb)) return -ENOMEM; @@ -658,7 +659,7 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, { struct sk_buff *skb; - skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); + skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC); if (unlikely(!skb)) return NULL; -- cgit v1.2.3 From 490a79faf95e705ba0ffd9ebf04a624b379e53c9 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 6 Mar 2024 16:00:30 +0000 Subject: net: introduce include/net/rps.h Move RPS related structures and helpers from include/linux/netdevice.h and include/net/sock.h to a new include file. Signed-off-by: Eric Dumazet Acked-by: Soheil Hassas Yeganeh Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240306160031.874438-18-edumazet@google.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/ice/ice_arfs.c | 1 + drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 1 + drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c | 1 + drivers/net/ethernet/sfc/rx_common.c | 1 + drivers/net/ethernet/sfc/siena/rx_common.c | 1 + drivers/net/tun.c | 1 + include/linux/netdevice.h | 82 -------------- include/net/rps.h | 127 ++++++++++++++++++++++ include/net/sock.h | 35 ------ net/core/dev.c | 1 + net/core/net-sysfs.c | 1 + net/core/sysctl_net_core.c | 1 + net/ipv4/af_inet.c | 1 + net/ipv4/tcp.c | 1 + net/ipv6/af_inet6.c | 1 + net/sctp/socket.c | 1 + 16 files changed, 140 insertions(+), 117 deletions(-) create mode 100644 include/net/rps.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.c b/drivers/net/ethernet/intel/ice/ice_arfs.c index cca0e753f38f..7cee365cc7d1 100644 --- a/drivers/net/ethernet/intel/ice/ice_arfs.c +++ b/drivers/net/ethernet/intel/ice/ice_arfs.c @@ -2,6 +2,7 @@ /* Copyright (C) 2018-2020, Intel Corporation. */ #include "ice.h" +#include /** * ice_is_arfs_active - helper to check is aRFS is active diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index d7da62cda821..5d3fde63b273 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index e66f486faafe..c7f542d0b8f0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "en.h" #define ARFS_HASH_SHIFT BITS_PER_BYTE diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index fac227d372db..dcd901eccfc8 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -11,6 +11,7 @@ #include "net_driver.h" #include #include +#include #include "efx.h" #include "nic.h" #include "rx_common.h" diff --git a/drivers/net/ethernet/sfc/siena/rx_common.c b/drivers/net/ethernet/sfc/siena/rx_common.c index 4579f43484c3..219fb358a646 100644 --- a/drivers/net/ethernet/sfc/siena/rx_common.c +++ b/drivers/net/ethernet/sfc/siena/rx_common.c @@ -11,6 +11,7 @@ #include "net_driver.h" #include #include +#include #include "efx.h" #include "nic.h" #include "rx_common.h" diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 8d258e263f54..0b3f21cba552 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -78,6 +78,7 @@ #include #include #include +#include #include #include diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index dd641297e807..416a800d72ba 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -225,12 +225,6 @@ struct net_device_core_stats { #include #include -#ifdef CONFIG_RPS -#include -extern struct static_key_false rps_needed; -extern struct static_key_false rfs_needed; -#endif - struct neighbour; struct neigh_parms; struct sk_buff; @@ -730,86 +724,10 @@ static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node #endif } -#ifdef CONFIG_RPS -/* - * This structure holds an RPS map which can be of variable length. The - * map is an array of CPUs. - */ -struct rps_map { - unsigned int len; - struct rcu_head rcu; - u16 cpus[]; -}; -#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16))) - -/* - * The rps_dev_flow structure contains the mapping of a flow to a CPU, the - * tail pointer for that CPU's input queue at the time of last enqueue, and - * a hardware filter index. - */ -struct rps_dev_flow { - u16 cpu; - u16 filter; - unsigned int last_qtail; -}; -#define RPS_NO_FILTER 0xffff - -/* - * The rps_dev_flow_table structure contains a table of flow mappings. - */ -struct rps_dev_flow_table { - unsigned int mask; - struct rcu_head rcu; - struct rps_dev_flow flows[]; -}; -#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \ - ((_num) * sizeof(struct rps_dev_flow))) - -/* - * The rps_sock_flow_table contains mappings of flows to the last CPU - * on which they were processed by the application (set in recvmsg). - * Each entry is a 32bit value. Upper part is the high-order bits - * of flow hash, lower part is CPU number. - * rps_cpu_mask is used to partition the space, depending on number of - * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1 - * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f, - * meaning we use 32-6=26 bits for the hash. - */ -struct rps_sock_flow_table { - u32 mask; - - u32 ents[] ____cacheline_aligned_in_smp; -}; -#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num])) - -#define RPS_NO_CPU 0xffff - -extern u32 rps_cpu_mask; -extern struct rps_sock_flow_table __rcu *rps_sock_flow_table; - -static inline void rps_record_sock_flow(struct rps_sock_flow_table *table, - u32 hash) -{ - if (table && hash) { - unsigned int index = hash & table->mask; - u32 val = hash & ~rps_cpu_mask; - - /* We only give a hint, preemption can change CPU under us */ - val |= raw_smp_processor_id(); - - /* The following WRITE_ONCE() is paired with the READ_ONCE() - * here, and another one in get_rps_cpu(). - */ - if (READ_ONCE(table->ents[index]) != val) - WRITE_ONCE(table->ents[index], val); - } -} - #ifdef CONFIG_RFS_ACCEL bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id, u16 filter_id); #endif -#endif /* CONFIG_RPS */ /* XPS map type and offset of the xps map within net_device->xps_maps[]. */ enum xps_map_type { diff --git a/include/net/rps.h b/include/net/rps.h new file mode 100644 index 000000000000..6081d817d245 --- /dev/null +++ b/include/net/rps.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _NET_RPS_H +#define _NET_RPS_H + +#include +#include +#include + +#ifdef CONFIG_RPS + +extern struct static_key_false rps_needed; +extern struct static_key_false rfs_needed; + +/* + * This structure holds an RPS map which can be of variable length. The + * map is an array of CPUs. + */ +struct rps_map { + unsigned int len; + struct rcu_head rcu; + u16 cpus[]; +}; +#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16))) + +/* + * The rps_dev_flow structure contains the mapping of a flow to a CPU, the + * tail pointer for that CPU's input queue at the time of last enqueue, and + * a hardware filter index. + */ +struct rps_dev_flow { + u16 cpu; + u16 filter; + unsigned int last_qtail; +}; +#define RPS_NO_FILTER 0xffff + +/* + * The rps_dev_flow_table structure contains a table of flow mappings. + */ +struct rps_dev_flow_table { + unsigned int mask; + struct rcu_head rcu; + struct rps_dev_flow flows[]; +}; +#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \ + ((_num) * sizeof(struct rps_dev_flow))) + +/* + * The rps_sock_flow_table contains mappings of flows to the last CPU + * on which they were processed by the application (set in recvmsg). + * Each entry is a 32bit value. Upper part is the high-order bits + * of flow hash, lower part is CPU number. + * rps_cpu_mask is used to partition the space, depending on number of + * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1 + * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f, + * meaning we use 32-6=26 bits for the hash. + */ +struct rps_sock_flow_table { + u32 mask; + + u32 ents[] ____cacheline_aligned_in_smp; +}; +#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num])) + +#define RPS_NO_CPU 0xffff + +extern u32 rps_cpu_mask; +extern struct rps_sock_flow_table __rcu *rps_sock_flow_table; + +static inline void rps_record_sock_flow(struct rps_sock_flow_table *table, + u32 hash) +{ + unsigned int index = hash & table->mask; + u32 val = hash & ~rps_cpu_mask; + + /* We only give a hint, preemption can change CPU under us */ + val |= raw_smp_processor_id(); + + /* The following WRITE_ONCE() is paired with the READ_ONCE() + * here, and another one in get_rps_cpu(). + */ + if (READ_ONCE(table->ents[index]) != val) + WRITE_ONCE(table->ents[index], val); +} + +#endif /* CONFIG_RPS */ + +static inline void sock_rps_record_flow_hash(__u32 hash) +{ +#ifdef CONFIG_RPS + struct rps_sock_flow_table *sock_flow_table; + + if (!hash) + return; + rcu_read_lock(); + sock_flow_table = rcu_dereference(rps_sock_flow_table); + if (sock_flow_table) + rps_record_sock_flow(sock_flow_table, hash); + rcu_read_unlock(); +#endif +} + +static inline void sock_rps_record_flow(const struct sock *sk) +{ +#ifdef CONFIG_RPS + if (static_branch_unlikely(&rfs_needed)) { + /* Reading sk->sk_rxhash might incur an expensive cache line + * miss. + * + * TCP_ESTABLISHED does cover almost all states where RFS + * might be useful, and is cheaper [1] than testing : + * IPv4: inet_sk(sk)->inet_daddr + * IPv6: ipv6_addr_any(&sk->sk_v6_daddr) + * OR an additional socket flag + * [1] : sk_state and sk_prot are in the same cache line. + */ + if (sk->sk_state == TCP_ESTABLISHED) { + /* This READ_ONCE() is paired with the WRITE_ONCE() + * from sock_rps_save_rxhash() and sock_rps_reset_rxhash(). + */ + sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash)); + } + } +#endif +} + +#endif /* _NET_RPS_H */ diff --git a/include/net/sock.h b/include/net/sock.h index 09a0cde8bf52..b5e00702acc1 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1117,41 +1117,6 @@ static inline void sk_incoming_cpu_update(struct sock *sk) WRITE_ONCE(sk->sk_incoming_cpu, cpu); } -static inline void sock_rps_record_flow_hash(__u32 hash) -{ -#ifdef CONFIG_RPS - struct rps_sock_flow_table *sock_flow_table; - - rcu_read_lock(); - sock_flow_table = rcu_dereference(rps_sock_flow_table); - rps_record_sock_flow(sock_flow_table, hash); - rcu_read_unlock(); -#endif -} - -static inline void sock_rps_record_flow(const struct sock *sk) -{ -#ifdef CONFIG_RPS - if (static_branch_unlikely(&rfs_needed)) { - /* Reading sk->sk_rxhash might incur an expensive cache line - * miss. - * - * TCP_ESTABLISHED does cover almost all states where RFS - * might be useful, and is cheaper [1] than testing : - * IPv4: inet_sk(sk)->inet_daddr - * IPv6: ipv6_addr_any(&sk->sk_v6_daddr) - * OR an additional socket flag - * [1] : sk_state and sk_prot are in the same cache line. - */ - if (sk->sk_state == TCP_ESTABLISHED) { - /* This READ_ONCE() is paired with the WRITE_ONCE() - * from sock_rps_save_rxhash() and sock_rps_reset_rxhash(). - */ - sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash)); - } - } -#endif -} static inline void sock_rps_save_rxhash(struct sock *sk, const struct sk_buff *skb) diff --git a/net/core/dev.c b/net/core/dev.c index 40ba02e04bcb..bcf49b0393d2 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -155,6 +155,7 @@ #include #include #include +#include #include "dev.h" #include "net-sysfs.h" diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index af238026ac3c..5560083774b1 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "dev.h" #include "net-sysfs.h" diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 8a4c698dad9c..4b93e27404e8 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "dev.h" diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 6f1cfd176e7b..55bd72997b31 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -119,6 +119,7 @@ #endif #include #include +#include #include diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 7e1b848398d0..c5b83875411a 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -279,6 +279,7 @@ #include #include #include +#include /* Track pending CMSGs. */ enum { diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index b90d46533cdc..8041dc181bd4 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 6b9fcdb0952a..c67679a41044 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -67,6 +67,7 @@ #include #include #include +#include /* Forward declarations for internal helper functions. */ static bool sctp_writeable(const struct sock *sk); -- cgit v1.2.3 From ab63a2387cb906d43b72a8effb611bbaecb2d0cd Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 6 Mar 2024 11:55:07 -0800 Subject: netdev: add per-queue statistics The ethtool-nl family does a good job exposing various protocol related and IEEE/IETF statistics which used to get dumped under ethtool -S, with creative names. Queue stats don't have a netlink API, yet, and remain a lion's share of ethtool -S output for new drivers. Not only is that bad because the names differ driver to driver but it's also bug-prone. Intuitively drivers try to report only the stats for active queues, but querying ethtool stats involves multiple system calls, and the number of stats is read separately from the stats themselves. Worse still when user space asks for values of the stats, it doesn't inform the kernel how big the buffer is. If number of stats increases in the meantime kernel will overflow user buffer. Add a netlink API for dumping queue stats. Queue information is exposed via the netdev-genl family, so add the stats there. Support per-queue and sum-for-device dumps. Latter will be useful when subsequent patches add more interesting common stats than just bytes and packets. The API does not currently distinguish between HW and SW stats. The expectation is that the source of the stats will either not matter much (good packets) or be obvious (skb alloc errors). Acked-by: Stanislav Fomichev Reviewed-by: Amritha Nambiar Reviewed-by: Xuan Zhuo Link: https://lore.kernel.org/r/20240306195509.1502746-2-kuba@kernel.org Signed-off-by: Jakub Kicinski --- Documentation/netlink/specs/netdev.yaml | 84 +++++++++++++ Documentation/networking/statistics.rst | 15 +++ include/linux/netdevice.h | 3 + include/net/netdev_queues.h | 54 ++++++++ include/uapi/linux/netdev.h | 19 +++ net/core/netdev-genl-gen.c | 12 ++ net/core/netdev-genl-gen.h | 2 + net/core/netdev-genl.c | 213 ++++++++++++++++++++++++++++++++ tools/include/uapi/linux/netdev.h | 19 +++ 9 files changed, 421 insertions(+) (limited to 'include/linux') diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index 3addac970680..a1e48c3c84c9 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -74,6 +74,10 @@ definitions: name: queue-type type: enum entries: [ rx, tx ] + - + name: qstats-scope + type: flags + entries: [ queue ] attribute-sets: - @@ -265,6 +269,66 @@ attribute-sets: doc: ID of the NAPI instance which services this queue. type: u32 + - + name: qstats + doc: | + Get device statistics, scoped to a device or a queue. + These statistics extend (and partially duplicate) statistics available + in struct rtnl_link_stats64. + Value of the `scope` attribute determines how statistics are + aggregated. When aggregated for the entire device the statistics + represent the total number of events since last explicit reset of + the device (i.e. not a reconfiguration like changing queue count). + When reported per-queue, however, the statistics may not add + up to the total number of events, will only be reported for currently + active objects, and will likely report the number of events since last + reconfiguration. + attributes: + - + name: ifindex + doc: ifindex of the netdevice to which stats belong. + type: u32 + checks: + min: 1 + - + name: queue-type + doc: Queue type as rx, tx, for queue-id. + type: u32 + enum: queue-type + - + name: queue-id + doc: Queue ID, if stats are scoped to a single queue instance. + type: u32 + - + name: scope + doc: | + What object type should be used to iterate over the stats. + type: uint + enum: qstats-scope + - + name: rx-packets + doc: | + Number of wire packets successfully received and passed to the stack. + For drivers supporting XDP, XDP is considered the first layer + of the stack, so packets consumed by XDP are still counted here. + type: uint + value: 8 # reserve some attr ids in case we need more metadata later + - + name: rx-bytes + doc: Successfully received bytes, see `rx-packets`. + type: uint + - + name: tx-packets + doc: | + Number of wire packets successfully sent. Packet is considered to be + successfully sent once it is in device memory (usually this means + the device has issued a DMA completion for the packet). + type: uint + - + name: tx-bytes + doc: Successfully sent bytes, see `tx-packets`. + type: uint + operations: list: - @@ -405,6 +469,26 @@ operations: attributes: - ifindex reply: *napi-get-op + - + name: qstats-get + doc: | + Get / dump fine grained statistics. Which statistics are reported + depends on the device and the driver, and whether the driver stores + software counters per-queue. + attribute-set: qstats + dump: + request: + attributes: + - scope + reply: + attributes: + - ifindex + - queue-type + - queue-id + - rx-packets + - rx-bytes + - tx-packets + - tx-bytes mcast-groups: list: diff --git a/Documentation/networking/statistics.rst b/Documentation/networking/statistics.rst index 551b3cc29a41..75e017dfa825 100644 --- a/Documentation/networking/statistics.rst +++ b/Documentation/networking/statistics.rst @@ -41,6 +41,15 @@ If `-s` is specified once the detailed errors won't be shown. `ip` supports JSON formatting via the `-j` option. +Queue statistics +~~~~~~~~~~~~~~~~ + +Queue statistics are accessible via the netdev netlink family. + +Currently no widely distributed CLI exists to access those statistics. +Kernel development tools (ynl) can be used to experiment with them, +see `Documentation/userspace-api/netlink/intro-specs.rst`. + Protocol-specific statistics ---------------------------- @@ -147,6 +156,12 @@ Statistics are reported both in the responses to link information requests (`RTM_GETLINK`) and statistic requests (`RTM_GETSTATS`, when `IFLA_STATS_LINK_64` bit is set in the `.filter_mask` of the request). +netdev (netlink) +~~~~~~~~~~~~~~~~ + +`netdev` generic netlink family allows accessing page pool and per queue +statistics. + ethtool ------- diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 416a800d72ba..4230c7f3b959 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1955,6 +1955,7 @@ enum netdev_reg_state { * * @sysfs_rx_queue_group: Space for optional per-rx queue attributes * @rtnl_link_ops: Rtnl_link_ops + * @stat_ops: Optional ops for queue-aware statistics * * @gso_max_size: Maximum size of generic segmentation offload * @tso_max_size: Device (as in HW) limit on the max TSO request size @@ -2335,6 +2336,8 @@ struct net_device { const struct rtnl_link_ops *rtnl_link_ops; + const struct netdev_stat_ops *stat_ops; + /* for setting kernel sock attribute on TCP connection setup */ #define GSO_MAX_SEGS 65535u #define GSO_LEGACY_MAX_SIZE 65536u diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h index 8b8ed4e13d74..d633347eeda5 100644 --- a/include/net/netdev_queues.h +++ b/include/net/netdev_queues.h @@ -4,6 +4,60 @@ #include +struct netdev_queue_stats_rx { + u64 bytes; + u64 packets; +}; + +struct netdev_queue_stats_tx { + u64 bytes; + u64 packets; +}; + +/** + * struct netdev_stat_ops - netdev ops for fine grained stats + * @get_queue_stats_rx: get stats for a given Rx queue + * @get_queue_stats_tx: get stats for a given Tx queue + * @get_base_stats: get base stats (not belonging to any live instance) + * + * Query stats for a given object. The values of the statistics are undefined + * on entry (specifically they are *not* zero-initialized). Drivers should + * assign values only to the statistics they collect. Statistics which are not + * collected must be left undefined. + * + * Queue objects are not necessarily persistent, and only currently active + * queues are queried by the per-queue callbacks. This means that per-queue + * statistics will not generally add up to the total number of events for + * the device. The @get_base_stats callback allows filling in the delta + * between events for currently live queues and overall device history. + * When the statistics for the entire device are queried, first @get_base_stats + * is issued to collect the delta, and then a series of per-queue callbacks. + * Only statistics which are set in @get_base_stats will be reported + * at the device level, meaning that unlike in queue callbacks, setting + * a statistic to zero in @get_base_stats is a legitimate thing to do. + * This is because @get_base_stats has a second function of designating which + * statistics are in fact correct for the entire device (e.g. when history + * for some of the events is not maintained, and reliable "total" cannot + * be provided). + * + * Device drivers can assume that when collecting total device stats, + * the @get_base_stats and subsequent per-queue calls are performed + * "atomically" (without releasing the rtnl_lock). + * + * Device drivers are encouraged to reset the per-queue statistics when + * number of queues change. This is because the primary use case for + * per-queue statistics is currently to detect traffic imbalance. + */ +struct netdev_stat_ops { + void (*get_queue_stats_rx)(struct net_device *dev, int idx, + struct netdev_queue_stats_rx *stats); + void (*get_queue_stats_tx)(struct net_device *dev, int idx, + struct netdev_queue_stats_tx *stats); + void (*get_base_stats)(struct net_device *dev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx); +}; + /** * DOC: Lockless queue stopping / waking helpers. * diff --git a/include/uapi/linux/netdev.h b/include/uapi/linux/netdev.h index 93cb411adf72..639ffa04c172 100644 --- a/include/uapi/linux/netdev.h +++ b/include/uapi/linux/netdev.h @@ -70,6 +70,10 @@ enum netdev_queue_type { NETDEV_QUEUE_TYPE_TX, }; +enum netdev_qstats_scope { + NETDEV_QSTATS_SCOPE_QUEUE = 1, +}; + enum { NETDEV_A_DEV_IFINDEX = 1, NETDEV_A_DEV_PAD, @@ -132,6 +136,20 @@ enum { NETDEV_A_QUEUE_MAX = (__NETDEV_A_QUEUE_MAX - 1) }; +enum { + NETDEV_A_QSTATS_IFINDEX = 1, + NETDEV_A_QSTATS_QUEUE_TYPE, + NETDEV_A_QSTATS_QUEUE_ID, + NETDEV_A_QSTATS_SCOPE, + NETDEV_A_QSTATS_RX_PACKETS = 8, + NETDEV_A_QSTATS_RX_BYTES, + NETDEV_A_QSTATS_TX_PACKETS, + NETDEV_A_QSTATS_TX_BYTES, + + __NETDEV_A_QSTATS_MAX, + NETDEV_A_QSTATS_MAX = (__NETDEV_A_QSTATS_MAX - 1) +}; + enum { NETDEV_CMD_DEV_GET = 1, NETDEV_CMD_DEV_ADD_NTF, @@ -144,6 +162,7 @@ enum { NETDEV_CMD_PAGE_POOL_STATS_GET, NETDEV_CMD_QUEUE_GET, NETDEV_CMD_NAPI_GET, + NETDEV_CMD_QSTATS_GET, __NETDEV_CMD_MAX, NETDEV_CMD_MAX = (__NETDEV_CMD_MAX - 1) diff --git a/net/core/netdev-genl-gen.c b/net/core/netdev-genl-gen.c index be7f2ebd61b2..8d8ace9ef87f 100644 --- a/net/core/netdev-genl-gen.c +++ b/net/core/netdev-genl-gen.c @@ -68,6 +68,11 @@ static const struct nla_policy netdev_napi_get_dump_nl_policy[NETDEV_A_NAPI_IFIN [NETDEV_A_NAPI_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1), }; +/* NETDEV_CMD_QSTATS_GET - dump */ +static const struct nla_policy netdev_qstats_get_nl_policy[NETDEV_A_QSTATS_SCOPE + 1] = { + [NETDEV_A_QSTATS_SCOPE] = NLA_POLICY_MASK(NLA_UINT, 0x1), +}; + /* Ops table for netdev */ static const struct genl_split_ops netdev_nl_ops[] = { { @@ -138,6 +143,13 @@ static const struct genl_split_ops netdev_nl_ops[] = { .maxattr = NETDEV_A_NAPI_IFINDEX, .flags = GENL_CMD_CAP_DUMP, }, + { + .cmd = NETDEV_CMD_QSTATS_GET, + .dumpit = netdev_nl_qstats_get_dumpit, + .policy = netdev_qstats_get_nl_policy, + .maxattr = NETDEV_A_QSTATS_SCOPE, + .flags = GENL_CMD_CAP_DUMP, + }, }; static const struct genl_multicast_group netdev_nl_mcgrps[] = { diff --git a/net/core/netdev-genl-gen.h b/net/core/netdev-genl-gen.h index a47f2bcbe4fa..4db40fd5b4a9 100644 --- a/net/core/netdev-genl-gen.h +++ b/net/core/netdev-genl-gen.h @@ -28,6 +28,8 @@ int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb); int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info); int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb); +int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); enum { NETDEV_NLGRP_MGMT, diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index 918b109e0cf4..7fa75e13dc6d 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "netdev-genl-gen.h" @@ -460,6 +461,218 @@ int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) return err; } +#define NETDEV_STAT_NOT_SET (~0ULL) + +static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) +{ + const u64 *add = _add; + u64 *sum = _sum; + + while (size) { + if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) + *sum += *add; + sum++; + add++; + size -= 8; + } +} + +static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) +{ + if (value == NETDEV_STAT_NOT_SET) + return 0; + return nla_put_uint(rsp, attr_id, value); +} + +static int +netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) +{ + if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || + netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes)) + return -EMSGSIZE; + return 0; +} + +static int +netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) +{ + if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || + netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes)) + return -EMSGSIZE; + return 0; +} + +static int +netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, + u32 q_type, int i, const struct genl_info *info) +{ + const struct netdev_stat_ops *ops = netdev->stat_ops; + struct netdev_queue_stats_rx rx; + struct netdev_queue_stats_tx tx; + void *hdr; + + hdr = genlmsg_iput(rsp, info); + if (!hdr) + return -EMSGSIZE; + if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || + nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || + nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) + goto nla_put_failure; + + switch (q_type) { + case NETDEV_QUEUE_TYPE_RX: + memset(&rx, 0xff, sizeof(rx)); + ops->get_queue_stats_rx(netdev, i, &rx); + if (!memchr_inv(&rx, 0xff, sizeof(rx))) + goto nla_cancel; + if (netdev_nl_stats_write_rx(rsp, &rx)) + goto nla_put_failure; + break; + case NETDEV_QUEUE_TYPE_TX: + memset(&tx, 0xff, sizeof(tx)); + ops->get_queue_stats_tx(netdev, i, &tx); + if (!memchr_inv(&tx, 0xff, sizeof(tx))) + goto nla_cancel; + if (netdev_nl_stats_write_tx(rsp, &tx)) + goto nla_put_failure; + break; + } + + genlmsg_end(rsp, hdr); + return 0; + +nla_cancel: + genlmsg_cancel(rsp, hdr); + return 0; +nla_put_failure: + genlmsg_cancel(rsp, hdr); + return -EMSGSIZE; +} + +static int +netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, + const struct genl_info *info, + struct netdev_nl_dump_ctx *ctx) +{ + const struct netdev_stat_ops *ops = netdev->stat_ops; + int i, err; + + if (!(netdev->flags & IFF_UP)) + return 0; + + i = ctx->rxq_idx; + while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { + err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, + i, info); + if (err) + return err; + ctx->rxq_idx = i++; + } + i = ctx->txq_idx; + while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { + err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, + i, info); + if (err) + return err; + ctx->txq_idx = i++; + } + + ctx->rxq_idx = 0; + ctx->txq_idx = 0; + return 0; +} + +static int +netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, + const struct genl_info *info) +{ + struct netdev_queue_stats_rx rx_sum, rx; + struct netdev_queue_stats_tx tx_sum, tx; + const struct netdev_stat_ops *ops; + void *hdr; + int i; + + ops = netdev->stat_ops; + /* Netdev can't guarantee any complete counters */ + if (!ops->get_base_stats) + return 0; + + memset(&rx_sum, 0xff, sizeof(rx_sum)); + memset(&tx_sum, 0xff, sizeof(tx_sum)); + + ops->get_base_stats(netdev, &rx_sum, &tx_sum); + + /* The op was there, but nothing reported, don't bother */ + if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && + !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) + return 0; + + hdr = genlmsg_iput(rsp, info); + if (!hdr) + return -EMSGSIZE; + if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) + goto nla_put_failure; + + for (i = 0; i < netdev->real_num_rx_queues; i++) { + memset(&rx, 0xff, sizeof(rx)); + if (ops->get_queue_stats_rx) + ops->get_queue_stats_rx(netdev, i, &rx); + netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx)); + } + for (i = 0; i < netdev->real_num_tx_queues; i++) { + memset(&tx, 0xff, sizeof(tx)); + if (ops->get_queue_stats_tx) + ops->get_queue_stats_tx(netdev, i, &tx); + netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx)); + } + + if (netdev_nl_stats_write_rx(rsp, &rx_sum) || + netdev_nl_stats_write_tx(rsp, &tx_sum)) + goto nla_put_failure; + + genlmsg_end(rsp, hdr); + return 0; + +nla_put_failure: + genlmsg_cancel(rsp, hdr); + return -EMSGSIZE; +} + +int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); + const struct genl_info *info = genl_info_dump(cb); + struct net *net = sock_net(skb->sk); + struct net_device *netdev; + unsigned int scope; + int err = 0; + + scope = 0; + if (info->attrs[NETDEV_A_QSTATS_SCOPE]) + scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); + + rtnl_lock(); + for_each_netdev_dump(net, netdev, ctx->ifindex) { + if (!netdev->stat_ops) + continue; + + switch (scope) { + case 0: + err = netdev_nl_stats_by_netdev(netdev, skb, info); + break; + case NETDEV_QSTATS_SCOPE_QUEUE: + err = netdev_nl_stats_by_queue(netdev, skb, info, ctx); + break; + } + if (err < 0) + break; + } + rtnl_unlock(); + + return err; +} + static int netdev_genl_netdevice_event(struct notifier_block *nb, unsigned long event, void *ptr) { diff --git a/tools/include/uapi/linux/netdev.h b/tools/include/uapi/linux/netdev.h index 93cb411adf72..639ffa04c172 100644 --- a/tools/include/uapi/linux/netdev.h +++ b/tools/include/uapi/linux/netdev.h @@ -70,6 +70,10 @@ enum netdev_queue_type { NETDEV_QUEUE_TYPE_TX, }; +enum netdev_qstats_scope { + NETDEV_QSTATS_SCOPE_QUEUE = 1, +}; + enum { NETDEV_A_DEV_IFINDEX = 1, NETDEV_A_DEV_PAD, @@ -132,6 +136,20 @@ enum { NETDEV_A_QUEUE_MAX = (__NETDEV_A_QUEUE_MAX - 1) }; +enum { + NETDEV_A_QSTATS_IFINDEX = 1, + NETDEV_A_QSTATS_QUEUE_TYPE, + NETDEV_A_QSTATS_QUEUE_ID, + NETDEV_A_QSTATS_SCOPE, + NETDEV_A_QSTATS_RX_PACKETS = 8, + NETDEV_A_QSTATS_RX_BYTES, + NETDEV_A_QSTATS_TX_PACKETS, + NETDEV_A_QSTATS_TX_BYTES, + + __NETDEV_A_QSTATS_MAX, + NETDEV_A_QSTATS_MAX = (__NETDEV_A_QSTATS_MAX - 1) +}; + enum { NETDEV_CMD_DEV_GET = 1, NETDEV_CMD_DEV_ADD_NTF, @@ -144,6 +162,7 @@ enum { NETDEV_CMD_PAGE_POOL_STATS_GET, NETDEV_CMD_QUEUE_GET, NETDEV_CMD_NAPI_GET, + NETDEV_CMD_QSTATS_GET, __NETDEV_CMD_MAX, NETDEV_CMD_MAX = (__NETDEV_CMD_MAX - 1) -- cgit v1.2.3 From 6025b9135f7a8b46826a5fcf947259da43bac281 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 4 Mar 2024 06:08:47 -0800 Subject: net: dqs: add NIC stall detector based on BQL softnet_data->time_squeeze is sometimes used as a proxy for host overload or indication of scheduling problems. In practice this statistic is very noisy and has hard to grasp units - e.g. is 10 squeezes a second to be expected, or high? Delaying network (NAPI) processing leads to drops on NIC queues but also RTT bloat, impacting pacing and CA decisions. Stalls are a little hard to detect on the Rx side, because there may simply have not been any packets received in given period of time. Packet timestamps help a little bit, but again we don't know if packets are stale because we're not keeping up or because someone (*cough* cgroups) disabled IRQs for a long time. We can, however, use Tx as a proxy for Rx stalls. Most drivers use combined Rx+Tx NAPIs so if Tx gets starved so will Rx. On the Tx side we know exactly when packets get queued, and completed, so there is no uncertainty. This patch adds stall checks to BQL. Why BQL? Because it's a convenient place to add such checks, already called by most drivers, and it has copious free space in its structures (this patch adds no extra cache references or dirtying to the fast path). The algorithm takes one parameter - max delay AKA stall threshold and increments a counter whenever NAPI got delayed for at least that amount of time. It also records the length of the longest stall. To be precise every time NAPI has not polled for at least stall thrs we check if there were any Tx packets queued between last NAPI run and now - stall_thrs/2. Unlike the classic Tx watchdog this mechanism does not ignore stalls caused by Tx being disabled, or loss of link. I don't think the check is worth the complexity, and stall is a stall, whether due to host overload, flow control, link down... doesn't matter much to the application. We have been running this detector in production at Meta for 2 years, with the threshold of 8ms. It's the lowest value where false positives become rare. There's still a constant stream of reported stalls (especially without the ksoftirqd deferral patches reverted), those who like their stall metrics to be 0 may prefer higher value. Signed-off-by: Jakub Kicinski Signed-off-by: Breno Leitao Signed-off-by: David S. Miller --- Documentation/ABI/testing/sysfs-class-net-queues | 23 ++++++++ include/linux/dynamic_queue_limits.h | 45 ++++++++++++++ include/trace/events/napi.h | 33 +++++++++++ lib/dynamic_queue_limits.c | 74 ++++++++++++++++++++++++ net/core/net-sysfs.c | 62 ++++++++++++++++++++ 5 files changed, 237 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-net-queues b/Documentation/ABI/testing/sysfs-class-net-queues index 5bff64d256c2..84aa25e0d14d 100644 --- a/Documentation/ABI/testing/sysfs-class-net-queues +++ b/Documentation/ABI/testing/sysfs-class-net-queues @@ -96,3 +96,26 @@ Description: Indicates the absolute minimum limit of bytes allowed to be queued on this network device transmit queue. Default value is 0. + +What: /sys/class/net//queues/tx-/byte_queue_limits/stall_thrs +Date: Jan 2024 +KernelVersion: 6.9 +Contact: netdev@vger.kernel.org +Description: + Tx completion stall detection threshold in ms. Kernel will + guarantee to detect all stalls longer than this threshold but + may also detect stalls longer than half of the threshold. + +What: /sys/class/net//queues/tx-/byte_queue_limits/stall_cnt +Date: Jan 2024 +KernelVersion: 6.9 +Contact: netdev@vger.kernel.org +Description: + Number of detected Tx completion stalls. + +What: /sys/class/net//queues/tx-/byte_queue_limits/stall_max +Date: Jan 2024 +KernelVersion: 6.9 +Contact: netdev@vger.kernel.org +Description: + Longest detected Tx completion stall. Write 0 to clear. diff --git a/include/linux/dynamic_queue_limits.h b/include/linux/dynamic_queue_limits.h index 407c2f281b64..5693a4be0d9a 100644 --- a/include/linux/dynamic_queue_limits.h +++ b/include/linux/dynamic_queue_limits.h @@ -38,14 +38,22 @@ #ifdef __KERNEL__ +#include #include +#define DQL_HIST_LEN 4 +#define DQL_HIST_ENT(dql, idx) ((dql)->history[(idx) % DQL_HIST_LEN]) + struct dql { /* Fields accessed in enqueue path (dql_queued) */ unsigned int num_queued; /* Total ever queued */ unsigned int adj_limit; /* limit + num_completed */ unsigned int last_obj_cnt; /* Count at last queuing */ + unsigned long history_head; /* top 58 bits of jiffies */ + /* stall entries, a bit per entry */ + unsigned long history[DQL_HIST_LEN]; + /* Fields accessed only by completion path (dql_completed) */ unsigned int limit ____cacheline_aligned_in_smp; /* Current limit */ @@ -62,6 +70,13 @@ struct dql { unsigned int max_limit; /* Max limit */ unsigned int min_limit; /* Minimum limit */ unsigned int slack_hold_time; /* Time to measure slack */ + + /* Stall threshold (in jiffies), defined by user */ + unsigned short stall_thrs; + /* Longest stall detected, reported to user */ + unsigned short stall_max; + unsigned long last_reap; /* Last reap (in jiffies) */ + unsigned long stall_cnt; /* Number of stalls */ }; /* Set some static maximums */ @@ -74,6 +89,8 @@ struct dql { */ static inline void dql_queued(struct dql *dql, unsigned int count) { + unsigned long map, now, now_hi, i; + BUG_ON(count > DQL_MAX_OBJECT); dql->last_obj_cnt = count; @@ -86,6 +103,34 @@ static inline void dql_queued(struct dql *dql, unsigned int count) barrier(); dql->num_queued += count; + + now = jiffies; + now_hi = now / BITS_PER_LONG; + + /* The following code set a bit in the ring buffer, where each + * bit trackes time the packet was queued. The dql->history buffer + * tracks DQL_HIST_LEN * BITS_PER_LONG time (jiffies) slot + */ + if (unlikely(now_hi != dql->history_head)) { + /* About to reuse slots, clear them */ + for (i = 0; i < DQL_HIST_LEN; i++) { + /* Multiplication masks high bits */ + if (now_hi * BITS_PER_LONG == + (dql->history_head + i) * BITS_PER_LONG) + break; + DQL_HIST_ENT(dql, dql->history_head + i + 1) = 0; + } + /* pairs with smp_rmb() in dql_check_stall() */ + smp_wmb(); + WRITE_ONCE(dql->history_head, now_hi); + } + + /* __set_bit() does not guarantee WRITE_ONCE() semantics */ + map = DQL_HIST_ENT(dql, now_hi); + + /* Populate the history with an entry (bit) per queued */ + if (!(map & BIT_MASK(now))) + WRITE_ONCE(DQL_HIST_ENT(dql, now_hi), map | BIT_MASK(now)); } /* Returns how many objects can be queued, < 0 indicates over limit. */ diff --git a/include/trace/events/napi.h b/include/trace/events/napi.h index 6678cf8b235b..dc03cf8e0369 100644 --- a/include/trace/events/napi.h +++ b/include/trace/events/napi.h @@ -36,6 +36,39 @@ TRACE_EVENT(napi_poll, __entry->work, __entry->budget) ); +TRACE_EVENT(dql_stall_detected, + + TP_PROTO(unsigned short thrs, unsigned int len, + unsigned long last_reap, unsigned long hist_head, + unsigned long now, unsigned long *hist), + + TP_ARGS(thrs, len, last_reap, hist_head, now, hist), + + TP_STRUCT__entry( + __field( unsigned short, thrs) + __field( unsigned int, len) + __field( unsigned long, last_reap) + __field( unsigned long, hist_head) + __field( unsigned long, now) + __array( unsigned long, hist, 4) + ), + + TP_fast_assign( + __entry->thrs = thrs; + __entry->len = len; + __entry->last_reap = last_reap; + __entry->hist_head = hist_head * BITS_PER_LONG; + __entry->now = now; + memcpy(__entry->hist, hist, sizeof(entry->hist)); + ), + + TP_printk("thrs %u len %u last_reap %lu hist_head %lu now %lu hist %016lx %016lx %016lx %016lx", + __entry->thrs, __entry->len, + __entry->last_reap, __entry->hist_head, __entry->now, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3]) +); + #undef NO_DEV #endif /* _TRACE_NAPI_H */ diff --git a/lib/dynamic_queue_limits.c b/lib/dynamic_queue_limits.c index fde0aa244148..a1389db1c30a 100644 --- a/lib/dynamic_queue_limits.c +++ b/lib/dynamic_queue_limits.c @@ -10,10 +10,77 @@ #include #include #include +#include #define POSDIFF(A, B) ((int)((A) - (B)) > 0 ? (A) - (B) : 0) #define AFTER_EQ(A, B) ((int)((A) - (B)) >= 0) +static void dql_check_stall(struct dql *dql) +{ + unsigned short stall_thrs; + unsigned long now; + + stall_thrs = READ_ONCE(dql->stall_thrs); + if (!stall_thrs) + return; + + now = jiffies; + /* Check for a potential stall */ + if (time_after_eq(now, dql->last_reap + stall_thrs)) { + unsigned long hist_head, t, start, end; + + /* We are trying to detect a period of at least @stall_thrs + * jiffies without any Tx completions, but during first half + * of which some Tx was posted. + */ +dqs_again: + hist_head = READ_ONCE(dql->history_head); + /* pairs with smp_wmb() in dql_queued() */ + smp_rmb(); + + /* Get the previous entry in the ring buffer, which is the + * oldest sample. + */ + start = (hist_head - DQL_HIST_LEN + 1) * BITS_PER_LONG; + + /* Advance start to continue from the last reap time */ + if (time_before(start, dql->last_reap + 1)) + start = dql->last_reap + 1; + + /* Newest sample we should have already seen a completion for */ + end = hist_head * BITS_PER_LONG + (BITS_PER_LONG - 1); + + /* Shrink the search space to [start, (now - start_thrs/2)] if + * `end` is beyond the stall zone + */ + if (time_before(now, end + stall_thrs / 2)) + end = now - stall_thrs / 2; + + /* Search for the queued time in [t, end] */ + for (t = start; time_before_eq(t, end); t++) + if (test_bit(t % (DQL_HIST_LEN * BITS_PER_LONG), + dql->history)) + break; + + /* Variable t contains the time of the queue */ + if (!time_before_eq(t, end)) + goto no_stall; + + /* The ring buffer was modified in the meantime, retry */ + if (hist_head != READ_ONCE(dql->history_head)) + goto dqs_again; + + dql->stall_cnt++; + dql->stall_max = max_t(unsigned short, dql->stall_max, now - t); + + trace_dql_stall_detected(dql->stall_thrs, now - t, + dql->last_reap, dql->history_head, + now, dql->history); + } +no_stall: + dql->last_reap = now; +} + /* Records completed count and recalculates the queue limit */ void dql_completed(struct dql *dql, unsigned int count) { @@ -110,6 +177,8 @@ void dql_completed(struct dql *dql, unsigned int count) dql->prev_last_obj_cnt = dql->last_obj_cnt; dql->num_completed = completed; dql->prev_num_queued = num_queued; + + dql_check_stall(dql); } EXPORT_SYMBOL(dql_completed); @@ -125,6 +194,10 @@ void dql_reset(struct dql *dql) dql->prev_ovlimit = 0; dql->lowest_slack = UINT_MAX; dql->slack_start_time = jiffies; + + dql->last_reap = jiffies; + dql->history_head = jiffies / BITS_PER_LONG; + memset(dql->history, 0, sizeof(dql->history)); } EXPORT_SYMBOL(dql_reset); @@ -133,6 +206,7 @@ void dql_init(struct dql *dql, unsigned int hold_time) dql->max_limit = DQL_MAX_LIMIT; dql->min_limit = 0; dql->slack_hold_time = hold_time; + dql->stall_thrs = 0; dql_reset(dql); } EXPORT_SYMBOL(dql_init); diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 5560083774b1..e3d7a8cfa20b 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1415,6 +1415,65 @@ static struct netdev_queue_attribute bql_hold_time_attribute __ro_after_init = __ATTR(hold_time, 0644, bql_show_hold_time, bql_set_hold_time); +static ssize_t bql_show_stall_thrs(struct netdev_queue *queue, char *buf) +{ + struct dql *dql = &queue->dql; + + return sprintf(buf, "%u\n", jiffies_to_msecs(dql->stall_thrs)); +} + +static ssize_t bql_set_stall_thrs(struct netdev_queue *queue, + const char *buf, size_t len) +{ + struct dql *dql = &queue->dql; + unsigned int value; + int err; + + err = kstrtouint(buf, 10, &value); + if (err < 0) + return err; + + value = msecs_to_jiffies(value); + if (value && (value < 4 || value > 4 / 2 * BITS_PER_LONG)) + return -ERANGE; + + if (!dql->stall_thrs && value) + dql->last_reap = jiffies; + /* Force last_reap to be live */ + smp_wmb(); + dql->stall_thrs = value; + + return len; +} + +static struct netdev_queue_attribute bql_stall_thrs_attribute __ro_after_init = + __ATTR(stall_thrs, 0644, bql_show_stall_thrs, bql_set_stall_thrs); + +static ssize_t bql_show_stall_max(struct netdev_queue *queue, char *buf) +{ + return sprintf(buf, "%u\n", READ_ONCE(queue->dql.stall_max)); +} + +static ssize_t bql_set_stall_max(struct netdev_queue *queue, + const char *buf, size_t len) +{ + WRITE_ONCE(queue->dql.stall_max, 0); + return len; +} + +static struct netdev_queue_attribute bql_stall_max_attribute __ro_after_init = + __ATTR(stall_max, 0644, bql_show_stall_max, bql_set_stall_max); + +static ssize_t bql_show_stall_cnt(struct netdev_queue *queue, char *buf) +{ + struct dql *dql = &queue->dql; + + return sprintf(buf, "%lu\n", dql->stall_cnt); +} + +static struct netdev_queue_attribute bql_stall_cnt_attribute __ro_after_init = + __ATTR(stall_cnt, 0444, bql_show_stall_cnt, NULL); + static ssize_t bql_show_inflight(struct netdev_queue *queue, char *buf) { @@ -1453,6 +1512,9 @@ static struct attribute *dql_attrs[] __ro_after_init = { &bql_limit_min_attribute.attr, &bql_hold_time_attribute.attr, &bql_inflight_attribute.attr, + &bql_stall_thrs_attribute.attr, + &bql_stall_cnt_attribute.attr, + &bql_stall_max_attribute.attr, NULL }; -- cgit v1.2.3 From 6b6ca096115e5b7a85e8313f4e68a72d52db91b3 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 5 Mar 2024 15:22:28 -0300 Subject: rtc: class: make rtc_class constant Since commit 43a7206b0963 ("driver core: class: make class_register() take a const *"), the driver core allows for struct class to be in read-only memory, so move the rtc_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Link: https://lore.kernel.org/r/20240305-class_cleanup-abelloni-v1-1-944c026137c8@marliere.net Signed-off-by: Alexandre Belloni --- drivers/rtc/class.c | 21 +++++++++++++-------- drivers/rtc/interface.c | 2 +- include/linux/rtc.h | 2 +- kernel/power/suspend_test.c | 2 +- kernel/time/alarmtimer.c | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c index 921ee1827974..e31fa0ad127e 100644 --- a/drivers/rtc/class.c +++ b/drivers/rtc/class.c @@ -21,7 +21,6 @@ #include "rtc-core.h" static DEFINE_IDA(rtc_ida); -struct class *rtc_class; static void rtc_device_release(struct device *dev) { @@ -199,6 +198,11 @@ static SIMPLE_DEV_PM_OPS(rtc_class_dev_pm_ops, rtc_suspend, rtc_resume); #define RTC_CLASS_DEV_PM_OPS NULL #endif +const struct class rtc_class = { + .name = "rtc", + .pm = RTC_CLASS_DEV_PM_OPS, +}; + /* Ensure the caller will set the id before releasing the device */ static struct rtc_device *rtc_allocate_device(void) { @@ -220,7 +224,7 @@ static struct rtc_device *rtc_allocate_device(void) rtc->irq_freq = 1; rtc->max_user_freq = 64; - rtc->dev.class = rtc_class; + rtc->dev.class = &rtc_class; rtc->dev.groups = rtc_get_dev_attribute_groups(); rtc->dev.release = rtc_device_release; @@ -475,13 +479,14 @@ EXPORT_SYMBOL_GPL(devm_rtc_device_register); static int __init rtc_init(void) { - rtc_class = class_create("rtc"); - if (IS_ERR(rtc_class)) { - pr_err("couldn't create class\n"); - return PTR_ERR(rtc_class); - } - rtc_class->pm = RTC_CLASS_DEV_PM_OPS; + int err; + + err = class_register(&rtc_class); + if (err) + return err; + rtc_dev_init(); + return 0; } subsys_initcall(rtc_init); diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index 1b63111cdda2..5faafb4aa55c 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -696,7 +696,7 @@ struct rtc_device *rtc_class_open(const char *name) struct device *dev; struct rtc_device *rtc = NULL; - dev = class_find_device_by_name(rtc_class, name); + dev = class_find_device_by_name(&rtc_class, name); if (dev) rtc = to_rtc_device(dev); diff --git a/include/linux/rtc.h b/include/linux/rtc.h index 5f8e438a0312..3f4d315aaec9 100644 --- a/include/linux/rtc.h +++ b/include/linux/rtc.h @@ -42,7 +42,7 @@ static inline time64_t rtc_tm_sub(struct rtc_time *lhs, struct rtc_time *rhs) #include #include -extern struct class *rtc_class; +extern const struct class rtc_class; /* * For these RTC methods the device parameter is the physical device diff --git a/kernel/power/suspend_test.c b/kernel/power/suspend_test.c index b663a97f5867..d4856ec61570 100644 --- a/kernel/power/suspend_test.c +++ b/kernel/power/suspend_test.c @@ -201,7 +201,7 @@ static int __init test_suspend(void) } /* RTCs have initialized by now too ... can we use one? */ - dev = class_find_device(rtc_class, NULL, NULL, has_wakealarm); + dev = class_find_device(&rtc_class, NULL, NULL, has_wakealarm); if (dev) { rtc = rtc_class_open(dev_name(dev)); put_device(dev); diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 4657cb8e8b1f..5abfa4390673 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -134,7 +134,7 @@ static struct class_interface alarmtimer_rtc_interface = { static int alarmtimer_rtc_interface_setup(void) { - alarmtimer_rtc_interface.class = rtc_class; + alarmtimer_rtc_interface.class = &rtc_class; return class_interface_register(&alarmtimer_rtc_interface); } static void alarmtimer_rtc_interface_remove(void) -- cgit v1.2.3 From 7b937cc243e5b1df8780a0aa743ce800df6c68d1 Mon Sep 17 00:00:00 2001 From: Frank Rowand Date: Fri, 16 Feb 2024 17:05:51 -0800 Subject: of: Create of_root if no dtb provided by firmware When enabling CONFIG_OF on a platform where 'of_root' is not populated by firmware, we end up without a root node. In order to apply overlays and create subnodes of the root node, we need one. Create this root node by unflattening an empty builtin dtb. If firmware provides a flattened device tree (FDT) then the FDT is unflattened via setup_arch(). Otherwise, the call to unflatten(_and_copy)?_device_tree() will create an empty root node. We make of_have_populated_dt() return true only if the DTB was loaded by firmware so that existing callers don't change behavior after this patch. The call in the of platform code is removed because it prevents overlays from creating platform devices when the empty root node is used. [sboyd@kernel.org: Update of_have_populated_dt() to treat this empty dtb as not populated. Drop setup_of() initcall] Signed-off-by: Frank Rowand Link: https://lore.kernel.org/r/20230317053415.2254616-2-frowand.list@gmail.com Cc: Rob Herring Signed-off-by: Stephen Boyd Link: https://lore.kernel.org/r/20240217010557.2381548-3-sboyd@kernel.org Signed-off-by: Rob Herring --- drivers/of/Kconfig | 5 ++--- drivers/of/Makefile | 2 +- drivers/of/empty_root.dts | 6 ++++++ drivers/of/fdt.c | 32 +++++++++++++++++++++++++++++++- drivers/of/platform.c | 3 --- include/linux/of.h | 25 +++++++++++++++---------- 6 files changed, 55 insertions(+), 18 deletions(-) create mode 100644 drivers/of/empty_root.dts (limited to 'include/linux') diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig index da9826accb1b..d738fbad9c36 100644 --- a/drivers/of/Kconfig +++ b/drivers/of/Kconfig @@ -14,9 +14,8 @@ if OF config OF_UNITTEST bool "Device Tree runtime unit tests" - depends on !SPARC + depends on OF_EARLY_FLATTREE select IRQ_DOMAIN - select OF_EARLY_FLATTREE select OF_RESOLVE help This option builds in test cases for the device tree infrastructure @@ -54,7 +53,7 @@ config OF_FLATTREE select CRC32 config OF_EARLY_FLATTREE - bool + def_bool OF && !(SPARC || ALPHA || HEXAGON || M68K || PARISC || S390) select DMA_DECLARE_COHERENT if HAS_DMA && HAS_IOMEM select OF_FLATTREE diff --git a/drivers/of/Makefile b/drivers/of/Makefile index eff624854575..df305348d1cb 100644 --- a/drivers/of/Makefile +++ b/drivers/of/Makefile @@ -2,7 +2,7 @@ obj-y = base.o cpu.o device.o module.o platform.o property.o obj-$(CONFIG_OF_KOBJ) += kobj.o obj-$(CONFIG_OF_DYNAMIC) += dynamic.o -obj-$(CONFIG_OF_FLATTREE) += fdt.o +obj-$(CONFIG_OF_FLATTREE) += fdt.o empty_root.dtb.o obj-$(CONFIG_OF_EARLY_FLATTREE) += fdt_address.o obj-$(CONFIG_OF_PROMTREE) += pdt.o obj-$(CONFIG_OF_ADDRESS) += address.o diff --git a/drivers/of/empty_root.dts b/drivers/of/empty_root.dts new file mode 100644 index 000000000000..cf9e97a60f48 --- /dev/null +++ b/drivers/of/empty_root.dts @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only +/dts-v1/; + +/ { + +}; diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index dfeba8b8ce94..e5a385285149 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -8,6 +8,7 @@ #define pr_fmt(fmt) "OF: fdt: " fmt +#include #include #include #include @@ -32,6 +33,13 @@ #include "of_private.h" +/* + * __dtb_empty_root_begin[] and __dtb_empty_root_end[] magically created by + * cmd_dt_S_dtb in scripts/Makefile.lib + */ +extern uint8_t __dtb_empty_root_begin[]; +extern uint8_t __dtb_empty_root_end[]; + /* * of_fdt_limit_memory - limit the number of regions in the /memory node * @limit: maximum entries @@ -1343,7 +1351,29 @@ static void *__init copy_device_tree(void *fdt) */ void __init unflatten_device_tree(void) { - __unflatten_device_tree(initial_boot_params, NULL, &of_root, + void *fdt = initial_boot_params; + + /* Don't use the bootloader provided DTB if ACPI is enabled */ + if (!acpi_disabled) + fdt = NULL; + + /* + * Populate an empty root node when ACPI is enabled or bootloader + * doesn't provide one. + */ + if (!fdt) { + fdt = (void *) __dtb_empty_root_begin; + /* fdt_totalsize() will be used for copy size */ + if (fdt_totalsize(fdt) > + __dtb_empty_root_end - __dtb_empty_root_begin) { + pr_err("invalid size in dtb_empty_root\n"); + return; + } + of_fdt_crc32 = crc32_be(~0, fdt, fdt_totalsize(fdt)); + fdt = copy_device_tree(fdt); + } + + __unflatten_device_tree(fdt, NULL, &of_root, early_init_dt_alloc_memory_arch, false); /* Get pointer to "/chosen" and "/aliases" nodes for use everywhere */ diff --git a/drivers/of/platform.c b/drivers/of/platform.c index ba964df6b6db..389d4ea6bfc1 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -512,9 +512,6 @@ static int __init of_platform_default_populate_init(void) device_links_supplier_sync_state_pause(); - if (!of_have_populated_dt()) - return -ENODEV; - if (IS_ENABLED(CONFIG_PPC)) { struct device_node *boot_display = NULL; struct platform_device *dev; diff --git a/include/linux/of.h b/include/linux/of.h index a3e8e429ad7f..d5e7acdc8c8e 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -180,11 +180,6 @@ static inline bool is_of_node(const struct fwnode_handle *fwnode) &__of_fwnode_handle_node->fwnode : NULL; \ }) -static inline bool of_have_populated_dt(void) -{ - return of_root != NULL; -} - static inline bool of_node_is_root(const struct device_node *node) { return node && (node->parent == NULL); @@ -546,11 +541,6 @@ static inline struct device_node *of_find_node_with_property( #define of_fwnode_handle(node) NULL -static inline bool of_have_populated_dt(void) -{ - return false; -} - static inline struct device_node *of_get_compatible_child(const struct device_node *parent, const char *compatible) { @@ -1634,6 +1624,21 @@ static inline bool of_device_is_system_power_controller(const struct device_node return of_property_read_bool(np, "system-power-controller"); } +/** + * of_have_populated_dt() - Has DT been populated by bootloader + * + * Return: True if a DTB has been populated by the bootloader and it isn't the + * empty builtin one. False otherwise. + */ +static inline bool of_have_populated_dt(void) +{ +#ifdef CONFIG_OF + return of_property_present(of_root, "compatible"); +#else + return false; +#endif +} + /* * Overlay support */ -- cgit v1.2.3 From 1cface552a5b5f6e53a855de1a503ff958e2e253 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 7 Mar 2024 12:34:46 +0000 Subject: net: add skb_data_unref() helper Similar to skb_unref(), add skb_data_unref() to save an expensive atomic operation (and cache line dirtying) when last reference on shinfo->dataref is released. I saw this opportunity on hosts with RAW sockets accidentally bound to UDP protocol, forcing an skb_clone() on all received packets. These RAW sockets had their receive queue full, so all clone packets were immediately dropped. When UDP recvmsg() consumes later the original skb, skb_release_data() is hitting atomic_sub_return() quite badly, because skb->clone has been set permanently. Note that this patch helps TCP TX performance, because TCP stack also use (fast) clones. This means that at least one of the two packets (the main skb or its clone) will no longer have to perform this atomic operation in skb_release_data(). Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20240307123446.2302230-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 18 ++++++++++++++++++ net/core/skbuff.c | 4 +--- 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d0508f90bed5..3023bc2be6a1 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1237,6 +1237,24 @@ static inline bool skb_unref(struct sk_buff *skb) return true; } +static inline bool skb_data_unref(const struct sk_buff *skb, + struct skb_shared_info *shinfo) +{ + int bias; + + if (!skb->cloned) + return true; + + bias = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1; + + if (atomic_read(&shinfo->dataref) == bias) + smp_rmb(); + else if (atomic_sub_return(bias, &shinfo->dataref)) + return false; + + return true; +} + void __fix_address kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 766219011aea..b99127712e67 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1115,9 +1115,7 @@ static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason, struct skb_shared_info *shinfo = skb_shinfo(skb); int i; - if (skb->cloned && - atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, - &shinfo->dataref)) + if (!skb_data_unref(skb, shinfo)) goto exit; if (skb_zcopy(skb)) { -- cgit v1.2.3 From 0a5a46a6a61be7b63c12c18495d427f91f3662a9 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Tue, 6 Feb 2024 15:57:15 +0200 Subject: PCI/AER: Generalize TLP Header Log reading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both AER and DPC RP PIO provide TLP Header Log registers (PCIe r6.1 secs 7.8.4 & 7.9.14) to convey error diagnostics but the struct is named after AER as the struct aer_header_log_regs. Also, not all places that handle TLP Header Log use the struct and the struct members are named individually. Generalize the struct name and members, and use it consistently where TLP Header Log is being handled so that a pcie_read_tlp_log() helper can be easily added. Link: https://lore.kernel.org/r/20240206135717.8565-3-ilpo.jarvinen@linux.intel.com Signed-off-by: Ilpo Järvinen [bhelgaas: drop ixgbe changes for now, tidy whitespace] Signed-off-by: Bjorn Helgaas --- drivers/firmware/efi/cper.c | 4 ++-- drivers/pci/pci.c | 28 ++++++++++++++++++++++++++++ drivers/pci/pci.h | 2 +- drivers/pci/pcie/aer.c | 14 +++----------- drivers/pci/pcie/dpc.c | 14 ++++---------- include/linux/aer.h | 11 +++++------ include/ras/ras_event.h | 10 +++++----- 7 files changed, 48 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index 35c37f667781..d3f98161171e 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -445,8 +445,8 @@ static void cper_print_pcie(const char *pfx, const struct cper_sec_pcie *pcie, printk("%saer_uncor_severity: 0x%08x\n", pfx, aer->uncor_severity); printk("%sTLP Header: %08x %08x %08x %08x\n", pfx, - aer->header_log.dw0, aer->header_log.dw1, - aer->header_log.dw2, aer->header_log.dw3); + aer->header_log.dw[0], aer->header_log.dw[1], + aer->header_log.dw[2], aer->header_log.dw[3]); } } diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index d8f11a078924..a0af94cfcf7d 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1067,6 +1067,34 @@ disable_acs_redir: pci_disable_acs_redir(dev); } +/** + * pcie_read_tlp_log - read TLP Header Log + * @dev: PCIe device + * @where: PCI Config offset of TLP Header Log + * @tlp_log: TLP Log structure to fill + * + * Fill @tlp_log from TLP Header Log registers, e.g., AER or DPC. + * + * Return: 0 on success and filled TLP Log structure, <0 on error. + */ +int pcie_read_tlp_log(struct pci_dev *dev, int where, + struct pcie_tlp_log *tlp_log) +{ + int i, ret; + + memset(tlp_log, 0, sizeof(*tlp_log)); + + for (i = 0; i < 4; i++) { + ret = pci_read_config_dword(dev, where + i * 4, + &tlp_log->dw[i]); + if (ret) + return pcibios_err_to_errno(ret); + } + + return 0; +} +EXPORT_SYMBOL_GPL(pcie_read_tlp_log); + /** * pci_restore_bars - restore a device's BAR values (e.g. after wake-up) * @dev: PCI device to have its BARs restored diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 2336a8d1edab..a59ba6fde2a0 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -409,7 +409,7 @@ struct aer_err_info { unsigned int status; /* COR/UNCOR Error Status */ unsigned int mask; /* COR/UNCOR Error Mask */ - struct aer_header_log_regs tlp; /* TLP Header */ + struct pcie_tlp_log tlp; /* TLP Header */ }; int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index e31e6a9a7773..ac6293c24976 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -664,11 +664,10 @@ static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, } } -static void __print_tlp_header(struct pci_dev *dev, - struct aer_header_log_regs *t) +static void __print_tlp_header(struct pci_dev *dev, struct pcie_tlp_log *t) { pci_err(dev, " TLP Header: %08x %08x %08x %08x\n", - t->dw0, t->dw1, t->dw2, t->dw3); + t->dw[0], t->dw[1], t->dw[2], t->dw[3]); } static void __aer_print_error(struct pci_dev *dev, @@ -1246,14 +1245,7 @@ int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) if (info->status & AER_LOG_TLP_MASKS) { info->tlp_header_valid = 1; - pci_read_config_dword(dev, - aer + PCI_ERR_HEADER_LOG, &info->tlp.dw0); - pci_read_config_dword(dev, - aer + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1); - pci_read_config_dword(dev, - aer + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2); - pci_read_config_dword(dev, - aer + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3); + pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, &info->tlp); } } diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index 94111e438241..c197bc7f7f2c 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -190,7 +190,8 @@ out: static void dpc_process_rp_pio_error(struct pci_dev *pdev) { u16 cap = pdev->dpc_cap, dpc_status, first_error; - u32 status, mask, sev, syserr, exc, dw0, dw1, dw2, dw3, log, prefix; + u32 status, mask, sev, syserr, exc, log, prefix; + struct pcie_tlp_log tlp_log; int i; pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_STATUS, &status); @@ -216,16 +217,9 @@ static void dpc_process_rp_pio_error(struct pci_dev *pdev) if (pdev->dpc_rp_log_size < 4) goto clear_status; - pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG, - &dw0); - pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 4, - &dw1); - pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 8, - &dw2); - pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 12, - &dw3); + pcie_read_tlp_log(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG, &tlp_log); pci_err(pdev, "TLP Header: %#010x %#010x %#010x %#010x\n", - dw0, dw1, dw2, dw3); + tlp_log.dw[0], tlp_log.dw[1], tlp_log.dw[2], tlp_log.dw[3]); if (pdev->dpc_rp_log_size < 5) goto clear_status; diff --git a/include/linux/aer.h b/include/linux/aer.h index ae0fae70d4bd..4b97f38f3fcf 100644 --- a/include/linux/aer.h +++ b/include/linux/aer.h @@ -18,11 +18,8 @@ struct pci_dev; -struct aer_header_log_regs { - u32 dw0; - u32 dw1; - u32 dw2; - u32 dw3; +struct pcie_tlp_log { + u32 dw[4]; }; struct aer_capability_regs { @@ -33,13 +30,15 @@ struct aer_capability_regs { u32 cor_status; u32 cor_mask; u32 cap_control; - struct aer_header_log_regs header_log; + struct pcie_tlp_log header_log; u32 root_command; u32 root_status; u16 cor_err_source; u16 uncor_err_source; }; +int pcie_read_tlp_log(struct pci_dev *dev, int where, struct pcie_tlp_log *log); + #if defined(CONFIG_PCIEAER) int pci_aer_clear_nonfatal_status(struct pci_dev *dev); int pcie_aer_is_native(struct pci_dev *dev); diff --git a/include/ras/ras_event.h b/include/ras/ras_event.h index cbd3ddd7c33d..c011ea236e9b 100644 --- a/include/ras/ras_event.h +++ b/include/ras/ras_event.h @@ -300,7 +300,7 @@ TRACE_EVENT(aer_event, const u32 status, const u8 severity, const u8 tlp_header_valid, - struct aer_header_log_regs *tlp), + struct pcie_tlp_log *tlp), TP_ARGS(dev_name, status, severity, tlp_header_valid, tlp), @@ -318,10 +318,10 @@ TRACE_EVENT(aer_event, __entry->severity = severity; __entry->tlp_header_valid = tlp_header_valid; if (tlp_header_valid) { - __entry->tlp_header[0] = tlp->dw0; - __entry->tlp_header[1] = tlp->dw1; - __entry->tlp_header[2] = tlp->dw2; - __entry->tlp_header[3] = tlp->dw3; + __entry->tlp_header[0] = tlp->dw[0]; + __entry->tlp_header[1] = tlp->dw[1]; + __entry->tlp_header[2] = tlp->dw[2]; + __entry->tlp_header[3] = tlp->dw[3]; } ), -- cgit v1.2.3 From 7a1381e8313f1f01cbecbe3fc2ddaa24fe37033a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 7 Mar 2024 15:56:10 +0100 Subject: efi/tpm: Use symbolic GUID name from spec for final events table The LINUX_EFI_ GUID identifiers are only intended to be used to refer to GUIDs that are part of the Linux implementation, and are not considered external ABI. (Famous last words). GUIDs that already have a symbolic name in the spec should use that name, to avoid confusion between firmware components. So use the official name EFI_TCG2_FINAL_EVENTS_TABLE_GUID for the TCG2 'final events' configuration table. Reviewed-by: Kuppuswamy Sathyanarayanan Reviewed-by: Ilias Apalodimas Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/efi.c | 2 +- drivers/firmware/efi/libstub/tpm.c | 2 +- include/linux/efi.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 4fcda50acfa4..f6cfd29308d9 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -597,7 +597,7 @@ static const efi_config_table_type_t common_tables[] __initconst = { {EFI_MEMORY_ATTRIBUTES_TABLE_GUID, &efi_mem_attr_table, "MEMATTR" }, {LINUX_EFI_RANDOM_SEED_TABLE_GUID, &efi_rng_seed, "RNG" }, {LINUX_EFI_TPM_EVENT_LOG_GUID, &efi.tpm_log, "TPMEventLog" }, - {LINUX_EFI_TPM_FINAL_LOG_GUID, &efi.tpm_final_log, "TPMFinalLog" }, + {EFI_TCG2_FINAL_EVENTS_TABLE_GUID, &efi.tpm_final_log, "TPMFinalLog" }, {LINUX_EFI_MEMRESERVE_TABLE_GUID, &mem_reserve, "MEMRESERVE" }, {LINUX_EFI_INITRD_MEDIA_GUID, &initrd, "INITRD" }, {EFI_RT_PROPERTIES_TABLE_GUID, &rt_prop, "RTPROP" }, diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c index 7acbac16eae0..a880f7374c27 100644 --- a/drivers/firmware/efi/libstub/tpm.c +++ b/drivers/firmware/efi/libstub/tpm.c @@ -128,7 +128,7 @@ void efi_retrieve_tpm2_eventlog(void) * final events structure, and if so how much space they take up */ if (version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) - final_events_table = get_efi_config_table(LINUX_EFI_TPM_FINAL_LOG_GUID); + final_events_table = get_efi_config_table(EFI_TCG2_FINAL_EVENTS_TABLE_GUID); if (final_events_table && final_events_table->nr_events) { struct tcg_pcr_event2_head *header; int offset; diff --git a/include/linux/efi.h b/include/linux/efi.h index c74f47711f0b..464fe16411b8 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -386,6 +386,7 @@ void efi_native_runtime_setup(void); #define EFI_CONSOLE_OUT_DEVICE_GUID EFI_GUID(0xd3b36f2c, 0xd551, 0x11d4, 0x9a, 0x46, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d) #define APPLE_PROPERTIES_PROTOCOL_GUID EFI_GUID(0x91bd12fe, 0xf6c3, 0x44fb, 0xa5, 0xb7, 0x51, 0x22, 0xab, 0x30, 0x3a, 0xe0) #define EFI_TCG2_PROTOCOL_GUID EFI_GUID(0x607f766c, 0x7455, 0x42be, 0x93, 0x0b, 0xe4, 0xd7, 0x6d, 0xb2, 0x72, 0x0f) +#define EFI_TCG2_FINAL_EVENTS_TABLE_GUID EFI_GUID(0x1e2ed096, 0x30e2, 0x4254, 0xbd, 0x89, 0x86, 0x3b, 0xbe, 0xf8, 0x23, 0x25) #define EFI_LOAD_FILE_PROTOCOL_GUID EFI_GUID(0x56ec3091, 0x954c, 0x11d2, 0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b) #define EFI_LOAD_FILE2_PROTOCOL_GUID EFI_GUID(0x4006c0c1, 0xfcb3, 0x403e, 0x99, 0x6d, 0x4a, 0x6c, 0x87, 0x24, 0xe0, 0x6d) #define EFI_RT_PROPERTIES_TABLE_GUID EFI_GUID(0xeb66918a, 0x7eef, 0x402a, 0x84, 0x2e, 0x93, 0x1d, 0x21, 0xc3, 0x8a, 0xe9) @@ -411,7 +412,6 @@ void efi_native_runtime_setup(void); #define LINUX_EFI_LOADER_ENTRY_GUID EFI_GUID(0x4a67b082, 0x0a4c, 0x41cf, 0xb6, 0xc7, 0x44, 0x0b, 0x29, 0xbb, 0x8c, 0x4f) #define LINUX_EFI_RANDOM_SEED_TABLE_GUID EFI_GUID(0x1ce1e5bc, 0x7ceb, 0x42f2, 0x81, 0xe5, 0x8a, 0xad, 0xf1, 0x80, 0xf5, 0x7b) #define LINUX_EFI_TPM_EVENT_LOG_GUID EFI_GUID(0xb7799cb0, 0xeca2, 0x4943, 0x96, 0x67, 0x1f, 0xae, 0x07, 0xb7, 0x47, 0xfa) -#define LINUX_EFI_TPM_FINAL_LOG_GUID EFI_GUID(0x1e2ed096, 0x30e2, 0x4254, 0xbd, 0x89, 0x86, 0x3b, 0xbe, 0xf8, 0x23, 0x25) #define LINUX_EFI_MEMRESERVE_TABLE_GUID EFI_GUID(0x888eb0c6, 0x8ede, 0x4ff5, 0xa8, 0xf0, 0x9a, 0xee, 0x5c, 0xb9, 0x77, 0xc2) #define LINUX_EFI_INITRD_MEDIA_GUID EFI_GUID(0x5568e427, 0x68fc, 0x4f3d, 0xac, 0x74, 0xca, 0x55, 0x52, 0x31, 0xcc, 0x68) #define LINUX_EFI_MOK_VARIABLE_TABLE_GUID EFI_GUID(0xc451ed2b, 0x9694, 0x45d3, 0xba, 0xba, 0xed, 0x9f, 0x89, 0x88, 0xa3, 0x89) -- cgit v1.2.3 From 0bbe5b0ea97aaaea6387bab89919a8654b07df27 Mon Sep 17 00:00:00 2001 From: Kuppuswamy Sathyanarayanan Date: Thu, 15 Feb 2024 03:00:01 +0000 Subject: efi/libstub: Add Confidential Computing (CC) measurement typedefs If the virtual firmware implements TPM support, TCG2 protocol will be used for kernel measurements and event logging support. But in CC environment, not all platforms support or enable the TPM feature. UEFI specification [1] exposes protocol and interfaces used for kernel measurements in CC platforms without TPM support. More details about the EFI CC measurements and logging can be found in [1]. Link: https://uefi.org/specs/UEFI/2.10/38_Confidential_Computing.html#efi-cc-measurement-protocol [1] Signed-off-by: Kuppuswamy Sathyanarayanan [ardb: Drop code changes, keep typedefs and #define's only] Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/libstub/efistub.h | 79 ++++++++++++++++++++++++++++++++++ include/linux/efi.h | 1 + 2 files changed, 80 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 043a3ff435f3..6b020aadcf94 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -882,6 +882,85 @@ union efi_tcg2_protocol { } mixed_mode; }; +typedef struct { + u8 major; + u8 minor; +} efi_cc_version_t; + +typedef struct { + u8 type; + u8 sub_type; +} efi_cc_type_t; + +/* EFI CC type/subtype defines */ +#define EFI_CC_TYPE_NONE 0 +#define EFI_CC_TYPE_AMD_SEV 1 +#define EFI_CC_TYPE_INTEL_TDX 2 + +typedef u32 efi_cc_mr_index_t; + +struct efi_cc_event { + u32 event_size; + struct { + u32 header_size; + u16 header_version; + u32 mr_index; + u32 event_type; + } __packed event_header; + /* u8[] event follows here */ +} __packed; + +typedef struct efi_cc_event efi_cc_event_t; + +typedef u32 efi_cc_event_log_bitmap_t; +typedef u32 efi_cc_event_log_format_t; +typedef u32 efi_cc_event_algorithm_bitmap_t; + +typedef struct { + u8 size; + efi_cc_version_t structure_version; + efi_cc_version_t protocol_version; + efi_cc_event_algorithm_bitmap_t hash_algorithm_bitmap; + efi_cc_event_log_bitmap_t supported_event_logs; + efi_cc_type_t cc_type; +} efi_cc_boot_service_cap_t; + +#define EFI_CC_EVENT_HEADER_VERSION 1 + +#define EFI_CC_BOOT_HASH_ALG_SHA384 0x00000004 + +typedef union efi_cc_protocol efi_cc_protocol_t; + +union efi_cc_protocol { + struct { + efi_status_t + (__efiapi *get_capability)(efi_cc_protocol_t *, + efi_cc_boot_service_cap_t *); + + efi_status_t + (__efiapi *get_event_log)(efi_cc_protocol_t *, + efi_cc_event_log_format_t, + efi_physical_addr_t *, + efi_physical_addr_t *, + efi_bool_t *); + + efi_status_t + (__efiapi *hash_log_extend_event)(efi_cc_protocol_t *, u64, + efi_physical_addr_t, u64, + const efi_cc_event_t *); + + efi_status_t + (__efiapi *map_pcr_to_mr_index)(efi_cc_protocol_t *, u32, + efi_cc_mr_index_t *); + }; + struct { + u32 get_capability; + u32 get_event_log; + u32 hash_log_extend_event; + u32 map_pcr_to_mr_index; + } mixed_mode; +}; + struct riscv_efi_boot_protocol { u64 revision; diff --git a/include/linux/efi.h b/include/linux/efi.h index 464fe16411b8..2493d3d4429b 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -401,6 +401,7 @@ void efi_native_runtime_setup(void); #define EFI_CERT_X509_GUID EFI_GUID(0xa5c059a1, 0x94e4, 0x4aa7, 0x87, 0xb5, 0xab, 0x15, 0x5c, 0x2b, 0xf0, 0x72) #define EFI_CERT_X509_SHA256_GUID EFI_GUID(0x3bd2a492, 0x96c0, 0x4079, 0xb4, 0x20, 0xfc, 0xf9, 0x8e, 0xf1, 0x03, 0xed) #define EFI_CC_BLOB_GUID EFI_GUID(0x067b1f5f, 0xcf26, 0x44c5, 0x85, 0x54, 0x93, 0xd7, 0x77, 0x91, 0x2d, 0x42) +#define EFI_CC_MEASUREMENT_PROTOCOL_GUID EFI_GUID(0x96751a3d, 0x72f4, 0x41a6, 0xa7, 0x94, 0xed, 0x5d, 0x0e, 0x67, 0xae, 0x6b) /* * This GUID is used to pass to the kernel proper the struct screen_info -- cgit v1.2.3 From d228814b1913444dfdd9a25519ed7b38a19653e2 Mon Sep 17 00:00:00 2001 From: Kuppuswamy Sathyanarayanan Date: Thu, 15 Feb 2024 03:00:02 +0000 Subject: efi/libstub: Add get_event_log() support for CC platforms To allow event log info access after boot, EFI boot stub extracts the event log information and installs it in an EFI configuration table. Currently, EFI boot stub only supports installation of event log only for TPM 1.2 and TPM 2.0 protocols. Extend the same support for CC protocol. Since CC platform also uses TCG2 format, reuse TPM2 support code as much as possible. Link: https://uefi.org/specs/UEFI/2.10/38_Confidential_Computing.html#efi-cc-measurement-protocol [1] Signed-off-by: Kuppuswamy Sathyanarayanan Link: https://lkml.kernel.org/r/0229a87e-fb19-4dad-99fc-4afd7ed4099a%40collabora.com [ardb: Split out final events table handling to avoid version confusion] Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/efi.c | 1 + drivers/firmware/efi/libstub/efi-stub.c | 2 +- drivers/firmware/efi/libstub/efistub.h | 4 +- drivers/firmware/efi/libstub/tpm.c | 82 ++++++++++++++++++++++----------- drivers/firmware/efi/libstub/x86-stub.c | 2 +- include/linux/efi.h | 1 + 6 files changed, 61 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index f6cfd29308d9..8859fb0b006d 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -598,6 +598,7 @@ static const efi_config_table_type_t common_tables[] __initconst = { {LINUX_EFI_RANDOM_SEED_TABLE_GUID, &efi_rng_seed, "RNG" }, {LINUX_EFI_TPM_EVENT_LOG_GUID, &efi.tpm_log, "TPMEventLog" }, {EFI_TCG2_FINAL_EVENTS_TABLE_GUID, &efi.tpm_final_log, "TPMFinalLog" }, + {EFI_CC_FINAL_EVENTS_TABLE_GUID, &efi.tpm_final_log, "CCFinalLog" }, {LINUX_EFI_MEMRESERVE_TABLE_GUID, &mem_reserve, "MEMRESERVE" }, {LINUX_EFI_INITRD_MEDIA_GUID, &initrd, "INITRD" }, {EFI_RT_PROPERTIES_TABLE_GUID, &rt_prop, "RTPROP" }, diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c index f9c1e8a2bd1d..958a680e0660 100644 --- a/drivers/firmware/efi/libstub/efi-stub.c +++ b/drivers/firmware/efi/libstub/efi-stub.c @@ -167,7 +167,7 @@ efi_status_t efi_stub_common(efi_handle_t handle, si = setup_graphics(); - efi_retrieve_tpm2_eventlog(); + efi_retrieve_eventlog(); /* Ask the firmware to clear memory on unclean shutdown */ efi_enable_reset_attack_mitigation(); diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 6b020aadcf94..df174edfc228 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -929,6 +929,8 @@ typedef struct { #define EFI_CC_BOOT_HASH_ALG_SHA384 0x00000004 +#define EFI_CC_EVENT_LOG_FORMAT_TCG_2 0x00000002 + typedef union efi_cc_protocol efi_cc_protocol_t; union efi_cc_protocol { @@ -1140,7 +1142,7 @@ static inline void efi_enable_reset_attack_mitigation(void) { } #endif -void efi_retrieve_tpm2_eventlog(void); +void efi_retrieve_eventlog(void); struct screen_info *alloc_screen_info(void); struct screen_info *__alloc_screen_info(void); diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c index a880f7374c27..df3182f2e63a 100644 --- a/drivers/firmware/efi/libstub/tpm.c +++ b/drivers/firmware/efi/libstub/tpm.c @@ -47,39 +47,18 @@ void efi_enable_reset_attack_mitigation(void) #endif -void efi_retrieve_tpm2_eventlog(void) +static void efi_retrieve_tcg2_eventlog(int version, efi_physical_addr_t log_location, + efi_physical_addr_t log_last_entry, + efi_bool_t truncated, + struct efi_tcg2_final_events_table *final_events_table) { - efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID; efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID; efi_status_t status; - efi_physical_addr_t log_location = 0, log_last_entry = 0; struct linux_efi_tpm_eventlog *log_tbl = NULL; - struct efi_tcg2_final_events_table *final_events_table = NULL; unsigned long first_entry_addr, last_entry_addr; size_t log_size, last_entry_size; - efi_bool_t truncated; - int version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_2; - efi_tcg2_protocol_t *tcg2_protocol = NULL; int final_events_size = 0; - status = efi_bs_call(locate_protocol, &tcg2_guid, NULL, - (void **)&tcg2_protocol); - if (status != EFI_SUCCESS) - return; - - status = efi_call_proto(tcg2_protocol, get_event_log, version, - &log_location, &log_last_entry, &truncated); - - if (status != EFI_SUCCESS || !log_location) { - version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2; - status = efi_call_proto(tcg2_protocol, get_event_log, version, - &log_location, &log_last_entry, - &truncated); - if (status != EFI_SUCCESS || !log_location) - return; - - } - first_entry_addr = (unsigned long) log_location; /* @@ -93,8 +72,10 @@ void efi_retrieve_tpm2_eventlog(void) * get_event_log only returns the address of the last entry. * We need to calculate its size to deduce the full size of * the logs. + * + * CC Event log also uses TCG2 format, handle it same as TPM2. */ - if (version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) { + if (version > EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2) { /* * The TCG2 log format has variable length entries, * and the information to decode the hash algorithms @@ -127,8 +108,6 @@ void efi_retrieve_tpm2_eventlog(void) * Figure out whether any events have already been logged to the * final events structure, and if so how much space they take up */ - if (version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) - final_events_table = get_efi_config_table(EFI_TCG2_FINAL_EVENTS_TABLE_GUID); if (final_events_table && final_events_table->nr_events) { struct tcg_pcr_event2_head *header; int offset; @@ -165,3 +144,50 @@ void efi_retrieve_tpm2_eventlog(void) err_free: efi_bs_call(free_pool, log_tbl); } + +void efi_retrieve_eventlog(void) +{ + struct efi_tcg2_final_events_table *final_events_table = NULL; + efi_physical_addr_t log_location = 0, log_last_entry = 0; + efi_guid_t tpm2_guid = EFI_TCG2_PROTOCOL_GUID; + int version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_2; + efi_tcg2_protocol_t *tpm2 = NULL; + efi_bool_t truncated; + efi_status_t status; + + status = efi_bs_call(locate_protocol, &tpm2_guid, NULL, (void **)&tpm2); + if (status == EFI_SUCCESS) { + status = efi_call_proto(tpm2, get_event_log, version, &log_location, + &log_last_entry, &truncated); + + if (status != EFI_SUCCESS || !log_location) { + version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2; + status = efi_call_proto(tpm2, get_event_log, version, + &log_location, &log_last_entry, + &truncated); + } else { + final_events_table = + get_efi_config_table(EFI_TCG2_FINAL_EVENTS_TABLE_GUID); + } + } else { + efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID; + efi_cc_protocol_t *cc = NULL; + + status = efi_bs_call(locate_protocol, &cc_guid, NULL, (void **)&cc); + if (status != EFI_SUCCESS) + return; + + version = EFI_CC_EVENT_LOG_FORMAT_TCG_2; + status = efi_call_proto(cc, get_event_log, version, &log_location, + &log_last_entry, &truncated); + + final_events_table = + get_efi_config_table(EFI_CC_FINAL_EVENTS_TABLE_GUID); + } + + if (status != EFI_SUCCESS || !log_location) + return; + + efi_retrieve_tcg2_eventlog(version, log_location, log_last_entry, + truncated, final_events_table); +} diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index 99429bc4b0c7..d09aa13c7ff0 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -923,7 +923,7 @@ void __noreturn efi_stub_entry(efi_handle_t handle, efi_random_get_seed(); - efi_retrieve_tpm2_eventlog(); + efi_retrieve_eventlog(); setup_graphics(boot_params); diff --git a/include/linux/efi.h b/include/linux/efi.h index 2493d3d4429b..f0d56f106b60 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -402,6 +402,7 @@ void efi_native_runtime_setup(void); #define EFI_CERT_X509_SHA256_GUID EFI_GUID(0x3bd2a492, 0x96c0, 0x4079, 0xb4, 0x20, 0xfc, 0xf9, 0x8e, 0xf1, 0x03, 0xed) #define EFI_CC_BLOB_GUID EFI_GUID(0x067b1f5f, 0xcf26, 0x44c5, 0x85, 0x54, 0x93, 0xd7, 0x77, 0x91, 0x2d, 0x42) #define EFI_CC_MEASUREMENT_PROTOCOL_GUID EFI_GUID(0x96751a3d, 0x72f4, 0x41a6, 0xa7, 0x94, 0xed, 0x5d, 0x0e, 0x67, 0xae, 0x6b) +#define EFI_CC_FINAL_EVENTS_TABLE_GUID EFI_GUID(0xdd4a4648, 0x2de7, 0x4665, 0x96, 0x4d, 0x21, 0xd9, 0xef, 0x5f, 0xb4, 0x46) /* * This GUID is used to pass to the kernel proper the struct screen_info -- cgit v1.2.3 From edc99a2dd3ce07f61c379e641e417c07226be5ec Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 15 Feb 2024 13:42:40 -0500 Subject: nfs: remove unused NFS_CALL macro Nothing uses this, and thank goodness, as the syntax looks horrid. Signed-off-by: Jeff Layton Signed-off-by: Trond Myklebust --- include/linux/nfs_xdr.h | 7 ------- 1 file changed, 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 539b57fbf3ce..d09b9773b20c 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1820,13 +1820,6 @@ struct nfs_rpc_ops { void (*disable_swap)(struct inode *inode); }; -/* - * NFS_CALL(getattr, inode, (fattr)); - * into - * NFS_PROTO(inode)->getattr(fattr); - */ -#define NFS_CALL(op, inode, args) NFS_PROTO(inode)->op args - /* * Function vectors etc. for the NFS client */ -- cgit v1.2.3 From 2057a48d0dd00c6a2a94ded7df2bf1d3f2a4a0da Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 15 Feb 2024 14:57:30 -0500 Subject: sunrpc: add a struct rpc_stats arg to rpc_create_args We want to be able to have our rpc stats handled in a per network namespace manner, so add an option to rpc_create_args to specify a different rpc_stats struct instead of using the one on the rpc_program. Signed-off-by: Josef Bacik Signed-off-by: Trond Myklebust --- include/linux/sunrpc/clnt.h | 1 + net/sunrpc/clnt.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 5e9d1469c6fa..5321585c778f 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -139,6 +139,7 @@ struct rpc_create_args { const char *servername; const char *nodename; const struct rpc_program *program; + struct rpc_stat *stats; u32 prognumber; /* overrides program->number */ u32 version; rpc_authflavor_t authflavor; diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index cda0935a68c9..28f3749f6dc6 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -405,7 +405,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args, clnt->cl_maxproc = version->nrprocs; clnt->cl_prog = args->prognumber ? : program->number; clnt->cl_vers = version->number; - clnt->cl_stats = program->stats; + clnt->cl_stats = args->stats ? : program->stats; clnt->cl_metrics = rpc_alloc_iostats(clnt); rpc_init_pipe_dir_head(&clnt->cl_pipedir_objects); err = -ENOMEM; @@ -691,6 +691,7 @@ struct rpc_clnt *rpc_clone_client(struct rpc_clnt *clnt) .version = clnt->cl_vers, .authflavor = clnt->cl_auth->au_flavor, .cred = clnt->cl_cred, + .stats = clnt->cl_stats, }; return __rpc_clone_client(&args, clnt); } @@ -713,6 +714,7 @@ rpc_clone_client_set_auth(struct rpc_clnt *clnt, rpc_authflavor_t flavor) .version = clnt->cl_vers, .authflavor = flavor, .cred = clnt->cl_cred, + .stats = clnt->cl_stats, }; return __rpc_clone_client(&args, clnt); } @@ -1068,6 +1070,7 @@ struct rpc_clnt *rpc_bind_new_program(struct rpc_clnt *old, .version = vers, .authflavor = old->cl_auth->au_flavor, .cred = old->cl_cred, + .stats = old->cl_stats, }; struct rpc_clnt *clnt; int err; -- cgit v1.2.3 From 17f46b803d4f23c66cacce81db35fef3adb8f2af Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 1 Mar 2024 11:49:57 -0500 Subject: nfs: fix UAF in direct writes In production we have been hitting the following warning consistently ------------[ cut here ]------------ refcount_t: underflow; use-after-free. WARNING: CPU: 17 PID: 1800359 at lib/refcount.c:28 refcount_warn_saturate+0x9c/0xe0 Workqueue: nfsiod nfs_direct_write_schedule_work [nfs] RIP: 0010:refcount_warn_saturate+0x9c/0xe0 PKRU: 55555554 Call Trace: ? __warn+0x9f/0x130 ? refcount_warn_saturate+0x9c/0xe0 ? report_bug+0xcc/0x150 ? handle_bug+0x3d/0x70 ? exc_invalid_op+0x16/0x40 ? asm_exc_invalid_op+0x16/0x20 ? refcount_warn_saturate+0x9c/0xe0 nfs_direct_write_schedule_work+0x237/0x250 [nfs] process_one_work+0x12f/0x4a0 worker_thread+0x14e/0x3b0 ? ZSTD_getCParams_internal+0x220/0x220 kthread+0xdc/0x120 ? __btf_name_valid+0xa0/0xa0 ret_from_fork+0x1f/0x30 This is because we're completing the nfs_direct_request twice in a row. The source of this is when we have our commit requests to submit, we process them and send them off, and then in the completion path for the commit requests we have if (nfs_commit_end(cinfo.mds)) nfs_direct_write_complete(dreq); However since we're submitting asynchronous requests we sometimes have one that completes before we submit the next one, so we end up calling complete on the nfs_direct_request twice. The only other place we use nfs_generic_commit_list() is in __nfs_commit_inode, which wraps this call in a nfs_commit_begin(); nfs_commit_end(); Which is a common pattern for this style of completion handling, one that is also repeated in the direct code with get_dreq()/put_dreq() calls around where we process events as well as in the completion paths. Fix this by using the same pattern for the commit requests. Before with my 200 node rocksdb stress running this warning would pop every 10ish minutes. With my patch the stress test has been running for several hours without popping. Signed-off-by: Josef Bacik Cc: stable@vger.kernel.org Signed-off-by: Trond Myklebust --- fs/nfs/direct.c | 11 +++++++++-- fs/nfs/write.c | 2 +- include/linux/nfs_fs.h | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index befcc167e25f..6b8798d01e3a 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -672,10 +672,17 @@ static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) LIST_HEAD(mds_list); nfs_init_cinfo_from_dreq(&cinfo, dreq); + nfs_commit_begin(cinfo.mds); nfs_scan_commit(dreq->inode, &mds_list, &cinfo); res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); - if (res < 0) /* res == -ENOMEM */ - nfs_direct_write_reschedule(dreq); + if (res < 0) { /* res == -ENOMEM */ + spin_lock(&dreq->lock); + if (dreq->flags == 0) + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + spin_unlock(&dreq->lock); + } + if (nfs_commit_end(cinfo.mds)) + nfs_direct_write_complete(dreq); } static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 58adbb7709ba..15359bbfa56b 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1646,7 +1646,7 @@ static int wait_on_commit(struct nfs_mds_commit_info *cinfo) !atomic_read(&cinfo->rpcs_out)); } -static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo) +void nfs_commit_begin(struct nfs_mds_commit_info *cinfo) { atomic_inc(&cinfo->rpcs_out); } diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index f5ce7b101146..d59116ac8209 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -611,6 +611,7 @@ int nfs_wb_folio_cancel(struct inode *inode, struct folio *folio); extern int nfs_commit_inode(struct inode *, int); extern struct nfs_commit_data *nfs_commitdata_alloc(void); extern void nfs_commit_free(struct nfs_commit_data *data); +void nfs_commit_begin(struct nfs_mds_commit_info *cinfo); bool nfs_commit_end(struct nfs_mds_commit_info *cinfo); static inline bool nfs_have_writebacks(const struct inode *inode) -- cgit v1.2.3 From 3f6d5e6a468d02676244b868b210433831846127 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 25 Jan 2024 19:00:24 -0500 Subject: mm: introduce memalloc_flags_{save,restore} Our proliferation of memalloc_*_{save,restore} APIs is getting a bit silly, this adds a generic version and converts the existing save/restore functions to wrappers. Signed-off-by: Kent Overstreet Cc: Vlastimil Babka Cc: Matthew Wilcox Cc: Michal Hocko Cc: Darrick J. Wong Cc: linux-mm@kvack.org Acked-by: Vlastimil Babka --- include/linux/sched/mm.h | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 9a19f1b42f64..f00d7ecc2adf 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -306,6 +306,24 @@ static inline void might_alloc(gfp_t gfp_mask) might_sleep_if(gfpflags_allow_blocking(gfp_mask)); } +/** + * memalloc_flags_save - Add a PF_* flag to current->flags, save old value + * + * This allows PF_* flags to be conveniently added, irrespective of current + * value, and then the old version restored with memalloc_flags_restore(). + */ +static inline unsigned memalloc_flags_save(unsigned flags) +{ + unsigned oldflags = ~current->flags & flags; + current->flags |= flags; + return oldflags; +} + +static inline void memalloc_flags_restore(unsigned flags) +{ + current->flags &= ~flags; +} + /** * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope. * @@ -319,9 +337,7 @@ static inline void might_alloc(gfp_t gfp_mask) */ static inline unsigned int memalloc_noio_save(void) { - unsigned int flags = current->flags & PF_MEMALLOC_NOIO; - current->flags |= PF_MEMALLOC_NOIO; - return flags; + return memalloc_flags_save(PF_MEMALLOC_NOIO); } /** @@ -334,7 +350,7 @@ static inline unsigned int memalloc_noio_save(void) */ static inline void memalloc_noio_restore(unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; + memalloc_flags_restore(flags); } /** @@ -350,9 +366,7 @@ static inline void memalloc_noio_restore(unsigned int flags) */ static inline unsigned int memalloc_nofs_save(void) { - unsigned int flags = current->flags & PF_MEMALLOC_NOFS; - current->flags |= PF_MEMALLOC_NOFS; - return flags; + return memalloc_flags_save(PF_MEMALLOC_NOFS); } /** @@ -365,32 +379,27 @@ static inline unsigned int memalloc_nofs_save(void) */ static inline void memalloc_nofs_restore(unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags; + memalloc_flags_restore(flags); } static inline unsigned int memalloc_noreclaim_save(void) { - unsigned int flags = current->flags & PF_MEMALLOC; - current->flags |= PF_MEMALLOC; - return flags; + return memalloc_flags_save(PF_MEMALLOC); } static inline void memalloc_noreclaim_restore(unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC) | flags; + memalloc_flags_restore(flags); } static inline unsigned int memalloc_pin_save(void) { - unsigned int flags = current->flags & PF_MEMALLOC_PIN; - - current->flags |= PF_MEMALLOC_PIN; - return flags; + return memalloc_flags_save(PF_MEMALLOC_PIN); } static inline void memalloc_pin_restore(unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_PIN) | flags; + memalloc_flags_restore(flags); } #ifdef CONFIG_MEMCG -- cgit v1.2.3 From eab0af905bfc3e9c05da2ca163d76a1513159aa4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 25 Jan 2024 19:00:24 -0500 Subject: mm: introduce PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN Introduce PF_MEMALLOC_* equivalents of some GFP_ flags: PF_MEMALLOC_NORECLAIM -> GFP_NOWAIT PF_MEMALLOC_NOWARN -> __GFP_NOWARN Cc: Vlastimil Babka Cc: Matthew Wilcox Cc: Michal Hocko Cc: Darrick J. Wong Cc: linux-mm@kvack.org Signed-off-by: Kent Overstreet --- include/linux/sched.h | 4 ++-- include/linux/sched/mm.h | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index ffe8f618ab86..192e2c892040 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1636,8 +1636,8 @@ extern struct pid *cad_pid; * I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ -#define PF__HOLE__00800000 0x00800000 -#define PF__HOLE__01000000 0x01000000 +#define PF_MEMALLOC_NORECLAIM 0x00800000 /* All allocation requests will clear __GFP_DIRECT_RECLAIM */ +#define PF_MEMALLOC_NOWARN 0x01000000 /* All allocation requests will inherit __GFP_NOWARN */ #define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index f00d7ecc2adf..c29059a76052 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -236,16 +236,25 @@ static inline gfp_t current_gfp_context(gfp_t flags) { unsigned int pflags = READ_ONCE(current->flags); - if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_PIN))) { + if (unlikely(pflags & (PF_MEMALLOC_NOIO | + PF_MEMALLOC_NOFS | + PF_MEMALLOC_NORECLAIM | + PF_MEMALLOC_NOWARN | + PF_MEMALLOC_PIN))) { /* - * NOIO implies both NOIO and NOFS and it is a weaker context - * so always make sure it makes precedence + * Stronger flags before weaker flags: + * NORECLAIM implies NOIO, which in turn implies NOFS */ - if (pflags & PF_MEMALLOC_NOIO) + if (pflags & PF_MEMALLOC_NORECLAIM) + flags &= ~__GFP_DIRECT_RECLAIM; + else if (pflags & PF_MEMALLOC_NOIO) flags &= ~(__GFP_IO | __GFP_FS); else if (pflags & PF_MEMALLOC_NOFS) flags &= ~__GFP_FS; + if (pflags & PF_MEMALLOC_NOWARN) + flags |= __GFP_NOWARN; + if (pflags & PF_MEMALLOC_PIN) flags &= ~__GFP_MOVABLE; } -- cgit v1.2.3 From a4735d40a5da96a637af6e5bf9f6ec8b9d996acd Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 9 Mar 2024 21:10:08 -0800 Subject: Input: make input_class constant Since commit 43a7206b0963 ("driver core: class: make class_register() take a const *"), the driver core allows for struct class to be in read-only memory, so move the input_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Link: https://lore.kernel.org/r/20240305-class_cleanup-input-v1-1-0c3d950c25db@marliere.net Signed-off-by: Dmitry Torokhov --- drivers/input/input.c | 2 +- include/linux/input.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/input/input.c b/drivers/input/input.c index de7884a5be39..711485437567 100644 --- a/drivers/input/input.c +++ b/drivers/input/input.c @@ -1918,7 +1918,7 @@ static char *input_devnode(const struct device *dev, umode_t *mode) return kasprintf(GFP_KERNEL, "input/%s", dev_name(dev)); } -struct class input_class = { +const struct class input_class = { .name = "input", .devnode = input_devnode, }; diff --git a/include/linux/input.h b/include/linux/input.h index de6503c0edb8..c22ac465254b 100644 --- a/include/linux/input.h +++ b/include/linux/input.h @@ -514,7 +514,7 @@ void input_enable_softrepeat(struct input_dev *dev, int delay, int period); bool input_device_enabled(struct input_dev *dev); -extern struct class input_class; +extern const struct class input_class; /** * struct ff_device - force-feedback part of an input device -- cgit v1.2.3 From abb3f9717a67a2666b2bc2f19543a657e3d4ad63 Mon Sep 17 00:00:00 2001 From: Sibi Sankar Date: Tue, 27 Feb 2024 23:04:32 +0530 Subject: OPP: Extend dev_pm_opp_data with turbo support Let's extend the dev_pm_opp_data with a turbo variable, to allow users to specify if it's a boost frequency for a dynamically added OPP. Signed-off-by: Sibi Sankar Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 1 + include/linux/pm_opp.h | 2 ++ 2 files changed, 3 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index c4e0432ae42a..e233734b7220 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2065,6 +2065,7 @@ int _opp_add_v1(struct opp_table *opp_table, struct device *dev, /* populate the opp table */ new_opp->rates[0] = data->freq; new_opp->level = data->level; + new_opp->turbo = data->turbo; tol = u_volt * opp_table->voltage_tolerance_v1 / 100; new_opp->supplies[0].u_volt = u_volt; new_opp->supplies[0].u_volt_min = u_volt - tol; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 76dcb7f37bcd..fa9b63c6bf0b 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -87,12 +87,14 @@ struct dev_pm_opp_config { /** * struct dev_pm_opp_data - The data to use to initialize an OPP. + * @turbo: Flag to indicate whether the OPP is to be marked turbo or not. * @level: The performance level for the OPP. Set level to OPP_LEVEL_UNSET if * level field isn't used. * @freq: The clock rate in Hz for the OPP. * @u_volt: The voltage in uV for the OPP. */ struct dev_pm_opp_data { + bool turbo; unsigned int level; unsigned long freq; unsigned long u_volt; -- cgit v1.2.3 From 838a4772bfc390a14b31c25dc4c9eb66de5f5b1a Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 18 Jan 2024 16:19:13 +0530 Subject: cpufreq: Move dev_pm_opp_{init|free}_cpufreq_table() to pm_opp.h Move the declaration of functions defined in the OPP core to pm_opp.h. These were added to cpufreq.h as it was the only user of the APIs, but that was a mistake perhaps. Fix it. Signed-off-by: Viresh Kumar --- include/linux/cpufreq.h | 20 -------------------- include/linux/pm_opp.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index afda5f24d3dd..8ff3e79727d8 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -694,26 +694,6 @@ struct cpufreq_frequency_table { * order */ }; -#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP) -int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table); -void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table); -#else -static inline int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table - **table) -{ - return -EINVAL; -} - -static inline void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table - **table) -{ -} -#endif - /* * cpufreq_for_each_entry - iterate over a cpufreq_frequency_table * @pos: the cpufreq_frequency_table * to use as a loop cursor. diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index fa9b63c6bf0b..065a47382302 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -16,6 +16,7 @@ #include struct clk; +struct cpufreq_frequency_table; struct regulator; struct dev_pm_opp; struct device; @@ -446,6 +447,21 @@ static inline int dev_pm_opp_sync_regulators(struct device *dev) #endif /* CONFIG_PM_OPP */ +#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP) +int dev_pm_opp_init_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table); +void dev_pm_opp_free_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table); +#else +static inline int dev_pm_opp_init_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table) +{ + return -EINVAL; +} + +static inline void dev_pm_opp_free_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table) +{ +} +#endif + + #if defined(CONFIG_PM_OPP) && defined(CONFIG_OF) int dev_pm_opp_of_add_table(struct device *dev); int dev_pm_opp_of_add_table_indexed(struct device *dev, int index); -- cgit v1.2.3 From de5f84338970815b9fdd3497a975fb572d11e0b5 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 7 Mar 2024 12:39:06 +0100 Subject: lib/bitmap: Introduce bitmap_scatter() and bitmap_gather() helpers These helpers scatters or gathers a bitmap with the help of the mask position bits parameter. bitmap_scatter() does the following: src: 0000000001011010 |||||| +------+||||| | +----+|||| | |+----+||| | || +-+|| | || | || mask: ...v..vv...v..vv ...0..11...0..10 dst: 0000001100000010 and bitmap_gather() performs this one: mask: ...v..vv...v..vv src: 0000001100000010 ^ ^^ ^ 0 | || | 10 | || > 010 | |+--> 1010 | +--> 11010 +----> 011010 dst: 0000000000011010 bitmap_gather() can the seen as the reverse bitmap_scatter() operation. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/lkml/20230926052007.3917389-3-andriy.shevchenko@linux.intel.com/ Co-developed-by: Herve Codina Signed-off-by: Herve Codina Acked-by: Yury Norov Signed-off-by: David S. Miller --- include/linux/bitmap.h | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/test_bitmap.c | 42 ++++++++++++++++++++ 2 files changed, 143 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 5814e9ee40ba..00b8c6d4355c 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -63,6 +63,8 @@ struct device; * bitmap_shift_left(dst, src, n, nbits) *dst = *src << n * bitmap_cut(dst, src, first, n, nbits) Cut n bits from first, copy rest * bitmap_replace(dst, old, new, mask, nbits) *dst = (*old & ~(*mask)) | (*new & *mask) + * bitmap_scatter(dst, src, mask, nbits) *dst = map(dense, sparse)(src) + * bitmap_gather(dst, src, mask, nbits) *dst = map(sparse, dense)(src) * bitmap_remap(dst, src, old, new, nbits) *dst = map(old, new)(src) * bitmap_bitremap(oldbit, old, new, nbits) newbit = map(old, new)(oldbit) * bitmap_onto(dst, orig, relmap, nbits) *dst = orig relative to relmap @@ -499,6 +501,105 @@ static inline void bitmap_replace(unsigned long *dst, __bitmap_replace(dst, old, new, mask, nbits); } +/** + * bitmap_scatter - Scatter a bitmap according to the given mask + * @dst: scattered bitmap + * @src: gathered bitmap + * @mask: mask representing bits to assign to in the scattered bitmap + * @nbits: number of bits in each of these bitmaps + * + * Scatters bitmap with sequential bits according to the given @mask. + * + * Example: + * If @src bitmap = 0x005a, with @mask = 0x1313, @dst will be 0x0302. + * + * Or in binary form + * @src @mask @dst + * 0000000001011010 0001001100010011 0000001100000010 + * + * (Bits 0, 1, 2, 3, 4, 5 are copied to the bits 0, 1, 4, 8, 9, 12) + * + * A more 'visual' description of the operation: + * src: 0000000001011010 + * |||||| + * +------+||||| + * | +----+|||| + * | |+----+||| + * | || +-+|| + * | || | || + * mask: ...v..vv...v..vv + * ...0..11...0..10 + * dst: 0000001100000010 + * + * A relationship exists between bitmap_scatter() and bitmap_gather(). + * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation. + * See bitmap_scatter() for details related to this relationship. + */ +static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, + const unsigned long *mask, unsigned int nbits) +{ + unsigned int n = 0; + unsigned int bit; + + bitmap_zero(dst, nbits); + + for_each_set_bit(bit, mask, nbits) + __assign_bit(bit, dst, test_bit(n++, src)); +} + +/** + * bitmap_gather - Gather a bitmap according to given mask + * @dst: gathered bitmap + * @src: scattered bitmap + * @mask: mask representing bits to extract from in the scattered bitmap + * @nbits: number of bits in each of these bitmaps + * + * Gathers bitmap with sparse bits according to the given @mask. + * + * Example: + * If @src bitmap = 0x0302, with @mask = 0x1313, @dst will be 0x001a. + * + * Or in binary form + * @src @mask @dst + * 0000001100000010 0001001100010011 0000000000011010 + * + * (Bits 0, 1, 4, 8, 9, 12 are copied to the bits 0, 1, 2, 3, 4, 5) + * + * A more 'visual' description of the operation: + * mask: ...v..vv...v..vv + * src: 0000001100000010 + * ^ ^^ ^ 0 + * | || | 10 + * | || > 010 + * | |+--> 1010 + * | +--> 11010 + * +----> 011010 + * dst: 0000000000011010 + * + * A relationship exists between bitmap_gather() and bitmap_scatter(). See + * bitmap_scatter() for the bitmap scatter detailed operations. + * Suppose scattered computed using bitmap_scatter(scattered, src, mask, n). + * The operation bitmap_gather(result, scattered, mask, n) leads to a result + * equal or equivalent to src. + * + * The result can be 'equivalent' because bitmap_scatter() and bitmap_gather() + * are not bijective. + * The result and src values are equivalent in that sense that a call to + * bitmap_scatter(res, src, mask, n) and a call to + * bitmap_scatter(res, result, mask, n) will lead to the same res value. + */ +static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, + const unsigned long *mask, unsigned int nbits) +{ + unsigned int n = 0; + unsigned int bit; + + bitmap_zero(dst, nbits); + + for_each_set_bit(bit, mask, nbits) + __assign_bit(n++, dst, test_bit(bit, src)); +} + static inline void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs, unsigned int *re, unsigned int end) diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 65f22c2578b0..6b2b33579f56 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -380,6 +380,47 @@ static void __init test_replace(void) expect_eq_bitmap(bmap, exp3_1_0, nbits); } +static const unsigned long sg_mask[] __initconst = { + BITMAP_FROM_U64(0x000000000000035aULL), +}; + +static const unsigned long sg_src[] __initconst = { + BITMAP_FROM_U64(0x0000000000000667ULL), +}; + +static const unsigned long sg_gather_exp[] __initconst = { + BITMAP_FROM_U64(0x0000000000000029ULL), +}; + +static const unsigned long sg_scatter_exp[] __initconst = { + BITMAP_FROM_U64(0x000000000000021aULL), +}; + +static void __init test_bitmap_sg(void) +{ + unsigned int nbits = 64; + DECLARE_BITMAP(bmap_gather, 100); + DECLARE_BITMAP(bmap_scatter, 100); + DECLARE_BITMAP(bmap_tmp, 100); + DECLARE_BITMAP(bmap_res, 100); + + /* Simple gather call */ + bitmap_zero(bmap_gather, 100); + bitmap_gather(bmap_gather, sg_src, sg_mask, nbits); + expect_eq_bitmap(sg_gather_exp, bmap_gather, nbits); + + /* Simple scatter call */ + bitmap_zero(bmap_scatter, 100); + bitmap_scatter(bmap_scatter, sg_src, sg_mask, nbits); + expect_eq_bitmap(sg_scatter_exp, bmap_scatter, nbits); + + /* Scatter/gather relationship */ + bitmap_zero(bmap_tmp, 100); + bitmap_gather(bmap_tmp, bmap_scatter, sg_mask, nbits); + bitmap_scatter(bmap_res, bmap_tmp, sg_mask, nbits); + expect_eq_bitmap(bmap_scatter, bmap_res, nbits); +} + #define PARSE_TIME 0x1 #define NO_LEN 0x2 @@ -1252,6 +1293,7 @@ static void __init selftest(void) test_copy(); test_bitmap_region(); test_replace(); + test_bitmap_sg(); test_bitmap_arr32(); test_bitmap_arr64(); test_bitmap_parse(); -- cgit v1.2.3 From cb8a2ef0848ca80d67d6d56e2df757cfdf6b3355 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 11 Mar 2024 22:23:47 +0800 Subject: LoongArch: Add ORC stack unwinder support The kernel CONFIG_UNWINDER_ORC option enables the ORC unwinder, which is similar in concept to a DWARF unwinder. The difference is that the format of the ORC data is much simpler than DWARF, which in turn allows the ORC unwinder to be much simpler and faster. The ORC data consists of unwind tables which are generated by objtool. After analyzing all the code paths of a .o file, it determines information about the stack state at each instruction address in the file and outputs that information to the .orc_unwind and .orc_unwind_ip sections. The per-object ORC sections are combined at link time and are sorted and post-processed at boot time. The unwinder uses the resulting data to correlate instruction addresses with their stack states at run time. Most of the logic are similar with x86, in order to get ra info before ra is saved into stack, add ra_reg and ra_offset into orc_entry. At the same time, modify some arch-specific code to silence the objtool warnings. Co-developed-by: Jinyang He Signed-off-by: Jinyang He Co-developed-by: Youling Tang Signed-off-by: Youling Tang Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 2 + arch/loongarch/Kconfig.debug | 11 + arch/loongarch/Makefile | 23 +- arch/loongarch/include/asm/Kbuild | 2 + arch/loongarch/include/asm/bug.h | 1 + arch/loongarch/include/asm/exception.h | 2 + arch/loongarch/include/asm/module.h | 7 + arch/loongarch/include/asm/orc_header.h | 18 + arch/loongarch/include/asm/orc_lookup.h | 31 ++ arch/loongarch/include/asm/orc_types.h | 58 ++++ arch/loongarch/include/asm/stackframe.h | 3 + arch/loongarch/include/asm/unwind.h | 20 +- arch/loongarch/include/asm/unwind_hints.h | 28 ++ arch/loongarch/kernel/Makefile | 4 + arch/loongarch/kernel/entry.S | 5 + arch/loongarch/kernel/fpu.S | 7 + arch/loongarch/kernel/genex.S | 6 + arch/loongarch/kernel/lbt.S | 3 + arch/loongarch/kernel/mcount_dyn.S | 6 + arch/loongarch/kernel/module.c | 22 +- arch/loongarch/kernel/relocate_kernel.S | 7 +- arch/loongarch/kernel/rethook_trampoline.S | 1 + arch/loongarch/kernel/setup.c | 2 + arch/loongarch/kernel/stacktrace.c | 1 + arch/loongarch/kernel/traps.c | 42 ++- arch/loongarch/kernel/unwind_orc.c | 528 +++++++++++++++++++++++++++++ arch/loongarch/kernel/vmlinux.lds.S | 3 + arch/loongarch/kvm/switch.S | 9 +- arch/loongarch/lib/clear_user.S | 3 + arch/loongarch/lib/copy_user.S | 3 + arch/loongarch/lib/memcpy.S | 3 + arch/loongarch/lib/memset.S | 3 + arch/loongarch/mm/tlb.c | 27 +- arch/loongarch/mm/tlbex.S | 9 + arch/loongarch/vdso/Makefile | 1 + include/linux/compiler.h | 9 + scripts/Makefile | 7 +- 37 files changed, 875 insertions(+), 42 deletions(-) create mode 100644 arch/loongarch/include/asm/orc_header.h create mode 100644 arch/loongarch/include/asm/orc_lookup.h create mode 100644 arch/loongarch/include/asm/orc_types.h create mode 100644 arch/loongarch/include/asm/unwind_hints.h create mode 100644 arch/loongarch/kernel/unwind_orc.c (limited to 'include/linux') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 929f68926b34..8d6725115ac6 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -136,6 +136,7 @@ config LOONGARCH select HAVE_KVM select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI + select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS @@ -148,6 +149,7 @@ config LOONGARCH select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SETUP_PER_CPU_AREA if NUMA + select HAVE_STACK_VALIDATION if HAVE_OBJTOOL select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select HAVE_TIF_NOHZ diff --git a/arch/loongarch/Kconfig.debug b/arch/loongarch/Kconfig.debug index 8d36aab53008..98d60630c3d4 100644 --- a/arch/loongarch/Kconfig.debug +++ b/arch/loongarch/Kconfig.debug @@ -26,4 +26,15 @@ config UNWINDER_PROLOGUE Some of the addresses it reports may be incorrect (but better than the Guess unwinder). +config UNWINDER_ORC + bool "ORC unwinder" + select OBJTOOL + help + This option enables the ORC (Oops Rewind Capability) unwinder for + unwinding kernel stack traces. It uses a custom data format which is + a simplified version of the DWARF Call Frame Information standard. + + Enabling this option will increase the kernel's runtime memory usage + by roughly 2-4MB, depending on your kernel config. + endchoice diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 983aa2b1629a..e3bc02fb7fdc 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -26,6 +26,18 @@ endif 32bit-emul = elf32loongarch 64bit-emul = elf64loongarch +ifdef CONFIG_UNWINDER_ORC +orc_hash_h := arch/$(SRCARCH)/include/generated/asm/orc_hash.h +orc_hash_sh := $(srctree)/scripts/orc_hash.sh +targets += $(orc_hash_h) +quiet_cmd_orc_hash = GEN $@ + cmd_orc_hash = mkdir -p $(dir $@); \ + $(CONFIG_SHELL) $(orc_hash_sh) < $< > $@ +$(orc_hash_h): $(srctree)/arch/loongarch/include/asm/orc_types.h $(orc_hash_sh) FORCE + $(call if_changed,orc_hash) +archprepare: $(orc_hash_h) +endif + ifdef CONFIG_DYNAMIC_FTRACE KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY CC_FLAGS_FTRACE := -fpatchable-function-entry=2 @@ -72,8 +84,6 @@ KBUILD_CFLAGS_KERNEL += $(call cc-option,-mdirect-extern-access) KBUILD_CFLAGS_KERNEL += $(call cc-option,-fdirect-access-external-data) KBUILD_AFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data) KBUILD_CFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data) -KBUILD_AFLAGS_MODULE += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) -KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) else cflags-y += $(call cc-option,-mno-explicit-relocs) KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel @@ -82,6 +92,15 @@ KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs endif +KBUILD_AFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) +KBUILD_CFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) +KBUILD_AFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub) +KBUILD_CFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub) + +ifdef CONFIG_OBJTOOL +KBUILD_CFLAGS += -fno-jump-tables +endif + KBUILD_RUSTFLAGS_MODULE += -Crelocation-model=pic ifeq ($(CONFIG_RELOCATABLE),y) diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index 93783fa24f6e..a97c0edbb866 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +generated-y += orc_hash.h + generic-y += dma-contiguous.h generic-y += mcs_spinlock.h generic-y += parport.h diff --git a/arch/loongarch/include/asm/bug.h b/arch/loongarch/include/asm/bug.h index d4ca3ba25418..08388876ade4 100644 --- a/arch/loongarch/include/asm/bug.h +++ b/arch/loongarch/include/asm/bug.h @@ -44,6 +44,7 @@ do { \ instrumentation_begin(); \ __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \ + annotate_reachable(); \ instrumentation_end(); \ } while (0) diff --git a/arch/loongarch/include/asm/exception.h b/arch/loongarch/include/asm/exception.h index af74a3fdcad1..c6d20736fd92 100644 --- a/arch/loongarch/include/asm/exception.h +++ b/arch/loongarch/include/asm/exception.h @@ -6,6 +6,8 @@ #include #include +extern void *exception_table[]; + void show_registers(struct pt_regs *regs); asmlinkage void cache_parity_error(void); diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h index 2ecd82bb64e1..f33f3fd32ecc 100644 --- a/arch/loongarch/include/asm/module.h +++ b/arch/loongarch/include/asm/module.h @@ -6,6 +6,7 @@ #define _ASM_MODULE_H #include +#include #include #define RELA_STACK_DEPTH 16 @@ -21,6 +22,12 @@ struct mod_arch_specific { struct mod_section plt; struct mod_section plt_idx; +#ifdef CONFIG_UNWINDER_ORC + unsigned int num_orcs; + int *orc_unwind_ip; + struct orc_entry *orc_unwind; +#endif + /* For CONFIG_DYNAMIC_FTRACE */ struct plt_entry *ftrace_trampolines; }; diff --git a/arch/loongarch/include/asm/orc_header.h b/arch/loongarch/include/asm/orc_header.h new file mode 100644 index 000000000000..f9d509c3fd70 --- /dev/null +++ b/arch/loongarch/include/asm/orc_header.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef _ORC_HEADER_H +#define _ORC_HEADER_H + +#include +#include +#include + +/* + * The header is currently a 20-byte hash of the ORC entry definition; see + * scripts/orc_hash.sh. + */ +#define ORC_HEADER \ + __used __section(".orc_header") __aligned(4) \ + static const u8 orc_header[] = { ORC_HASH } + +#endif /* _ORC_HEADER_H */ diff --git a/arch/loongarch/include/asm/orc_lookup.h b/arch/loongarch/include/asm/orc_lookup.h new file mode 100644 index 000000000000..b02e6357def4 --- /dev/null +++ b/arch/loongarch/include/asm/orc_lookup.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ORC_LOOKUP_H +#define _ORC_LOOKUP_H + +/* + * This is a lookup table for speeding up access to the .orc_unwind table. + * Given an input address offset, the corresponding lookup table entry + * specifies a subset of the .orc_unwind table to search. + * + * Each block represents the end of the previous range and the start of the + * next range. An extra block is added to give the last range an end. + * + * The block size should be a power of 2 to avoid a costly 'div' instruction. + * + * A block size of 256 was chosen because it roughly doubles unwinder + * performance while only adding ~5% to the ORC data footprint. + */ +#define LOOKUP_BLOCK_ORDER 8 +#define LOOKUP_BLOCK_SIZE (1 << LOOKUP_BLOCK_ORDER) + +#ifndef LINKER_SCRIPT + +extern unsigned int orc_lookup[]; +extern unsigned int orc_lookup_end[]; + +#define LOOKUP_START_IP (unsigned long)_stext +#define LOOKUP_STOP_IP (unsigned long)_etext + +#endif /* LINKER_SCRIPT */ + +#endif /* _ORC_LOOKUP_H */ diff --git a/arch/loongarch/include/asm/orc_types.h b/arch/loongarch/include/asm/orc_types.h new file mode 100644 index 000000000000..caf1f71a1057 --- /dev/null +++ b/arch/loongarch/include/asm/orc_types.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ORC_TYPES_H +#define _ORC_TYPES_H + +#include + +/* + * The ORC_REG_* registers are base registers which are used to find other + * registers on the stack. + * + * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the + * address of the previous frame: the caller's SP before it called the current + * function. + * + * ORC_REG_UNDEFINED means the corresponding register's value didn't change in + * the current frame. + * + * The most commonly used base registers are SP and FP -- which the previous SP + * is usually based on -- and PREV_SP and UNDEFINED -- which the previous FP is + * usually based on. + * + * The rest of the base registers are needed for special cases like entry code + * and GCC realigned stacks. + */ +#define ORC_REG_UNDEFINED 0 +#define ORC_REG_PREV_SP 1 +#define ORC_REG_SP 2 +#define ORC_REG_FP 3 +#define ORC_REG_MAX 4 + +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 + +#ifndef __ASSEMBLY__ +/* + * This struct is more or less a vastly simplified version of the DWARF Call + * Frame Information standard. It contains only the necessary parts of DWARF + * CFI, simplified for ease of access by the in-kernel unwinder. It tells the + * unwinder how to find the previous SP and FP (and sometimes entry regs) on + * the stack for a given code address. Each instance of the struct corresponds + * to one or more code locations. + */ +struct orc_entry { + s16 sp_offset; + s16 fp_offset; + s16 ra_offset; + unsigned int sp_reg:4; + unsigned int fp_reg:4; + unsigned int ra_reg:4; + unsigned int type:3; + unsigned int signal:1; +}; +#endif /* __ASSEMBLY__ */ + +#endif /* _ORC_TYPES_H */ diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 4fb1e6408b98..45b507a7b06f 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -13,6 +13,7 @@ #include #include #include +#include /* Make the addition of cfi info a little easier. */ .macro cfi_rel_offset reg offset=0 docfi=0 @@ -162,6 +163,7 @@ li.w t0, CSR_CRMD_WE csrxchg t0, t0, LOONGARCH_CSR_CRMD #endif + UNWIND_HINT_REGS .endm .macro SAVE_ALL docfi=0 @@ -219,6 +221,7 @@ .macro RESTORE_SP_AND_RET docfi=0 cfi_ld sp, PT_R3, \docfi + UNWIND_HINT_FUNC ertn .endm diff --git a/arch/loongarch/include/asm/unwind.h b/arch/loongarch/include/asm/unwind.h index b9dce87afd2e..40a6763c5aec 100644 --- a/arch/loongarch/include/asm/unwind.h +++ b/arch/loongarch/include/asm/unwind.h @@ -16,6 +16,7 @@ enum unwinder_type { UNWINDER_GUESS, UNWINDER_PROLOGUE, + UNWINDER_ORC, }; struct unwind_state { @@ -24,7 +25,7 @@ struct unwind_state { struct task_struct *task; bool first, error, reset; int graph_idx; - unsigned long sp, pc, ra; + unsigned long sp, fp, pc, ra; }; bool default_next_frame(struct unwind_state *state); @@ -61,14 +62,17 @@ static __always_inline void __unwind_start(struct unwind_state *state, state->sp = regs->regs[3]; state->pc = regs->csr_era; state->ra = regs->regs[1]; + state->fp = regs->regs[22]; } else if (task && task != current) { state->sp = thread_saved_fp(task); state->pc = thread_saved_ra(task); state->ra = 0; + state->fp = 0; } else { state->sp = (unsigned long)__builtin_frame_address(0); state->pc = (unsigned long)__builtin_return_address(0); state->ra = 0; + state->fp = 0; } state->task = task; get_stack_info(state->sp, state->task, &state->stack_info); @@ -77,6 +81,18 @@ static __always_inline void __unwind_start(struct unwind_state *state, static __always_inline unsigned long __unwind_get_return_address(struct unwind_state *state) { - return unwind_done(state) ? 0 : state->pc; + if (unwind_done(state)) + return 0; + + return __kernel_text_address(state->pc) ? state->pc : 0; } + +#ifdef CONFIG_UNWINDER_ORC +void unwind_init(void); +void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size); +#else +static inline void unwind_init(void) {} +static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {} +#endif + #endif /* _ASM_UNWIND_H */ diff --git a/arch/loongarch/include/asm/unwind_hints.h b/arch/loongarch/include/asm/unwind_hints.h new file mode 100644 index 000000000000..a01086ad9dde --- /dev/null +++ b/arch/loongarch/include/asm/unwind_hints.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_UNWIND_HINTS_H +#define _ASM_LOONGARCH_UNWIND_HINTS_H + +#include +#include + +#ifdef __ASSEMBLY__ + +.macro UNWIND_HINT_UNDEFINED + UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED +.endm + +.macro UNWIND_HINT_END_OF_STACK + UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK +.endm + +.macro UNWIND_HINT_REGS + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_REGS +.endm + +.macro UNWIND_HINT_FUNC + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_CALL +.endm + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_LOONGARCH_UNWIND_HINTS_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 3c808c680370..3a7620b66bc6 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -3,6 +3,8 @@ # Makefile for the Linux/LoongArch kernel. # +OBJECT_FILES_NON_STANDARD_head.o := y + extra-y := vmlinux.lds obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \ @@ -21,6 +23,7 @@ obj-$(CONFIG_ARCH_STRICT_ALIGN) += unaligned.o CFLAGS_module.o += $(call cc-option,-Wno-override-init,) CFLAGS_syscall.o += $(call cc-option,-Wno-override-init,) +CFLAGS_traps.o += $(call cc-option,-Wno-override-init,) CFLAGS_perf_event.o += $(call cc-option,-Wno-override-init,) ifdef CONFIG_FUNCTION_TRACER @@ -62,6 +65,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o +obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S index 1ec8e4c4cc2b..48e7e34e355e 100644 --- a/arch/loongarch/kernel/entry.S +++ b/arch/loongarch/kernel/entry.S @@ -14,11 +14,13 @@ #include #include #include +#include .text .cfi_sections .debug_frame .align 5 SYM_CODE_START(handle_syscall) + UNWIND_HINT_UNDEFINED csrrd t0, PERCPU_BASE_KS la.pcrel t1, kernelsp add.d t1, t1, t0 @@ -57,6 +59,7 @@ SYM_CODE_START(handle_syscall) cfi_st fp, PT_R22 SAVE_STATIC + UNWIND_HINT_REGS #ifdef CONFIG_KGDB li.w t1, CSR_CRMD_WE @@ -75,6 +78,7 @@ SYM_CODE_END(handle_syscall) _ASM_NOKPROBE(handle_syscall) SYM_CODE_START(ret_from_fork) + UNWIND_HINT_REGS bl schedule_tail # a0 = struct task_struct *prev move a0, sp bl syscall_exit_to_user_mode @@ -84,6 +88,7 @@ SYM_CODE_START(ret_from_fork) SYM_CODE_END(ret_from_fork) SYM_CODE_START(ret_from_kernel_thread) + UNWIND_HINT_REGS bl schedule_tail # a0 = struct task_struct *prev move a0, s1 jirl ra, s0, 0 diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S index 4382e36ae3d4..69a85f2479fb 100644 --- a/arch/loongarch/kernel/fpu.S +++ b/arch/loongarch/kernel/fpu.S @@ -15,6 +15,7 @@ #include #include #include +#include #define FPU_REG_WIDTH 8 #define LSX_REG_WIDTH 16 @@ -526,3 +527,9 @@ SYM_FUNC_END(_restore_lasx_context) .L_fpu_fault: li.w a0, -EFAULT # failure jr ra + +#ifdef CONFIG_CPU_HAS_LBT +STACK_FRAME_NON_STANDARD _restore_fp +STACK_FRAME_NON_STANDARD _restore_lsx +STACK_FRAME_NON_STANDARD _restore_lasx +#endif diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S index 2bb3aa2dcfcb..86d5d90ebefe 100644 --- a/arch/loongarch/kernel/genex.S +++ b/arch/loongarch/kernel/genex.S @@ -32,6 +32,7 @@ SYM_FUNC_START(__arch_cpu_idle) SYM_FUNC_END(__arch_cpu_idle) SYM_CODE_START(handle_vint) + UNWIND_HINT_UNDEFINED BACKUP_T0T1 SAVE_ALL la_abs t1, __arch_cpu_idle @@ -49,6 +50,7 @@ SYM_CODE_START(handle_vint) SYM_CODE_END(handle_vint) SYM_CODE_START(except_vec_cex) + UNWIND_HINT_UNDEFINED b cache_parity_error SYM_CODE_END(except_vec_cex) @@ -67,6 +69,7 @@ SYM_CODE_END(except_vec_cex) .macro BUILD_HANDLER exception handler prep .align 5 SYM_CODE_START(handle_\exception) + UNWIND_HINT_UNDEFINED 666: BACKUP_T0T1 SAVE_ALL @@ -77,7 +80,9 @@ SYM_CODE_END(except_vec_cex) 668: RESTORE_ALL_AND_RET SYM_CODE_END(handle_\exception) + .pushsection ".data", "aw", %progbits SYM_DATA(unwind_hint_\exception, .word 668b - 666b) + .popsection .endm BUILD_HANDLER ade ade badv @@ -94,6 +99,7 @@ SYM_CODE_END(except_vec_cex) BUILD_HANDLER reserved reserved none /* others */ SYM_CODE_START(handle_sys) + UNWIND_HINT_UNDEFINED la_abs t0, handle_syscall jr t0 SYM_CODE_END(handle_sys) diff --git a/arch/loongarch/kernel/lbt.S b/arch/loongarch/kernel/lbt.S index 9c75120a26d8..001f061d226a 100644 --- a/arch/loongarch/kernel/lbt.S +++ b/arch/loongarch/kernel/lbt.S @@ -11,6 +11,7 @@ #include #include #include +#include #define SCR_REG_WIDTH 8 @@ -153,3 +154,5 @@ SYM_FUNC_END(_restore_ftop_context) .L_lbt_fault: li.w a0, -EFAULT # failure jr ra + +STACK_FRAME_NON_STANDARD _restore_ftop_context diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S index 482aa553aa2d..0c65cf09110c 100644 --- a/arch/loongarch/kernel/mcount_dyn.S +++ b/arch/loongarch/kernel/mcount_dyn.S @@ -73,6 +73,7 @@ SYM_FUNC_START(ftrace_stub) SYM_FUNC_END(ftrace_stub) SYM_CODE_START(ftrace_common) + UNWIND_HINT_UNDEFINED PTR_ADDI a0, ra, -8 /* arg0: ip */ move a1, t0 /* arg1: parent_ip */ la.pcrel t1, function_trace_op @@ -113,12 +114,14 @@ ftrace_common_return: SYM_CODE_END(ftrace_common) SYM_CODE_START(ftrace_caller) + UNWIND_HINT_UNDEFINED ftrace_regs_entry allregs=0 b ftrace_common SYM_CODE_END(ftrace_caller) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS SYM_CODE_START(ftrace_regs_caller) + UNWIND_HINT_UNDEFINED ftrace_regs_entry allregs=1 b ftrace_common SYM_CODE_END(ftrace_regs_caller) @@ -126,6 +129,7 @@ SYM_CODE_END(ftrace_regs_caller) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_CODE_START(ftrace_graph_caller) + UNWIND_HINT_UNDEFINED PTR_L a0, sp, PT_ERA PTR_ADDI a0, a0, -8 /* arg0: self_addr */ PTR_ADDI a1, sp, PT_R1 /* arg1: parent */ @@ -134,6 +138,7 @@ SYM_CODE_START(ftrace_graph_caller) SYM_CODE_END(ftrace_graph_caller) SYM_CODE_START(return_to_handler) + UNWIND_HINT_UNDEFINED /* Save return value regs */ PTR_ADDI sp, sp, -FGRET_REGS_SIZE PTR_S a0, sp, FGRET_REGS_A0 @@ -155,6 +160,7 @@ SYM_CODE_END(return_to_handler) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_CODE_START(ftrace_stub_direct_tramp) + UNWIND_HINT_UNDEFINED jr t0 SYM_CODE_END(ftrace_stub_direct_tramp) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c index b13b2858fe39..c7d0338d12c1 100644 --- a/arch/loongarch/kernel/module.c +++ b/arch/loongarch/kernel/module.c @@ -20,6 +20,7 @@ #include #include #include +#include static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) { @@ -515,15 +516,28 @@ static void module_init_ftrace_plt(const Elf_Ehdr *hdr, int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) { - const Elf_Shdr *s, *se; const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL; - for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { + for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { if (!strcmp(".altinstructions", secstrs + s->sh_name)) - apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size); + alt = s; + if (!strcmp(".orc_unwind", secstrs + s->sh_name)) + orc = s; + if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name)) + orc_ip = s; if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name)) - module_init_ftrace_plt(hdr, s, mod); + ftrace = s; } + if (alt) + apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size); + + if (orc && orc_ip) + unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size); + + if (ftrace) + module_init_ftrace_plt(hdr, ftrace, mod); + return 0; } diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S index f49f6b053763..84e6de2fd973 100644 --- a/arch/loongarch/kernel/relocate_kernel.S +++ b/arch/loongarch/kernel/relocate_kernel.S @@ -15,6 +15,7 @@ #include SYM_CODE_START(relocate_new_kernel) + UNWIND_HINT_UNDEFINED /* * a0: EFI boot flag for the new kernel * a1: Command line pointer for the new kernel @@ -90,6 +91,7 @@ SYM_CODE_END(relocate_new_kernel) * then start at the entry point from LOONGARCH_IOCSR_MBUF0. */ SYM_CODE_START(kexec_smp_wait) + UNWIND_HINT_UNDEFINED 1: li.w t0, 0x100 /* wait for init loop */ 2: addi.w t0, t0, -1 /* limit mailbox access */ bnez t0, 2b @@ -106,6 +108,5 @@ SYM_CODE_END(kexec_smp_wait) relocate_new_kernel_end: -SYM_DATA_START(relocate_new_kernel_size) - PTR relocate_new_kernel_end - relocate_new_kernel -SYM_DATA_END(relocate_new_kernel_size) + .section ".data" +SYM_DATA(relocate_new_kernel_size, .long relocate_new_kernel_end - relocate_new_kernel) diff --git a/arch/loongarch/kernel/rethook_trampoline.S b/arch/loongarch/kernel/rethook_trampoline.S index bd5772c96338..d4ceb2fa2a5c 100644 --- a/arch/loongarch/kernel/rethook_trampoline.S +++ b/arch/loongarch/kernel/rethook_trampoline.S @@ -76,6 +76,7 @@ .endm SYM_CODE_START(arch_rethook_trampoline) + UNWIND_HINT_UNDEFINED addi.d sp, sp, -PT_SIZE save_all_base_regs diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 634ef17fd38b..7bf9afaeea00 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -47,6 +47,7 @@ #include #include #include +#include #define SMBIOS_BIOSSIZE_OFFSET 0x09 #define SMBIOS_BIOSEXTERN_OFFSET 0x13 @@ -587,6 +588,7 @@ static void __init prefill_possible_map(void) void __init setup_arch(char **cmdline_p) { cpu_probe(); + unwind_init(); init_environ(); efi_init(); diff --git a/arch/loongarch/kernel/stacktrace.c b/arch/loongarch/kernel/stacktrace.c index f623feb2129f..eaec82e02c92 100644 --- a/arch/loongarch/kernel/stacktrace.c +++ b/arch/loongarch/kernel/stacktrace.c @@ -29,6 +29,7 @@ void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, regs->csr_era = thread_saved_ra(task); } regs->regs[1] = 0; + regs->regs[22] = 0; } for (unwind_start(&state, task, regs); diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index aebfc3733a76..f9f4eb00c92e 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -53,6 +53,32 @@ #include "access-helper.h" +void *exception_table[EXCCODE_INT_START] = { + [0 ... EXCCODE_INT_START - 1] = handle_reserved, + + [EXCCODE_TLBI] = handle_tlb_load, + [EXCCODE_TLBL] = handle_tlb_load, + [EXCCODE_TLBS] = handle_tlb_store, + [EXCCODE_TLBM] = handle_tlb_modify, + [EXCCODE_TLBNR] = handle_tlb_protect, + [EXCCODE_TLBNX] = handle_tlb_protect, + [EXCCODE_TLBPE] = handle_tlb_protect, + [EXCCODE_ADE] = handle_ade, + [EXCCODE_ALE] = handle_ale, + [EXCCODE_BCE] = handle_bce, + [EXCCODE_SYS] = handle_sys, + [EXCCODE_BP] = handle_bp, + [EXCCODE_INE] = handle_ri, + [EXCCODE_IPE] = handle_ri, + [EXCCODE_FPDIS] = handle_fpu, + [EXCCODE_LSXDIS] = handle_lsx, + [EXCCODE_LASXDIS] = handle_lasx, + [EXCCODE_FPE] = handle_fpe, + [EXCCODE_WATCH] = handle_watch, + [EXCCODE_BTDIS] = handle_lbt, +}; +EXPORT_SYMBOL_GPL(exception_table); + static void show_backtrace(struct task_struct *task, const struct pt_regs *regs, const char *loglvl, bool user) { @@ -1150,19 +1176,9 @@ void __init trap_init(void) for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++) set_handler(i * VECSIZE, handle_vint, VECSIZE); - set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE); - set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE); - set_handler(EXCCODE_BCE * VECSIZE, handle_bce, VECSIZE); - set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE); - set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE); - set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE); - set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE); - set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE); - set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE); - set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE); - set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE); - set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE); - set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE); + /* Set exception vector handler */ + for (i = EXCCODE_ADE; i <= EXCCODE_BTDIS; i++) + set_handler(i * VECSIZE, exception_table[i], VECSIZE); cache_error_setup(); diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c new file mode 100644 index 000000000000..b25722876331 --- /dev/null +++ b/arch/loongarch/kernel/unwind_orc.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ORC_HEADER; + +#define orc_warn(fmt, ...) \ + printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) + +extern int __start_orc_unwind_ip[]; +extern int __stop_orc_unwind_ip[]; +extern struct orc_entry __start_orc_unwind[]; +extern struct orc_entry __stop_orc_unwind[]; + +static bool orc_init __ro_after_init; +static unsigned int lookup_num_blocks __ro_after_init; + +/* Fake frame pointer entry -- used as a fallback for generated code */ +static struct orc_entry orc_fp_entry = { + .sp_reg = ORC_REG_FP, + .sp_offset = 16, + .fp_reg = ORC_REG_PREV_SP, + .fp_offset = -16, + .ra_reg = ORC_REG_PREV_SP, + .ra_offset = -8, + .type = ORC_TYPE_CALL +}; + +/* + * If we crash with IP==0, the last successfully executed instruction + * was probably an indirect function call with a NULL function pointer, + * and we don't have unwind information for NULL. + * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function + * pointer into its parent and then continue normally from there. + */ +static struct orc_entry orc_null_entry = { + .sp_reg = ORC_REG_SP, + .sp_offset = sizeof(long), + .fp_reg = ORC_REG_UNDEFINED, + .type = ORC_TYPE_CALL +}; + +static inline unsigned long orc_ip(const int *ip) +{ + return (unsigned long)ip + *ip; +} + +static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, + unsigned int num_entries, unsigned long ip) +{ + int *first = ip_table; + int *mid = first, *found = first; + int *last = ip_table + num_entries - 1; + + if (!num_entries) + return NULL; + + /* + * Do a binary range search to find the rightmost duplicate of a given + * starting address. Some entries are section terminators which are + * "weak" entries for ensuring there are no gaps. They should be + * ignored when they conflict with a real entry. + */ + while (first <= last) { + mid = first + ((last - first) / 2); + + if (orc_ip(mid) <= ip) { + found = mid; + first = mid + 1; + } else + last = mid - 1; + } + + return u_table + (found - ip_table); +} + +#ifdef CONFIG_MODULES +static struct orc_entry *orc_module_find(unsigned long ip) +{ + struct module *mod; + + mod = __module_address(ip); + if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) + return NULL; + + return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, mod->arch.num_orcs, ip); +} +#else +static struct orc_entry *orc_module_find(unsigned long ip) +{ + return NULL; +} +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE +static struct orc_entry *orc_find(unsigned long ip); + +/* + * Ftrace dynamic trampolines do not have orc entries of their own. + * But they are copies of the ftrace entries that are static and + * defined in ftrace_*.S, which do have orc entries. + * + * If the unwinder comes across a ftrace trampoline, then find the + * ftrace function that was used to create it, and use that ftrace + * function's orc entry, as the placement of the return code in + * the stack will be identical. + */ +static struct orc_entry *orc_ftrace_find(unsigned long ip) +{ + struct ftrace_ops *ops; + unsigned long tramp_addr, offset; + + ops = ftrace_ops_trampoline(ip); + if (!ops) + return NULL; + + /* Set tramp_addr to the start of the code copied by the trampoline */ + if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) + tramp_addr = (unsigned long)ftrace_regs_caller; + else + tramp_addr = (unsigned long)ftrace_caller; + + /* Now place tramp_addr to the location within the trampoline ip is at */ + offset = ip - ops->trampoline; + tramp_addr += offset; + + /* Prevent unlikely recursion */ + if (ip == tramp_addr) + return NULL; + + return orc_find(tramp_addr); +} +#else +static struct orc_entry *orc_ftrace_find(unsigned long ip) +{ + return NULL; +} +#endif + +static struct orc_entry *orc_find(unsigned long ip) +{ + static struct orc_entry *orc; + + if (ip == 0) + return &orc_null_entry; + + /* For non-init vmlinux addresses, use the fast lookup table: */ + if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { + unsigned int idx, start, stop; + + idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; + + if (unlikely((idx >= lookup_num_blocks-1))) { + orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n", + idx, lookup_num_blocks, (void *)ip); + return NULL; + } + + start = orc_lookup[idx]; + stop = orc_lookup[idx + 1] + 1; + + if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || + (__start_orc_unwind + stop > __stop_orc_unwind))) { + orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n", + idx, lookup_num_blocks, start, stop, (void *)ip); + return NULL; + } + + return __orc_find(__start_orc_unwind_ip + start, + __start_orc_unwind + start, stop - start, ip); + } + + /* vmlinux .init slow lookup: */ + if (is_kernel_inittext(ip)) + return __orc_find(__start_orc_unwind_ip, __start_orc_unwind, + __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); + + /* Module lookup: */ + orc = orc_module_find(ip); + if (orc) + return orc; + + return orc_ftrace_find(ip); +} + +#ifdef CONFIG_MODULES + +static DEFINE_MUTEX(sort_mutex); +static int *cur_orc_ip_table = __start_orc_unwind_ip; +static struct orc_entry *cur_orc_table = __start_orc_unwind; + +static void orc_sort_swap(void *_a, void *_b, int size) +{ + int delta = _b - _a; + int *a = _a, *b = _b, tmp; + struct orc_entry *orc_a, *orc_b; + + /* Swap the .orc_unwind_ip entries: */ + tmp = *a; + *a = *b + delta; + *b = tmp - delta; + + /* Swap the corresponding .orc_unwind entries: */ + orc_a = cur_orc_table + (a - cur_orc_ip_table); + orc_b = cur_orc_table + (b - cur_orc_ip_table); + swap(*orc_a, *orc_b); +} + +static int orc_sort_cmp(const void *_a, const void *_b) +{ + const int *a = _a, *b = _b; + unsigned long a_val = orc_ip(a); + unsigned long b_val = orc_ip(b); + struct orc_entry *orc_a; + + if (a_val > b_val) + return 1; + if (a_val < b_val) + return -1; + + /* + * The "weak" section terminator entries need to always be first + * to ensure the lookup code skips them in favor of real entries. + * These terminator entries exist to handle any gaps created by + * whitelisted .o files which didn't get objtool generation. + */ + orc_a = cur_orc_table + (a - cur_orc_ip_table); + + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; +} + +void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, + void *_orc, size_t orc_size) +{ + int *orc_ip = _orc_ip; + struct orc_entry *orc = _orc; + unsigned int num_entries = orc_ip_size / sizeof(int); + + WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || + orc_size % sizeof(*orc) != 0 || + num_entries != orc_size / sizeof(*orc)); + + /* + * The 'cur_orc_*' globals allow the orc_sort_swap() callback to + * associate an .orc_unwind_ip table entry with its corresponding + * .orc_unwind entry so they can both be swapped. + */ + mutex_lock(&sort_mutex); + cur_orc_ip_table = orc_ip; + cur_orc_table = orc; + sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap); + mutex_unlock(&sort_mutex); + + mod->arch.orc_unwind_ip = orc_ip; + mod->arch.orc_unwind = orc; + mod->arch.num_orcs = num_entries; +} +#endif + +void __init unwind_init(void) +{ + int i; + size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; + size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; + size_t num_entries = orc_ip_size / sizeof(int); + struct orc_entry *orc; + + if (!num_entries || orc_ip_size % sizeof(int) != 0 || + orc_size % sizeof(struct orc_entry) != 0 || + num_entries != orc_size / sizeof(struct orc_entry)) { + orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n"); + return; + } + + /* + * Note, the orc_unwind and orc_unwind_ip tables were already + * sorted at build time via the 'sorttable' tool. + * It's ready for binary search straight away, no need to sort it. + */ + + /* Initialize the fast lookup table: */ + lookup_num_blocks = orc_lookup_end - orc_lookup; + for (i = 0; i < lookup_num_blocks-1; i++) { + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, + num_entries, LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); + if (!orc) { + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); + return; + } + + orc_lookup[i] = orc - __start_orc_unwind; + } + + /* Initialize the ending block: */ + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, LOOKUP_STOP_IP); + if (!orc) { + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); + return; + } + orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; + + orc_init = true; +} + +static inline bool on_stack(struct stack_info *info, unsigned long addr, size_t len) +{ + unsigned long begin = info->begin; + unsigned long end = info->end; + + return (info->type != STACK_TYPE_UNKNOWN && + addr >= begin && addr < end && addr + len > begin && addr + len <= end); +} + +static bool stack_access_ok(struct unwind_state *state, unsigned long addr, size_t len) +{ + struct stack_info *info = &state->stack_info; + + if (on_stack(info, addr, len)) + return true; + + return !get_stack_info(addr, state->task, info) && on_stack(info, addr, len); +} + +unsigned long unwind_get_return_address(struct unwind_state *state) +{ + return __unwind_get_return_address(state); +} +EXPORT_SYMBOL_GPL(unwind_get_return_address); + +void unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs) +{ + __unwind_start(state, task, regs); + state->type = UNWINDER_ORC; + if (!unwind_done(state) && !__kernel_text_address(state->pc)) + unwind_next_frame(state); +} +EXPORT_SYMBOL_GPL(unwind_start); + +static bool is_entry_func(unsigned long addr) +{ + extern u32 kernel_entry; + extern u32 kernel_entry_end; + + return addr >= (unsigned long)&kernel_entry && addr < (unsigned long)&kernel_entry_end; +} + +static inline unsigned long bt_address(unsigned long ra) +{ + extern unsigned long eentry; + + if (__kernel_text_address(ra)) + return ra; + + if (__module_text_address(ra)) + return ra; + + if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { + unsigned long func; + unsigned long type = (ra - eentry) / VECSIZE; + unsigned long offset = (ra - eentry) % VECSIZE; + + switch (type) { + case 0 ... EXCCODE_INT_START - 1: + func = (unsigned long)exception_table[type]; + break; + case EXCCODE_INT_START ... EXCCODE_INT_END: + func = (unsigned long)handle_vint; + break; + default: + func = (unsigned long)handle_reserved; + break; + } + + return func + offset; + } + + return ra; +} + +bool unwind_next_frame(struct unwind_state *state) +{ + unsigned long *p, pc; + struct pt_regs *regs; + struct orc_entry *orc; + struct stack_info *info = &state->stack_info; + + if (unwind_done(state)) + return false; + + /* Don't let modules unload while we're reading their ORC data. */ + preempt_disable(); + + if (is_entry_func(state->pc)) + goto end; + + orc = orc_find(state->pc); + if (!orc) { + /* + * As a fallback, try to assume this code uses a frame pointer. + * This is useful for generated code, like BPF, which ORC + * doesn't know about. This is just a guess, so the rest of + * the unwind is no longer considered reliable. + */ + orc = &orc_fp_entry; + state->error = true; + } else { + if (orc->type == ORC_TYPE_UNDEFINED) + goto err; + + if (orc->type == ORC_TYPE_END_OF_STACK) + goto end; + } + + switch (orc->sp_reg) { + case ORC_REG_SP: + if (info->type == STACK_TYPE_IRQ && state->sp == info->end) + orc->type = ORC_TYPE_REGS; + else + state->sp = state->sp + orc->sp_offset; + break; + case ORC_REG_FP: + state->sp = state->fp; + break; + default: + orc_warn("unknown SP base reg %d at %pB\n", orc->sp_reg, (void *)state->pc); + goto err; + } + + switch (orc->fp_reg) { + case ORC_REG_PREV_SP: + p = (unsigned long *)(state->sp + orc->fp_offset); + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long))) + goto err; + + state->fp = *p; + break; + case ORC_REG_UNDEFINED: + /* Nothing. */ + break; + default: + orc_warn("unknown FP base reg %d at %pB\n", orc->fp_reg, (void *)state->pc); + goto err; + } + + switch (orc->type) { + case ORC_TYPE_CALL: + if (orc->ra_reg == ORC_REG_PREV_SP) { + p = (unsigned long *)(state->sp + orc->ra_offset); + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long))) + goto err; + + pc = unwind_graph_addr(state, *p, state->sp); + pc -= LOONGARCH_INSN_SIZE; + } else if (orc->ra_reg == ORC_REG_UNDEFINED) { + if (!state->ra || state->ra == state->pc) + goto err; + + pc = unwind_graph_addr(state, state->ra, state->sp); + pc -= LOONGARCH_INSN_SIZE; + state->ra = 0; + } else { + orc_warn("unknown ra base reg %d at %pB\n", orc->ra_reg, (void *)state->pc); + goto err; + } + break; + case ORC_TYPE_REGS: + if (info->type == STACK_TYPE_IRQ && state->sp == info->end) + regs = (struct pt_regs *)info->next_sp; + else + regs = (struct pt_regs *)state->sp; + + if (!stack_access_ok(state, (unsigned long)regs, sizeof(*regs))) + goto err; + + if ((info->end == (unsigned long)regs + sizeof(*regs)) && + !regs->regs[3] && !regs->regs[1]) + goto end; + + if (user_mode(regs)) + goto end; + + pc = regs->csr_era; + if (!__kernel_text_address(pc)) + goto err; + + state->sp = regs->regs[3]; + state->ra = regs->regs[1]; + state->fp = regs->regs[22]; + get_stack_info(state->sp, state->task, info); + + break; + default: + orc_warn("unknown .orc_unwind entry type %d at %pB\n", orc->type, (void *)state->pc); + goto err; + } + + state->pc = bt_address(pc); + if (!state->pc) { + pr_err("cannot find unwind pc at %pK\n", (void *)pc); + goto err; + } + + if (!__kernel_text_address(state->pc)) + goto err; + + preempt_enable(); + return true; + +err: + state->error = true; + +end: + preempt_enable(); + state->stack_info.type = STACK_TYPE_UNKNOWN; + return false; +} +EXPORT_SYMBOL_GPL(unwind_next_frame); diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index a5d0cd2035da..e8e97dbf9ca4 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -2,6 +2,7 @@ #include #include #include +#include #define PAGE_SIZE _PAGE_SIZE #define RO_EXCEPTION_TABLE_ALIGN 4 @@ -122,6 +123,8 @@ SECTIONS } #endif + ORC_UNWIND_TABLE + .sdata : { *(.sdata) } diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index ba976509bfe8..1fcc4b7eda32 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -8,7 +8,7 @@ #include #include #include -#include +#include #define HGPR_OFFSET(x) (PT_R0 + 8*x) #define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x) @@ -112,6 +112,7 @@ .text .cfi_sections .debug_frame SYM_CODE_START(kvm_exc_entry) + UNWIND_HINT_UNDEFINED csrwr a2, KVM_TEMP_KS csrrd a2, KVM_VCPU_KS addi.d a2, a2, KVM_VCPU_ARCH @@ -279,3 +280,9 @@ SYM_FUNC_END(kvm_restore_lasx) .section ".rodata" SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry) SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) + +#ifdef CONFIG_CPU_HAS_LBT +STACK_FRAME_NON_STANDARD kvm_restore_fpu +STACK_FRAME_NON_STANDARD kvm_restore_lsx +STACK_FRAME_NON_STANDARD kvm_restore_lasx +#endif diff --git a/arch/loongarch/lib/clear_user.S b/arch/loongarch/lib/clear_user.S index be741544e62b..7a0db643b286 100644 --- a/arch/loongarch/lib/clear_user.S +++ b/arch/loongarch/lib/clear_user.S @@ -10,6 +10,7 @@ #include #include #include +#include SYM_FUNC_START(__clear_user) /* @@ -204,3 +205,5 @@ SYM_FUNC_START(__clear_user_fast) _asm_extable 28b, .Lsmall_fixup _asm_extable 29b, .Lexit SYM_FUNC_END(__clear_user_fast) + +STACK_FRAME_NON_STANDARD __clear_user_fast diff --git a/arch/loongarch/lib/copy_user.S b/arch/loongarch/lib/copy_user.S index feec3d362803..095ce9181c6c 100644 --- a/arch/loongarch/lib/copy_user.S +++ b/arch/loongarch/lib/copy_user.S @@ -10,6 +10,7 @@ #include #include #include +#include SYM_FUNC_START(__copy_user) /* @@ -278,3 +279,5 @@ SYM_FUNC_START(__copy_user_fast) _asm_extable 58b, .Lexit _asm_extable 59b, .Lexit SYM_FUNC_END(__copy_user_fast) + +STACK_FRAME_NON_STANDARD __copy_user_fast diff --git a/arch/loongarch/lib/memcpy.S b/arch/loongarch/lib/memcpy.S index fa1148878d2b..9517a2f961af 100644 --- a/arch/loongarch/lib/memcpy.S +++ b/arch/loongarch/lib/memcpy.S @@ -9,6 +9,7 @@ #include #include #include +#include .section .noinstr.text, "ax" @@ -197,3 +198,5 @@ SYM_FUNC_START(__memcpy_fast) jr ra SYM_FUNC_END(__memcpy_fast) _ASM_NOKPROBE(__memcpy_fast) + +STACK_FRAME_NON_STANDARD __memcpy_small diff --git a/arch/loongarch/lib/memset.S b/arch/loongarch/lib/memset.S index 06d3ca54cbfe..df3846620553 100644 --- a/arch/loongarch/lib/memset.S +++ b/arch/loongarch/lib/memset.S @@ -9,6 +9,7 @@ #include #include #include +#include .macro fill_to_64 r0 bstrins.d \r0, \r0, 15, 8 @@ -166,3 +167,5 @@ SYM_FUNC_START(__memset_fast) jr ra SYM_FUNC_END(__memset_fast) _ASM_NOKPROBE(__memset_fast) + +STACK_FRAME_NON_STANDARD __memset_fast diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 0b95d32b30c9..5ac9beb5f093 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -9,8 +9,9 @@ #include #include -#include #include +#include +#include #include #include #include @@ -266,24 +267,20 @@ static void setup_tlb_handler(int cpu) setup_ptwalker(); local_flush_tlb_all(); + if (cpu_has_ptw) { + exception_table[EXCCODE_TLBI] = handle_tlb_load_ptw; + exception_table[EXCCODE_TLBL] = handle_tlb_load_ptw; + exception_table[EXCCODE_TLBS] = handle_tlb_store_ptw; + exception_table[EXCCODE_TLBM] = handle_tlb_modify_ptw; + } + /* The tlb handlers are generated only once */ if (cpu == 0) { memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); local_flush_icache_range(tlbrentry, tlbrentry + 0x80); - if (!cpu_has_ptw) { - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); - } else { - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE); - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE); - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE); - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE); - } - set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE); - set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE); - set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE); + + for (int i = EXCCODE_TLBL; i <= EXCCODE_TLBPE; i++) + set_handler(i * VECSIZE, exception_table[i], VECSIZE); } else { int vec_sz __maybe_unused; void *addr __maybe_unused; diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index d5d682f3d29f..a44387b838af 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -18,6 +18,7 @@ .macro tlb_do_page_fault, write SYM_CODE_START(tlb_do_page_fault_\write) + UNWIND_HINT_UNDEFINED SAVE_ALL csrrd a2, LOONGARCH_CSR_BADV move a0, sp @@ -32,6 +33,7 @@ tlb_do_page_fault 1 SYM_CODE_START(handle_tlb_protect) + UNWIND_HINT_UNDEFINED BACKUP_T0T1 SAVE_ALL move a0, sp @@ -44,6 +46,7 @@ SYM_CODE_START(handle_tlb_protect) SYM_CODE_END(handle_tlb_protect) SYM_CODE_START(handle_tlb_load) + UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 @@ -190,6 +193,7 @@ nopage_tlb_load: SYM_CODE_END(handle_tlb_load) SYM_CODE_START(handle_tlb_load_ptw) + UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_0 @@ -197,6 +201,7 @@ SYM_CODE_START(handle_tlb_load_ptw) SYM_CODE_END(handle_tlb_load_ptw) SYM_CODE_START(handle_tlb_store) + UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 @@ -346,6 +351,7 @@ nopage_tlb_store: SYM_CODE_END(handle_tlb_store) SYM_CODE_START(handle_tlb_store_ptw) + UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_1 @@ -353,6 +359,7 @@ SYM_CODE_START(handle_tlb_store_ptw) SYM_CODE_END(handle_tlb_store_ptw) SYM_CODE_START(handle_tlb_modify) + UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 @@ -500,6 +507,7 @@ nopage_tlb_modify: SYM_CODE_END(handle_tlb_modify) SYM_CODE_START(handle_tlb_modify_ptw) + UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_1 @@ -507,6 +515,7 @@ SYM_CODE_START(handle_tlb_modify_ptw) SYM_CODE_END(handle_tlb_modify_ptw) SYM_CODE_START(handle_tlb_refill) + UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_TLBRSAVE csrrd t0, LOONGARCH_CSR_PGD lddir t0, t0, 3 diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index f597cd08a96b..75c6726382c3 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -4,6 +4,7 @@ KASAN_SANITIZE := n UBSAN_SANITIZE := n KCOV_INSTRUMENT := n +OBJECT_FILES_NON_STANDARD := y # Include the generic Makefile to check the built vdso. include $(srctree)/lib/vdso/Makefile diff --git a/include/linux/compiler.h b/include/linux/compiler.h index bb1339c7057b..39f2d4a05208 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -116,6 +116,14 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, */ #define __stringify_label(n) #n +#define __annotate_reachable(c) ({ \ + asm volatile(__stringify_label(c) ":\n\t" \ + ".pushsection .discard.reachable\n\t" \ + ".long " __stringify_label(c) "b - .\n\t" \ + ".popsection\n\t"); \ +}) +#define annotate_reachable() __annotate_reachable(__COUNTER__) + #define __annotate_unreachable(c) ({ \ asm volatile(__stringify_label(c) ":\n\t" \ ".pushsection .discard.unreachable\n\t" \ @@ -128,6 +136,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #define __annotate_jump_table __section(".rodata..c_jump_table") #else /* !CONFIG_OBJTOOL */ +#define annotate_reachable() #define annotate_unreachable() #define __annotate_jump_table #endif /* CONFIG_OBJTOOL */ diff --git a/scripts/Makefile b/scripts/Makefile index 576cf64be667..e4cca53d2285 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -31,9 +31,12 @@ HOSTLDLIBS_sign-file = $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null | ifdef CONFIG_UNWINDER_ORC ifeq ($(ARCH),x86_64) -ARCH := x86 +SRCARCH := x86 endif -HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include +ifeq ($(ARCH),loongarch) +SRCARCH := loongarch +endif +HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/$(SRCARCH)/include HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED endif -- cgit v1.2.3 From d7bca9199a27b8690ae1c71dc11f825154af7234 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 8 Mar 2024 09:12:54 -0800 Subject: mm: Introduce vmap_page_range() to map pages in PCI address space ioremap_page_range() should be used for ranges within vmalloc range only. The vmalloc ranges are allocated by get_vm_area(). PCI has "resource" allocator that manages PCI_IOBASE, IO_SPACE_LIMIT address range, hence introduce vmap_page_range() to be used exclusively to map pages in PCI address space. Fixes: 3e49a866c9dc ("mm: Enforce VM_IOREMAP flag and range in ioremap_page_range.") Reported-by: Miguel Ojeda Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Reviewed-by: Christoph Hellwig Tested-by: Miguel Ojeda Link: https://lore.kernel.org/bpf/CANiq72ka4rir+RTN2FQoT=Vvprp_Ao-CvoYEkSNqtSY+RZj+AA@mail.gmail.com --- arch/arm/mm/ioremap.c | 8 ++++---- arch/loongarch/kernel/setup.c | 2 +- arch/mips/loongson64/init.c | 2 +- arch/powerpc/kernel/isa-bridge.c | 4 ++-- drivers/pci/pci.c | 4 ++-- include/linux/io.h | 7 +++++++ mm/vmalloc.c | 23 +++++++++++++++-------- 7 files changed, 32 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 2129070065c3..794cfea9f9d4 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -110,8 +110,8 @@ void __init add_static_vm_early(struct static_vm *svm) int ioremap_page(unsigned long virt, unsigned long phys, const struct mem_type *mtype) { - return ioremap_page_range(virt, virt + PAGE_SIZE, phys, - __pgprot(mtype->prot_pte)); + return vmap_page_range(virt, virt + PAGE_SIZE, phys, + __pgprot(mtype->prot_pte)); } EXPORT_SYMBOL(ioremap_page); @@ -466,8 +466,8 @@ int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) if (res->end > IO_SPACE_LIMIT) return -EINVAL; - return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr, - __pgprot(get_mem_type(pci_ioremap_mem_type)->prot_pte)); + return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr, + __pgprot(get_mem_type(pci_ioremap_mem_type)->prot_pte)); } EXPORT_SYMBOL(pci_remap_iospace); diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 634ef17fd38b..fd915ad69c09 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -490,7 +490,7 @@ static int __init add_legacy_isa_io(struct fwnode_handle *fwnode, } vaddr = (unsigned long)(PCI_IOBASE + range->io_start); - ioremap_page_range(vaddr, vaddr + size, hw_start, pgprot_device(PAGE_KERNEL)); + vmap_page_range(vaddr, vaddr + size, hw_start, pgprot_device(PAGE_KERNEL)); return 0; } diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index 553142c1f14f..a35dd7311795 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -180,7 +180,7 @@ static int __init add_legacy_isa_io(struct fwnode_handle *fwnode, resource_size_ vaddr = PCI_IOBASE + range->io_start; - ioremap_page_range(vaddr, vaddr + size, hw_start, pgprot_device(PAGE_KERNEL)); + vmap_page_range(vaddr, vaddr + size, hw_start, pgprot_device(PAGE_KERNEL)); return 0; } diff --git a/arch/powerpc/kernel/isa-bridge.c b/arch/powerpc/kernel/isa-bridge.c index 48e0eaf1ad61..5c064485197a 100644 --- a/arch/powerpc/kernel/isa-bridge.c +++ b/arch/powerpc/kernel/isa-bridge.c @@ -46,8 +46,8 @@ static void remap_isa_base(phys_addr_t pa, unsigned long size) WARN_ON_ONCE(size & ~PAGE_MASK); if (slab_is_available()) { - if (ioremap_page_range(ISA_IO_BASE, ISA_IO_BASE + size, pa, - pgprot_noncached(PAGE_KERNEL))) + if (vmap_page_range(ISA_IO_BASE, ISA_IO_BASE + size, pa, + pgprot_noncached(PAGE_KERNEL))) vunmap_range(ISA_IO_BASE, ISA_IO_BASE + size); } else { early_ioremap_range(ISA_IO_BASE, pa, size, diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index c3585229c12a..ccee56615f78 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4353,8 +4353,8 @@ int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) if (res->end > IO_SPACE_LIMIT) return -EINVAL; - return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr, - pgprot_device(PAGE_KERNEL)); + return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr, + pgprot_device(PAGE_KERNEL)); #else /* * This architecture does not have memory mapped I/O space, diff --git a/include/linux/io.h b/include/linux/io.h index 7304f2a69960..235ba7d80a8f 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -23,12 +23,19 @@ void __iowrite64_copy(void __iomem *to, const void *from, size_t count); #ifdef CONFIG_MMU int ioremap_page_range(unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot); +int vmap_page_range(unsigned long addr, unsigned long end, + phys_addr_t phys_addr, pgprot_t prot); #else static inline int ioremap_page_range(unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot) { return 0; } +static inline int vmap_page_range(unsigned long addr, unsigned long end, + phys_addr_t phys_addr, pgprot_t prot) +{ + return 0; +} #endif /* diff --git a/mm/vmalloc.c b/mm/vmalloc.c index e5b8c70950bc..1e36322d83d8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -304,11 +304,24 @@ static int vmap_range_noflush(unsigned long addr, unsigned long end, return err; } +int vmap_page_range(unsigned long addr, unsigned long end, + phys_addr_t phys_addr, pgprot_t prot) +{ + int err; + + err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), + ioremap_max_page_shift); + flush_cache_vmap(addr, end); + if (!err) + err = kmsan_ioremap_page_range(addr, end, phys_addr, prot, + ioremap_max_page_shift); + return err; +} + int ioremap_page_range(unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot) { struct vm_struct *area; - int err; area = find_vm_area((void *)addr); if (!area || !(area->flags & VM_IOREMAP)) { @@ -322,13 +335,7 @@ int ioremap_page_range(unsigned long addr, unsigned long end, (long)area->addr + get_vm_area_size(area)); return -ERANGE; } - err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), - ioremap_max_page_shift); - flush_cache_vmap(addr, end); - if (!err) - err = kmsan_ioremap_page_range(addr, end, phys_addr, prot, - ioremap_max_page_shift); - return err; + return vmap_page_range(addr, end, phys_addr, prot); } static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, -- cgit v1.2.3 From b620ecbd17a03cacd06f014a5d3f3a11285ce053 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Fri, 8 Mar 2024 16:05:24 -0700 Subject: vfio: Introduce interface to flush virqfd inject workqueue In order to synchronize changes that can affect the thread callback, introduce an interface to force a flush of the inject workqueue. The irqfd pointer is only valid under spinlock, but the workqueue cannot be flushed under spinlock. Therefore the flush work for the irqfd is queued under spinlock. The vfio_irqfd_cleanup_wq workqueue is re-used for queuing this work such that flushing the workqueue is also ordered relative to shutdown. Reviewed-by: Kevin Tian Reviewed-by: Reinette Chatre Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20240308230557.805580-4-alex.williamson@redhat.com Signed-off-by: Alex Williamson --- drivers/vfio/virqfd.c | 21 +++++++++++++++++++++ include/linux/vfio.h | 2 ++ 2 files changed, 23 insertions(+) (limited to 'include/linux') diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c index 29c564b7a6e1..532269133801 100644 --- a/drivers/vfio/virqfd.c +++ b/drivers/vfio/virqfd.c @@ -101,6 +101,13 @@ static void virqfd_inject(struct work_struct *work) virqfd->thread(virqfd->opaque, virqfd->data); } +static void virqfd_flush_inject(struct work_struct *work) +{ + struct virqfd *virqfd = container_of(work, struct virqfd, flush_inject); + + flush_work(&virqfd->inject); +} + int vfio_virqfd_enable(void *opaque, int (*handler)(void *, void *), void (*thread)(void *, void *), @@ -124,6 +131,7 @@ int vfio_virqfd_enable(void *opaque, INIT_WORK(&virqfd->shutdown, virqfd_shutdown); INIT_WORK(&virqfd->inject, virqfd_inject); + INIT_WORK(&virqfd->flush_inject, virqfd_flush_inject); irqfd = fdget(fd); if (!irqfd.file) { @@ -213,3 +221,16 @@ void vfio_virqfd_disable(struct virqfd **pvirqfd) flush_workqueue(vfio_irqfd_cleanup_wq); } EXPORT_SYMBOL_GPL(vfio_virqfd_disable); + +void vfio_virqfd_flush_thread(struct virqfd **pvirqfd) +{ + unsigned long flags; + + spin_lock_irqsave(&virqfd_lock, flags); + if (*pvirqfd && (*pvirqfd)->thread) + queue_work(vfio_irqfd_cleanup_wq, &(*pvirqfd)->flush_inject); + spin_unlock_irqrestore(&virqfd_lock, flags); + + flush_workqueue(vfio_irqfd_cleanup_wq); +} +EXPORT_SYMBOL_GPL(vfio_virqfd_flush_thread); diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 89b265bc6ec3..8b1a29820409 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -356,6 +356,7 @@ struct virqfd { wait_queue_entry_t wait; poll_table pt; struct work_struct shutdown; + struct work_struct flush_inject; struct virqfd **pvirqfd; }; @@ -363,5 +364,6 @@ int vfio_virqfd_enable(void *opaque, int (*handler)(void *, void *), void (*thread)(void *, void *), void *data, struct virqfd **pvirqfd, int fd); void vfio_virqfd_disable(struct virqfd **pvirqfd); +void vfio_virqfd_flush_thread(struct virqfd **pvirqfd); #endif /* VFIO_H */ -- cgit v1.2.3 From 8076fcde016c9c0e0660543e67bff86cb48a7c9c Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 11 Mar 2024 12:29:43 -0700 Subject: x86/rfds: Mitigate Register File Data Sampling (RFDS) RFDS is a CPU vulnerability that may allow userspace to infer kernel stale data previously used in floating point registers, vector registers and integer registers. RFDS only affects certain Intel Atom processors. Intel released a microcode update that uses VERW instruction to clear the affected CPU buffers. Unlike MDS, none of the affected cores support SMT. Add RFDS bug infrastructure and enable the VERW based mitigation by default, that clears the affected buffers just before exiting to userspace. Also add sysfs reporting and cmdline parameter "reg_file_data_sampling" to control the mitigation. For details see: Documentation/admin-guide/hw-vuln/reg-file-data-sampling.rst Signed-off-by: Pawan Gupta Signed-off-by: Dave Hansen Reviewed-by: Thomas Gleixner Acked-by: Josh Poimboeuf --- Documentation/ABI/testing/sysfs-devices-system-cpu | 1 + Documentation/admin-guide/kernel-parameters.txt | 21 ++++++ arch/x86/Kconfig | 11 +++ arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/msr-index.h | 8 +++ arch/x86/kernel/cpu/bugs.c | 78 +++++++++++++++++++++- arch/x86/kernel/cpu/common.c | 38 ++++++++++- drivers/base/cpu.c | 3 + include/linux/cpu.h | 2 + 9 files changed, 157 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index a1db6db47505..710d47be11e0 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -516,6 +516,7 @@ What: /sys/devices/system/cpu/vulnerabilities /sys/devices/system/cpu/vulnerabilities/mds /sys/devices/system/cpu/vulnerabilities/meltdown /sys/devices/system/cpu/vulnerabilities/mmio_stale_data + /sys/devices/system/cpu/vulnerabilities/reg_file_data_sampling /sys/devices/system/cpu/vulnerabilities/retbleed /sys/devices/system/cpu/vulnerabilities/spec_store_bypass /sys/devices/system/cpu/vulnerabilities/spectre_v1 diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 31b3a25680d0..73062d47a462 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1150,6 +1150,26 @@ The filter can be disabled or changed to another driver later using sysfs. + reg_file_data_sampling= + [X86] Controls mitigation for Register File Data + Sampling (RFDS) vulnerability. RFDS is a CPU + vulnerability which may allow userspace to infer + kernel data values previously stored in floating point + registers, vector registers, or integer registers. + RFDS only affects Intel Atom processors. + + on: Turns ON the mitigation. + off: Turns OFF the mitigation. + + This parameter overrides the compile time default set + by CONFIG_MITIGATION_RFDS. Mitigation cannot be + disabled when other VERW based mitigations (like MDS) + are enabled. In order to disable RFDS mitigation all + VERW based mitigations need to be disabled. + + For details see: + Documentation/admin-guide/hw-vuln/reg-file-data-sampling.rst + driver_async_probe= [KNL] List of driver names to be probed asynchronously. * matches with all driver names. If * is specified, the @@ -3398,6 +3418,7 @@ nospectre_bhb [ARM64] nospectre_v1 [X86,PPC] nospectre_v2 [X86,PPC,S390,ARM64] + reg_file_data_sampling=off [X86] retbleed=off [X86] spec_store_bypass_disable=off [X86,PPC] spectre_v2_user=off [X86] diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5edec175b9bf..637e337c332e 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2614,6 +2614,17 @@ config GDS_FORCE_MITIGATION If in doubt, say N. +config MITIGATION_RFDS + bool "RFDS Mitigation" + depends on CPU_SUP_INTEL + default y + help + Enable mitigation for Register File Data Sampling (RFDS) by default. + RFDS is a hardware vulnerability which affects Intel Atom CPUs. It + allows unprivileged speculative access to stale data previously + stored in floating point, vector and integer registers. + See also + endif config ARCH_HAS_ADD_PAGES diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 2b62cdd8dd12..8511aad59581 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -503,4 +503,5 @@ /* BUG word 2 */ #define X86_BUG_SRSO X86_BUG(1*32 + 0) /* AMD SRSO bug */ #define X86_BUG_DIV0 X86_BUG(1*32 + 1) /* AMD DIV0 speculation bug */ +#define X86_BUG_RFDS X86_BUG(1*32 + 2) /* CPU is vulnerable to Register File Data Sampling */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index f1bd7b91b3c6..d1b5edaf6c34 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -165,6 +165,14 @@ * CPU is not vulnerable to Gather * Data Sampling (GDS). */ +#define ARCH_CAP_RFDS_NO BIT(27) /* + * Not susceptible to Register + * File Data Sampling. + */ +#define ARCH_CAP_RFDS_CLEAR BIT(28) /* + * VERW clears CPU Register + * File. + */ #define ARCH_CAP_XAPIC_DISABLE BIT(21) /* * IA32_XAPIC_DISABLE_STATUS MSR diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index cd6ac89c1a0d..01ac18f56147 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -480,6 +480,57 @@ static int __init mmio_stale_data_parse_cmdline(char *str) } early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); +#undef pr_fmt +#define pr_fmt(fmt) "Register File Data Sampling: " fmt + +enum rfds_mitigations { + RFDS_MITIGATION_OFF, + RFDS_MITIGATION_VERW, + RFDS_MITIGATION_UCODE_NEEDED, +}; + +/* Default mitigation for Register File Data Sampling */ +static enum rfds_mitigations rfds_mitigation __ro_after_init = + IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_VERW : RFDS_MITIGATION_OFF; + +static const char * const rfds_strings[] = { + [RFDS_MITIGATION_OFF] = "Vulnerable", + [RFDS_MITIGATION_VERW] = "Mitigation: Clear Register File", + [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", +}; + +static void __init rfds_select_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) { + rfds_mitigation = RFDS_MITIGATION_OFF; + return; + } + if (rfds_mitigation == RFDS_MITIGATION_OFF) + return; + + if (x86_read_arch_cap_msr() & ARCH_CAP_RFDS_CLEAR) + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); + else + rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; +} + +static __init int rfds_parse_cmdline(char *str) +{ + if (!str) + return -EINVAL; + + if (!boot_cpu_has_bug(X86_BUG_RFDS)) + return 0; + + if (!strcmp(str, "off")) + rfds_mitigation = RFDS_MITIGATION_OFF; + else if (!strcmp(str, "on")) + rfds_mitigation = RFDS_MITIGATION_VERW; + + return 0; +} +early_param("reg_file_data_sampling", rfds_parse_cmdline); + #undef pr_fmt #define pr_fmt(fmt) "" fmt @@ -513,6 +564,11 @@ static void __init md_clear_update_mitigation(void) mmio_mitigation = MMIO_MITIGATION_VERW; mmio_select_mitigation(); } + if (rfds_mitigation == RFDS_MITIGATION_OFF && + boot_cpu_has_bug(X86_BUG_RFDS)) { + rfds_mitigation = RFDS_MITIGATION_VERW; + rfds_select_mitigation(); + } out: if (boot_cpu_has_bug(X86_BUG_MDS)) pr_info("MDS: %s\n", mds_strings[mds_mitigation]); @@ -522,6 +578,8 @@ out: pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) pr_info("MMIO Stale Data: Unknown: No mitigations\n"); + if (boot_cpu_has_bug(X86_BUG_RFDS)) + pr_info("Register File Data Sampling: %s\n", rfds_strings[rfds_mitigation]); } static void __init md_clear_select_mitigation(void) @@ -529,11 +587,12 @@ static void __init md_clear_select_mitigation(void) mds_select_mitigation(); taa_select_mitigation(); mmio_select_mitigation(); + rfds_select_mitigation(); /* - * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update - * and print their mitigation after MDS, TAA and MMIO Stale Data - * mitigation selection is done. + * As these mitigations are inter-related and rely on VERW instruction + * to clear the microarchitural buffers, update and print their status + * after mitigation selection is done for each of these vulnerabilities. */ md_clear_update_mitigation(); } @@ -2622,6 +2681,11 @@ static ssize_t mmio_stale_data_show_state(char *buf) sched_smt_active() ? "vulnerable" : "disabled"); } +static ssize_t rfds_show_state(char *buf) +{ + return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); +} + static char *stibp_state(void) { if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && @@ -2781,6 +2845,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr case X86_BUG_GDS: return gds_show_state(buf); + case X86_BUG_RFDS: + return rfds_show_state(buf); + default: break; } @@ -2855,4 +2922,9 @@ ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *bu { return cpu_show_common(dev, attr, buf, X86_BUG_GDS); } + +ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); +} #endif diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index fbc4e60d027c..40d8c110bb32 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1267,6 +1267,8 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define SRSO BIT(5) /* CPU is affected by GDS */ #define GDS BIT(6) +/* CPU is affected by Register File Data Sampling */ +#define RFDS BIT(7) static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS), @@ -1294,9 +1296,18 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPPINGS(TIGERLAKE, X86_STEPPING_ANY, GDS), VULNBL_INTEL_STEPPINGS(LAKEFIELD, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RETBLEED), VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPING_ANY, MMIO | RETBLEED | GDS), - VULNBL_INTEL_STEPPINGS(ATOM_TREMONT, X86_STEPPING_ANY, MMIO | MMIO_SBDS), - VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPING_ANY, MMIO), - VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS), + VULNBL_INTEL_STEPPINGS(ALDERLAKE, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ALDERLAKE_L, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE_P, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(RAPTORLAKE_S, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_GRACEMONT, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_TREMONT, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPING_ANY, MMIO | RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L, X86_STEPPING_ANY, MMIO | MMIO_SBDS | RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_D, X86_STEPPING_ANY, RFDS), + VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_PLUS, X86_STEPPING_ANY, RFDS), VULNBL_AMD(0x15, RETBLEED), VULNBL_AMD(0x16, RETBLEED), @@ -1330,6 +1341,24 @@ static bool arch_cap_mmio_immune(u64 ia32_cap) ia32_cap & ARCH_CAP_SBDR_SSDP_NO); } +static bool __init vulnerable_to_rfds(u64 ia32_cap) +{ + /* The "immunity" bit trumps everything else: */ + if (ia32_cap & ARCH_CAP_RFDS_NO) + return false; + + /* + * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to + * indicate that mitigation is needed because guest is running on a + * vulnerable hardware or may migrate to such hardware: + */ + if (ia32_cap & ARCH_CAP_RFDS_CLEAR) + return true; + + /* Only consult the blacklist when there is no enumeration: */ + return cpu_matches(cpu_vuln_blacklist, RFDS); +} + static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) { u64 ia32_cap = x86_read_arch_cap_msr(); @@ -1441,6 +1470,9 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) boot_cpu_has(X86_FEATURE_AVX)) setup_force_cpu_bug(X86_BUG_GDS); + if (vulnerable_to_rfds(ia32_cap)) + setup_force_cpu_bug(X86_BUG_RFDS); + if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN)) return; diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 47de0f140ba6..0b33e81f9c9b 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -588,6 +588,7 @@ CPU_SHOW_VULN_FALLBACK(mmio_stale_data); CPU_SHOW_VULN_FALLBACK(retbleed); CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow); CPU_SHOW_VULN_FALLBACK(gds); +CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling); static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL); static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL); @@ -602,6 +603,7 @@ static DEVICE_ATTR(mmio_stale_data, 0444, cpu_show_mmio_stale_data, NULL); static DEVICE_ATTR(retbleed, 0444, cpu_show_retbleed, NULL); static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NULL); static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL); +static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL); static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_meltdown.attr, @@ -617,6 +619,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_retbleed.attr, &dev_attr_spec_rstack_overflow.attr, &dev_attr_gather_data_sampling.attr, + &dev_attr_reg_file_data_sampling.attr, NULL }; diff --git a/include/linux/cpu.h b/include/linux/cpu.h index dcb89c987164..8654714421a0 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -75,6 +75,8 @@ extern ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf); extern ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf); +extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev, + struct device_attribute *attr, char *buf); extern __printf(4, 5) struct device *cpu_device_create(struct device *parent, void *drvdata, -- cgit v1.2.3 From e5b7aefe38f7f6258935d8a10c36552dd957048a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 8 Mar 2024 10:22:30 +0000 Subject: net: gro: move two declarations to include/net/gro.h Move gro_find_receive_by_type() and gro_find_complete_by_type() to include/net/gro.h where they belong. Also use _NET_GRO_H instead of _NET_IPV6_GRO_H to protect include/net/gro.h from multiple inclusions. Signed-off-by: Eric Dumazet Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20240308102230.296224-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 -- include/net/gro.h | 9 ++++++--- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 4230c7f3b959..c6f6ac779b34 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3901,8 +3901,6 @@ void napi_gro_flush(struct napi_struct *napi, bool flush_old); struct sk_buff *napi_get_frags(struct napi_struct *napi); void napi_get_frags_check(struct napi_struct *napi); gro_result_t napi_gro_frags(struct napi_struct *napi); -struct packet_offload *gro_find_receive_by_type(__be16 type); -struct packet_offload *gro_find_complete_by_type(__be16 type); static inline void napi_free_frags(struct napi_struct *napi) { diff --git a/include/net/gro.h b/include/net/gro.h index d6fc8fbd3730..50f1e403dbbb 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _NET_IPV6_GRO_H -#define _NET_IPV6_GRO_H +#ifndef _NET_GRO_H +#define _NET_GRO_H #include #include @@ -494,4 +494,7 @@ static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int * #endif } -#endif /* _NET_IPV6_GRO_H */ +struct packet_offload *gro_find_receive_by_type(__be16 type); +struct packet_offload *gro_find_complete_by_type(__be16 type); + +#endif /* _NET_GRO_H */ -- cgit v1.2.3 From 317460317a02a1af512697e6e964298dedd8a163 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 7 Mar 2024 17:07:59 -0800 Subject: bpf: Introduce bpf_arena. Introduce bpf_arena, which is a sparse shared memory region between the bpf program and user space. Use cases: 1. User space mmap-s bpf_arena and uses it as a traditional mmap-ed anonymous region, like memcached or any key/value storage. The bpf program implements an in-kernel accelerator. XDP prog can search for a key in bpf_arena and return a value without going to user space. 2. The bpf program builds arbitrary data structures in bpf_arena (hash tables, rb-trees, sparse arrays), while user space consumes it. 3. bpf_arena is a "heap" of memory from the bpf program's point of view. The user space may mmap it, but bpf program will not convert pointers to user base at run-time to improve bpf program speed. Initially, the kernel vm_area and user vma are not populated. User space can fault in pages within the range. While servicing a page fault, bpf_arena logic will insert a new page into the kernel and user vmas. The bpf program can allocate pages from that region via bpf_arena_alloc_pages(). This kernel function will insert pages into the kernel vm_area. The subsequent fault-in from user space will populate that page into the user vma. The BPF_F_SEGV_ON_FAULT flag at arena creation time can be used to prevent fault-in from user space. In such a case, if a page is not allocated by the bpf program and not present in the kernel vm_area, the user process will segfault. This is useful for use cases 2 and 3 above. bpf_arena_alloc_pages() is similar to user space mmap(). It allocates pages either at a specific address within the arena or allocates a range with the maple tree. bpf_arena_free_pages() is analogous to munmap(), which frees pages and removes the range from the kernel vm_area and from user process vmas. bpf_arena can be used as a bpf program "heap" of up to 4GB. The speed of bpf program is more important than ease of sharing with user space. This is use case 3. In such a case, the BPF_F_NO_USER_CONV flag is recommended. It will tell the verifier to treat the rX = bpf_arena_cast_user(rY) instruction as a 32-bit move wX = wY, which will improve bpf prog performance. Otherwise, bpf_arena_cast_user is translated by JIT to conditionally add the upper 32 bits of user vm_start (if the pointer is not NULL) to arena pointers before they are stored into memory. This way, user space sees them as valid 64-bit pointers. Diff https://github.com/llvm/llvm-project/pull/84410 enables LLVM BPF backend generate the bpf_addr_space_cast() instruction to cast pointers between address_space(1) which is reserved for bpf_arena pointers and default address space zero. All arena pointers in a bpf program written in C language are tagged as __attribute__((address_space(1))). Hence, clang provides helpful diagnostics when pointers cross address space. Libbpf and the kernel support only address_space == 1. All other address space identifiers are reserved. rX = bpf_addr_space_cast(rY, /* dst_as */ 1, /* src_as */ 0) tells the verifier that rX->type = PTR_TO_ARENA. Any further operations on PTR_TO_ARENA register have to be in the 32-bit domain. The verifier will mark load/store through PTR_TO_ARENA with PROBE_MEM32. JIT will generate them as kern_vm_start + 32bit_addr memory accesses. The behavior is similar to copy_from_kernel_nofault() except that no address checks are necessary. The address is guaranteed to be in the 4GB range. If the page is not present, the destination register is zeroed on read, and the operation is ignored on write. rX = bpf_addr_space_cast(rY, 0, 1) tells the verifier that rX->type = unknown scalar. If arena->map_flags has BPF_F_NO_USER_CONV set, then the verifier converts such cast instructions to mov32. Otherwise, JIT will emit native code equivalent to: rX = (u32)rY; if (rY) rX |= clear_lo32_bits(arena->user_vm_start); /* replace hi32 bits in rX */ After such conversion, the pointer becomes a valid user pointer within bpf_arena range. The user process can access data structures created in bpf_arena without any additional computations. For example, a linked list built by a bpf program can be walked natively by user space. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Reviewed-by: Barret Rhoden Link: https://lore.kernel.org/bpf/20240308010812.89848-2-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 7 +- include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 10 + kernel/bpf/Makefile | 3 + kernel/bpf/arena.c | 558 +++++++++++++++++++++++++++++++++++++++++ kernel/bpf/core.c | 11 + kernel/bpf/syscall.c | 36 +++ kernel/bpf/verifier.c | 1 + tools/include/uapi/linux/bpf.h | 10 + 9 files changed, 635 insertions(+), 2 deletions(-) create mode 100644 kernel/bpf/arena.c (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 95e07673cdc1..ea6ab6e0eef9 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -37,6 +37,7 @@ struct perf_event; struct bpf_prog; struct bpf_prog_aux; struct bpf_map; +struct bpf_arena; struct sock; struct seq_file; struct btf; @@ -528,8 +529,8 @@ void bpf_list_head_free(const struct btf_field *field, void *list_head, struct bpf_spin_lock *spin_lock); void bpf_rb_root_free(const struct btf_field *field, void *rb_root, struct bpf_spin_lock *spin_lock); - - +u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena); +u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena); int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); struct bpf_offload_dev; @@ -2215,6 +2216,8 @@ int generic_map_delete_batch(struct bpf_map *map, struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); +int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid, + unsigned long nr_pages, struct page **page_array); #ifdef CONFIG_MEMCG_KMEM void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, int node); diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 94baced5a1ad..9f2a6b83b49e 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -132,6 +132,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_RINGBUF, ringbuf_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_BLOOM_FILTER, bloom_filter_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_USER_RINGBUF, user_ringbuf_map_ops) +BPF_MAP_TYPE(BPF_MAP_TYPE_ARENA, arena_map_ops) BPF_LINK_TYPE(BPF_LINK_TYPE_RAW_TRACEPOINT, raw_tracepoint) BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING, tracing) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 85ec7fc799d7..e30d943db8a4 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1009,6 +1009,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + BPF_MAP_TYPE_ARENA, __MAX_BPF_MAP_TYPE }; @@ -1396,6 +1397,12 @@ enum { /* BPF token FD is passed in a corresponding command's token_fd field */ BPF_F_TOKEN_FD = (1U << 16), + +/* When user space page faults in bpf_arena send SIGSEGV instead of inserting new page */ + BPF_F_SEGV_ON_FAULT = (1U << 17), + +/* Do not translate kernel bpf_arena pointers to user pointers */ + BPF_F_NO_USER_CONV = (1U << 18), }; /* Flags for BPF_PROG_QUERY. */ @@ -1467,6 +1474,9 @@ union bpf_attr { * BPF_MAP_TYPE_BLOOM_FILTER - the lowest 4 bits indicate the * number of hash functions (if 0, the bloom filter will default * to using 5 hash functions). + * + * BPF_MAP_TYPE_ARENA - contains the address where user space + * is going to mmap() the arena. It has to be page aligned. */ __u64 map_extra; diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 4ce95acfcaa7..368c5d86b5b7 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -15,6 +15,9 @@ obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o obj-$(CONFIG_BPF_JIT) += trampoline.o obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o +ifeq ($(CONFIG_MMU)$(CONFIG_64BIT),yy) +obj-$(CONFIG_BPF_SYSCALL) += arena.o +endif obj-$(CONFIG_BPF_JIT) += dispatcher.o ifeq ($(CONFIG_NET),y) obj-$(CONFIG_BPF_SYSCALL) += devmap.o diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c new file mode 100644 index 000000000000..86571e760dd6 --- /dev/null +++ b/kernel/bpf/arena.c @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include +#include +#include +#include + +/* + * bpf_arena is a sparsely populated shared memory region between bpf program and + * user space process. + * + * For example on x86-64 the values could be: + * user_vm_start 7f7d26200000 // picked by mmap() + * kern_vm_start ffffc90001e69000 // picked by get_vm_area() + * For user space all pointers within the arena are normal 8-byte addresses. + * In this example 7f7d26200000 is the address of the first page (pgoff=0). + * The bpf program will access it as: kern_vm_start + lower_32bit_of_user_ptr + * (u32)7f7d26200000 -> 26200000 + * hence + * ffffc90001e69000 + 26200000 == ffffc90028069000 is "pgoff=0" within 4Gb + * kernel memory region. + * + * BPF JITs generate the following code to access arena: + * mov eax, eax // eax has lower 32-bit of user pointer + * mov word ptr [rax + r12 + off], bx + * where r12 == kern_vm_start and off is s16. + * Hence allocate 4Gb + GUARD_SZ/2 on each side. + * + * Initially kernel vm_area and user vma are not populated. + * User space can fault-in any address which will insert the page + * into kernel and user vma. + * bpf program can allocate a page via bpf_arena_alloc_pages() kfunc + * which will insert it into kernel vm_area. + * The later fault-in from user space will populate that page into user vma. + */ + +/* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */ +#define GUARD_SZ (1ull << sizeof(((struct bpf_insn *)0)->off) * 8) +#define KERN_VM_SZ ((1ull << 32) + GUARD_SZ) + +struct bpf_arena { + struct bpf_map map; + u64 user_vm_start; + u64 user_vm_end; + struct vm_struct *kern_vm; + struct maple_tree mt; + struct list_head vma_list; + struct mutex lock; +}; + +u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena) +{ + return arena ? (u64) (long) arena->kern_vm->addr + GUARD_SZ / 2 : 0; +} + +u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) +{ + return arena ? arena->user_vm_start : 0; +} + +static long arena_map_peek_elem(struct bpf_map *map, void *value) +{ + return -EOPNOTSUPP; +} + +static long arena_map_push_elem(struct bpf_map *map, void *value, u64 flags) +{ + return -EOPNOTSUPP; +} + +static long arena_map_pop_elem(struct bpf_map *map, void *value) +{ + return -EOPNOTSUPP; +} + +static long arena_map_delete_elem(struct bpf_map *map, void *value) +{ + return -EOPNOTSUPP; +} + +static int arena_map_get_next_key(struct bpf_map *map, void *key, void *next_key) +{ + return -EOPNOTSUPP; +} + +static long compute_pgoff(struct bpf_arena *arena, long uaddr) +{ + return (u32)(uaddr - (u32)arena->user_vm_start) >> PAGE_SHIFT; +} + +static struct bpf_map *arena_map_alloc(union bpf_attr *attr) +{ + struct vm_struct *kern_vm; + int numa_node = bpf_map_attr_numa_node(attr); + struct bpf_arena *arena; + u64 vm_range; + int err = -ENOMEM; + + if (attr->key_size || attr->value_size || attr->max_entries == 0 || + /* BPF_F_MMAPABLE must be set */ + !(attr->map_flags & BPF_F_MMAPABLE) || + /* No unsupported flags present */ + (attr->map_flags & ~(BPF_F_SEGV_ON_FAULT | BPF_F_MMAPABLE | BPF_F_NO_USER_CONV))) + return ERR_PTR(-EINVAL); + + if (attr->map_extra & ~PAGE_MASK) + /* If non-zero the map_extra is an expected user VMA start address */ + return ERR_PTR(-EINVAL); + + vm_range = (u64)attr->max_entries * PAGE_SIZE; + if (vm_range > (1ull << 32)) + return ERR_PTR(-E2BIG); + + if ((attr->map_extra >> 32) != ((attr->map_extra + vm_range - 1) >> 32)) + /* user vma must not cross 32-bit boundary */ + return ERR_PTR(-ERANGE); + + kern_vm = get_vm_area(KERN_VM_SZ, VM_SPARSE | VM_USERMAP); + if (!kern_vm) + return ERR_PTR(-ENOMEM); + + arena = bpf_map_area_alloc(sizeof(*arena), numa_node); + if (!arena) + goto err; + + arena->kern_vm = kern_vm; + arena->user_vm_start = attr->map_extra; + if (arena->user_vm_start) + arena->user_vm_end = arena->user_vm_start + vm_range; + + INIT_LIST_HEAD(&arena->vma_list); + bpf_map_init_from_attr(&arena->map, attr); + mt_init_flags(&arena->mt, MT_FLAGS_ALLOC_RANGE); + mutex_init(&arena->lock); + + return &arena->map; +err: + free_vm_area(kern_vm); + return ERR_PTR(err); +} + +static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data) +{ + struct page *page; + pte_t pte; + + pte = ptep_get(ptep); + if (!pte_present(pte)) /* sanity check */ + return 0; + page = pte_page(pte); + /* + * We do not update pte here: + * 1. Nobody should be accessing bpf_arena's range outside of a kernel bug + * 2. TLB flushing is batched or deferred. Even if we clear pte, + * the TLB entries can stick around and continue to permit access to + * the freed page. So it all relies on 1. + */ + __free_page(page); + return 0; +} + +static void arena_map_free(struct bpf_map *map) +{ + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + /* + * Check that user vma-s are not around when bpf map is freed. + * mmap() holds vm_file which holds bpf_map refcnt. + * munmap() must have happened on vma followed by arena_vm_close() + * which would clear arena->vma_list. + */ + if (WARN_ON_ONCE(!list_empty(&arena->vma_list))) + return; + + /* + * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). + * It unmaps everything from vmalloc area and clears pgtables. + * Call apply_to_existing_page_range() first to find populated ptes and + * free those pages. + */ + apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), + KERN_VM_SZ - GUARD_SZ, existing_page_cb, NULL); + free_vm_area(arena->kern_vm); + mtree_destroy(&arena->mt); + bpf_map_area_free(arena); +} + +static void *arena_map_lookup_elem(struct bpf_map *map, void *key) +{ + return ERR_PTR(-EINVAL); +} + +static long arena_map_update_elem(struct bpf_map *map, void *key, + void *value, u64 flags) +{ + return -EOPNOTSUPP; +} + +static int arena_map_check_btf(const struct bpf_map *map, const struct btf *btf, + const struct btf_type *key_type, const struct btf_type *value_type) +{ + return 0; +} + +static u64 arena_map_mem_usage(const struct bpf_map *map) +{ + return 0; +} + +struct vma_list { + struct vm_area_struct *vma; + struct list_head head; +}; + +static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) +{ + struct vma_list *vml; + + vml = kmalloc(sizeof(*vml), GFP_KERNEL); + if (!vml) + return -ENOMEM; + vma->vm_private_data = vml; + vml->vma = vma; + list_add(&vml->head, &arena->vma_list); + return 0; +} + +static void arena_vm_close(struct vm_area_struct *vma) +{ + struct bpf_map *map = vma->vm_file->private_data; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + struct vma_list *vml; + + guard(mutex)(&arena->lock); + vml = vma->vm_private_data; + list_del(&vml->head); + vma->vm_private_data = NULL; + kfree(vml); +} + +#define MT_ENTRY ((void *)&arena_map_ops) /* unused. has to be valid pointer */ + +static vm_fault_t arena_vm_fault(struct vm_fault *vmf) +{ + struct bpf_map *map = vmf->vma->vm_file->private_data; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + struct page *page; + long kbase, kaddr; + int ret; + + kbase = bpf_arena_get_kern_vm_start(arena); + kaddr = kbase + (u32)(vmf->address & PAGE_MASK); + + guard(mutex)(&arena->lock); + page = vmalloc_to_page((void *)kaddr); + if (page) + /* already have a page vmap-ed */ + goto out; + + if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) + /* User space requested to segfault when page is not allocated by bpf prog */ + return VM_FAULT_SIGSEGV; + + ret = mtree_insert(&arena->mt, vmf->pgoff, MT_ENTRY, GFP_KERNEL); + if (ret) + return VM_FAULT_SIGSEGV; + + /* Account into memcg of the process that created bpf_arena */ + ret = bpf_map_alloc_pages(map, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 1, &page); + if (ret) { + mtree_erase(&arena->mt, vmf->pgoff); + return VM_FAULT_SIGSEGV; + } + + ret = vm_area_map_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE, &page); + if (ret) { + mtree_erase(&arena->mt, vmf->pgoff); + __free_page(page); + return VM_FAULT_SIGSEGV; + } +out: + page_ref_add(page, 1); + vmf->page = page; + return 0; +} + +static const struct vm_operations_struct arena_vm_ops = { + .close = arena_vm_close, + .fault = arena_vm_fault, +}; + +static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + struct bpf_map *map = filp->private_data; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + long ret; + + if (pgoff) + return -EINVAL; + if (len > (1ull << 32)) + return -E2BIG; + + /* if user_vm_start was specified at arena creation time */ + if (arena->user_vm_start) { + if (len > arena->user_vm_end - arena->user_vm_start) + return -E2BIG; + if (len != arena->user_vm_end - arena->user_vm_start) + return -EINVAL; + if (addr != arena->user_vm_start) + return -EINVAL; + } + + ret = current->mm->get_unmapped_area(filp, addr, len * 2, 0, flags); + if (IS_ERR_VALUE(ret)) + return ret; + if ((ret >> 32) == ((ret + len - 1) >> 32)) + return ret; + if (WARN_ON_ONCE(arena->user_vm_start)) + /* checks at map creation time should prevent this */ + return -EFAULT; + return round_up(ret, 1ull << 32); +} + +static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) +{ + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + guard(mutex)(&arena->lock); + if (arena->user_vm_start && arena->user_vm_start != vma->vm_start) + /* + * If map_extra was not specified at arena creation time then + * 1st user process can do mmap(NULL, ...) to pick user_vm_start + * 2nd user process must pass the same addr to mmap(addr, MAP_FIXED..); + * or + * specify addr in map_extra and + * use the same addr later with mmap(addr, MAP_FIXED..); + */ + return -EBUSY; + + if (arena->user_vm_end && arena->user_vm_end != vma->vm_end) + /* all user processes must have the same size of mmap-ed region */ + return -EBUSY; + + /* Earlier checks should prevent this */ + if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > (1ull << 32) || vma->vm_pgoff)) + return -EFAULT; + + if (remember_vma(arena, vma)) + return -ENOMEM; + + arena->user_vm_start = vma->vm_start; + arena->user_vm_end = vma->vm_end; + /* + * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and + * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid + * potential change of user_vm_start. + */ + vm_flags_set(vma, VM_DONTEXPAND); + vma->vm_ops = &arena_vm_ops; + return 0; +} + +static int arena_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off) +{ + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + if ((u64)off > arena->user_vm_end - arena->user_vm_start) + return -ERANGE; + *imm = (unsigned long)arena->user_vm_start; + return 0; +} + +BTF_ID_LIST_SINGLE(bpf_arena_map_btf_ids, struct, bpf_arena) +const struct bpf_map_ops arena_map_ops = { + .map_meta_equal = bpf_map_meta_equal, + .map_alloc = arena_map_alloc, + .map_free = arena_map_free, + .map_direct_value_addr = arena_map_direct_value_addr, + .map_mmap = arena_map_mmap, + .map_get_unmapped_area = arena_get_unmapped_area, + .map_get_next_key = arena_map_get_next_key, + .map_push_elem = arena_map_push_elem, + .map_peek_elem = arena_map_peek_elem, + .map_pop_elem = arena_map_pop_elem, + .map_lookup_elem = arena_map_lookup_elem, + .map_update_elem = arena_map_update_elem, + .map_delete_elem = arena_map_delete_elem, + .map_check_btf = arena_map_check_btf, + .map_mem_usage = arena_map_mem_usage, + .map_btf_id = &bpf_arena_map_btf_ids[0], +}; + +static u64 clear_lo32(u64 val) +{ + return val & ~(u64)~0U; +} + +/* + * Allocate pages and vmap them into kernel vmalloc area. + * Later the pages will be mmaped into user space vma. + */ +static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt, int node_id) +{ + /* user_vm_end/start are fixed before bpf prog runs */ + long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >> PAGE_SHIFT; + u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena); + struct page **pages; + long pgoff = 0; + u32 uaddr32; + int ret, i; + + if (page_cnt > page_cnt_max) + return 0; + + if (uaddr) { + if (uaddr & ~PAGE_MASK) + return 0; + pgoff = compute_pgoff(arena, uaddr); + if (pgoff + page_cnt > page_cnt_max) + /* requested address will be outside of user VMA */ + return 0; + } + + /* zeroing is needed, since alloc_pages_bulk_array() only fills in non-zero entries */ + pages = kvcalloc(page_cnt, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return 0; + + guard(mutex)(&arena->lock); + + if (uaddr) + ret = mtree_insert_range(&arena->mt, pgoff, pgoff + page_cnt - 1, + MT_ENTRY, GFP_KERNEL); + else + ret = mtree_alloc_range(&arena->mt, &pgoff, MT_ENTRY, + page_cnt, 0, page_cnt_max - 1, GFP_KERNEL); + if (ret) + goto out_free_pages; + + ret = bpf_map_alloc_pages(&arena->map, GFP_KERNEL | __GFP_ZERO, + node_id, page_cnt, pages); + if (ret) + goto out; + + uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); + /* Earlier checks make sure that uaddr32 + page_cnt * PAGE_SIZE will not overflow 32-bit */ + ret = vm_area_map_pages(arena->kern_vm, kern_vm_start + uaddr32, + kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE, pages); + if (ret) { + for (i = 0; i < page_cnt; i++) + __free_page(pages[i]); + goto out; + } + kvfree(pages); + return clear_lo32(arena->user_vm_start) + uaddr32; +out: + mtree_erase(&arena->mt, pgoff); +out_free_pages: + kvfree(pages); + return 0; +} + +/* + * If page is present in vmalloc area, unmap it from vmalloc area, + * unmap it from all user space vma-s, + * and free it. + */ +static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) +{ + struct vma_list *vml; + + list_for_each_entry(vml, &arena->vma_list, head) + zap_page_range_single(vml->vma, uaddr, + PAGE_SIZE * page_cnt, NULL); +} + +static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt) +{ + u64 full_uaddr, uaddr_end; + long kaddr, pgoff, i; + struct page *page; + + /* only aligned lower 32-bit are relevant */ + uaddr = (u32)uaddr; + uaddr &= PAGE_MASK; + full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); + if (full_uaddr >= uaddr_end) + return; + + page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; + + guard(mutex)(&arena->lock); + + pgoff = compute_pgoff(arena, uaddr); + /* clear range */ + mtree_store_range(&arena->mt, pgoff, pgoff + page_cnt - 1, NULL, GFP_KERNEL); + + if (page_cnt > 1) + /* bulk zap if multiple pages being freed */ + zap_pages(arena, full_uaddr, page_cnt); + + kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; + for (i = 0; i < page_cnt; i++, kaddr += PAGE_SIZE, full_uaddr += PAGE_SIZE) { + page = vmalloc_to_page((void *)kaddr); + if (!page) + continue; + if (page_cnt == 1 && page_mapped(page)) /* mapped by some user process */ + zap_pages(arena, full_uaddr, 1); + vm_area_unmap_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE); + __free_page(page); + } +} + +__bpf_kfunc_start_defs(); + +__bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__ign, u32 page_cnt, + int node_id, u64 flags) +{ + struct bpf_map *map = p__map; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + if (map->map_type != BPF_MAP_TYPE_ARENA || flags || !page_cnt) + return NULL; + + return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id); +} + +__bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt) +{ + struct bpf_map *map = p__map; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt || !ptr__ign) + return; + arena_free_pages(arena, (long)ptr__ign, page_cnt); +} +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(arena_kfuncs) +BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE) +BTF_KFUNCS_END(arena_kfuncs) + +static const struct btf_kfunc_id_set common_kfunc_set = { + .owner = THIS_MODULE, + .set = &arena_kfuncs, +}; + +static int __init kfunc_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &common_kfunc_set); +} +late_initcall(kfunc_init); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 134b7979f537..a8ecf69c7754 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2976,6 +2976,17 @@ void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, { } +/* for configs without MMU or 32-bit */ +__weak const struct bpf_map_ops arena_map_ops; +__weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) +{ + return 0; +} +__weak u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena) +{ + return 0; +} + #ifdef CONFIG_BPF_SYSCALL static int __init bpf_global_ma_init(void) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f63f4da4db5e..67923e41a07e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -164,6 +164,7 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, if (bpf_map_is_offloaded(map)) { return bpf_map_offload_update_elem(map, key, value, flags); } else if (map->map_type == BPF_MAP_TYPE_CPUMAP || + map->map_type == BPF_MAP_TYPE_ARENA || map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { return map->ops->map_update_elem(map, key, value, flags); } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH || @@ -479,6 +480,39 @@ static void bpf_map_release_memcg(struct bpf_map *map) } #endif +int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid, + unsigned long nr_pages, struct page **pages) +{ + unsigned long i, j; + struct page *pg; + int ret = 0; +#ifdef CONFIG_MEMCG_KMEM + struct mem_cgroup *memcg, *old_memcg; + + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); +#endif + for (i = 0; i < nr_pages; i++) { + pg = alloc_pages_node(nid, gfp | __GFP_ACCOUNT, 0); + + if (pg) { + pages[i] = pg; + continue; + } + for (j = 0; j < i; j++) + __free_page(pages[j]); + ret = -ENOMEM; + break; + } + +#ifdef CONFIG_MEMCG_KMEM + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); +#endif + return ret; +} + + static int btf_field_cmp(const void *a, const void *b) { const struct btf_field *f1 = a, *f2 = b; @@ -1176,6 +1210,7 @@ static int map_create(union bpf_attr *attr) } if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER && + attr->map_type != BPF_MAP_TYPE_ARENA && attr->map_extra != 0) return -EINVAL; @@ -1265,6 +1300,7 @@ static int map_create(union bpf_attr *attr) case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_STRUCT_OPS: case BPF_MAP_TYPE_CPUMAP: + case BPF_MAP_TYPE_ARENA: if (!bpf_token_capable(token, CAP_BPF)) goto put_token; break; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bf084c693507..fbcf2e5e635a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18108,6 +18108,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, case BPF_MAP_TYPE_CGRP_STORAGE: case BPF_MAP_TYPE_QUEUE: case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_ARENA: break; default: verbose(env, diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 85ec7fc799d7..e30d943db8a4 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1009,6 +1009,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + BPF_MAP_TYPE_ARENA, __MAX_BPF_MAP_TYPE }; @@ -1396,6 +1397,12 @@ enum { /* BPF token FD is passed in a corresponding command's token_fd field */ BPF_F_TOKEN_FD = (1U << 16), + +/* When user space page faults in bpf_arena send SIGSEGV instead of inserting new page */ + BPF_F_SEGV_ON_FAULT = (1U << 17), + +/* Do not translate kernel bpf_arena pointers to user pointers */ + BPF_F_NO_USER_CONV = (1U << 18), }; /* Flags for BPF_PROG_QUERY. */ @@ -1467,6 +1474,9 @@ union bpf_attr { * BPF_MAP_TYPE_BLOOM_FILTER - the lowest 4 bits indicate the * number of hash functions (if 0, the bloom filter will default * to using 5 hash functions). + * + * BPF_MAP_TYPE_ARENA - contains the address where user space + * is going to mmap() the arena. It has to be page aligned. */ __u64 map_extra; -- cgit v1.2.3 From 2fe99eb0ccf2bb73df65ebcbbf2f2ff70e63547b Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 7 Mar 2024 17:08:01 -0800 Subject: bpf: Add x86-64 JIT support for PROBE_MEM32 pseudo instructions. Add support for [LDX | STX | ST], PROBE_MEM32, [B | H | W | DW] instructions. They are similar to PROBE_MEM instructions with the following differences: - PROBE_MEM has to check that the address is in the kernel range with src_reg + insn->off >= TASK_SIZE_MAX + PAGE_SIZE check - PROBE_MEM doesn't support store - PROBE_MEM32 relies on the verifier to clear upper 32-bit in the register - PROBE_MEM32 adds 64-bit kern_vm_start address (which is stored in %r12 in the prologue) Due to bpf_arena constructions such %r12 + %reg + off16 access is guaranteed to be within arena virtual range, so no address check at run-time. - PROBE_MEM32 allows STX and ST. If they fault the store is a nop. When LDX faults the destination register is zeroed. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20240308010812.89848-4-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 191 +++++++++++++++++++++++++++++++++++++++++++- include/linux/bpf.h | 1 + include/linux/filter.h | 3 + 3 files changed, 194 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e1390d1e331b..38705a1abe62 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -113,6 +113,7 @@ static int bpf_size_to_x86_bytes(int bpf_size) /* Pick a register outside of BPF range for JIT internal work */ #define AUX_REG (MAX_BPF_JIT_REG + 1) #define X86_REG_R9 (MAX_BPF_JIT_REG + 2) +#define X86_REG_R12 (MAX_BPF_JIT_REG + 3) /* * The following table maps BPF registers to x86-64 registers. @@ -139,6 +140,7 @@ static const int reg2hex[] = { [BPF_REG_AX] = 2, /* R10 temp register */ [AUX_REG] = 3, /* R11 temp register */ [X86_REG_R9] = 1, /* R9 register, 6th function argument */ + [X86_REG_R12] = 4, /* R12 callee saved */ }; static const int reg2pt_regs[] = { @@ -167,6 +169,7 @@ static bool is_ereg(u32 reg) BIT(BPF_REG_8) | BIT(BPF_REG_9) | BIT(X86_REG_R9) | + BIT(X86_REG_R12) | BIT(BPF_REG_AX)); } @@ -205,6 +208,17 @@ static u8 add_2mod(u8 byte, u32 r1, u32 r2) return byte; } +static u8 add_3mod(u8 byte, u32 r1, u32 r2, u32 index) +{ + if (is_ereg(r1)) + byte |= 1; + if (is_ereg(index)) + byte |= 2; + if (is_ereg(r2)) + byte |= 4; + return byte; +} + /* Encode 'dst_reg' register into x86-64 opcode 'byte' */ static u8 add_1reg(u8 byte, u32 dst_reg) { @@ -645,6 +659,8 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); + if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + pop_r12(&prog); } EMIT1(0x58); /* pop rax */ @@ -704,6 +720,8 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); + if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + pop_r12(&prog); } EMIT1(0x58); /* pop rax */ @@ -887,6 +905,18 @@ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) *pprog = prog; } +static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off) +{ + u8 *prog = *pprog; + + if (is_imm8(off)) { + EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + } else { + EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + } + *pprog = prog; +} + /* * Emit a REX byte if it will be necessary to address these registers */ @@ -968,6 +998,37 @@ static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) *pprog = prog; } +static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off) +{ + u8 *prog = *pprog; + + switch (size) { + case BPF_B: + /* movzx rax, byte ptr [rax + r12 + off] */ + EMIT3(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x0F, 0xB6); + break; + case BPF_H: + /* movzx rax, word ptr [rax + r12 + off] */ + EMIT3(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x0F, 0xB7); + break; + case BPF_W: + /* mov eax, dword ptr [rax + r12 + off] */ + EMIT2(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x8B); + break; + case BPF_DW: + /* mov rax, qword ptr [rax + r12 + off] */ + EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x8B); + break; + } + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + *pprog = prog; +} + +static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) +{ + emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off); +} + /* STX: *(u8*)(dst_reg + off) = src_reg */ static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -1002,6 +1063,71 @@ static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) *pprog = prog; } +/* STX: *(u8*)(dst_reg + index_reg + off) = src_reg */ +static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off) +{ + u8 *prog = *pprog; + + switch (size) { + case BPF_B: + /* mov byte ptr [rax + r12 + off], al */ + EMIT2(add_3mod(0x40, dst_reg, src_reg, index_reg), 0x88); + break; + case BPF_H: + /* mov word ptr [rax + r12 + off], ax */ + EMIT3(0x66, add_3mod(0x40, dst_reg, src_reg, index_reg), 0x89); + break; + case BPF_W: + /* mov dword ptr [rax + r12 + 1], eax */ + EMIT2(add_3mod(0x40, dst_reg, src_reg, index_reg), 0x89); + break; + case BPF_DW: + /* mov qword ptr [rax + r12 + 1], rax */ + EMIT2(add_3mod(0x48, dst_reg, src_reg, index_reg), 0x89); + break; + } + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + *pprog = prog; +} + +static void emit_stx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) +{ + emit_stx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off); +} + +/* ST: *(u8*)(dst_reg + index_reg + off) = imm32 */ +static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int off, int imm) +{ + u8 *prog = *pprog; + + switch (size) { + case BPF_B: + /* mov byte ptr [rax + r12 + off], imm8 */ + EMIT2(add_3mod(0x40, dst_reg, 0, index_reg), 0xC6); + break; + case BPF_H: + /* mov word ptr [rax + r12 + off], imm16 */ + EMIT3(0x66, add_3mod(0x40, dst_reg, 0, index_reg), 0xC7); + break; + case BPF_W: + /* mov dword ptr [rax + r12 + 1], imm32 */ + EMIT2(add_3mod(0x40, dst_reg, 0, index_reg), 0xC7); + break; + case BPF_DW: + /* mov qword ptr [rax + r12 + 1], imm32 */ + EMIT2(add_3mod(0x48, dst_reg, 0, index_reg), 0xC7); + break; + } + emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off); + EMIT(imm, bpf_size_to_x86_bytes(size)); + *pprog = prog; +} + +static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) +{ + emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); +} + static int emit_atomic(u8 **pprog, u8 atomic_op, u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size) { @@ -1043,12 +1169,15 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, return 0; } +#define DONT_CLEAR 1 + bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) { u32 reg = x->fixup >> 8; /* jump over faulting load and clear dest register */ - *(unsigned long *)((void *)regs + reg) = 0; + if (reg != DONT_CLEAR) + *(unsigned long *)((void *)regs + reg) = 0; regs->ip += x->fixup & 0xff; return true; } @@ -1147,11 +1276,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image bool tail_call_seen = false; bool seen_exit = false; u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; + u64 arena_vm_start; int i, excnt = 0; int ilen, proglen = 0; u8 *prog = temp; int err; + arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); + detect_reg_usage(insn, insn_cnt, callee_regs_used, &tail_call_seen); @@ -1172,8 +1304,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image push_r12(&prog); push_callee_regs(&prog, all_callee_regs_used); } else { + if (arena_vm_start) + push_r12(&prog); push_callee_regs(&prog, callee_regs_used); } + if (arena_vm_start) + emit_mov_imm64(&prog, X86_REG_R12, + arena_vm_start >> 32, (u32) arena_vm_start); ilen = prog - temp; if (rw_image) @@ -1564,6 +1701,56 @@ st: if (is_imm8(insn->off)) emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); break; + case BPF_ST | BPF_PROBE_MEM32 | BPF_B: + case BPF_ST | BPF_PROBE_MEM32 | BPF_H: + case BPF_ST | BPF_PROBE_MEM32 | BPF_W: + case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: + start_of_ldx = prog; + emit_st_r12(&prog, BPF_SIZE(insn->code), dst_reg, insn->off, insn->imm); + goto populate_extable; + + /* LDX: dst_reg = *(u8*)(src_reg + r12 + off) */ + case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + case BPF_STX | BPF_PROBE_MEM32 | BPF_B: + case BPF_STX | BPF_PROBE_MEM32 | BPF_H: + case BPF_STX | BPF_PROBE_MEM32 | BPF_W: + case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: + start_of_ldx = prog; + if (BPF_CLASS(insn->code) == BPF_LDX) + emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); + else + emit_stx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); +populate_extable: + { + struct exception_table_entry *ex; + u8 *_insn = image + proglen + (start_of_ldx - temp); + s64 delta; + + if (!bpf_prog->aux->extable) + break; + + if (excnt >= bpf_prog->aux->num_exentries) { + pr_err("mem32 extable bug\n"); + return -EFAULT; + } + ex = &bpf_prog->aux->extable[excnt++]; + + delta = _insn - (u8 *)&ex->insn; + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + + ex->insn = delta; + + ex->data = EX_TYPE_BPF; + + ex->fixup = (prog - start_of_ldx) | + ((BPF_CLASS(insn->code) == BPF_LDX ? reg2pt_regs[dst_reg] : DONT_CLEAR) << 8); + } + break; + /* LDX: dst_reg = *(u8*)(src_reg + off) */ case BPF_LDX | BPF_MEM | BPF_B: case BPF_LDX | BPF_PROBE_MEM | BPF_B: @@ -2036,6 +2223,8 @@ emit_jmp: pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); + if (arena_vm_start) + pop_r12(&prog); } EMIT1(0xC9); /* leave */ emit_return(&prog, image + addrs[i - 1] + (prog - temp)); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ea6ab6e0eef9..8904d1606125 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1458,6 +1458,7 @@ struct bpf_prog_aux { bool xdp_has_frags; bool exception_cb; bool exception_boundary; + struct bpf_arena *arena; /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto; /* function name for valid attach_btf_id */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 36cc29a2934c..b119f04ecb0b 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -72,6 +72,9 @@ struct ctl_table_header; /* unused opcode to mark special ldsx instruction. Same as BPF_IND */ #define BPF_PROBE_MEMSX 0x40 +/* unused opcode to mark special load instruction. Same as BPF_MSH */ +#define BPF_PROBE_MEM32 0xa0 + /* unused opcode to mark call to interpreter with arguments */ #define BPF_CALL_ARGS 0xe0 -- cgit v1.2.3 From 142fd4d2dcf58b1720a6af644f31de1a5551f219 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 7 Mar 2024 17:08:02 -0800 Subject: bpf: Add x86-64 JIT support for bpf_addr_space_cast instruction. LLVM generates bpf_addr_space_cast instruction while translating pointers between native (zero) address space and __attribute__((address_space(N))). The addr_space=1 is reserved as bpf_arena address space. rY = addr_space_cast(rX, 0, 1) is processed by the verifier and converted to normal 32-bit move: wX = wY rY = addr_space_cast(rX, 1, 0) has to be converted by JIT: aux_reg = upper_32_bits of arena->user_vm_start aux_reg <<= 32 wX = wY // clear upper 32 bits of dst register if (wX) // if not zero add upper bits of user_vm_start wX |= aux_reg JIT can do it more efficiently: mov dst_reg32, src_reg32 // 32-bit move shl dst_reg, 32 or dst_reg, user_vm_start rol dst_reg, 32 xor r11, r11 test dst_reg32, dst_reg32 // check if lower 32-bit are zero cmove r11, dst_reg // if so, set dst_reg to zero // Intel swapped src/dst register encoding in CMOVcc Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20240308010812.89848-5-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 42 +++++++++++++++++++++++++++++++++++++++++- include/linux/filter.h | 1 + kernel/bpf/core.c | 5 +++++ 3 files changed, 47 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 38705a1abe62..27058d7395f6 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1276,13 +1276,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image bool tail_call_seen = false; bool seen_exit = false; u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; - u64 arena_vm_start; + u64 arena_vm_start, user_vm_start; int i, excnt = 0; int ilen, proglen = 0; u8 *prog = temp; int err; arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); + user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); detect_reg_usage(insn, insn_cnt, callee_regs_used, &tail_call_seen); @@ -1350,6 +1351,40 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image break; case BPF_ALU64 | BPF_MOV | BPF_X: + if (insn->off == BPF_ADDR_SPACE_CAST && + insn->imm == 1U << 16) { + if (dst_reg != src_reg) + /* 32-bit mov */ + emit_mov_reg(&prog, false, dst_reg, src_reg); + /* shl dst_reg, 32 */ + maybe_emit_1mod(&prog, dst_reg, true); + EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32); + + /* or dst_reg, user_vm_start */ + maybe_emit_1mod(&prog, dst_reg, true); + if (is_axreg(dst_reg)) + EMIT1_off32(0x0D, user_vm_start >> 32); + else + EMIT2_off32(0x81, add_1reg(0xC8, dst_reg), user_vm_start >> 32); + + /* rol dst_reg, 32 */ + maybe_emit_1mod(&prog, dst_reg, true); + EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32); + + /* xor r11, r11 */ + EMIT3(0x4D, 0x31, 0xDB); + + /* test dst_reg32, dst_reg32; check if lower 32-bit are zero */ + maybe_emit_mod(&prog, dst_reg, dst_reg, false); + EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); + + /* cmove r11, dst_reg; if so, set dst_reg to zero */ + /* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */ + maybe_emit_mod(&prog, AUX_REG, dst_reg, true); + EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg)); + break; + } + fallthrough; case BPF_ALU | BPF_MOV | BPF_X: if (insn->off == 0) emit_mov_reg(&prog, @@ -3432,6 +3467,11 @@ void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke, } } +bool bpf_jit_supports_arena(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; diff --git a/include/linux/filter.h b/include/linux/filter.h index b119f04ecb0b..c99bc3df2d28 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -962,6 +962,7 @@ bool bpf_jit_supports_kfunc_call(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); +bool bpf_jit_supports_arena(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); bool bpf_helper_changes_pkt_data(void *func); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a8ecf69c7754..bdbdc75cdcd5 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2932,6 +2932,11 @@ bool __weak bpf_jit_supports_far_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_arena(void) +{ + return false; +} + /* Return TRUE if the JIT backend satisfies the following two conditions: * 1) JIT backend supports atomic_xchg() on pointer-sized words. * 2) Under the specific arch, the implementation of xchg() is the same -- cgit v1.2.3 From 6082b6c328b5486da2b356eae94b8b83c98b5565 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 7 Mar 2024 17:08:03 -0800 Subject: bpf: Recognize addr_space_cast instruction in the verifier. rY = addr_space_cast(rX, 0, 1) tells the verifier that rY->type = PTR_TO_ARENA. Any further operations on PTR_TO_ARENA register have to be in 32-bit domain. The verifier will mark load/store through PTR_TO_ARENA with PROBE_MEM32. JIT will generate them as kern_vm_start + 32bit_addr memory accesses. rY = addr_space_cast(rX, 1, 0) tells the verifier that rY->type = unknown scalar. If arena->map_flags has BPF_F_NO_USER_CONV set then convert cast_user to mov32 as well. Otherwise JIT will convert it to: rY = (u32)rX; if (rY) rY |= arena->user_vm_start & ~(u64)~0U; Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20240308010812.89848-6-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 1 + include/linux/bpf_verifier.h | 1 + kernel/bpf/log.c | 3 ++ kernel/bpf/syscall.c | 6 +++ kernel/bpf/verifier.c | 107 +++++++++++++++++++++++++++++++++++++++---- 5 files changed, 109 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 8904d1606125..d0c836ba009d 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -883,6 +883,7 @@ enum bpf_reg_type { * an explicit null check is required for this struct. */ PTR_TO_MEM, /* reg points to valid memory region */ + PTR_TO_ARENA, PTR_TO_BUF, /* reg points to a read/write buffer */ PTR_TO_FUNC, /* reg points to a bpf program function */ CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 4b0f6600e499..7cb1b75eee38 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -548,6 +548,7 @@ struct bpf_insn_aux_data { u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ bool sanitize_stack_spill; /* subject to Spectre v4 sanitation */ bool zext_dst; /* this insn zero extends dst reg */ + bool needs_zext; /* alu op needs to clear upper bits */ bool storage_get_func_atomic; /* bpf_*_storage_get() with atomic memory alloc */ bool is_iter_next; /* bpf_iter__next() kfunc call */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 63c34e7b0715..2a243cf37c60 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -458,6 +458,7 @@ const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type) [PTR_TO_XDP_SOCK] = "xdp_sock", [PTR_TO_BTF_ID] = "ptr_", [PTR_TO_MEM] = "mem", + [PTR_TO_ARENA] = "arena", [PTR_TO_BUF] = "buf", [PTR_TO_FUNC] = "func", [PTR_TO_MAP_KEY] = "map_key", @@ -693,6 +694,8 @@ static void print_reg_state(struct bpf_verifier_env *env, } verbose(env, "%s", reg_type_str(env, t)); + if (t == PTR_TO_ARENA) + return; if (t == PTR_TO_STACK) { if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 67923e41a07e..07f2a4db4511 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4453,6 +4453,12 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, continue; } + if ((BPF_CLASS(code) == BPF_LDX || BPF_CLASS(code) == BPF_STX || + BPF_CLASS(code) == BPF_ST) && BPF_MODE(code) == BPF_PROBE_MEM32) { + insns[i].code = BPF_CLASS(code) | BPF_SIZE(code) | BPF_MEM; + continue; + } + if (code != (BPF_LD | BPF_IMM | BPF_DW)) continue; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fbcf2e5e635a..1358e20d315a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4386,6 +4386,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) case PTR_TO_MEM: case PTR_TO_FUNC: case PTR_TO_MAP_KEY: + case PTR_TO_ARENA: return true; default: return false; @@ -5828,6 +5829,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, case PTR_TO_XDP_SOCK: pointer_desc = "xdp_sock "; break; + case PTR_TO_ARENA: + return 0; default: break; } @@ -6937,6 +6940,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); + } else if (reg->type == PTR_TO_ARENA) { + if (t == BPF_READ && value_regno >= 0) + mark_reg_unknown(env, regs, value_regno); } else { verbose(env, "R%d invalid mem access '%s'\n", regno, reg_type_str(env, reg->type)); @@ -8408,6 +8414,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_MEM | MEM_RINGBUF: case PTR_TO_BUF: case PTR_TO_BUF | MEM_RDONLY: + case PTR_TO_ARENA: case SCALAR_VALUE: return 0; /* All the rest must be rejected, except PTR_TO_BTF_ID which allows @@ -13852,6 +13859,21 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, dst_reg = ®s[insn->dst_reg]; src_reg = NULL; + + if (dst_reg->type == PTR_TO_ARENA) { + struct bpf_insn_aux_data *aux = cur_aux(env); + + if (BPF_CLASS(insn->code) == BPF_ALU64) + /* + * 32-bit operations zero upper bits automatically. + * 64-bit operations need to be converted to 32. + */ + aux->needs_zext = true; + + /* Any arithmetic operations are allowed on arena pointers */ + return 0; + } + if (dst_reg->type != SCALAR_VALUE) ptr_reg = dst_reg; else @@ -13969,19 +13991,20 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) } else if (opcode == BPF_MOV) { if (BPF_SRC(insn->code) == BPF_X) { - if (insn->imm != 0) { - verbose(env, "BPF_MOV uses reserved fields\n"); - return -EINVAL; - } - if (BPF_CLASS(insn->code) == BPF_ALU) { - if (insn->off != 0 && insn->off != 8 && insn->off != 16) { + if ((insn->off != 0 && insn->off != 8 && insn->off != 16) || + insn->imm) { verbose(env, "BPF_MOV uses reserved fields\n"); return -EINVAL; } + } else if (insn->off == BPF_ADDR_SPACE_CAST) { + if (insn->imm != 1 && insn->imm != 1u << 16) { + verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n"); + return -EINVAL; + } } else { - if (insn->off != 0 && insn->off != 8 && insn->off != 16 && - insn->off != 32) { + if ((insn->off != 0 && insn->off != 8 && insn->off != 16 && + insn->off != 32) || insn->imm) { verbose(env, "BPF_MOV uses reserved fields\n"); return -EINVAL; } @@ -14008,7 +14031,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) struct bpf_reg_state *dst_reg = regs + insn->dst_reg; if (BPF_CLASS(insn->code) == BPF_ALU64) { - if (insn->off == 0) { + if (insn->imm) { + /* off == BPF_ADDR_SPACE_CAST */ + mark_reg_unknown(env, regs, insn->dst_reg); + if (insn->imm == 1) /* cast from as(1) to as(0) */ + dst_reg->type = PTR_TO_ARENA; + } else if (insn->off == 0) { /* case: R1 = R2 * copy register state to dest reg */ @@ -15182,6 +15210,10 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { + if (map->map_type == BPF_MAP_TYPE_ARENA) { + __mark_reg_unknown(env, dst_reg); + return 0; + } dst_reg->type = PTR_TO_MAP_VALUE; dst_reg->off = aux->map_off; WARN_ON_ONCE(map->max_entries != 1); @@ -16568,6 +16600,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, * the same stack frame, since fp-8 in foo != fp-8 in bar */ return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; + case PTR_TO_ARENA: + return true; default: return regs_exact(rold, rcur, idmap); } @@ -17443,6 +17477,7 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) case PTR_TO_TCP_SOCK: case PTR_TO_XDP_SOCK: case PTR_TO_BTF_ID: + case PTR_TO_ARENA: return false; default: return true; @@ -18296,6 +18331,31 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) fdput(f); return -EBUSY; } + if (map->map_type == BPF_MAP_TYPE_ARENA) { + if (env->prog->aux->arena) { + verbose(env, "Only one arena per program\n"); + fdput(f); + return -EBUSY; + } + if (!env->allow_ptr_leaks || !env->bpf_capable) { + verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); + fdput(f); + return -EPERM; + } + if (!env->prog->jit_requested) { + verbose(env, "JIT is required to use arena\n"); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_arena()) { + verbose(env, "JIT doesn't support arena\n"); + return -EOPNOTSUPP; + } + env->prog->aux->arena = (void *)map; + if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { + verbose(env, "arena's user address must be set via map_extra or mmap()\n"); + return -EINVAL; + } + } fdput(f); next_insn: @@ -18917,6 +18977,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) env->prog->aux->num_exentries++; } continue; + case PTR_TO_ARENA: + if (BPF_MODE(insn->code) == BPF_MEMSX) { + verbose(env, "sign extending loads from arena are not supported yet\n"); + return -EOPNOTSUPP; + } + insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); + env->prog->aux->num_exentries++; + continue; default: continue; } @@ -19102,13 +19170,19 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->nr_linfo = prog->aux->nr_linfo; func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; + func[i]->aux->arena = prog->aux->arena; num_exentries = 0; insn = func[i]->insnsi; for (j = 0; j < func[i]->len; j++, insn++) { if (BPF_CLASS(insn->code) == BPF_LDX && (BPF_MODE(insn->code) == BPF_PROBE_MEM || + BPF_MODE(insn->code) == BPF_PROBE_MEM32 || BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) num_exentries++; + if ((BPF_CLASS(insn->code) == BPF_STX || + BPF_CLASS(insn->code) == BPF_ST) && + BPF_MODE(insn->code) == BPF_PROBE_MEM32) + num_exentries++; } func[i]->aux->num_exentries = num_exentries; func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; @@ -19507,6 +19581,21 @@ static int do_misc_fixups(struct bpf_verifier_env *env) } for (i = 0; i < insn_cnt;) { + if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) { + if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) || + (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { + /* convert to 32-bit mov that clears upper 32-bit */ + insn->code = BPF_ALU | BPF_MOV | BPF_X; + /* clear off, so it's a normal 'wX = wY' from JIT pov */ + insn->off = 0; + } /* cast from as(0) to as(1) should be handled by JIT */ + goto next_insn; + } + + if (env->insn_aux_data[i + delta].needs_zext) + /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ + insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); + /* Make divide-by-zero exceptions impossible. */ if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || -- cgit v1.2.3 From 2edc3de6fb650924a87fffebebc3b7572cbf6e38 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 7 Mar 2024 17:08:04 -0800 Subject: bpf: Recognize btf_decl_tag("arg: Arena") as PTR_TO_ARENA. In global bpf functions recognize btf_decl_tag("arg:arena") as PTR_TO_ARENA. Note, when the verifier sees: __weak void foo(struct bar *p) it recognizes 'p' as PTR_TO_MEM and 'struct bar' has to be a struct with scalars. Hence the only way to use arena pointers in global functions is to tag them with "arg:arena". Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20240308010812.89848-7-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 1 + kernel/bpf/btf.c | 19 +++++++++++++++---- kernel/bpf/verifier.c | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d0c836ba009d..08ad265cb195 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -712,6 +712,7 @@ enum bpf_arg_type { * on eBPF program stack */ ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ + ARG_PTR_TO_ARENA, ARG_CONST_SIZE, /* number of bytes accessed from memory */ ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 170d017e8e4a..90c4a32d89ff 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7111,10 +7111,11 @@ cand_cache_unlock: } enum btf_arg_tag { - ARG_TAG_CTX = 0x1, - ARG_TAG_NONNULL = 0x2, - ARG_TAG_TRUSTED = 0x4, - ARG_TAG_NULLABLE = 0x8, + ARG_TAG_CTX = BIT_ULL(0), + ARG_TAG_NONNULL = BIT_ULL(1), + ARG_TAG_TRUSTED = BIT_ULL(2), + ARG_TAG_NULLABLE = BIT_ULL(3), + ARG_TAG_ARENA = BIT_ULL(4), }; /* Process BTF of a function to produce high-level expectation of function @@ -7226,6 +7227,8 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) tags |= ARG_TAG_NONNULL; } else if (strcmp(tag, "nullable") == 0) { tags |= ARG_TAG_NULLABLE; + } else if (strcmp(tag, "arena") == 0) { + tags |= ARG_TAG_ARENA; } else { bpf_log(log, "arg#%d has unsupported set of tags\n", i); return -EOPNOTSUPP; @@ -7280,6 +7283,14 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) sub->args[i].btf_id = kern_type_id; continue; } + if (tags & ARG_TAG_ARENA) { + if (tags & ~ARG_TAG_ARENA) { + bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); + return -EINVAL; + } + sub->args[i].arg_type = ARG_PTR_TO_ARENA; + continue; + } if (is_global) { /* generic user data pointer */ u32 mem_size; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1358e20d315a..d64f7a9b60e8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9379,6 +9379,18 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, bpf_log(log, "arg#%d is expected to be non-NULL\n", i); return -EINVAL; } + } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { + /* + * Can pass any value and the kernel won't crash, but + * only PTR_TO_ARENA or SCALAR make sense. Everything + * else is a bug in the bpf program. Point it out to + * the user at the verification time instead of + * run-time debug nightmare. + */ + if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { + bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno); + return -EINVAL; + } } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) { ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0); if (ret) @@ -20448,6 +20460,9 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */ reg->btf_id = arg->btf_id; reg->id = ++env->id_gen; + } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { + /* caller can pass either PTR_TO_ARENA or SCALAR */ + mark_reg_unknown(env, regs, i); } else { WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n", i - BPF_REG_1, arg->arg_type); -- cgit v1.2.3 From 66c8473135c62f478301a0e5b3012f203562dfa6 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 8 Mar 2024 16:47:39 -0800 Subject: bpf: move sleepable flag from bpf_prog_aux to bpf_prog prog->aux->sleepable is checked very frequently as part of (some) BPF program run hot paths. So this extra aux indirection seems wasteful and on busy systems might cause unnecessary memory cache misses. Let's move sleepable flag into prog itself to eliminate unnecessary pointer dereference. Signed-off-by: Andrii Nakryiko Acked-by: Jiri Olsa Message-ID: <20240309004739.2961431-1-andrii@kernel.org> Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 8 ++++---- kernel/bpf/bpf_iter.c | 4 ++-- kernel/bpf/core.c | 2 +- kernel/bpf/syscall.c | 6 +++--- kernel/bpf/trampoline.c | 4 ++-- kernel/bpf/verifier.c | 12 ++++++------ kernel/events/core.c | 2 +- kernel/trace/bpf_trace.c | 2 +- net/bpf/bpf_dummy_struct_ops.c | 2 +- 9 files changed, 21 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 08ad265cb195..4f20f62f9d63 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1455,7 +1455,6 @@ struct bpf_prog_aux { bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool func_proto_unreliable; - bool sleepable; bool tail_call_reachable; bool xdp_has_frags; bool exception_cb; @@ -1541,7 +1540,8 @@ struct bpf_prog { enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ call_get_func_ip:1, /* Do we call get_func_ip() */ - tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */ + tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */ + sleepable:1; /* BPF program is sleepable */ enum bpf_prog_type type; /* Type of BPF program */ enum bpf_attach_type expected_attach_type; /* For some prog types */ u32 len; /* Number of filter blocks */ @@ -2112,14 +2112,14 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu, old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); item = &array->items[0]; while ((prog = READ_ONCE(item->prog))) { - if (!prog->aux->sleepable) + if (!prog->sleepable) rcu_read_lock(); run_ctx.bpf_cookie = item->bpf_cookie; ret &= run_prog(prog, ctx); item++; - if (!prog->aux->sleepable) + if (!prog->sleepable) rcu_read_unlock(); } bpf_reset_run_ctx(old_run_ctx); diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 0fae79164187..112581cf97e7 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -548,7 +548,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, return -ENOENT; /* Only allow sleepable program for resched-able iterator */ - if (prog->aux->sleepable && !bpf_iter_target_support_resched(tinfo)) + if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) return -EINVAL; link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN); @@ -697,7 +697,7 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) struct bpf_run_ctx run_ctx, *old_run_ctx; int ret; - if (prog->aux->sleepable) { + if (prog->sleepable) { rcu_read_lock_trace(); migrate_disable(); might_fault(); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 297184baeb2d..696bc55de8e8 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2706,7 +2706,7 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, bool sleepable; u32 i; - sleepable = aux->sleepable; + sleepable = aux->prog->sleepable; for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 07f2a4db4511..ae2ff73bde7e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2248,7 +2248,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) btf_put(prog->aux->attach_btf); if (deferred) { - if (prog->aux->sleepable) + if (prog->sleepable) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); @@ -2813,11 +2813,11 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) } prog->expected_attach_type = attr->expected_attach_type; + prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = attr->attach_btf_id; prog->aux->dst_prog = dst_prog; prog->aux->dev_bound = !!attr->prog_ifindex; - prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; /* move token into prog->aux, reuse taken refcnt */ @@ -5554,7 +5554,7 @@ static int bpf_prog_bind_map(union bpf_attr *attr) /* The bpf program will not access the bpf map, but for the sake of * simplicity, increase sleepable_refcnt for sleepable program as well. */ - if (prog->aux->sleepable) + if (prog->sleepable) atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index d382f5ebe06c..db7599c59c78 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1014,7 +1014,7 @@ void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr) bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog) { - bool sleepable = prog->aux->sleepable; + bool sleepable = prog->sleepable; if (bpf_prog_check_recur(prog)) return sleepable ? __bpf_prog_enter_sleepable_recur : @@ -1029,7 +1029,7 @@ bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog) bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog) { - bool sleepable = prog->aux->sleepable; + bool sleepable = prog->sleepable; if (bpf_prog_check_recur(prog)) return sleepable ? __bpf_prog_exit_sleepable_recur : diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d64f7a9b60e8..d501c90f8ab5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5274,7 +5274,7 @@ bad_type: static bool in_sleepable(struct bpf_verifier_env *env) { - return env->prog->aux->sleepable; + return env->prog->sleepable; } /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock() @@ -18137,7 +18137,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EINVAL; } - if (prog->aux->sleepable) + if (prog->sleepable) switch (map->map_type) { case BPF_MAP_TYPE_HASH: case BPF_MAP_TYPE_LRU_HASH: @@ -18325,7 +18325,7 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) return -E2BIG; } - if (env->prog->aux->sleepable) + if (env->prog->sleepable) atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, * the map will be released by release_maps() or it @@ -20938,7 +20938,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, } } - if (prog->aux->sleepable) { + if (prog->sleepable) { ret = -EINVAL; switch (prog->type) { case BPF_PROG_TYPE_TRACING: @@ -21049,14 +21049,14 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) u64 key; if (prog->type == BPF_PROG_TYPE_SYSCALL) { - if (prog->aux->sleepable) + if (prog->sleepable) /* attach_btf_id checked to be zero already */ return 0; verbose(env, "Syscall programs can only be sleepable\n"); return -EINVAL; } - if (prog->aux->sleepable && !can_be_sleepable(prog)) { + if (prog->sleepable && !can_be_sleepable(prog)) { verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n"); return -EINVAL; } diff --git a/kernel/events/core.c b/kernel/events/core.c index 5ecfa57e3b97..724e6d7e128f 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -10553,7 +10553,7 @@ int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) return -EINVAL; - if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe) + if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe) /* only uprobe programs are allowed to be sleepable */ return -EINVAL; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 241ddf5e3895..0a5c4efc73c3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3256,7 +3256,7 @@ static int uprobe_prog_run(struct bpf_uprobe *uprobe, .uprobe = uprobe, }; struct bpf_prog *prog = link->link.prog; - bool sleepable = prog->aux->sleepable; + bool sleepable = prog->sleepable; struct bpf_run_ctx *old_run_ctx; int err = 0; diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 1b5f812e6972..de33dc1b0daa 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -174,7 +174,7 @@ static int bpf_dummy_ops_check_member(const struct btf_type *t, case offsetof(struct bpf_dummy_ops, test_sleepable): break; default: - if (prog->aux->sleepable) + if (prog->sleepable) return -EINVAL; } -- cgit v1.2.3 From a66ccfc2535418b536b1203b65f87c4f501f6bdd Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Mon, 26 Feb 2024 20:35:56 +0100 Subject: platform/x86: wmi: Do not instantiate older WMI drivers multiple times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many older WMI drivers cannot be instantiated multiple times for two reasons: - they are using the legacy GUID-based WMI API - they are singletons (with global state) Prevent such WMI drivers from binding to WMI devices with a duplicated GUID, as this would mean that the WMI driver will be instantiated at least two times (one for the original GUID and one for the duplicated GUID). WMI drivers which can be instantiated multiple times can signal this by setting a flag inside struct wmi_driver. Tested on a ASUS Prime B650-Plus. Signed-off-by: Armin Wolf Link: https://lore.kernel.org/r/20240226193557.2888-2-W_Armin@gmx.de Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen --- drivers/hwmon/dell-smm-hwmon.c | 1 + drivers/platform/x86/dell/dell-wmi-ddv.c | 1 + drivers/platform/x86/intel/wmi/sbl-fw-update.c | 1 + drivers/platform/x86/intel/wmi/thunderbolt.c | 1 + drivers/platform/x86/wmi-bmof.c | 1 + drivers/platform/x86/wmi.c | 12 ++++++++++++ include/linux/wmi.h | 2 ++ 7 files changed, 19 insertions(+) (limited to 'include/linux') diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 6d8c0f328b7b..168d669c4eca 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1587,6 +1587,7 @@ static struct wmi_driver dell_smm_wmi_driver = { }, .id_table = dell_smm_wmi_id_table, .probe = dell_smm_wmi_probe, + .no_singleton = true, }; /* diff --git a/drivers/platform/x86/dell/dell-wmi-ddv.c b/drivers/platform/x86/dell/dell-wmi-ddv.c index db1e9240dd02..0b2299f7a2de 100644 --- a/drivers/platform/x86/dell/dell-wmi-ddv.c +++ b/drivers/platform/x86/dell/dell-wmi-ddv.c @@ -882,6 +882,7 @@ static struct wmi_driver dell_wmi_ddv_driver = { }, .id_table = dell_wmi_ddv_id_table, .probe = dell_wmi_ddv_probe, + .no_singleton = true, }; module_wmi_driver(dell_wmi_ddv_driver); diff --git a/drivers/platform/x86/intel/wmi/sbl-fw-update.c b/drivers/platform/x86/intel/wmi/sbl-fw-update.c index 040153ad67c1..75c82c08117f 100644 --- a/drivers/platform/x86/intel/wmi/sbl-fw-update.c +++ b/drivers/platform/x86/intel/wmi/sbl-fw-update.c @@ -131,6 +131,7 @@ static struct wmi_driver intel_wmi_sbl_fw_update_driver = { .probe = intel_wmi_sbl_fw_update_probe, .remove = intel_wmi_sbl_fw_update_remove, .id_table = intel_wmi_sbl_id_table, + .no_singleton = true, }; module_wmi_driver(intel_wmi_sbl_fw_update_driver); diff --git a/drivers/platform/x86/intel/wmi/thunderbolt.c b/drivers/platform/x86/intel/wmi/thunderbolt.c index e2ad3f46f356..08df560a2c7a 100644 --- a/drivers/platform/x86/intel/wmi/thunderbolt.c +++ b/drivers/platform/x86/intel/wmi/thunderbolt.c @@ -63,6 +63,7 @@ static struct wmi_driver intel_wmi_thunderbolt_driver = { .dev_groups = tbt_groups, }, .id_table = intel_wmi_thunderbolt_id_table, + .no_singleton = true, }; module_wmi_driver(intel_wmi_thunderbolt_driver); diff --git a/drivers/platform/x86/wmi-bmof.c b/drivers/platform/x86/wmi-bmof.c index 644d2fd889c0..df6f0ae6e6c7 100644 --- a/drivers/platform/x86/wmi-bmof.c +++ b/drivers/platform/x86/wmi-bmof.c @@ -94,6 +94,7 @@ static struct wmi_driver wmi_bmof_driver = { .probe = wmi_bmof_probe, .remove = wmi_bmof_remove, .id_table = wmi_bmof_id_table, + .no_singleton = true, }; module_wmi_driver(wmi_bmof_driver); diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c index 29dfe52eb802..349deced87e8 100644 --- a/drivers/platform/x86/wmi.c +++ b/drivers/platform/x86/wmi.c @@ -883,6 +883,18 @@ static int wmi_dev_probe(struct device *dev) struct wmi_driver *wdriver = drv_to_wdrv(dev->driver); int ret = 0; + /* Some older WMI drivers will break if instantiated multiple times, + * so they are blocked from probing WMI devices with a duplicated GUID. + * + * New WMI drivers should support being instantiated multiple times. + */ + if (test_bit(WMI_GUID_DUPLICATED, &wblock->flags) && !wdriver->no_singleton) { + dev_warn(dev, "Legacy driver %s cannot be instantiated multiple times\n", + dev->driver->name); + + return -ENODEV; + } + if (wdriver->notify) { if (test_bit(WMI_NO_EVENT_DATA, &wblock->flags) && !wdriver->no_notify_data) return -ENODEV; diff --git a/include/linux/wmi.h b/include/linux/wmi.h index 781958310bfb..63cca3b58d6d 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -49,6 +49,7 @@ u8 wmidev_instance_count(struct wmi_device *wdev); * @driver: Driver model structure * @id_table: List of WMI GUIDs supported by this driver * @no_notify_data: Driver supports WMI events which provide no event data + * @no_singleton: Driver can be instantiated multiple times * @probe: Callback for device binding * @remove: Callback for device unbinding * @notify: Callback for receiving WMI events @@ -59,6 +60,7 @@ struct wmi_driver { struct device_driver driver; const struct wmi_device_id *id_table; bool no_notify_data; + bool no_singleton; int (*probe)(struct wmi_device *wdev, const void *context); void (*remove)(struct wmi_device *wdev); -- cgit v1.2.3 From dbab9afe8640a51ffcce87bfdb59a814e0dc7780 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 5 Mar 2024 11:59:11 +0100 Subject: clk: x86: Move clk-pmc-atom register defines to include/linux/platform_data/x86/pmc_atom.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the register defines for the Atom (Bay Trail, Cherry Trail) PMC clocks to include/linux/platform_data/x86/pmc_atom.h. This is a preparation patch to extend the S0i3 readiness checks in drivers/platform/x86/pmc_atom.c with checking that the PMC clocks are off on suspend entry. Note these are added to include/linux/platform_data/x86/pmc_atom.h rather then to include/linux/platform_data/x86/clk-pmc-atom.h because the former already has all the other Atom PMC register defines. Reviewed-by: Ilpo Järvinen Acked-by: Stephen Boyd Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20240305105915.76242-2-hdegoede@redhat.com Signed-off-by: Ilpo Järvinen --- drivers/clk/x86/clk-pmc-atom.c | 13 +------------ include/linux/platform_data/x86/pmc_atom.h | 13 +++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/clk/x86/clk-pmc-atom.c b/drivers/clk/x86/clk-pmc-atom.c index 2974dd0ec6f4..5ec9255e33fa 100644 --- a/drivers/clk/x86/clk-pmc-atom.c +++ b/drivers/clk/x86/clk-pmc-atom.c @@ -11,23 +11,12 @@ #include #include #include +#include #include #include #define PLT_CLK_NAME_BASE "pmc_plt_clk" -#define PMC_CLK_CTL_OFFSET 0x60 -#define PMC_CLK_CTL_SIZE 4 -#define PMC_CLK_NUM 6 -#define PMC_CLK_CTL_GATED_ON_D3 0x0 -#define PMC_CLK_CTL_FORCE_ON 0x1 -#define PMC_CLK_CTL_FORCE_OFF 0x2 -#define PMC_CLK_CTL_RESERVED 0x3 -#define PMC_MASK_CLK_CTL GENMASK(1, 0) -#define PMC_MASK_CLK_FREQ BIT(2) -#define PMC_CLK_FREQ_XTAL (0 << 2) /* 25 MHz */ -#define PMC_CLK_FREQ_PLL (1 << 2) /* 19.2 MHz */ - struct clk_plt_fixed { struct clk_hw *clk; struct clk_lookup *lookup; diff --git a/include/linux/platform_data/x86/pmc_atom.h b/include/linux/platform_data/x86/pmc_atom.h index b8a701c77fd0..557622ef0390 100644 --- a/include/linux/platform_data/x86/pmc_atom.h +++ b/include/linux/platform_data/x86/pmc_atom.h @@ -43,6 +43,19 @@ BIT_ORED_DEDICATED_IRQ_GPSC | \ BIT_SHARED_IRQ_GPSS) +/* External clk generator settings */ +#define PMC_CLK_CTL_OFFSET 0x60 +#define PMC_CLK_CTL_SIZE 4 +#define PMC_CLK_NUM 6 +#define PMC_CLK_CTL_GATED_ON_D3 0x0 +#define PMC_CLK_CTL_FORCE_ON 0x1 +#define PMC_CLK_CTL_FORCE_OFF 0x2 +#define PMC_CLK_CTL_RESERVED 0x3 +#define PMC_MASK_CLK_CTL GENMASK(1, 0) +#define PMC_MASK_CLK_FREQ BIT(2) +#define PMC_CLK_FREQ_XTAL (0 << 2) /* 25 MHz */ +#define PMC_CLK_FREQ_PLL (1 << 2) /* 19.2 MHz */ + /* The timers accumulate time spent in sleep state */ #define PMC_S0IR_TMR 0x80 #define PMC_S0I1_TMR 0x84 -- cgit v1.2.3 From a21ff5a0a7948b6ef1364f8f6d07eda49426d09a Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 5 Mar 2024 11:59:12 +0100 Subject: platform/x86: pmc_atom: Annotate d3_sts register bit defines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The include/linux/platform_data/x86/pmc_atom.h d3_sts register bit defines are named after how these bits are used on Bay Trail devices. On Cherry Trail (CHT) devices some of these bits have a different meaning according to the datasheet. At a comment to the defines for bits which have a different meaning on Cherry Trail devices. Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20240305105915.76242-3-hdegoede@redhat.com Reviewed-by: Ilpo Järvinen Signed-off-by: Ilpo Järvinen --- include/linux/platform_data/x86/pmc_atom.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/x86/pmc_atom.h b/include/linux/platform_data/x86/pmc_atom.h index 557622ef0390..161e4bc1c9ee 100644 --- a/include/linux/platform_data/x86/pmc_atom.h +++ b/include/linux/platform_data/x86/pmc_atom.h @@ -117,14 +117,14 @@ #define BIT_SCC_SDIO BIT(9) #define BIT_SCC_SDCARD BIT(10) #define BIT_SCC_MIPI BIT(11) -#define BIT_HDA BIT(12) +#define BIT_HDA BIT(12) /* CHT datasheet: reserved */ #define BIT_LPE BIT(13) #define BIT_OTG BIT(14) -#define BIT_USH BIT(15) -#define BIT_GBE BIT(16) -#define BIT_SATA BIT(17) -#define BIT_USB_EHCI BIT(18) -#define BIT_SEC BIT(19) +#define BIT_USH BIT(15) /* CHT datasheet: reserved */ +#define BIT_GBE BIT(16) /* CHT datasheet: reserved */ +#define BIT_SATA BIT(17) /* CHT datasheet: reserved */ +#define BIT_USB_EHCI BIT(18) /* CHT datasheet: XHCI! */ +#define BIT_SEC BIT(19) /* BYT datasheet: reserved */ #define BIT_PCIE_PORT0 BIT(20) #define BIT_PCIE_PORT1 BIT(21) #define BIT_PCIE_PORT2 BIT(22) -- cgit v1.2.3 From fa63587f94a77a49b53274dc0fd1ea41dfde5966 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 11 Mar 2024 14:32:23 +0100 Subject: drivers/gpio/nomadik: move dummy nmk_gpio_dbg_show_one() to header When `CONFIG_DEBUG_FS` is disabled, nmk_gpio_dbg_show_one() is an empty dummy function; this however triggers a `-Wmissing-prototypes` warning and later a linker error because the function is also used by drivers/pinctrl/nomadik/pinctrl-nomadik.c, therefore it needs to be non-static. To allow both sources to access this dummy function, this patch moves it to the header, adding the `#ifdef CONFIG_DEBUG_FS` there as well. Signed-off-by: Max Kellermann Link: https://lore.kernel.org/r/20240311133223.3429428-1-max.kellermann@ionos.com Signed-off-by: Linus Walleij --- drivers/gpio/gpio-nomadik.c | 8 -------- include/linux/gpio/gpio-nomadik.h | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c index 5e2f9b51ece3..836f1cc760c2 100644 --- a/drivers/gpio/gpio-nomadik.c +++ b/drivers/gpio/gpio-nomadik.c @@ -486,14 +486,6 @@ static void nmk_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) #else -static inline void nmk_gpio_dbg_show_one(struct seq_file *s, - struct pinctrl_dev *pctldev, - struct gpio_chip *chip, - unsigned int offset, - unsigned int gpio) -{ -} - #define nmk_gpio_dbg_show NULL #endif diff --git a/include/linux/gpio/gpio-nomadik.h b/include/linux/gpio/gpio-nomadik.h index 4a95ea7935fb..b5a84864650d 100644 --- a/include/linux/gpio/gpio-nomadik.h +++ b/include/linux/gpio/gpio-nomadik.h @@ -253,6 +253,8 @@ nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc) struct platform_device; +#ifdef CONFIG_DEBUG_FS + /* * Symbols declared in gpio-nomadik used by pinctrl-nomadik. If pinctrl-nomadik * is enabled, then gpio-nomadik is enabled as well; the reverse if not always @@ -261,6 +263,19 @@ struct platform_device; void nmk_gpio_dbg_show_one(struct seq_file *s, struct pinctrl_dev *pctldev, struct gpio_chip *chip, unsigned int offset, unsigned int gpio); + +#else + +static inline void nmk_gpio_dbg_show_one(struct seq_file *s, + struct pinctrl_dev *pctldev, + struct gpio_chip *chip, + unsigned int offset, + unsigned int gpio) +{ +} + +#endif + void __nmk_gpio_make_output(struct nmk_gpio_chip *nmk_chip, unsigned int offset, int val); void __nmk_gpio_set_slpm(struct nmk_gpio_chip *nmk_chip, unsigned int offset, -- cgit v1.2.3 From 7af9ded0c2caac0a95f33df5cb04706b0f502588 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 12 Mar 2024 08:15:07 -0400 Subject: ring-buffer: Use wait_event_interruptible() in ring_buffer_wait() Convert ring_buffer_wait() over to wait_event_interruptible(). The default condition is to execute the wait loop inside __wait_event() just once. This does not change the ring_buffer_wait() prototype yet, but restructures the code so that it can take a "cond" and "data" parameter and will call wait_event_interruptible() with a helper function as the condition. The helper function (rb_wait_cond) takes the cond function and data parameters. It will first check if the buffer hit the watermark defined by the "full" parameter and then call the passed in condition parameter. If either are true, it returns true. If rb_wait_cond() does not return true, it will set the appropriate "waiters_pending" flag and returns false. Link: https://lore.kernel.org/linux-trace-kernel/CAHk-=wgsNgewHFxZAJiAQznwPMqEtQmi1waeS2O1v6L4c_Um5A@mail.gmail.com/ Link: https://lore.kernel.org/linux-trace-kernel/20240312121703.399598519@goodmis.org Cc: stable@vger.kernel.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Linus Torvalds Cc: linke li Cc: Rabin Vincent Fixes: f3ddb74ad0790 ("tracing: Wake up ring buffer waiters on closing of the file") Signed-off-by: Steven Rostedt (Google) --- include/linux/ring_buffer.h | 1 + kernel/trace/ring_buffer.c | 116 ++++++++++++++++++++++++++------------------ 2 files changed, 69 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index fa802db216f9..338a33db1577 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -98,6 +98,7 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k __ring_buffer_alloc((size), (flags), &__key); \ }) +typedef bool (*ring_buffer_cond_fn)(void *data); int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full); __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, struct file *filp, poll_table *poll_table, int full); diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 8c3730a88662..f4c34b7c7e1e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -843,43 +843,15 @@ static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full) return ret; } -/** - * ring_buffer_wait - wait for input to the ring buffer - * @buffer: buffer to wait on - * @cpu: the cpu buffer to wait on - * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS - * - * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon - * as data is added to any of the @buffer's cpu buffers. Otherwise - * it will wait for data to be added to a specific cpu buffer. - */ -int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) +static inline bool +rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer, + int cpu, int full, ring_buffer_cond_fn cond, void *data) { - struct ring_buffer_per_cpu *cpu_buffer; - DEFINE_WAIT(wait); - struct rb_irq_work *work; - int ret = 0; - - /* - * Depending on what the caller is waiting for, either any - * data in any cpu buffer, or a specific buffer, put the - * caller on the appropriate wait queue. - */ - if (cpu == RING_BUFFER_ALL_CPUS) { - work = &buffer->irq_work; - /* Full only makes sense on per cpu reads */ - full = 0; - } else { - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return -ENODEV; - cpu_buffer = buffer->buffers[cpu]; - work = &cpu_buffer->irq_work; - } + if (rb_watermark_hit(buffer, cpu, full)) + return true; - if (full) - prepare_to_wait(&work->full_waiters, &wait, TASK_INTERRUPTIBLE); - else - prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE); + if (cond(data)) + return true; /* * The events can happen in critical sections where @@ -902,27 +874,75 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) * a task has been queued. It's OK for spurious wake ups. */ if (full) - work->full_waiters_pending = true; + rbwork->full_waiters_pending = true; else - work->waiters_pending = true; + rbwork->waiters_pending = true; - if (rb_watermark_hit(buffer, cpu, full)) - goto out; + return false; +} - if (signal_pending(current)) { - ret = -EINTR; - goto out; +/* + * The default wait condition for ring_buffer_wait() is to just to exit the + * wait loop the first time it is woken up. + */ +static bool rb_wait_once(void *data) +{ + long *once = data; + + /* wait_event() actually calls this twice before scheduling*/ + if (*once > 1) + return true; + + (*once)++; + return false; +} + +/** + * ring_buffer_wait - wait for input to the ring buffer + * @buffer: buffer to wait on + * @cpu: the cpu buffer to wait on + * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS + * + * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon + * as data is added to any of the @buffer's cpu buffers. Otherwise + * it will wait for data to be added to a specific cpu buffer. + */ +int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) +{ + struct ring_buffer_per_cpu *cpu_buffer; + struct wait_queue_head *waitq; + ring_buffer_cond_fn cond; + struct rb_irq_work *rbwork; + void *data; + long once = 0; + int ret = 0; + + cond = rb_wait_once; + data = &once; + + /* + * Depending on what the caller is waiting for, either any + * data in any cpu buffer, or a specific buffer, put the + * caller on the appropriate wait queue. + */ + if (cpu == RING_BUFFER_ALL_CPUS) { + rbwork = &buffer->irq_work; + /* Full only makes sense on per cpu reads */ + full = 0; + } else { + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + return -ENODEV; + cpu_buffer = buffer->buffers[cpu]; + rbwork = &cpu_buffer->irq_work; } - schedule(); - out: if (full) - finish_wait(&work->full_waiters, &wait); + waitq = &rbwork->full_waiters; else - finish_wait(&work->waiters, &wait); + waitq = &rbwork->waiters; - if (!ret && !rb_watermark_hit(buffer, cpu, full) && signal_pending(current)) - ret = -EINTR; + ret = wait_event_interruptible((*waitq), + rb_wait_cond(rbwork, buffer, cpu, full, cond, data)); return ret; } -- cgit v1.2.3 From 2aa043a55b9a764c9cbde5a8c654eeaaffe224cf Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 12 Mar 2024 08:15:08 -0400 Subject: tracing/ring-buffer: Fix wait_on_pipe() race When the trace_pipe_raw file is closed, there should be no new readers on the file descriptor. This is mostly handled with the waking and wait_index fields of the iterator. But there's still a slight race. CPU 0 CPU 1 ----- ----- wait_index++; index = wait_index; ring_buffer_wake_waiters(); wait_on_pipe() ring_buffer_wait(); The ring_buffer_wait() will miss the wakeup from CPU 1. The problem is that the ring_buffer_wait() needs the logic of: prepare_to_wait(); if (!condition) schedule(); Where the missing condition check is the iter->wait_index update. Have the ring_buffer_wait() take a conditional callback function and a data parameter that can be used within the wait_event_interruptible() of the ring_buffer_wait() function. In wait_on_pipe(), pass a condition function that will check if the wait_index has been updated, if it has, it will return true to break out of the wait_event_interruptible() loop. Create a new field "closed" in the trace_iterator and set it in the .flush() callback before calling ring_buffer_wake_waiters(). This will keep any new readers from waiting on a closed file descriptor. Have the wait_on_pipe() condition callback also check the closed field. Change the wait_index field of the trace_iterator to atomic_t. There's no reason it needs to be 'long' and making it atomic and using atomic_read_acquire() and atomic_fetch_inc_release() will provide the necessary memory barriers. Add a "woken" flag to tracing_buffers_splice_read() to exit the loop after one more try to fetch data. That is, if it waited for data and something woke it up, it should try to collect any new data and then exit back to user space. Link: https://lore.kernel.org/linux-trace-kernel/CAHk-=wgsNgewHFxZAJiAQznwPMqEtQmi1waeS2O1v6L4c_Um5A@mail.gmail.com/ Link: https://lore.kernel.org/linux-trace-kernel/20240312121703.557950713@goodmis.org Cc: stable@vger.kernel.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Linus Torvalds Cc: linke li Cc: Rabin Vincent Fixes: f3ddb74ad0790 ("tracing: Wake up ring buffer waiters on closing of the file") Signed-off-by: Steven Rostedt (Google) --- include/linux/ring_buffer.h | 3 ++- include/linux/trace_events.h | 5 ++++- kernel/trace/ring_buffer.c | 13 ++++++++----- kernel/trace/trace.c | 43 +++++++++++++++++++++++++++++++------------ 4 files changed, 45 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 338a33db1577..dc5ae4e96aee 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -99,7 +99,8 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k }) typedef bool (*ring_buffer_cond_fn)(void *data); -int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full); +int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full, + ring_buffer_cond_fn cond, void *data); __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, struct file *filp, poll_table *poll_table, int full); void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu); diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index d68ff9b1247f..fc6d0af56bb1 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -103,13 +103,16 @@ struct trace_iterator { unsigned int temp_size; char *fmt; /* modified format holder */ unsigned int fmt_size; - long wait_index; + atomic_t wait_index; /* trace_seq for __print_flags() and __print_symbolic() etc. */ struct trace_seq tmp_seq; cpumask_var_t started; + /* Set when the file is closed to prevent new waiters */ + bool closed; + /* it's true when current open file is snapshot */ bool snapshot; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index f4c34b7c7e1e..350607cce869 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -902,23 +902,26 @@ static bool rb_wait_once(void *data) * @buffer: buffer to wait on * @cpu: the cpu buffer to wait on * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS + * @cond: condition function to break out of wait (NULL to run once) + * @data: the data to pass to @cond. * * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon * as data is added to any of the @buffer's cpu buffers. Otherwise * it will wait for data to be added to a specific cpu buffer. */ -int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) +int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full, + ring_buffer_cond_fn cond, void *data) { struct ring_buffer_per_cpu *cpu_buffer; struct wait_queue_head *waitq; - ring_buffer_cond_fn cond; struct rb_irq_work *rbwork; - void *data; long once = 0; int ret = 0; - cond = rb_wait_once; - data = &once; + if (!cond) { + cond = rb_wait_once; + data = &once; + } /* * Depending on what the caller is waiting for, either any diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c9c898307348..d390fea3a6a5 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1955,15 +1955,36 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) #endif /* CONFIG_TRACER_MAX_TRACE */ +struct pipe_wait { + struct trace_iterator *iter; + int wait_index; +}; + +static bool wait_pipe_cond(void *data) +{ + struct pipe_wait *pwait = data; + struct trace_iterator *iter = pwait->iter; + + if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index) + return true; + + return iter->closed; +} + static int wait_on_pipe(struct trace_iterator *iter, int full) { + struct pipe_wait pwait; int ret; /* Iterators are static, they should be filled or empty */ if (trace_buffer_iter(iter, iter->cpu_file)) return 0; - ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full); + pwait.wait_index = atomic_read_acquire(&iter->wait_index); + pwait.iter = iter; + + ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full, + wait_pipe_cond, &pwait); #ifdef CONFIG_TRACER_MAX_TRACE /* @@ -8398,9 +8419,9 @@ static int tracing_buffers_flush(struct file *file, fl_owner_t id) struct ftrace_buffer_info *info = file->private_data; struct trace_iterator *iter = &info->iter; - iter->wait_index++; + iter->closed = true; /* Make sure the waiters see the new wait_index */ - smp_wmb(); + (void)atomic_fetch_inc_release(&iter->wait_index); ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); @@ -8500,6 +8521,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, .spd_release = buffer_spd_release, }; struct buffer_ref *ref; + bool woken = false; int page_size; int entries, i; ssize_t ret = 0; @@ -8573,17 +8595,17 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, /* did we read anything? */ if (!spd.nr_pages) { - long wait_index; if (ret) goto out; + if (woken) + goto out; + ret = -EAGAIN; if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) goto out; - wait_index = READ_ONCE(iter->wait_index); - ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent); if (ret) goto out; @@ -8592,10 +8614,8 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, if (!tracer_tracing_is_on(iter->tr)) goto out; - /* Make sure we see the new wait_index */ - smp_rmb(); - if (wait_index != iter->wait_index) - goto out; + /* Iterate one more time to collect any new data then exit */ + woken = true; goto again; } @@ -8618,9 +8638,8 @@ static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned mutex_lock(&trace_types_lock); - iter->wait_index++; /* Make sure the waiters see the new wait_index */ - smp_wmb(); + (void)atomic_fetch_inc_release(&iter->wait_index); ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); -- cgit v1.2.3 From 17423360a27ae58c1850f588bdd8013bbfcd250b Mon Sep 17 00:00:00 2001 From: "David E. Box" Date: Fri, 23 Feb 2024 14:58:50 -0600 Subject: PCI/ASPM: Save L1 PM Substates Capability for suspend/resume 4ff116d0d5fd ("PCI/ASPM: Save L1 PM Substates Capability for suspend/resume") restored the L1 PM Substates Capability after resume, which reduced power consumption by making the ASPM L1.x states work after resume. a7152be79b62 ("Revert "PCI/ASPM: Save L1 PM Substates Capability for suspend/resume"") reverted 4ff116d0d5fd because resume failed on some systems, so power consumption after resume increased again. a7152be79b62 mentioned that we restore L1 PM substate configuration even though ASPM L1 may already be enabled. This is due the fact that the pci_restore_aspm_l1ss_state() was called before pci_restore_pcie_state(). Save and restore the L1 PM Substates Capability, following PCIe r6.1, sec 5.5.4 more closely by: 1) Do not restore ASPM configuration in pci_restore_pcie_state() but do that after PCIe capability is restored in pci_restore_aspm_state() following PCIe r6.1, sec 5.5.4. 2) If BIOS reenables L1SS, particularly L1.2, we need to clear the enables in the right order, downstream before upstream. Defer restoring the L1SS config until we are at the downstream component. Then update the config for both ends of the link in the prescribed order. 3) Program ASPM L1 PM substate configuration before L1 enables. 4) Program ASPM L1 PM substate enables last, after rest of the fields in the capability are programmed. [bhelgaas: commit log, squash L1SS-related patches, do both LNKCTL restores in pci_restore_pcie_state()] Link: https://lore.kernel.org/r/20240128233212.1139663-3-david.e.box@linux.intel.com Link: https://lore.kernel.org/r/20240128233212.1139663-4-david.e.box@linux.intel.com Link: https://lore.kernel.org/r/20240223205851.114931-5-helgaas@kernel.org Closes: https://bugzilla.kernel.org/show_bug.cgi?id=217321 Link: https://bugzilla.kernel.org/show_bug.cgi?id=216782 Link: https://bugzilla.kernel.org/show_bug.cgi?id=216877 Co-developed-by: Mika Westerberg Co-developed-by: David E. Box Reported-by: Koba Ko Signed-off-by: Mika Westerberg Signed-off-by: David E. Box Signed-off-by: Bjorn Helgaas Tested-by: Tasev Nikola # Asus UX305FA Cc: Mark Enriquez Cc: Thomas Witt Cc: Werner Sembach Cc: Vidya Sagar --- drivers/pci/pci.c | 17 +++++++- drivers/pci/pci.h | 3 ++ drivers/pci/pcie/aspm.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++-- drivers/pci/probe.c | 1 + include/linux/pci.h | 2 +- 5 files changed, 119 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 564e2cf2dde5..ca6673588bc0 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1623,6 +1623,8 @@ static int pci_save_pcie_state(struct pci_dev *dev) pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]); pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]); + pci_save_aspm_l1ss_state(dev); + return 0; } @@ -1630,7 +1632,7 @@ static void pci_restore_pcie_state(struct pci_dev *dev) { int i = 0; struct pci_cap_saved_state *save_state; - u16 *cap; + u16 *cap, lnkctl; save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); if (!save_state) @@ -1645,12 +1647,23 @@ static void pci_restore_pcie_state(struct pci_dev *dev) cap = (u16 *)&save_state->cap.data[0]; pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); + + /* Restore LNKCTL register with ASPM control field clear */ + lnkctl = cap[i++]; + pcie_capability_write_word(dev, PCI_EXP_LNKCTL, + lnkctl & ~PCI_EXP_LNKCTL_ASPMC); + pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]); pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]); pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]); pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]); pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]); + + pci_restore_aspm_l1ss_state(dev); + + /* Restore ASPM control after restoring L1SS state */ + pcie_capability_set_word(dev, PCI_EXP_LNKCTL, + lnkctl & PCI_EXP_LNKCTL_ASPMC); } static int pci_save_pcix_state(struct pci_dev *dev) diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index ad3add45345c..eca5938deb07 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -571,6 +571,9 @@ int pcie_retrain_link(struct pci_dev *pdev, bool use_lt); /* ASPM-related functionality we need even without CONFIG_PCIEASPM */ void pci_save_ltr_state(struct pci_dev *dev); void pci_restore_ltr_state(struct pci_dev *dev); +void pci_configure_aspm_l1ss(struct pci_dev *dev); +void pci_save_aspm_l1ss_state(struct pci_dev *dev); +void pci_restore_aspm_l1ss_state(struct pci_dev *dev); #ifdef CONFIG_PCIEASPM void pcie_aspm_init_link_state(struct pci_dev *pdev); diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 21731b232fb8..977eca893b2a 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -64,6 +64,105 @@ void pci_restore_ltr_state(struct pci_dev *dev) pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap); } +void pci_configure_aspm_l1ss(struct pci_dev *pdev) +{ + int rc; + + pdev->l1ss = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); + + rc = pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_L1SS, + 2 * sizeof(u32)); + if (rc) + pci_err(pdev, "unable to allocate ASPM L1SS save buffer (%pe)\n", + ERR_PTR(rc)); +} + +void pci_save_aspm_l1ss_state(struct pci_dev *pdev) +{ + struct pci_cap_saved_state *save_state; + u16 l1ss = pdev->l1ss; + u32 *cap; + + /* + * Save L1 substate configuration. The ASPM L0s/L1 configuration + * in PCI_EXP_LNKCTL_ASPMC is saved by pci_save_pcie_state(). + */ + if (!l1ss) + return; + + save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_L1SS); + if (!save_state) + return; + + cap = &save_state->cap.data[0]; + pci_read_config_dword(pdev, l1ss + PCI_L1SS_CTL2, cap++); + pci_read_config_dword(pdev, l1ss + PCI_L1SS_CTL1, cap++); +} + +void pci_restore_aspm_l1ss_state(struct pci_dev *pdev) +{ + struct pci_cap_saved_state *pl_save_state, *cl_save_state; + struct pci_dev *parent = pdev->bus->self; + u32 *cap, pl_ctl1, pl_ctl2, pl_l1_2_enable; + u32 cl_ctl1, cl_ctl2, cl_l1_2_enable; + + /* + * In case BIOS enabled L1.2 when resuming, we need to disable it first + * on the downstream component before the upstream. So, don't attempt to + * restore either until we are at the downstream component. + */ + if (pcie_downstream_port(pdev) || !parent) + return; + + if (!pdev->l1ss || !parent->l1ss) + return; + + cl_save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_L1SS); + pl_save_state = pci_find_saved_ext_cap(parent, PCI_EXT_CAP_ID_L1SS); + if (!cl_save_state || !pl_save_state) + return; + + cap = &cl_save_state->cap.data[0]; + cl_ctl2 = *cap++; + cl_ctl1 = *cap; + cap = &pl_save_state->cap.data[0]; + pl_ctl2 = *cap++; + pl_ctl1 = *cap; + + /* + * Disable L1.2 on this downstream endpoint device first, followed + * by the upstream + */ + pci_clear_and_set_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_L1_2_MASK, 0); + pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_L1_2_MASK, 0); + + /* + * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD + * in PCI_L1SS_CTL1 must be programmed *before* setting the L1.2 + * enable bits, even though they're all in PCI_L1SS_CTL1. + */ + pl_l1_2_enable = pl_ctl1 & PCI_L1SS_CTL1_L1_2_MASK; + pl_ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK; + cl_l1_2_enable = cl_ctl1 & PCI_L1SS_CTL1_L1_2_MASK; + cl_ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK; + + /* Write back without enables first (above we cleared them in ctl1) */ + pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, pl_ctl2); + pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL2, cl_ctl2); + pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, pl_ctl1); + pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, cl_ctl1); + + /* Then write back the enables */ + if (pl_l1_2_enable || cl_l1_2_enable) { + pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + pl_ctl1 | pl_l1_2_enable); + pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, + cl_ctl1 | cl_l1_2_enable); + } +} + #ifdef CONFIG_PCIEASPM #ifdef MODULE_PARAM_PREFIX @@ -1005,9 +1104,6 @@ void pci_configure_ltr(struct pci_dev *pdev) if (!pci_is_pcie(pdev)) return; - /* Read L1 PM substate capabilities */ - pdev->l1ss = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); - pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &cap); if (!(cap & PCI_EXP_DEVCAP2_LTR)) return; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index b809c0b0e0e5..1434bf495db3 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2259,6 +2259,7 @@ static void pci_configure_device(struct pci_dev *dev) pci_configure_extended_tags(dev, NULL); pci_configure_relaxed_ordering(dev); pci_configure_ltr(dev); + pci_configure_aspm_l1ss(dev); pci_configure_eetlp_prefix(dev); pci_configure_serr(dev); diff --git a/include/linux/pci.h b/include/linux/pci.h index add9368e6314..6967ae7b4115 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -390,9 +390,9 @@ struct pci_dev { unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ + u16 l1ss; /* L1SS Capability pointer */ #ifdef CONFIG_PCIEASPM struct pcie_link_state *link_state; /* ASPM link state */ - u16 l1ss; /* L1SS Capability pointer */ unsigned int ltr_path:1; /* Latency Tolerance Reporting supported from root to here */ #endif -- cgit v1.2.3 From 11270e526276ffad4c4237acb393da82a3287487 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:21 -0700 Subject: base/node / ACPI: Enumerate node access class for 'struct access_coordinate' Both generic node and HMAT handling code have been using magic numbers to indicate access classes for 'struct access_coordinate'. Introduce enums to enumerate the access0 and access1 classes shared by the two subsystems. Update the function parameters and callers as appropriate to utilize the new enum. Access0 is named to ACCESS_COORDINATE_LOCAL in order to indicate that the access class is for 'struct access_coordinate' between a target node and the nearest initiator node. Access1 is named to ACCESS_COORDINATE_CPU in order to indicate that the access class is for 'struct access_coordinate' between a target node and the nearest CPU node. Cc: Greg Kroah-Hartman Cc: Rafael J. Wysocki Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Acked-by: Greg Kroah-Hartman Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-3-dave.jiang@intel.com Signed-off-by: Dan Williams --- drivers/acpi/numa/hmat.c | 26 ++++++++++++++------------ drivers/base/node.c | 6 +++--- include/linux/node.h | 18 +++++++++++++++--- 3 files changed, 32 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index a26e7793ec4e..e0144cfbf1f3 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -59,9 +59,7 @@ struct target_cache { }; enum { - NODE_ACCESS_CLASS_0 = 0, - NODE_ACCESS_CLASS_1, - NODE_ACCESS_CLASS_GENPORT_SINK, + NODE_ACCESS_CLASS_GENPORT_SINK = ACCESS_COORDINATE_MAX, NODE_ACCESS_CLASS_MAX, }; @@ -374,11 +372,11 @@ static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_px if (target && target->processor_pxm == init_pxm) { hmat_update_target_access(target, type, value, - NODE_ACCESS_CLASS_0); + ACCESS_COORDINATE_LOCAL); /* If the node has a CPU, update access 1 */ if (node_state(pxm_to_node(init_pxm), N_CPU)) hmat_update_target_access(target, type, value, - NODE_ACCESS_CLASS_1); + ACCESS_COORDINATE_CPU); } } @@ -709,7 +707,8 @@ static void hmat_update_target_attrs(struct memory_target *target, */ if (target->processor_pxm != PXM_INVAL) { cpu_nid = pxm_to_node(target->processor_pxm); - if (access == 0 || node_state(cpu_nid, N_CPU)) { + if (access == ACCESS_COORDINATE_LOCAL || + node_state(cpu_nid, N_CPU)) { set_bit(target->processor_pxm, p_nodes); return; } @@ -737,7 +736,8 @@ static void hmat_update_target_attrs(struct memory_target *target, list_for_each_entry(initiator, &initiators, node) { u32 value; - if (access == 1 && !initiator->has_cpu) { + if (access == ACCESS_COORDINATE_CPU && + !initiator->has_cpu) { clear_bit(initiator->processor_pxm, p_nodes); continue; } @@ -782,8 +782,10 @@ static void hmat_register_target_initiators(struct memory_target *target) { static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); - __hmat_register_target_initiators(target, p_nodes, 0); - __hmat_register_target_initiators(target, p_nodes, 1); + __hmat_register_target_initiators(target, p_nodes, + ACCESS_COORDINATE_LOCAL); + __hmat_register_target_initiators(target, p_nodes, + ACCESS_COORDINATE_CPU); } static void hmat_register_target_cache(struct memory_target *target) @@ -854,8 +856,8 @@ static void hmat_register_target(struct memory_target *target) if (!target->registered) { hmat_register_target_initiators(target); hmat_register_target_cache(target); - hmat_register_target_perf(target, NODE_ACCESS_CLASS_0); - hmat_register_target_perf(target, NODE_ACCESS_CLASS_1); + hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL); + hmat_register_target_perf(target, ACCESS_COORDINATE_CPU); target->registered = true; } mutex_unlock(&target_lock); @@ -927,7 +929,7 @@ static int hmat_calculate_adistance(struct notifier_block *self, return NOTIFY_OK; mutex_lock(&target_lock); - hmat_update_target_attrs(target, p_nodes, 1); + hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU); mutex_unlock(&target_lock); perf = &target->coord[1]; diff --git a/drivers/base/node.c b/drivers/base/node.c index 1c05640461dd..a73b0c9a401a 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -126,7 +126,7 @@ static void node_access_release(struct device *dev) } static struct node_access_nodes *node_init_node_access(struct node *node, - unsigned int access) + enum access_coordinate_class access) { struct node_access_nodes *access_node; struct device *dev; @@ -191,7 +191,7 @@ static struct attribute *access_attrs[] = { * @access: The access class the for the given attributes */ void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, - unsigned int access) + enum access_coordinate_class access) { struct node_access_nodes *c; struct node *node; @@ -689,7 +689,7 @@ int register_cpu_under_node(unsigned int cpu, unsigned int nid) */ int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, - unsigned int access) + enum access_coordinate_class access) { struct node *init_node, *targ_node; struct node_access_nodes *initiator, *target; diff --git a/include/linux/node.h b/include/linux/node.h index 25b66d705ee2..dfc004e4bee7 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -34,6 +34,18 @@ struct access_coordinate { unsigned int write_latency; }; +/* + * ACCESS_COORDINATE_LOCAL correlates to ACCESS CLASS 0 + * - access_coordinate between target node and nearest initiator node + * ACCESS_COORDINATE_CPU correlates to ACCESS CLASS 1 + * - access_coordinate between target node and nearest CPU node + */ +enum access_coordinate_class { + ACCESS_COORDINATE_LOCAL, + ACCESS_COORDINATE_CPU, + ACCESS_COORDINATE_MAX +}; + enum cache_indexing { NODE_CACHE_DIRECT_MAP, NODE_CACHE_INDEXED, @@ -66,7 +78,7 @@ struct node_cache_attrs { #ifdef CONFIG_HMEM_REPORTING void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs); void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, - unsigned access); + enum access_coordinate_class access); #else static inline void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) @@ -75,7 +87,7 @@ static inline void node_add_cache(unsigned int nid, static inline void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, - unsigned access) + enum access_coordinate_class access) { } #endif @@ -137,7 +149,7 @@ extern void unregister_memory_block_under_nodes(struct memory_block *mem_blk); extern int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, - unsigned access); + enum access_coordinate_class access); #else static inline void node_dev_init(void) { -- cgit v1.2.3 From 067353a46d8ccdac279ebab97c038c3658e97541 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:30 -0700 Subject: cxl/region: Add memory hotplug notifier for cxl region When the CXL region is formed, the driver computes the performance data for the region. However this data is not available at the node data collection that has been populated by the HMAT during kernel initialization. Add a memory hotplug notifier to update the access coordinates to the 'struct memory_target' context kept by the HMAT_REPORTING code. Add CXL_CALLBACK_PRI for a memory hotplug callback priority. Set the priority number to be called before HMAT_CALLBACK_PRI. The CXL update must happen before hmat_callback(). A new HMAT_REPORTING helper hmat_update_target_coordinates() is added in order to allow CXL to update the memory_target access coordinates. A new ext_updated member is added to the memory_target to indicate that the access coordinates within the memory_target has been updated by an external agent such as CXL. This prevents data being overwritten by the hmat_update_target_attrs() triggered by hmat_callback(). Cc: Andrew Morton Cc: Rafael J. Wysocki Reviewed-by: Huang, Ying Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-12-dave.jiang@intel.com Signed-off-by: Dan Williams --- drivers/acpi/numa/hmat.c | 34 ++++++++++++++++++++++++ drivers/cxl/core/cdat.c | 6 +++++ drivers/cxl/core/core.h | 3 +++ drivers/cxl/core/region.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 3 +++ include/linux/acpi.h | 12 +++++++++ include/linux/memory.h | 1 + 7 files changed, 127 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 75e9aac43228..2c8ccc91ebe6 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -74,6 +74,7 @@ struct memory_target { struct node_cache_attrs cache_attrs; u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; bool registered; + bool ext_updated; /* externally updated */ }; struct memory_initiator { @@ -328,6 +329,35 @@ static void hmat_update_target_access(struct memory_target *target, } } +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct memory_target *target; + int pxm; + + if (nid == NUMA_NO_NODE) + return -EINVAL; + + pxm = node_to_pxm(nid); + guard(mutex)(&target_lock); + target = find_mem_target(pxm); + if (!target) + return -ENODEV; + + hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY, + coord->read_latency, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY, + coord->write_latency, access); + hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH, + coord->read_bandwidth, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, + coord->write_bandwidth, access); + target->ext_updated = true; + + return 0; +} +EXPORT_SYMBOL_GPL(hmat_update_target_coordinates); + static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) { struct memory_locality *loc; @@ -699,6 +729,10 @@ static void hmat_update_target_attrs(struct memory_target *target, u32 best = 0; int i; + /* Don't update if an external agent has changed the data. */ + if (target->ext_updated) + return; + /* Don't update for generic port if there's no device handle */ if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 40052666ebf1..ee1bc8fa396b 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -580,3 +580,9 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000); } } + +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access) +{ + return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 3b64fb1b9ed0..e19800a7ce06 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -90,4 +90,7 @@ enum cxl_poison_trace_type { long cxl_pci_get_latency(struct pci_dev *pdev); +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access); + #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index d4d39536e6c7..535492ec8529 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -116,12 +117,22 @@ static const struct attribute_group cxl_region_access0_coordinate_group = { .is_visible = cxl_region_access0_coordinate_visible, }; +static const struct attribute_group *get_cxl_region_access0_group(void) +{ + return &cxl_region_access0_coordinate_group; +} + static const struct attribute_group cxl_region_access1_coordinate_group = { .name = "access1", .attrs = access1_coordinate_attrs, .is_visible = cxl_region_access1_coordinate_visible, }; +static const struct attribute_group *get_cxl_region_access1_group(void) +{ + return &cxl_region_access1_coordinate_group; +} + static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2216,6 +2227,7 @@ static void unregister_region(void *_cxlr) struct cxl_region_params *p = &cxlr->params; int i; + unregister_memory_notifier(&cxlr->memory_notifier); device_del(&cxlr->dev); /* @@ -2260,6 +2272,58 @@ static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int i return cxlr; } +static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) +{ + int cset = 0; + int rc; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + if (cxlr->coord[i].read_bandwidth) { + rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + if (rc == 0) + cset++; + } + } + + if (!cset) + return false; + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); + + return true; +} + +static int cxl_region_perf_attrs_callback(struct notifier_block *nb, + unsigned long action, void *arg) +{ + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + memory_notifier); + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled = p->targets[0]; + struct cxl_decoder *cxld = &cxled->cxld; + struct memory_notify *mnb = arg; + int nid = mnb->status_change_nid; + int region_nid; + + if (nid == NUMA_NO_NODE || action != MEM_ONLINE) + return NOTIFY_DONE; + + region_nid = phys_to_target_node(cxld->hpa_range.start); + if (nid != region_nid) + return NOTIFY_DONE; + + if (!cxl_region_update_coordinates(cxlr, nid)) + return NOTIFY_DONE; + + return NOTIFY_OK; +} + /** * devm_cxl_add_region - Adds a region to a decoder * @cxlrd: root decoder @@ -2307,6 +2371,10 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; + cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; + cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; + register_memory_notifier(&cxlr->memory_notifier); + rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); if (rc) return ERR_PTR(rc); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 95864ce7b394..534e25e2f0a4 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -518,6 +519,7 @@ struct cxl_region_params { * @flags: Region state flags * @params: active + config params for the region * @coord: QoS access coordinates for the region + * @memory_notifier: notifier for setting the access coordinates to node */ struct cxl_region { struct device dev; @@ -529,6 +531,7 @@ struct cxl_region { unsigned long flags; struct cxl_region_params params; struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct notifier_block memory_notifier; }; struct cxl_nvdimm_bridge { diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b7165e52b3c6..c84c2f34b8ee 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1547,4 +1547,16 @@ static inline void acpi_use_parent_companion(struct device *dev) ACPI_COMPANION_SET(dev, ACPI_COMPANION(dev->parent)); } +#ifdef CONFIG_ACPI_HMAT +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access); +#else +static inline int hmat_update_target_coordinates(int nid, + struct access_coordinate *coord, + enum access_coordinate_class access) +{ + return -EOPNOTSUPP; +} +#endif + #endif /*_LINUX_ACPI_H*/ diff --git a/include/linux/memory.h b/include/linux/memory.h index f53cfdaaaa41..d8588256578a 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -114,6 +114,7 @@ struct mem_section; #define DEFAULT_CALLBACK_PRI 0 #define SLAB_CALLBACK_PRI 1 #define HMAT_CALLBACK_PRI 2 +#define CXL_CALLBACK_PRI 5 #define MM_COMPUTE_BATCH_PRI 10 #define CPUSET_CALLBACK_PRI 10 #define MEMTIER_HOTPLUG_PRI 100 -- cgit v1.2.3 From 75060b6ead0e93b7b43a451ba1e13c49b1aa2025 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 6 Mar 2024 07:49:16 +0100 Subject: watchdog/core: remove sysctl handlers from public header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The functions are only used in the file where they are defined. Remove them from the header and make them static. Also guard proc_soft_watchdog with a #define-guard as it is not used otherwise. Link: https://lkml.kernel.org/r/20240306-const-sysctl-prep-watchdog-v1-1-bd45da3a41cf@weissschuh.net Signed-off-by: Thomas Weißschuh Signed-off-by: Andrew Morton --- include/linux/nmi.h | 7 ------- kernel/watchdog.c | 22 ++++++++++++---------- 2 files changed, 12 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index e92e378df000..f53438eae815 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -216,13 +216,6 @@ void watchdog_update_hrtimer_threshold(u64 period); static inline void watchdog_update_hrtimer_threshold(u64 period) { } #endif -struct ctl_table; -int proc_watchdog(struct ctl_table *, int, void *, size_t *, loff_t *); -int proc_nmi_watchdog(struct ctl_table *, int , void *, size_t *, loff_t *); -int proc_soft_watchdog(struct ctl_table *, int , void *, size_t *, loff_t *); -int proc_watchdog_thresh(struct ctl_table *, int , void *, size_t *, loff_t *); -int proc_watchdog_cpumask(struct ctl_table *, int, void *, size_t *, loff_t *); - #ifdef CONFIG_HAVE_ACPI_APEI_NMI #include #endif diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 81a8862295d6..d7b2125503af 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -796,8 +796,8 @@ static int proc_watchdog_common(int which, struct ctl_table *table, int write, /* * /proc/sys/kernel/watchdog */ -int proc_watchdog(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_watchdog(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) { return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED | WATCHDOG_SOFTOCKUP_ENABLED, @@ -807,8 +807,8 @@ int proc_watchdog(struct ctl_table *table, int write, /* * /proc/sys/kernel/nmi_watchdog */ -int proc_nmi_watchdog(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_nmi_watchdog(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) { if (!watchdog_hardlockup_available && write) return -ENOTSUPP; @@ -816,21 +816,23 @@ int proc_nmi_watchdog(struct ctl_table *table, int write, table, write, buffer, lenp, ppos); } +#ifdef CONFIG_SOFTLOCKUP_DETECTOR /* * /proc/sys/kernel/soft_watchdog */ -int proc_soft_watchdog(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_soft_watchdog(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) { return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED, table, write, buffer, lenp, ppos); } +#endif /* * /proc/sys/kernel/watchdog_thresh */ -int proc_watchdog_thresh(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_watchdog_thresh(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) { int err, old; @@ -852,8 +854,8 @@ int proc_watchdog_thresh(struct ctl_table *table, int write, * user to specify a mask that will include cpus that have not yet * been brought online, if desired. */ -int proc_watchdog_cpumask(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_watchdog_cpumask(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) { int err; -- cgit v1.2.3 From debdce20c4f28b7e5aa48512e7abf270a00e9051 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:31 -0700 Subject: cxl/region: Deal with numa nodes not enumerated by SRAT For the numa nodes that are not created by SRAT, no memory_target is allocated and is not managed by the HMAT_REPORTING code. Therefore hmat_callback() memory hotplug notifier will exit early on those NUMA nodes. The CXL memory hotplug notifier will need to call node_set_perf_attrs() directly in order to setup the access sysfs attributes. In acpi_numa_init(), the last proximity domain (pxm) id created by SRAT is stored. Add a helper function acpi_node_backed_by_real_pxm() in order to check if a NUMA node id is defined by SRAT or created by CFMWS. node_set_perf_attrs() symbol is exported to allow update of perf attribs for a node. The sysfs path of /sys/devices/system/node/nodeX/access0/initiators/* is created by node_set_perf_attrs() for the various attributes where nodeX is matched to the NUMA node of the CXL region. Cc: Rafael J. Wysocki Reviewed-by: Alison Schofield Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-13-dave.jiang@intel.com Signed-off-by: Dan Williams --- drivers/acpi/numa/srat.c | 11 +++++++++++ drivers/base/node.c | 1 + drivers/cxl/core/cdat.c | 5 +++++ drivers/cxl/core/core.h | 1 + drivers/cxl/core/region.c | 7 ++++++- include/linux/acpi.h | 9 +++++++++ 6 files changed, 33 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 0214518fc582..e45e64993c50 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -29,6 +29,8 @@ static int node_to_pxm_map[MAX_NUMNODES] unsigned char acpi_srat_revision __initdata; static int acpi_numa __initdata; +static int last_real_pxm; + void __init disable_srat(void) { acpi_numa = -1; @@ -536,6 +538,7 @@ int __init acpi_numa_init(void) if (node_to_pxm_map[i] > fake_pxm) fake_pxm = node_to_pxm_map[i]; } + last_real_pxm = fake_pxm; fake_pxm++; acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws, &fake_pxm); @@ -547,6 +550,14 @@ int __init acpi_numa_init(void) return 0; } +bool acpi_node_backed_by_real_pxm(int nid) +{ + int pxm = node_to_pxm(nid); + + return pxm <= last_real_pxm; +} +EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm); + static int acpi_get_pxm(acpi_handle h) { unsigned long long pxm; diff --git a/drivers/base/node.c b/drivers/base/node.c index a73b0c9a401a..eb72580288e6 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -215,6 +215,7 @@ void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, } } } +EXPORT_SYMBOL_GPL(node_set_perf_attrs); /** * struct node_cache_info - Internal tracking for memory node caches diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index ee1bc8fa396b..10d4f1d7dad1 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -586,3 +586,8 @@ int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, { return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); } + +bool cxl_need_node_perf_attrs_update(int nid) +{ + return !acpi_node_backed_by_real_pxm(nid); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index e19800a7ce06..bc5a95665aa0 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -92,5 +92,6 @@ long cxl_pci_get_latency(struct pci_dev *pdev); int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, enum access_coordinate_class access); +bool cxl_need_node_perf_attrs_update(int nid); #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 535492ec8529..5c186e0a39b9 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2279,7 +2279,12 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { if (cxlr->coord[i].read_bandwidth) { - rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + rc = 0; + if (cxl_need_node_perf_attrs_update(nid)) + node_set_perf_attrs(nid, &cxlr->coord[i], i); + else + rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + if (rc == 0) cset++; } diff --git a/include/linux/acpi.h b/include/linux/acpi.h index c84c2f34b8ee..2a7c4b90d589 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1559,4 +1559,13 @@ static inline int hmat_update_target_coordinates(int nid, } #endif +#ifdef CONFIG_ACPI_NUMA +bool acpi_node_backed_by_real_pxm(int nid); +#else +static inline bool acpi_node_backed_by_real_pxm(int nid) +{ + return false; +} +#endif + #endif /*_LINUX_ACPI_H*/ -- cgit v1.2.3 From 9f0c4a46be1fe9b97dbe66d49204c1371e3ece65 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 8 Mar 2024 09:08:34 +0800 Subject: f2fs: fix to truncate meta inode pages forcely Below race case can cause data corruption: Thread A GC thread - gc_data_segment - ra_data_block - locked meta_inode page - f2fs_inplace_write_data - invalidate_mapping_pages : fail to invalidate meta_inode page due to lock failure or dirty|writeback status - f2fs_submit_page_bio : write last dirty data to old blkaddr - move_data_block - load old data from meta_inode page - f2fs_submit_page_write : write old data to new blkaddr Because invalidate_mapping_pages() will skip invalidating page which has unclear status including locked, dirty, writeback and so on, so we need to use truncate_inode_pages_range() instead of invalidate_mapping_pages() to make sure meta_inode page will be dropped. Fixes: 6aa58d8ad20a ("f2fs: readahead encrypted block during GC") Fixes: e3b49ea36802 ("f2fs: invalidate META_MAPPING before IPU/DIO write") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/checkpoint.c | 5 +++-- fs/f2fs/f2fs.h | 28 +++++++++++++++++++++++++++- fs/f2fs/segment.c | 5 ++--- include/linux/f2fs_fs.h | 1 + 4 files changed, 33 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index a09a9609e228..55b7d2cf030f 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -1598,8 +1598,9 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) */ if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) || f2fs_sb_has_compression(sbi)) - invalidate_mapping_pages(META_MAPPING(sbi), - MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1); + f2fs_bug_on(sbi, + invalidate_inode_pages2_range(META_MAPPING(sbi), + MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1)); f2fs_release_ino_entry(sbi, false); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 4836e7cb0efe..9814e5981a6a 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -4655,10 +4655,36 @@ static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi) return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sbi->sb); } +static inline void f2fs_truncate_meta_inode_pages(struct f2fs_sb_info *sbi, + block_t blkaddr, unsigned int cnt) +{ + bool need_submit = false; + int i = 0; + + do { + struct page *page; + + page = find_get_page(META_MAPPING(sbi), blkaddr + i); + if (page) { + if (PageWriteback(page)) + need_submit = true; + f2fs_put_page(page, 0); + } + } while (++i < cnt && !need_submit); + + if (need_submit) + f2fs_submit_merged_write_cond(sbi, sbi->meta_inode, + NULL, 0, DATA); + + truncate_inode_pages_range(META_MAPPING(sbi), + F2FS_BLK_TO_BYTES((loff_t)blkaddr), + F2FS_BLK_END_BYTES((loff_t)(blkaddr + cnt - 1))); +} + static inline void f2fs_invalidate_internal_cache(struct f2fs_sb_info *sbi, block_t blkaddr) { - invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr); + f2fs_truncate_meta_inode_pages(sbi, blkaddr, 1); f2fs_invalidate_compress_page(sbi, blkaddr); } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 4ff3b2d14ddf..20af48d7f784 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3741,8 +3741,7 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio) } if (fio->post_read) - invalidate_mapping_pages(META_MAPPING(sbi), - fio->new_blkaddr, fio->new_blkaddr); + f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1); stat_inc_inplace_blocks(fio->sbi); @@ -3932,7 +3931,7 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, for (i = 0; i < len; i++) f2fs_wait_on_block_writeback(inode, blkaddr + i); - invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1); + f2fs_truncate_meta_inode_pages(sbi, blkaddr, len); } static int read_compacted_summaries(struct f2fs_sb_info *sbi) diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 755e9a41b196..a357287eac1e 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -27,6 +27,7 @@ #define F2FS_BYTES_TO_BLK(bytes) ((bytes) >> F2FS_BLKSIZE_BITS) #define F2FS_BLK_TO_BYTES(blk) ((blk) << F2FS_BLKSIZE_BITS) +#define F2FS_BLK_END_BYTES(blk) (F2FS_BLK_TO_BYTES(blk + 1) - 1) /* 0, 1(node nid), 2(meta nid) are reserved node id */ #define F2FS_RESERVED_NODE_NUM 3 -- cgit v1.2.3 From f88c3fb81c4badb46c2fef7d168ff138043e86bb Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Tue, 12 Mar 2024 20:32:19 -0700 Subject: mm, slab: remove last vestiges of SLAB_MEM_SPREAD Yes, yes, I know the slab people were planning on going slow and letting every subsystem fight this thing on their own. But let's just rip off the band-aid and get it over and done with. I don't want to see a number of unnecessary pull requests just to get rid of a flag that no longer has any meaning. This was mainly done with a couple of 'sed' scripts and then some manual cleanup of the end result. Link: https://lore.kernel.org/all/CAHk-=wji0u+OOtmAOD-5JV3SXcRJF___k_+8XNKmak0yd5vW1Q@mail.gmail.com/ Signed-off-by: Linus Torvalds --- drivers/dax/super.c | 3 +-- drivers/usb/isp1760/isp1760-hcd.c | 8 +++----- fs/9p/v9fs.c | 2 +- fs/adfs/super.c | 2 +- fs/befs/linuxvfs.c | 3 +-- fs/bfs/inode.c | 2 +- fs/ceph/super.c | 18 +++++++++--------- fs/coda/inode.c | 4 ++-- fs/erofs/super.c | 2 +- fs/exfat/cache.c | 2 +- fs/exfat/super.c | 2 +- fs/ext2/super.c | 3 +-- fs/ext4/super.c | 3 +-- fs/fat/cache.c | 2 +- fs/fat/inode.c | 2 +- fs/freevxfs/vxfs_super.c | 2 +- fs/gfs2/main.c | 1 - fs/hpfs/super.c | 2 +- fs/isofs/inode.c | 2 +- fs/jffs2/super.c | 2 +- fs/nfs/direct.c | 3 +-- fs/nfs/inode.c | 2 +- fs/nfs/nfs42xattr.c | 2 +- fs/ntfs3/super.c | 2 +- fs/ocfs2/dlmfs/dlmfs.c | 2 +- fs/ocfs2/super.c | 7 +++---- fs/overlayfs/super.c | 2 +- fs/qnx4/inode.c | 2 +- fs/quota/dquot.c | 2 +- fs/smb/client/cifsfs.c | 2 +- fs/tracefs/inode.c | 1 - fs/ubifs/super.c | 4 ++-- fs/udf/super.c | 1 - fs/ufs/super.c | 3 +-- fs/vboxsf/super.c | 3 +-- fs/xfs/xfs_super.c | 7 +++---- fs/zonefs/super.c | 2 +- include/linux/slab.h | 3 --- net/sunrpc/rpc_pipe.c | 2 +- 39 files changed, 51 insertions(+), 68 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index f4b635526345..a0244f6bb44b 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -532,8 +532,7 @@ static int dax_fs_init(void) int rc; dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0, - (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, init_once); if (!dax_cache) return -ENOMEM; diff --git a/drivers/usb/isp1760/isp1760-hcd.c b/drivers/usb/isp1760/isp1760-hcd.c index 76862ba40f35..0e5e4cb74c87 100644 --- a/drivers/usb/isp1760/isp1760-hcd.c +++ b/drivers/usb/isp1760/isp1760-hcd.c @@ -2521,21 +2521,19 @@ static const struct hc_driver isp1760_hc_driver = { int __init isp1760_init_kmem_once(void) { urb_listitem_cachep = kmem_cache_create("isp1760_urb_listitem", - sizeof(struct urb_listitem), 0, SLAB_TEMPORARY | - SLAB_MEM_SPREAD, NULL); + sizeof(struct urb_listitem), 0, SLAB_TEMPORARY, NULL); if (!urb_listitem_cachep) return -ENOMEM; qtd_cachep = kmem_cache_create("isp1760_qtd", - sizeof(struct isp1760_qtd), 0, SLAB_TEMPORARY | - SLAB_MEM_SPREAD, NULL); + sizeof(struct isp1760_qtd), 0, SLAB_TEMPORARY, NULL); if (!qtd_cachep) goto destroy_urb_listitem; qh_cachep = kmem_cache_create("isp1760_qh", sizeof(struct isp1760_qh), - 0, SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); + 0, SLAB_TEMPORARY, NULL); if (!qh_cachep) goto destroy_qtd; diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index 61dbe52bb3a3..281a1ed03a04 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -637,7 +637,7 @@ static int v9fs_init_inode_cache(void) v9fs_inode_cache = kmem_cache_create("v9fs_inode_cache", sizeof(struct v9fs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), v9fs_inode_init_once); if (!v9fs_inode_cache) return -ENOMEM; diff --git a/fs/adfs/super.c b/fs/adfs/super.c index e8bfc38239cd..9354b14bbfe3 100644 --- a/fs/adfs/super.c +++ b/fs/adfs/super.c @@ -249,7 +249,7 @@ static int __init init_inodecache(void) adfs_inode_cachep = kmem_cache_create("adfs_inode_cache", sizeof(struct adfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (adfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c index 2b4dda047450..d76f406d3b2e 100644 --- a/fs/befs/linuxvfs.c +++ b/fs/befs/linuxvfs.c @@ -435,8 +435,7 @@ befs_init_inodecache(void) { befs_inode_cachep = kmem_cache_create_usercopy("befs_inode_cache", sizeof(struct befs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct befs_inode_info, i_data.symlink), sizeof_field(struct befs_inode_info, diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c index 355957dbce39..db81570c9637 100644 --- a/fs/bfs/inode.c +++ b/fs/bfs/inode.c @@ -259,7 +259,7 @@ static int __init init_inodecache(void) bfs_inode_cachep = kmem_cache_create("bfs_inode_cache", sizeof(struct bfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (bfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/ceph/super.c b/fs/ceph/super.c index 5ec102f6b1ac..885cb5d4e771 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -928,36 +928,36 @@ static int __init init_caches(void) ceph_inode_cachep = kmem_cache_create("ceph_inode_info", sizeof(struct ceph_inode_info), __alignof__(struct ceph_inode_info), - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT, ceph_inode_init_once); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + ceph_inode_init_once); if (!ceph_inode_cachep) return -ENOMEM; - ceph_cap_cachep = KMEM_CACHE(ceph_cap, SLAB_MEM_SPREAD); + ceph_cap_cachep = KMEM_CACHE(ceph_cap, 0); if (!ceph_cap_cachep) goto bad_cap; - ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, SLAB_MEM_SPREAD); + ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, 0); if (!ceph_cap_snap_cachep) goto bad_cap_snap; ceph_cap_flush_cachep = KMEM_CACHE(ceph_cap_flush, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD); + SLAB_RECLAIM_ACCOUNT); if (!ceph_cap_flush_cachep) goto bad_cap_flush; ceph_dentry_cachep = KMEM_CACHE(ceph_dentry_info, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD); + SLAB_RECLAIM_ACCOUNT); if (!ceph_dentry_cachep) goto bad_dentry; - ceph_file_cachep = KMEM_CACHE(ceph_file_info, SLAB_MEM_SPREAD); + ceph_file_cachep = KMEM_CACHE(ceph_file_info, 0); if (!ceph_file_cachep) goto bad_file; - ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, SLAB_MEM_SPREAD); + ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, 0); if (!ceph_dir_file_cachep) goto bad_dir_file; - ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, SLAB_MEM_SPREAD); + ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, 0); if (!ceph_mds_request_cachep) goto bad_mds_req; diff --git a/fs/coda/inode.c b/fs/coda/inode.c index a50356c541f6..6898dc621011 100644 --- a/fs/coda/inode.c +++ b/fs/coda/inode.c @@ -72,8 +72,8 @@ int __init coda_init_inodecache(void) { coda_inode_cachep = kmem_cache_create("coda_inode_cache", sizeof(struct coda_inode_info), 0, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT, init_once); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); if (coda_inode_cachep == NULL) return -ENOMEM; return 0; diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 6fbb1fba2d31..69308fd73e4a 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -857,7 +857,7 @@ static int __init erofs_module_init(void) erofs_inode_cachep = kmem_cache_create("erofs_inode", sizeof(struct erofs_inode), 0, - SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, erofs_inode_init_once); if (!erofs_inode_cachep) return -ENOMEM; diff --git a/fs/exfat/cache.c b/fs/exfat/cache.c index 5a2f119b7e8c..7cc200d89821 100644 --- a/fs/exfat/cache.c +++ b/fs/exfat/cache.c @@ -46,7 +46,7 @@ int exfat_cache_init(void) { exfat_cachep = kmem_cache_create("exfat_cache", sizeof(struct exfat_cache), - 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, exfat_cache_init_once); if (!exfat_cachep) return -ENOMEM; diff --git a/fs/exfat/super.c b/fs/exfat/super.c index fcb658267765..3d5ea2cfad66 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -813,7 +813,7 @@ static int __init init_exfat_fs(void) exfat_inode_cachep = kmem_cache_create("exfat_inode_cache", sizeof(struct exfat_inode_info), - 0, SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, exfat_inode_init_once); if (!exfat_inode_cachep) { err = -ENOMEM; diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 01f9addc8b1f..cabea887314d 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -213,8 +213,7 @@ static int __init init_inodecache(void) { ext2_inode_cachep = kmem_cache_create_usercopy("ext2_inode_cache", sizeof(struct ext2_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct ext2_inode_info, i_data), sizeof_field(struct ext2_inode_info, i_data), init_once); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index a8ba84eabab2..59c72b6dd153 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1500,8 +1500,7 @@ static int __init init_inodecache(void) { ext4_inode_cachep = kmem_cache_create_usercopy("ext4_inode_cache", sizeof(struct ext4_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct ext4_inode_info, i_data), sizeof_field(struct ext4_inode_info, i_data), init_once); diff --git a/fs/fat/cache.c b/fs/fat/cache.c index 738e427e2d21..2af424e200b3 100644 --- a/fs/fat/cache.c +++ b/fs/fat/cache.c @@ -47,7 +47,7 @@ int __init fat_cache_init(void) { fat_cache_cachep = kmem_cache_create("fat_cache", sizeof(struct fat_cache), - 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, init_once); if (fat_cache_cachep == NULL) return -ENOMEM; diff --git a/fs/fat/inode.c b/fs/fat/inode.c index 5c813696d1ff..d9e6fbb6f246 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -787,7 +787,7 @@ static int __init fat_init_inodecache(void) fat_inode_cachep = kmem_cache_create("fat_inode_cache", sizeof(struct msdos_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (fat_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/freevxfs/vxfs_super.c b/fs/freevxfs/vxfs_super.c index e6e2a2185e7c..42e03b6b1cc7 100644 --- a/fs/freevxfs/vxfs_super.c +++ b/fs/freevxfs/vxfs_super.c @@ -307,7 +307,7 @@ vxfs_init(void) vxfs_inode_cachep = kmem_cache_create_usercopy("vxfs_inode", sizeof(struct vxfs_inode_info), 0, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, offsetof(struct vxfs_inode_info, vii_immed.vi_immed), sizeof_field(struct vxfs_inode_info, vii_immed.vi_immed), diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c index 79be0cdc730c..04cadc02e5a6 100644 --- a/fs/gfs2/main.c +++ b/fs/gfs2/main.c @@ -111,7 +111,6 @@ static int __init init_gfs2_fs(void) gfs2_inode_cachep = kmem_cache_create("gfs2_inode", sizeof(struct gfs2_inode), 0, SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD| SLAB_ACCOUNT, gfs2_init_inode_once); if (!gfs2_inode_cachep) diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c index 6b0ba3c1efba..314834a078e9 100644 --- a/fs/hpfs/super.c +++ b/fs/hpfs/super.c @@ -255,7 +255,7 @@ static int init_inodecache(void) hpfs_inode_cachep = kmem_cache_create("hpfs_inode_cache", sizeof(struct hpfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (hpfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index 3e4d53e26f94..25fca44149dd 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -93,7 +93,7 @@ static int __init init_inodecache(void) isofs_inode_cachep = kmem_cache_create("isofs_inode_cache", sizeof(struct iso_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (!isofs_inode_cachep) return -ENOMEM; diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c index f99591a634b4..aede1be4dc0c 100644 --- a/fs/jffs2/super.c +++ b/fs/jffs2/super.c @@ -387,7 +387,7 @@ static int __init init_jffs2_fs(void) jffs2_inode_cachep = kmem_cache_create("jffs2_i", sizeof(struct jffs2_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), jffs2_i_init_once); if (!jffs2_inode_cachep) { pr_err("error: Failed to initialise inode cache\n"); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index c03926a1cc73..7af5d270de28 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -1037,8 +1037,7 @@ int __init nfs_init_directcache(void) { nfs_direct_cachep = kmem_cache_create("nfs_direct_cache", sizeof(struct nfs_direct_req), - 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD), + 0, SLAB_RECLAIM_ACCOUNT, NULL); if (nfs_direct_cachep == NULL) return -ENOMEM; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index ebb8d60e1152..93ea49a7eb61 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -2372,7 +2372,7 @@ static int __init nfs_init_inodecache(void) nfs_inode_cachep = kmem_cache_create("nfs_inode_cache", sizeof(struct nfs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (nfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/nfs/nfs42xattr.c b/fs/nfs/nfs42xattr.c index 49aaf28a6950..b6e3d8f77b91 100644 --- a/fs/nfs/nfs42xattr.c +++ b/fs/nfs/nfs42xattr.c @@ -1017,7 +1017,7 @@ int __init nfs4_xattr_cache_init(void) nfs4_xattr_cache_cachep = kmem_cache_create("nfs4_xattr_cache_cache", sizeof(struct nfs4_xattr_cache), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD), + (SLAB_RECLAIM_ACCOUNT), nfs4_xattr_cache_init_once); if (nfs4_xattr_cache_cachep == NULL) return -ENOMEM; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index cef5467fd928..9df7c20d066f 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -1825,7 +1825,7 @@ static int __init init_ntfs_fs(void) ntfs_inode_cachep = kmem_cache_create( "ntfs_inode_cache", sizeof(struct ntfs_inode), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), init_once); if (!ntfs_inode_cachep) { err = -ENOMEM; diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index 85215162c9dd..7fc0e920eda7 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -578,7 +578,7 @@ static int __init init_dlmfs_fs(void) dlmfs_inode_cache = kmem_cache_create("dlmfs_inode_cache", sizeof(struct dlmfs_inode_private), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), dlmfs_init_once); if (!dlmfs_inode_cache) { status = -ENOMEM; diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index a70aff17d455..b3f860888e93 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1706,18 +1706,17 @@ static int ocfs2_initialize_mem_caches(void) sizeof(struct ocfs2_inode_info), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), ocfs2_inode_init_once); ocfs2_dquot_cachep = kmem_cache_create("ocfs2_dquot_cache", sizeof(struct ocfs2_dquot), 0, - (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD), + (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT), NULL); ocfs2_qf_chunk_cachep = kmem_cache_create("ocfs2_qf_chunk_cache", sizeof(struct ocfs2_quota_chunk), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD), + (SLAB_RECLAIM_ACCOUNT), NULL); if (!ocfs2_inode_cachep || !ocfs2_dquot_cachep || !ocfs2_qf_chunk_cachep) { diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 36d4b8b1f784..a40fc7e05525 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1503,7 +1503,7 @@ static int __init ovl_init(void) ovl_inode_cachep = kmem_cache_create("ovl_inode", sizeof(struct ovl_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), ovl_inode_init_once); if (ovl_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c index 7b5711f76709..d79841e94428 100644 --- a/fs/qnx4/inode.c +++ b/fs/qnx4/inode.c @@ -378,7 +378,7 @@ static int init_inodecache(void) qnx4_inode_cachep = kmem_cache_create("qnx4_inode_cache", sizeof(struct qnx4_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (qnx4_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 1f0c754416b6..eb6e9d95dea1 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -2984,7 +2984,7 @@ static int __init dquot_init(void) dquot_cachep = kmem_cache_create("dquot", sizeof(struct dquot), sizeof(unsigned long) * 4, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_PANIC), + SLAB_PANIC), NULL); order = 0; diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index fb368b191eef..e0d8c79cdde1 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -1664,7 +1664,7 @@ cifs_init_inodecache(void) cifs_inode_cachep = kmem_cache_create("cifs_inode_cache", sizeof(struct cifsInodeInfo), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), cifs_init_once); if (cifs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index d65ffad4c327..5545e6bf7d26 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -731,7 +731,6 @@ static int __init tracefs_init(void) tracefs_inode_cachep = kmem_cache_create("tracefs_inode_cache", sizeof(struct tracefs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD| SLAB_ACCOUNT), init_once); if (!tracefs_inode_cachep) diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index d2881041b393..7f4031a15f4d 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -2434,8 +2434,8 @@ static int __init ubifs_init(void) ubifs_inode_slab = kmem_cache_create("ubifs_inode_slab", sizeof(struct ubifs_inode), 0, - SLAB_MEM_SPREAD | SLAB_RECLAIM_ACCOUNT | - SLAB_ACCOUNT, &inode_slab_ctor); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + &inode_slab_ctor); if (!ubifs_inode_slab) return -ENOMEM; diff --git a/fs/udf/super.c b/fs/udf/super.c index 928a04d9d9e0..6f420f4ca005 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -177,7 +177,6 @@ static int __init init_inodecache(void) udf_inode_cachep = kmem_cache_create("udf_inode_cache", sizeof(struct udf_inode_info), 0, (SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD | SLAB_ACCOUNT), init_once); if (!udf_inode_cachep) diff --git a/fs/ufs/super.c b/fs/ufs/super.c index a480810cd4e3..44666afc6209 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1470,8 +1470,7 @@ static int __init init_inodecache(void) { ufs_inode_cachep = kmem_cache_create_usercopy("ufs_inode_cache", sizeof(struct ufs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), offsetof(struct ufs_inode_info, i_u1.i_symlink), sizeof_field(struct ufs_inode_info, i_u1.i_symlink), diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c index 1fb8f4df60cb..cabe8ac4fefc 100644 --- a/fs/vboxsf/super.c +++ b/fs/vboxsf/super.c @@ -339,8 +339,7 @@ static int vboxsf_setup(void) vboxsf_inode_cachep = kmem_cache_create("vboxsf_inode_cache", sizeof(struct vboxsf_inode), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, vboxsf_inode_init_once); if (!vboxsf_inode_cachep) { err = -ENOMEM; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 00fbd5b6e582..59c8c0541bdd 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -2043,8 +2043,7 @@ xfs_init_caches(void) xfs_buf_cache = kmem_cache_create("xfs_buf", sizeof(struct xfs_buf), 0, SLAB_HWCACHE_ALIGN | - SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, NULL); if (!xfs_buf_cache) goto out; @@ -2109,14 +2108,14 @@ xfs_init_caches(void) sizeof(struct xfs_inode), 0, (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD | SLAB_ACCOUNT), + SLAB_ACCOUNT), xfs_fs_inode_init_once); if (!xfs_inode_cache) goto out_destroy_efi_cache; xfs_ili_cache = kmem_cache_create("xfs_ili", sizeof(struct xfs_inode_log_item), 0, - SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, NULL); if (!xfs_ili_cache) goto out_destroy_inode_cache; diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 236a6d88306f..c6a124e8d565 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1422,7 +1422,7 @@ static int __init zonefs_init_inodecache(void) { zonefs_inode_cachep = kmem_cache_create("zonefs_inode_cache", sizeof(struct zonefs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, NULL); if (zonefs_inode_cachep == NULL) return -ENOMEM; diff --git a/include/linux/slab.h b/include/linux/slab.h index f6323763cd61..e53cbfa18325 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -202,9 +202,6 @@ enum _slab_flag_bits { #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ -/* Obsolete unused flag, to be removed */ -#define SLAB_MEM_SPREAD __SLAB_FLAG_UNUSED - /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c index dcc2b4f49e77..910a5d850d04 100644 --- a/net/sunrpc/rpc_pipe.c +++ b/net/sunrpc/rpc_pipe.c @@ -1490,7 +1490,7 @@ int register_rpc_pipefs(void) rpc_inode_cachep = kmem_cache_create("rpc_inode_cache", sizeof(struct rpc_inode), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (!rpc_inode_cachep) return -ENOMEM; -- cgit v1.2.3 From 12fb28ea6b1cf24bde27c406eb38ee3c108d73f9 Mon Sep 17 00:00:00 2001 From: Ben Cheatham Date: Mon, 11 Mar 2024 09:25:06 -0500 Subject: EINJ: Add CXL error type support Move CXL protocol error types from einj.c (now einj-core.c) to einj-cxl.c. einj-cxl.c implements the necessary handling for CXL protocol error injection and exposes an API for the CXL core to use said functionality, while also allowing the EINJ module to be built without CXL support. Because CXL error types targeting CXL 1.0/1.1 ports require special handling, only allow them to be injected through the new cxl debugfs interface (next commit) and return an error when attempting to inject through the legacy interface. Reviewed-by: Jonathan Cameron Signed-off-by: Ben Cheatham Link: https://lore.kernel.org/r/20240311142508.31717-3-Benjamin.Cheatham@amd.com Signed-off-by: Dan Williams --- MAINTAINERS | 1 + drivers/acpi/apei/Kconfig | 13 + drivers/acpi/apei/Makefile | 2 + drivers/acpi/apei/apei-internal.h | 18 + drivers/acpi/apei/einj-core.c | 914 ++++++++++++++++++++++++++++++++++++++ drivers/acpi/apei/einj-cxl.c | 113 +++++ drivers/acpi/apei/einj.c | 874 ------------------------------------ include/linux/einj-cxl.h | 44 ++ 8 files changed, 1105 insertions(+), 874 deletions(-) create mode 100644 drivers/acpi/apei/einj-core.c create mode 100644 drivers/acpi/apei/einj-cxl.c delete mode 100644 drivers/acpi/apei/einj.c create mode 100644 include/linux/einj-cxl.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 2ecaaec6a6bf..90cf8403dd17 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5289,6 +5289,7 @@ M: Dan Williams L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ +F: include/linux/cxl-einj.h F: include/linux/cxl-event.h F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ diff --git a/drivers/acpi/apei/Kconfig b/drivers/acpi/apei/Kconfig index 6b18f8bc7be3..3cfe7e7475f2 100644 --- a/drivers/acpi/apei/Kconfig +++ b/drivers/acpi/apei/Kconfig @@ -60,6 +60,19 @@ config ACPI_APEI_EINJ mainly used for debugging and testing the other parts of APEI and some other RAS features. +config ACPI_APEI_EINJ_CXL + bool "CXL Error INJection Support" + default ACPI_APEI_EINJ + depends on ACPI_APEI_EINJ + depends on CXL_BUS && CXL_BUS <= ACPI_APEI_EINJ + help + Support for CXL protocol Error INJection through debugfs/cxl. + Availability and which errors are supported is dependent on + the host platform. Look to ACPI v6.5 section 18.6.4 and kernel + EINJ documentation for more information. + + If unsure say 'n' + config ACPI_APEI_ERST_DEBUG tristate "APEI Error Record Serialization Table (ERST) Debug Support" depends on ACPI_APEI diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 4dfac2128737..2c474e6477e1 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -2,6 +2,8 @@ obj-$(CONFIG_ACPI_APEI) += apei.o obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o +einj-y := einj-core.o +einj-$(CONFIG_ACPI_APEI_EINJ_CXL) += einj-cxl.o obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o apei-y := apei-base.o hest.o erst.o bert.o diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h index 67c2c3b959e1..cd2766c69d78 100644 --- a/drivers/acpi/apei/apei-internal.h +++ b/drivers/acpi/apei/apei-internal.h @@ -130,4 +130,22 @@ static inline u32 cper_estatus_len(struct acpi_hest_generic_status *estatus) } int apei_osc_setup(void); + +int einj_get_available_error_type(u32 *type); +int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, + u64 param4); +int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4); +bool einj_is_cxl_error_type(u64 type); +int einj_validate_error_type(u64 type); + +#ifndef ACPI_EINJ_CXL_CACHE_CORRECTABLE +#define ACPI_EINJ_CXL_CACHE_CORRECTABLE BIT(12) +#define ACPI_EINJ_CXL_CACHE_UNCORRECTABLE BIT(13) +#define ACPI_EINJ_CXL_CACHE_FATAL BIT(14) +#define ACPI_EINJ_CXL_MEM_CORRECTABLE BIT(15) +#define ACPI_EINJ_CXL_MEM_UNCORRECTABLE BIT(16) +#define ACPI_EINJ_CXL_MEM_FATAL BIT(17) +#endif + #endif diff --git a/drivers/acpi/apei/einj-core.c b/drivers/acpi/apei/einj-core.c new file mode 100644 index 000000000000..66e7f529e92f --- /dev/null +++ b/drivers/acpi/apei/einj-core.c @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * APEI Error INJection support + * + * EINJ provides a hardware error injection mechanism, this is useful + * for debugging and testing of other APEI and RAS features. + * + * For more information about EINJ, please refer to ACPI Specification + * version 4.0, section 17.5. + * + * Copyright 2009-2010 Intel Corp. + * Author: Huang Ying + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apei-internal.h" + +#undef pr_fmt +#define pr_fmt(fmt) "EINJ: " fmt + +#define SLEEP_UNIT_MIN 1000 /* 1ms */ +#define SLEEP_UNIT_MAX 5000 /* 5ms */ +/* Firmware should respond within 1 seconds */ +#define FIRMWARE_TIMEOUT (1 * USEC_PER_SEC) +#define ACPI5_VENDOR_BIT BIT(31) +#define MEM_ERROR_MASK (ACPI_EINJ_MEMORY_CORRECTABLE | \ + ACPI_EINJ_MEMORY_UNCORRECTABLE | \ + ACPI_EINJ_MEMORY_FATAL) +#define CXL_ERROR_MASK (ACPI_EINJ_CXL_CACHE_CORRECTABLE | \ + ACPI_EINJ_CXL_CACHE_UNCORRECTABLE | \ + ACPI_EINJ_CXL_CACHE_FATAL | \ + ACPI_EINJ_CXL_MEM_CORRECTABLE | \ + ACPI_EINJ_CXL_MEM_UNCORRECTABLE | \ + ACPI_EINJ_CXL_MEM_FATAL) + +/* + * ACPI version 5 provides a SET_ERROR_TYPE_WITH_ADDRESS action. + */ +static int acpi5; + +struct set_error_type_with_address { + u32 type; + u32 vendor_extension; + u32 flags; + u32 apicid; + u64 memory_address; + u64 memory_address_range; + u32 pcie_sbdf; +}; +enum { + SETWA_FLAGS_APICID = 1, + SETWA_FLAGS_MEM = 2, + SETWA_FLAGS_PCIE_SBDF = 4, +}; + +/* + * Vendor extensions for platform specific operations + */ +struct vendor_error_type_extension { + u32 length; + u32 pcie_sbdf; + u16 vendor_id; + u16 device_id; + u8 rev_id; + u8 reserved[3]; +}; + +static u32 notrigger; + +static u32 vendor_flags; +static struct debugfs_blob_wrapper vendor_blob; +static struct debugfs_blob_wrapper vendor_errors; +static char vendor_dev[64]; + +/* + * Some BIOSes allow parameters to the SET_ERROR_TYPE entries in the + * EINJ table through an unpublished extension. Use with caution as + * most will ignore the parameter and make their own choice of address + * for error injection. This extension is used only if + * param_extension module parameter is specified. + */ +struct einj_parameter { + u64 type; + u64 reserved1; + u64 reserved2; + u64 param1; + u64 param2; +}; + +#define EINJ_OP_BUSY 0x1 +#define EINJ_STATUS_SUCCESS 0x0 +#define EINJ_STATUS_FAIL 0x1 +#define EINJ_STATUS_INVAL 0x2 + +#define EINJ_TAB_ENTRY(tab) \ + ((struct acpi_whea_header *)((char *)(tab) + \ + sizeof(struct acpi_table_einj))) + +static bool param_extension; +module_param(param_extension, bool, 0); + +static struct acpi_table_einj *einj_tab; + +static struct apei_resources einj_resources; + +static struct apei_exec_ins_type einj_ins_type[] = { + [ACPI_EINJ_READ_REGISTER] = { + .flags = APEI_EXEC_INS_ACCESS_REGISTER, + .run = apei_exec_read_register, + }, + [ACPI_EINJ_READ_REGISTER_VALUE] = { + .flags = APEI_EXEC_INS_ACCESS_REGISTER, + .run = apei_exec_read_register_value, + }, + [ACPI_EINJ_WRITE_REGISTER] = { + .flags = APEI_EXEC_INS_ACCESS_REGISTER, + .run = apei_exec_write_register, + }, + [ACPI_EINJ_WRITE_REGISTER_VALUE] = { + .flags = APEI_EXEC_INS_ACCESS_REGISTER, + .run = apei_exec_write_register_value, + }, + [ACPI_EINJ_NOOP] = { + .flags = 0, + .run = apei_exec_noop, + }, +}; + +/* + * Prevent EINJ interpreter to run simultaneously, because the + * corresponding firmware implementation may not work properly when + * invoked simultaneously. + */ +static DEFINE_MUTEX(einj_mutex); + +/* + * Exported APIs use this flag to exit early if einj_probe() failed. + */ +bool einj_initialized __ro_after_init; + +static void *einj_param; + +static void einj_exec_ctx_init(struct apei_exec_context *ctx) +{ + apei_exec_ctx_init(ctx, einj_ins_type, ARRAY_SIZE(einj_ins_type), + EINJ_TAB_ENTRY(einj_tab), einj_tab->entries); +} + +static int __einj_get_available_error_type(u32 *type) +{ + struct apei_exec_context ctx; + int rc; + + einj_exec_ctx_init(&ctx); + rc = apei_exec_run(&ctx, ACPI_EINJ_GET_ERROR_TYPE); + if (rc) + return rc; + *type = apei_exec_ctx_get_output(&ctx); + + return 0; +} + +/* Get error injection capabilities of the platform */ +int einj_get_available_error_type(u32 *type) +{ + int rc; + + mutex_lock(&einj_mutex); + rc = __einj_get_available_error_type(type); + mutex_unlock(&einj_mutex); + + return rc; +} + +static int einj_timedout(u64 *t) +{ + if ((s64)*t < SLEEP_UNIT_MIN) { + pr_warn(FW_WARN "Firmware does not respond in time\n"); + return 1; + } + *t -= SLEEP_UNIT_MIN; + usleep_range(SLEEP_UNIT_MIN, SLEEP_UNIT_MAX); + + return 0; +} + +static void get_oem_vendor_struct(u64 paddr, int offset, + struct vendor_error_type_extension *v) +{ + unsigned long vendor_size; + u64 target_pa = paddr + offset + sizeof(struct vendor_error_type_extension); + + vendor_size = v->length - sizeof(struct vendor_error_type_extension); + + if (vendor_size) + vendor_errors.data = acpi_os_map_memory(target_pa, vendor_size); + + if (vendor_errors.data) + vendor_errors.size = vendor_size; +} + +static void check_vendor_extension(u64 paddr, + struct set_error_type_with_address *v5param) +{ + int offset = v5param->vendor_extension; + struct vendor_error_type_extension *v; + u32 sbdf; + + if (!offset) + return; + v = acpi_os_map_iomem(paddr + offset, sizeof(*v)); + if (!v) + return; + get_oem_vendor_struct(paddr, offset, v); + sbdf = v->pcie_sbdf; + sprintf(vendor_dev, "%x:%x:%x.%x vendor_id=%x device_id=%x rev_id=%x\n", + sbdf >> 24, (sbdf >> 16) & 0xff, + (sbdf >> 11) & 0x1f, (sbdf >> 8) & 0x7, + v->vendor_id, v->device_id, v->rev_id); + acpi_os_unmap_iomem(v, sizeof(*v)); +} + +static void *einj_get_parameter_address(void) +{ + int i; + u64 pa_v4 = 0, pa_v5 = 0; + struct acpi_whea_header *entry; + + entry = EINJ_TAB_ENTRY(einj_tab); + for (i = 0; i < einj_tab->entries; i++) { + if (entry->action == ACPI_EINJ_SET_ERROR_TYPE && + entry->instruction == ACPI_EINJ_WRITE_REGISTER && + entry->register_region.space_id == + ACPI_ADR_SPACE_SYSTEM_MEMORY) + pa_v4 = get_unaligned(&entry->register_region.address); + if (entry->action == ACPI_EINJ_SET_ERROR_TYPE_WITH_ADDRESS && + entry->instruction == ACPI_EINJ_WRITE_REGISTER && + entry->register_region.space_id == + ACPI_ADR_SPACE_SYSTEM_MEMORY) + pa_v5 = get_unaligned(&entry->register_region.address); + entry++; + } + if (pa_v5) { + struct set_error_type_with_address *v5param; + + v5param = acpi_os_map_iomem(pa_v5, sizeof(*v5param)); + if (v5param) { + acpi5 = 1; + check_vendor_extension(pa_v5, v5param); + return v5param; + } + } + if (param_extension && pa_v4) { + struct einj_parameter *v4param; + + v4param = acpi_os_map_iomem(pa_v4, sizeof(*v4param)); + if (!v4param) + return NULL; + if (v4param->reserved1 || v4param->reserved2) { + acpi_os_unmap_iomem(v4param, sizeof(*v4param)); + return NULL; + } + return v4param; + } + + return NULL; +} + +/* do sanity check to trigger table */ +static int einj_check_trigger_header(struct acpi_einj_trigger *trigger_tab) +{ + if (trigger_tab->header_size != sizeof(struct acpi_einj_trigger)) + return -EINVAL; + if (trigger_tab->table_size > PAGE_SIZE || + trigger_tab->table_size < trigger_tab->header_size) + return -EINVAL; + if (trigger_tab->entry_count != + (trigger_tab->table_size - trigger_tab->header_size) / + sizeof(struct acpi_einj_entry)) + return -EINVAL; + + return 0; +} + +static struct acpi_generic_address *einj_get_trigger_parameter_region( + struct acpi_einj_trigger *trigger_tab, u64 param1, u64 param2) +{ + int i; + struct acpi_whea_header *entry; + + entry = (struct acpi_whea_header *) + ((char *)trigger_tab + sizeof(struct acpi_einj_trigger)); + for (i = 0; i < trigger_tab->entry_count; i++) { + if (entry->action == ACPI_EINJ_TRIGGER_ERROR && + entry->instruction <= ACPI_EINJ_WRITE_REGISTER_VALUE && + entry->register_region.space_id == + ACPI_ADR_SPACE_SYSTEM_MEMORY && + (entry->register_region.address & param2) == (param1 & param2)) + return &entry->register_region; + entry++; + } + + return NULL; +} +/* Execute instructions in trigger error action table */ +static int __einj_error_trigger(u64 trigger_paddr, u32 type, + u64 param1, u64 param2) +{ + struct acpi_einj_trigger *trigger_tab = NULL; + struct apei_exec_context trigger_ctx; + struct apei_resources trigger_resources; + struct acpi_whea_header *trigger_entry; + struct resource *r; + u32 table_size; + int rc = -EIO; + struct acpi_generic_address *trigger_param_region = NULL; + + r = request_mem_region(trigger_paddr, sizeof(*trigger_tab), + "APEI EINJ Trigger Table"); + if (!r) { + pr_err("Can not request [mem %#010llx-%#010llx] for Trigger table\n", + (unsigned long long)trigger_paddr, + (unsigned long long)trigger_paddr + + sizeof(*trigger_tab) - 1); + goto out; + } + trigger_tab = ioremap_cache(trigger_paddr, sizeof(*trigger_tab)); + if (!trigger_tab) { + pr_err("Failed to map trigger table!\n"); + goto out_rel_header; + } + rc = einj_check_trigger_header(trigger_tab); + if (rc) { + pr_warn(FW_BUG "Invalid trigger error action table.\n"); + goto out_rel_header; + } + + /* No action structures in the TRIGGER_ERROR table, nothing to do */ + if (!trigger_tab->entry_count) + goto out_rel_header; + + rc = -EIO; + table_size = trigger_tab->table_size; + r = request_mem_region(trigger_paddr + sizeof(*trigger_tab), + table_size - sizeof(*trigger_tab), + "APEI EINJ Trigger Table"); + if (!r) { + pr_err("Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n", + (unsigned long long)trigger_paddr + sizeof(*trigger_tab), + (unsigned long long)trigger_paddr + table_size - 1); + goto out_rel_header; + } + iounmap(trigger_tab); + trigger_tab = ioremap_cache(trigger_paddr, table_size); + if (!trigger_tab) { + pr_err("Failed to map trigger table!\n"); + goto out_rel_entry; + } + trigger_entry = (struct acpi_whea_header *) + ((char *)trigger_tab + sizeof(struct acpi_einj_trigger)); + apei_resources_init(&trigger_resources); + apei_exec_ctx_init(&trigger_ctx, einj_ins_type, + ARRAY_SIZE(einj_ins_type), + trigger_entry, trigger_tab->entry_count); + rc = apei_exec_collect_resources(&trigger_ctx, &trigger_resources); + if (rc) + goto out_fini; + rc = apei_resources_sub(&trigger_resources, &einj_resources); + if (rc) + goto out_fini; + /* + * Some firmware will access target address specified in + * param1 to trigger the error when injecting memory error. + * This will cause resource conflict with regular memory. So + * remove it from trigger table resources. + */ + if ((param_extension || acpi5) && (type & MEM_ERROR_MASK) && param2) { + struct apei_resources addr_resources; + + apei_resources_init(&addr_resources); + trigger_param_region = einj_get_trigger_parameter_region( + trigger_tab, param1, param2); + if (trigger_param_region) { + rc = apei_resources_add(&addr_resources, + trigger_param_region->address, + trigger_param_region->bit_width/8, true); + if (rc) + goto out_fini; + rc = apei_resources_sub(&trigger_resources, + &addr_resources); + } + apei_resources_fini(&addr_resources); + if (rc) + goto out_fini; + } + rc = apei_resources_request(&trigger_resources, "APEI EINJ Trigger"); + if (rc) + goto out_fini; + rc = apei_exec_pre_map_gars(&trigger_ctx); + if (rc) + goto out_release; + + rc = apei_exec_run(&trigger_ctx, ACPI_EINJ_TRIGGER_ERROR); + + apei_exec_post_unmap_gars(&trigger_ctx); +out_release: + apei_resources_release(&trigger_resources); +out_fini: + apei_resources_fini(&trigger_resources); +out_rel_entry: + release_mem_region(trigger_paddr + sizeof(*trigger_tab), + table_size - sizeof(*trigger_tab)); +out_rel_header: + release_mem_region(trigger_paddr, sizeof(*trigger_tab)); +out: + if (trigger_tab) + iounmap(trigger_tab); + + return rc; +} + +static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4) +{ + struct apei_exec_context ctx; + u64 val, trigger_paddr, timeout = FIRMWARE_TIMEOUT; + int rc; + + einj_exec_ctx_init(&ctx); + + rc = apei_exec_run_optional(&ctx, ACPI_EINJ_BEGIN_OPERATION); + if (rc) + return rc; + apei_exec_ctx_set_input(&ctx, type); + if (acpi5) { + struct set_error_type_with_address *v5param = einj_param; + + v5param->type = type; + if (type & ACPI5_VENDOR_BIT) { + switch (vendor_flags) { + case SETWA_FLAGS_APICID: + v5param->apicid = param1; + break; + case SETWA_FLAGS_MEM: + v5param->memory_address = param1; + v5param->memory_address_range = param2; + break; + case SETWA_FLAGS_PCIE_SBDF: + v5param->pcie_sbdf = param1; + break; + } + v5param->flags = vendor_flags; + } else if (flags) { + v5param->flags = flags; + v5param->memory_address = param1; + v5param->memory_address_range = param2; + v5param->apicid = param3; + v5param->pcie_sbdf = param4; + } else { + switch (type) { + case ACPI_EINJ_PROCESSOR_CORRECTABLE: + case ACPI_EINJ_PROCESSOR_UNCORRECTABLE: + case ACPI_EINJ_PROCESSOR_FATAL: + v5param->apicid = param1; + v5param->flags = SETWA_FLAGS_APICID; + break; + case ACPI_EINJ_MEMORY_CORRECTABLE: + case ACPI_EINJ_MEMORY_UNCORRECTABLE: + case ACPI_EINJ_MEMORY_FATAL: + v5param->memory_address = param1; + v5param->memory_address_range = param2; + v5param->flags = SETWA_FLAGS_MEM; + break; + case ACPI_EINJ_PCIX_CORRECTABLE: + case ACPI_EINJ_PCIX_UNCORRECTABLE: + case ACPI_EINJ_PCIX_FATAL: + v5param->pcie_sbdf = param1; + v5param->flags = SETWA_FLAGS_PCIE_SBDF; + break; + } + } + } else { + rc = apei_exec_run(&ctx, ACPI_EINJ_SET_ERROR_TYPE); + if (rc) + return rc; + if (einj_param) { + struct einj_parameter *v4param = einj_param; + + v4param->param1 = param1; + v4param->param2 = param2; + } + } + rc = apei_exec_run(&ctx, ACPI_EINJ_EXECUTE_OPERATION); + if (rc) + return rc; + for (;;) { + rc = apei_exec_run(&ctx, ACPI_EINJ_CHECK_BUSY_STATUS); + if (rc) + return rc; + val = apei_exec_ctx_get_output(&ctx); + if (!(val & EINJ_OP_BUSY)) + break; + if (einj_timedout(&timeout)) + return -EIO; + } + rc = apei_exec_run(&ctx, ACPI_EINJ_GET_COMMAND_STATUS); + if (rc) + return rc; + val = apei_exec_ctx_get_output(&ctx); + if (val == EINJ_STATUS_FAIL) + return -EBUSY; + else if (val == EINJ_STATUS_INVAL) + return -EINVAL; + + /* + * The error is injected into the platform successfully, then it needs + * to trigger the error. + */ + rc = apei_exec_run(&ctx, ACPI_EINJ_GET_TRIGGER_TABLE); + if (rc) + return rc; + trigger_paddr = apei_exec_ctx_get_output(&ctx); + if (notrigger == 0) { + rc = __einj_error_trigger(trigger_paddr, type, param1, param2); + if (rc) + return rc; + } + rc = apei_exec_run_optional(&ctx, ACPI_EINJ_END_OPERATION); + + return rc; +} + +/* Inject the specified hardware error */ +int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, + u64 param4) +{ + int rc; + u64 base_addr, size; + + /* If user manually set "flags", make sure it is legal */ + if (flags && (flags & + ~(SETWA_FLAGS_APICID|SETWA_FLAGS_MEM|SETWA_FLAGS_PCIE_SBDF))) + return -EINVAL; + + /* + * We need extra sanity checks for memory errors. + * Other types leap directly to injection. + */ + + /* ensure param1/param2 existed */ + if (!(param_extension || acpi5)) + goto inject; + + /* ensure injection is memory related */ + if (type & ACPI5_VENDOR_BIT) { + if (vendor_flags != SETWA_FLAGS_MEM) + goto inject; + } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) { + goto inject; + } + + /* + * Injections targeting a CXL 1.0/1.1 port have to be injected + * via the einj_cxl_rch_error_inject() path as that does the proper + * validation of the given RCRB base (MMIO) address. + */ + if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM)) + return -EINVAL; + + /* + * Disallow crazy address masks that give BIOS leeway to pick + * injection address almost anywhere. Insist on page or + * better granularity and that target address is normal RAM or + * NVDIMM. + */ + base_addr = param1 & param2; + size = ~param2 + 1; + + if (((param2 & PAGE_MASK) != PAGE_MASK) || + ((region_intersects(base_addr, size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) + != REGION_INTERSECTS) && + (region_intersects(base_addr, size, IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY) + != REGION_INTERSECTS) && + (region_intersects(base_addr, size, IORESOURCE_MEM, IORES_DESC_SOFT_RESERVED) + != REGION_INTERSECTS) && + !arch_is_platform_page(base_addr))) + return -EINVAL; + + if (is_zero_pfn(base_addr >> PAGE_SHIFT)) + return -EADDRINUSE; + +inject: + mutex_lock(&einj_mutex); + rc = __einj_error_inject(type, flags, param1, param2, param3, param4); + mutex_unlock(&einj_mutex); + + return rc; +} + +int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4) +{ + int rc; + + if (!(einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM))) + return -EINVAL; + + mutex_lock(&einj_mutex); + rc = __einj_error_inject(type, flags, param1, param2, param3, param4); + mutex_unlock(&einj_mutex); + + return rc; +} + +static u32 error_type; +static u32 error_flags; +static u64 error_param1; +static u64 error_param2; +static u64 error_param3; +static u64 error_param4; +static struct dentry *einj_debug_dir; +static struct { u32 mask; const char *str; } const einj_error_type_string[] = { + { BIT(0), "Processor Correctable" }, + { BIT(1), "Processor Uncorrectable non-fatal" }, + { BIT(2), "Processor Uncorrectable fatal" }, + { BIT(3), "Memory Correctable" }, + { BIT(4), "Memory Uncorrectable non-fatal" }, + { BIT(5), "Memory Uncorrectable fatal" }, + { BIT(6), "PCI Express Correctable" }, + { BIT(7), "PCI Express Uncorrectable non-fatal" }, + { BIT(8), "PCI Express Uncorrectable fatal" }, + { BIT(9), "Platform Correctable" }, + { BIT(10), "Platform Uncorrectable non-fatal" }, + { BIT(11), "Platform Uncorrectable fatal"}, + { BIT(31), "Vendor Defined Error Types" }, +}; + +static int available_error_type_show(struct seq_file *m, void *v) +{ + int rc; + u32 error_type = 0; + + rc = einj_get_available_error_type(&error_type); + if (rc) + return rc; + for (int pos = 0; pos < ARRAY_SIZE(einj_error_type_string); pos++) + if (error_type & einj_error_type_string[pos].mask) + seq_printf(m, "0x%08x\t%s\n", einj_error_type_string[pos].mask, + einj_error_type_string[pos].str); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(available_error_type); + +static int error_type_get(void *data, u64 *val) +{ + *val = error_type; + + return 0; +} + +bool einj_is_cxl_error_type(u64 type) +{ + return (type & CXL_ERROR_MASK) && (!(type & ACPI5_VENDOR_BIT)); +} + +int einj_validate_error_type(u64 type) +{ + u32 tval, vendor, available_error_type = 0; + int rc; + + /* Only low 32 bits for error type are valid */ + if (type & GENMASK_ULL(63, 32)) + return -EINVAL; + + /* + * Vendor defined types have 0x80000000 bit set, and + * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE + */ + vendor = type & ACPI5_VENDOR_BIT; + tval = type & GENMASK(30, 0); + + /* Only one error type can be specified */ + if (tval & (tval - 1)) + return -EINVAL; + if (!vendor) { + rc = einj_get_available_error_type(&available_error_type); + if (rc) + return rc; + if (!(type & available_error_type)) + return -EINVAL; + } + + return 0; +} + +static int error_type_set(void *data, u64 val) +{ + int rc; + + rc = einj_validate_error_type(val); + if (rc) + return rc; + + error_type = val; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(error_type_fops, error_type_get, error_type_set, + "0x%llx\n"); + +static int error_inject_set(void *data, u64 val) +{ + if (!error_type) + return -EINVAL; + + return einj_error_inject(error_type, error_flags, error_param1, error_param2, + error_param3, error_param4); +} + +DEFINE_DEBUGFS_ATTRIBUTE(error_inject_fops, NULL, error_inject_set, "%llu\n"); + +static int einj_check_table(struct acpi_table_einj *einj_tab) +{ + if ((einj_tab->header_length != + (sizeof(struct acpi_table_einj) - sizeof(einj_tab->header))) + && (einj_tab->header_length != sizeof(struct acpi_table_einj))) + return -EINVAL; + if (einj_tab->header.length < sizeof(struct acpi_table_einj)) + return -EINVAL; + if (einj_tab->entries != + (einj_tab->header.length - sizeof(struct acpi_table_einj)) / + sizeof(struct acpi_einj_entry)) + return -EINVAL; + + return 0; +} + +static int __init einj_probe(struct platform_device *pdev) +{ + int rc; + acpi_status status; + struct apei_exec_context ctx; + + if (acpi_disabled) { + pr_debug("ACPI disabled.\n"); + return -ENODEV; + } + + status = acpi_get_table(ACPI_SIG_EINJ, 0, + (struct acpi_table_header **)&einj_tab); + if (status == AE_NOT_FOUND) { + pr_debug("EINJ table not found.\n"); + return -ENODEV; + } else if (ACPI_FAILURE(status)) { + pr_err("Failed to get EINJ table: %s\n", + acpi_format_exception(status)); + return -EINVAL; + } + + rc = einj_check_table(einj_tab); + if (rc) { + pr_warn(FW_BUG "Invalid EINJ table.\n"); + goto err_put_table; + } + + rc = -ENOMEM; + einj_debug_dir = debugfs_create_dir("einj", apei_get_debugfs_dir()); + + debugfs_create_file("available_error_type", S_IRUSR, einj_debug_dir, + NULL, &available_error_type_fops); + debugfs_create_file_unsafe("error_type", 0600, einj_debug_dir, + NULL, &error_type_fops); + debugfs_create_file_unsafe("error_inject", 0200, einj_debug_dir, + NULL, &error_inject_fops); + + apei_resources_init(&einj_resources); + einj_exec_ctx_init(&ctx); + rc = apei_exec_collect_resources(&ctx, &einj_resources); + if (rc) { + pr_err("Error collecting EINJ resources.\n"); + goto err_fini; + } + + rc = apei_resources_request(&einj_resources, "APEI EINJ"); + if (rc) { + pr_err("Error requesting memory/port resources.\n"); + goto err_fini; + } + + rc = apei_exec_pre_map_gars(&ctx); + if (rc) { + pr_err("Error pre-mapping GARs.\n"); + goto err_release; + } + + einj_param = einj_get_parameter_address(); + if ((param_extension || acpi5) && einj_param) { + debugfs_create_x32("flags", S_IRUSR | S_IWUSR, einj_debug_dir, + &error_flags); + debugfs_create_x64("param1", S_IRUSR | S_IWUSR, einj_debug_dir, + &error_param1); + debugfs_create_x64("param2", S_IRUSR | S_IWUSR, einj_debug_dir, + &error_param2); + debugfs_create_x64("param3", S_IRUSR | S_IWUSR, einj_debug_dir, + &error_param3); + debugfs_create_x64("param4", S_IRUSR | S_IWUSR, einj_debug_dir, + &error_param4); + debugfs_create_x32("notrigger", S_IRUSR | S_IWUSR, + einj_debug_dir, ¬rigger); + } + + if (vendor_dev[0]) { + vendor_blob.data = vendor_dev; + vendor_blob.size = strlen(vendor_dev); + debugfs_create_blob("vendor", S_IRUSR, einj_debug_dir, + &vendor_blob); + debugfs_create_x32("vendor_flags", S_IRUSR | S_IWUSR, + einj_debug_dir, &vendor_flags); + } + + if (vendor_errors.size) + debugfs_create_blob("oem_error", 0600, einj_debug_dir, + &vendor_errors); + + pr_info("Error INJection is initialized.\n"); + + return 0; + +err_release: + apei_resources_release(&einj_resources); +err_fini: + apei_resources_fini(&einj_resources); + debugfs_remove_recursive(einj_debug_dir); +err_put_table: + acpi_put_table((struct acpi_table_header *)einj_tab); + + return rc; +} + +static void __exit einj_remove(struct platform_device *pdev) +{ + struct apei_exec_context ctx; + + if (einj_param) { + acpi_size size = (acpi5) ? + sizeof(struct set_error_type_with_address) : + sizeof(struct einj_parameter); + + acpi_os_unmap_iomem(einj_param, size); + if (vendor_errors.size) + acpi_os_unmap_memory(vendor_errors.data, vendor_errors.size); + } + einj_exec_ctx_init(&ctx); + apei_exec_post_unmap_gars(&ctx); + apei_resources_release(&einj_resources); + apei_resources_fini(&einj_resources); + debugfs_remove_recursive(einj_debug_dir); + acpi_put_table((struct acpi_table_header *)einj_tab); +} + +static struct platform_device *einj_dev; +static struct platform_driver einj_driver = { + .remove_new = einj_remove, + .driver = { + .name = "acpi-einj", + }, +}; + +static int __init einj_init(void) +{ + struct platform_device_info einj_dev_info = { + .name = "acpi-einj", + .id = -1, + }; + int rc; + + einj_dev = platform_device_register_full(&einj_dev_info); + if (IS_ERR(einj_dev)) + return PTR_ERR(einj_dev); + + rc = platform_driver_probe(&einj_driver, einj_probe); + einj_initialized = rc == 0; + + return 0; +} + +static void __exit einj_exit(void) +{ + if (einj_initialized) + platform_driver_unregister(&einj_driver); + + platform_device_del(einj_dev); +} + +module_init(einj_init); +module_exit(einj_exit); + +MODULE_AUTHOR("Huang Ying"); +MODULE_DESCRIPTION("APEI Error INJection support"); +MODULE_LICENSE("GPL"); diff --git a/drivers/acpi/apei/einj-cxl.c b/drivers/acpi/apei/einj-cxl.c new file mode 100644 index 000000000000..8b8be0c90709 --- /dev/null +++ b/drivers/acpi/apei/einj-cxl.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CXL Error INJection support. Used by CXL core to inject + * protocol errors into CXL ports. + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Ben Cheatham + */ +#include +#include +#include + +#include "apei-internal.h" + +/* Defined in einj-core.c */ +extern bool einj_initialized; + +static struct { u32 mask; const char *str; } const einj_cxl_error_type_string[] = { + { ACPI_EINJ_CXL_CACHE_CORRECTABLE, "CXL.cache Protocol Correctable" }, + { ACPI_EINJ_CXL_CACHE_UNCORRECTABLE, "CXL.cache Protocol Uncorrectable non-fatal" }, + { ACPI_EINJ_CXL_CACHE_FATAL, "CXL.cache Protocol Uncorrectable fatal" }, + { ACPI_EINJ_CXL_MEM_CORRECTABLE, "CXL.mem Protocol Correctable" }, + { ACPI_EINJ_CXL_MEM_UNCORRECTABLE, "CXL.mem Protocol Uncorrectable non-fatal" }, + { ACPI_EINJ_CXL_MEM_FATAL, "CXL.mem Protocol Uncorrectable fatal" }, +}; + +int einj_cxl_available_error_type_show(struct seq_file *m, void *v) +{ + int cxl_err, rc; + u32 available_error_type = 0; + + rc = einj_get_available_error_type(&available_error_type); + if (rc) + return rc; + + for (int pos = 0; pos < ARRAY_SIZE(einj_cxl_error_type_string); pos++) { + cxl_err = ACPI_EINJ_CXL_CACHE_CORRECTABLE << pos; + + if (available_error_type & cxl_err) + seq_printf(m, "0x%08x\t%s\n", + einj_cxl_error_type_string[pos].mask, + einj_cxl_error_type_string[pos].str); + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_available_error_type_show, CXL); + +static int cxl_dport_get_sbdf(struct pci_dev *dport_dev, u64 *sbdf) +{ + struct pci_bus *pbus; + struct pci_host_bridge *bridge; + u64 seg = 0, bus; + + pbus = dport_dev->bus; + bridge = pci_find_host_bridge(pbus); + + if (!bridge) + return -ENODEV; + + if (bridge->domain_nr != PCI_DOMAIN_NR_NOT_SET) + seg = bridge->domain_nr; + + bus = pbus->number; + *sbdf = (seg << 24) | (bus << 16) | dport_dev->devfn; + + return 0; +} + +int einj_cxl_inject_rch_error(u64 rcrb, u64 type) +{ + int rc; + + /* Only CXL error types can be specified */ + if (!einj_is_cxl_error_type(type)) + return -EINVAL; + + rc = einj_validate_error_type(type); + if (rc) + return rc; + + return einj_cxl_rch_error_inject(type, 0x2, rcrb, GENMASK_ULL(63, 0), + 0, 0); +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_inject_rch_error, CXL); + +int einj_cxl_inject_error(struct pci_dev *dport, u64 type) +{ + u64 param4 = 0; + int rc; + + /* Only CXL error types can be specified */ + if (!einj_is_cxl_error_type(type)) + return -EINVAL; + + rc = einj_validate_error_type(type); + if (rc) + return rc; + + rc = cxl_dport_get_sbdf(dport, ¶m4); + if (rc) + return rc; + + return einj_error_inject(type, 0x4, 0, 0, 0, param4); +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_inject_error, CXL); + +bool einj_cxl_is_initialized(void) +{ + return einj_initialized; +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_is_initialized, CXL); diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c deleted file mode 100644 index 937c69844dac..000000000000 --- a/drivers/acpi/apei/einj.c +++ /dev/null @@ -1,874 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * APEI Error INJection support - * - * EINJ provides a hardware error injection mechanism, this is useful - * for debugging and testing of other APEI and RAS features. - * - * For more information about EINJ, please refer to ACPI Specification - * version 4.0, section 17.5. - * - * Copyright 2009-2010 Intel Corp. - * Author: Huang Ying - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "apei-internal.h" - -#undef pr_fmt -#define pr_fmt(fmt) "EINJ: " fmt - -#define SLEEP_UNIT_MIN 1000 /* 1ms */ -#define SLEEP_UNIT_MAX 5000 /* 5ms */ -/* Firmware should respond within 1 seconds */ -#define FIRMWARE_TIMEOUT (1 * USEC_PER_SEC) -#define ACPI5_VENDOR_BIT BIT(31) -#define MEM_ERROR_MASK (ACPI_EINJ_MEMORY_CORRECTABLE | \ - ACPI_EINJ_MEMORY_UNCORRECTABLE | \ - ACPI_EINJ_MEMORY_FATAL) - -/* - * ACPI version 5 provides a SET_ERROR_TYPE_WITH_ADDRESS action. - */ -static int acpi5; - -struct set_error_type_with_address { - u32 type; - u32 vendor_extension; - u32 flags; - u32 apicid; - u64 memory_address; - u64 memory_address_range; - u32 pcie_sbdf; -}; -enum { - SETWA_FLAGS_APICID = 1, - SETWA_FLAGS_MEM = 2, - SETWA_FLAGS_PCIE_SBDF = 4, -}; - -/* - * Vendor extensions for platform specific operations - */ -struct vendor_error_type_extension { - u32 length; - u32 pcie_sbdf; - u16 vendor_id; - u16 device_id; - u8 rev_id; - u8 reserved[3]; -}; - -static u32 notrigger; - -static u32 vendor_flags; -static struct debugfs_blob_wrapper vendor_blob; -static struct debugfs_blob_wrapper vendor_errors; -static char vendor_dev[64]; - -/* - * Some BIOSes allow parameters to the SET_ERROR_TYPE entries in the - * EINJ table through an unpublished extension. Use with caution as - * most will ignore the parameter and make their own choice of address - * for error injection. This extension is used only if - * param_extension module parameter is specified. - */ -struct einj_parameter { - u64 type; - u64 reserved1; - u64 reserved2; - u64 param1; - u64 param2; -}; - -#define EINJ_OP_BUSY 0x1 -#define EINJ_STATUS_SUCCESS 0x0 -#define EINJ_STATUS_FAIL 0x1 -#define EINJ_STATUS_INVAL 0x2 - -#define EINJ_TAB_ENTRY(tab) \ - ((struct acpi_whea_header *)((char *)(tab) + \ - sizeof(struct acpi_table_einj))) - -static bool param_extension; -module_param(param_extension, bool, 0); - -static struct acpi_table_einj *einj_tab; - -static struct apei_resources einj_resources; - -static struct apei_exec_ins_type einj_ins_type[] = { - [ACPI_EINJ_READ_REGISTER] = { - .flags = APEI_EXEC_INS_ACCESS_REGISTER, - .run = apei_exec_read_register, - }, - [ACPI_EINJ_READ_REGISTER_VALUE] = { - .flags = APEI_EXEC_INS_ACCESS_REGISTER, - .run = apei_exec_read_register_value, - }, - [ACPI_EINJ_WRITE_REGISTER] = { - .flags = APEI_EXEC_INS_ACCESS_REGISTER, - .run = apei_exec_write_register, - }, - [ACPI_EINJ_WRITE_REGISTER_VALUE] = { - .flags = APEI_EXEC_INS_ACCESS_REGISTER, - .run = apei_exec_write_register_value, - }, - [ACPI_EINJ_NOOP] = { - .flags = 0, - .run = apei_exec_noop, - }, -}; - -/* - * Prevent EINJ interpreter to run simultaneously, because the - * corresponding firmware implementation may not work properly when - * invoked simultaneously. - */ -static DEFINE_MUTEX(einj_mutex); - -/* - * Exported APIs use this flag to exit early if einj_probe() failed. - */ -static bool einj_initialized __ro_after_init; - -static void *einj_param; - -static void einj_exec_ctx_init(struct apei_exec_context *ctx) -{ - apei_exec_ctx_init(ctx, einj_ins_type, ARRAY_SIZE(einj_ins_type), - EINJ_TAB_ENTRY(einj_tab), einj_tab->entries); -} - -static int __einj_get_available_error_type(u32 *type) -{ - struct apei_exec_context ctx; - int rc; - - einj_exec_ctx_init(&ctx); - rc = apei_exec_run(&ctx, ACPI_EINJ_GET_ERROR_TYPE); - if (rc) - return rc; - *type = apei_exec_ctx_get_output(&ctx); - - return 0; -} - -/* Get error injection capabilities of the platform */ -static int einj_get_available_error_type(u32 *type) -{ - int rc; - - mutex_lock(&einj_mutex); - rc = __einj_get_available_error_type(type); - mutex_unlock(&einj_mutex); - - return rc; -} - -static int einj_timedout(u64 *t) -{ - if ((s64)*t < SLEEP_UNIT_MIN) { - pr_warn(FW_WARN "Firmware does not respond in time\n"); - return 1; - } - *t -= SLEEP_UNIT_MIN; - usleep_range(SLEEP_UNIT_MIN, SLEEP_UNIT_MAX); - - return 0; -} - -static void get_oem_vendor_struct(u64 paddr, int offset, - struct vendor_error_type_extension *v) -{ - unsigned long vendor_size; - u64 target_pa = paddr + offset + sizeof(struct vendor_error_type_extension); - - vendor_size = v->length - sizeof(struct vendor_error_type_extension); - - if (vendor_size) - vendor_errors.data = acpi_os_map_memory(target_pa, vendor_size); - - if (vendor_errors.data) - vendor_errors.size = vendor_size; -} - -static void check_vendor_extension(u64 paddr, - struct set_error_type_with_address *v5param) -{ - int offset = v5param->vendor_extension; - struct vendor_error_type_extension *v; - u32 sbdf; - - if (!offset) - return; - v = acpi_os_map_iomem(paddr + offset, sizeof(*v)); - if (!v) - return; - get_oem_vendor_struct(paddr, offset, v); - sbdf = v->pcie_sbdf; - sprintf(vendor_dev, "%x:%x:%x.%x vendor_id=%x device_id=%x rev_id=%x\n", - sbdf >> 24, (sbdf >> 16) & 0xff, - (sbdf >> 11) & 0x1f, (sbdf >> 8) & 0x7, - v->vendor_id, v->device_id, v->rev_id); - acpi_os_unmap_iomem(v, sizeof(*v)); -} - -static void *einj_get_parameter_address(void) -{ - int i; - u64 pa_v4 = 0, pa_v5 = 0; - struct acpi_whea_header *entry; - - entry = EINJ_TAB_ENTRY(einj_tab); - for (i = 0; i < einj_tab->entries; i++) { - if (entry->action == ACPI_EINJ_SET_ERROR_TYPE && - entry->instruction == ACPI_EINJ_WRITE_REGISTER && - entry->register_region.space_id == - ACPI_ADR_SPACE_SYSTEM_MEMORY) - pa_v4 = get_unaligned(&entry->register_region.address); - if (entry->action == ACPI_EINJ_SET_ERROR_TYPE_WITH_ADDRESS && - entry->instruction == ACPI_EINJ_WRITE_REGISTER && - entry->register_region.space_id == - ACPI_ADR_SPACE_SYSTEM_MEMORY) - pa_v5 = get_unaligned(&entry->register_region.address); - entry++; - } - if (pa_v5) { - struct set_error_type_with_address *v5param; - - v5param = acpi_os_map_iomem(pa_v5, sizeof(*v5param)); - if (v5param) { - acpi5 = 1; - check_vendor_extension(pa_v5, v5param); - return v5param; - } - } - if (param_extension && pa_v4) { - struct einj_parameter *v4param; - - v4param = acpi_os_map_iomem(pa_v4, sizeof(*v4param)); - if (!v4param) - return NULL; - if (v4param->reserved1 || v4param->reserved2) { - acpi_os_unmap_iomem(v4param, sizeof(*v4param)); - return NULL; - } - return v4param; - } - - return NULL; -} - -/* do sanity check to trigger table */ -static int einj_check_trigger_header(struct acpi_einj_trigger *trigger_tab) -{ - if (trigger_tab->header_size != sizeof(struct acpi_einj_trigger)) - return -EINVAL; - if (trigger_tab->table_size > PAGE_SIZE || - trigger_tab->table_size < trigger_tab->header_size) - return -EINVAL; - if (trigger_tab->entry_count != - (trigger_tab->table_size - trigger_tab->header_size) / - sizeof(struct acpi_einj_entry)) - return -EINVAL; - - return 0; -} - -static struct acpi_generic_address *einj_get_trigger_parameter_region( - struct acpi_einj_trigger *trigger_tab, u64 param1, u64 param2) -{ - int i; - struct acpi_whea_header *entry; - - entry = (struct acpi_whea_header *) - ((char *)trigger_tab + sizeof(struct acpi_einj_trigger)); - for (i = 0; i < trigger_tab->entry_count; i++) { - if (entry->action == ACPI_EINJ_TRIGGER_ERROR && - entry->instruction <= ACPI_EINJ_WRITE_REGISTER_VALUE && - entry->register_region.space_id == - ACPI_ADR_SPACE_SYSTEM_MEMORY && - (entry->register_region.address & param2) == (param1 & param2)) - return &entry->register_region; - entry++; - } - - return NULL; -} -/* Execute instructions in trigger error action table */ -static int __einj_error_trigger(u64 trigger_paddr, u32 type, - u64 param1, u64 param2) -{ - struct acpi_einj_trigger *trigger_tab = NULL; - struct apei_exec_context trigger_ctx; - struct apei_resources trigger_resources; - struct acpi_whea_header *trigger_entry; - struct resource *r; - u32 table_size; - int rc = -EIO; - struct acpi_generic_address *trigger_param_region = NULL; - - r = request_mem_region(trigger_paddr, sizeof(*trigger_tab), - "APEI EINJ Trigger Table"); - if (!r) { - pr_err("Can not request [mem %#010llx-%#010llx] for Trigger table\n", - (unsigned long long)trigger_paddr, - (unsigned long long)trigger_paddr + - sizeof(*trigger_tab) - 1); - goto out; - } - trigger_tab = ioremap_cache(trigger_paddr, sizeof(*trigger_tab)); - if (!trigger_tab) { - pr_err("Failed to map trigger table!\n"); - goto out_rel_header; - } - rc = einj_check_trigger_header(trigger_tab); - if (rc) { - pr_warn(FW_BUG "Invalid trigger error action table.\n"); - goto out_rel_header; - } - - /* No action structures in the TRIGGER_ERROR table, nothing to do */ - if (!trigger_tab->entry_count) - goto out_rel_header; - - rc = -EIO; - table_size = trigger_tab->table_size; - r = request_mem_region(trigger_paddr + sizeof(*trigger_tab), - table_size - sizeof(*trigger_tab), - "APEI EINJ Trigger Table"); - if (!r) { - pr_err("Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n", - (unsigned long long)trigger_paddr + sizeof(*trigger_tab), - (unsigned long long)trigger_paddr + table_size - 1); - goto out_rel_header; - } - iounmap(trigger_tab); - trigger_tab = ioremap_cache(trigger_paddr, table_size); - if (!trigger_tab) { - pr_err("Failed to map trigger table!\n"); - goto out_rel_entry; - } - trigger_entry = (struct acpi_whea_header *) - ((char *)trigger_tab + sizeof(struct acpi_einj_trigger)); - apei_resources_init(&trigger_resources); - apei_exec_ctx_init(&trigger_ctx, einj_ins_type, - ARRAY_SIZE(einj_ins_type), - trigger_entry, trigger_tab->entry_count); - rc = apei_exec_collect_resources(&trigger_ctx, &trigger_resources); - if (rc) - goto out_fini; - rc = apei_resources_sub(&trigger_resources, &einj_resources); - if (rc) - goto out_fini; - /* - * Some firmware will access target address specified in - * param1 to trigger the error when injecting memory error. - * This will cause resource conflict with regular memory. So - * remove it from trigger table resources. - */ - if ((param_extension || acpi5) && (type & MEM_ERROR_MASK) && param2) { - struct apei_resources addr_resources; - - apei_resources_init(&addr_resources); - trigger_param_region = einj_get_trigger_parameter_region( - trigger_tab, param1, param2); - if (trigger_param_region) { - rc = apei_resources_add(&addr_resources, - trigger_param_region->address, - trigger_param_region->bit_width/8, true); - if (rc) - goto out_fini; - rc = apei_resources_sub(&trigger_resources, - &addr_resources); - } - apei_resources_fini(&addr_resources); - if (rc) - goto out_fini; - } - rc = apei_resources_request(&trigger_resources, "APEI EINJ Trigger"); - if (rc) - goto out_fini; - rc = apei_exec_pre_map_gars(&trigger_ctx); - if (rc) - goto out_release; - - rc = apei_exec_run(&trigger_ctx, ACPI_EINJ_TRIGGER_ERROR); - - apei_exec_post_unmap_gars(&trigger_ctx); -out_release: - apei_resources_release(&trigger_resources); -out_fini: - apei_resources_fini(&trigger_resources); -out_rel_entry: - release_mem_region(trigger_paddr + sizeof(*trigger_tab), - table_size - sizeof(*trigger_tab)); -out_rel_header: - release_mem_region(trigger_paddr, sizeof(*trigger_tab)); -out: - if (trigger_tab) - iounmap(trigger_tab); - - return rc; -} - -static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, - u64 param3, u64 param4) -{ - struct apei_exec_context ctx; - u64 val, trigger_paddr, timeout = FIRMWARE_TIMEOUT; - int rc; - - einj_exec_ctx_init(&ctx); - - rc = apei_exec_run_optional(&ctx, ACPI_EINJ_BEGIN_OPERATION); - if (rc) - return rc; - apei_exec_ctx_set_input(&ctx, type); - if (acpi5) { - struct set_error_type_with_address *v5param = einj_param; - - v5param->type = type; - if (type & ACPI5_VENDOR_BIT) { - switch (vendor_flags) { - case SETWA_FLAGS_APICID: - v5param->apicid = param1; - break; - case SETWA_FLAGS_MEM: - v5param->memory_address = param1; - v5param->memory_address_range = param2; - break; - case SETWA_FLAGS_PCIE_SBDF: - v5param->pcie_sbdf = param1; - break; - } - v5param->flags = vendor_flags; - } else if (flags) { - v5param->flags = flags; - v5param->memory_address = param1; - v5param->memory_address_range = param2; - v5param->apicid = param3; - v5param->pcie_sbdf = param4; - } else { - switch (type) { - case ACPI_EINJ_PROCESSOR_CORRECTABLE: - case ACPI_EINJ_PROCESSOR_UNCORRECTABLE: - case ACPI_EINJ_PROCESSOR_FATAL: - v5param->apicid = param1; - v5param->flags = SETWA_FLAGS_APICID; - break; - case ACPI_EINJ_MEMORY_CORRECTABLE: - case ACPI_EINJ_MEMORY_UNCORRECTABLE: - case ACPI_EINJ_MEMORY_FATAL: - v5param->memory_address = param1; - v5param->memory_address_range = param2; - v5param->flags = SETWA_FLAGS_MEM; - break; - case ACPI_EINJ_PCIX_CORRECTABLE: - case ACPI_EINJ_PCIX_UNCORRECTABLE: - case ACPI_EINJ_PCIX_FATAL: - v5param->pcie_sbdf = param1; - v5param->flags = SETWA_FLAGS_PCIE_SBDF; - break; - } - } - } else { - rc = apei_exec_run(&ctx, ACPI_EINJ_SET_ERROR_TYPE); - if (rc) - return rc; - if (einj_param) { - struct einj_parameter *v4param = einj_param; - - v4param->param1 = param1; - v4param->param2 = param2; - } - } - rc = apei_exec_run(&ctx, ACPI_EINJ_EXECUTE_OPERATION); - if (rc) - return rc; - for (;;) { - rc = apei_exec_run(&ctx, ACPI_EINJ_CHECK_BUSY_STATUS); - if (rc) - return rc; - val = apei_exec_ctx_get_output(&ctx); - if (!(val & EINJ_OP_BUSY)) - break; - if (einj_timedout(&timeout)) - return -EIO; - } - rc = apei_exec_run(&ctx, ACPI_EINJ_GET_COMMAND_STATUS); - if (rc) - return rc; - val = apei_exec_ctx_get_output(&ctx); - if (val == EINJ_STATUS_FAIL) - return -EBUSY; - else if (val == EINJ_STATUS_INVAL) - return -EINVAL; - - /* - * The error is injected into the platform successfully, then it needs - * to trigger the error. - */ - rc = apei_exec_run(&ctx, ACPI_EINJ_GET_TRIGGER_TABLE); - if (rc) - return rc; - trigger_paddr = apei_exec_ctx_get_output(&ctx); - if (notrigger == 0) { - rc = __einj_error_trigger(trigger_paddr, type, param1, param2); - if (rc) - return rc; - } - rc = apei_exec_run_optional(&ctx, ACPI_EINJ_END_OPERATION); - - return rc; -} - -/* Inject the specified hardware error */ -static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, - u64 param3, u64 param4) -{ - int rc; - u64 base_addr, size; - - /* If user manually set "flags", make sure it is legal */ - if (flags && (flags & - ~(SETWA_FLAGS_APICID|SETWA_FLAGS_MEM|SETWA_FLAGS_PCIE_SBDF))) - return -EINVAL; - - /* - * We need extra sanity checks for memory errors. - * Other types leap directly to injection. - */ - - /* ensure param1/param2 existed */ - if (!(param_extension || acpi5)) - goto inject; - - /* ensure injection is memory related */ - if (type & ACPI5_VENDOR_BIT) { - if (vendor_flags != SETWA_FLAGS_MEM) - goto inject; - } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) - goto inject; - - /* - * Disallow crazy address masks that give BIOS leeway to pick - * injection address almost anywhere. Insist on page or - * better granularity and that target address is normal RAM or - * NVDIMM. - */ - base_addr = param1 & param2; - size = ~param2 + 1; - - if (((param2 & PAGE_MASK) != PAGE_MASK) || - ((region_intersects(base_addr, size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) - != REGION_INTERSECTS) && - (region_intersects(base_addr, size, IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY) - != REGION_INTERSECTS) && - (region_intersects(base_addr, size, IORESOURCE_MEM, IORES_DESC_SOFT_RESERVED) - != REGION_INTERSECTS) && - !arch_is_platform_page(base_addr))) - return -EINVAL; - - if (is_zero_pfn(base_addr >> PAGE_SHIFT)) - return -EADDRINUSE; - -inject: - mutex_lock(&einj_mutex); - rc = __einj_error_inject(type, flags, param1, param2, param3, param4); - mutex_unlock(&einj_mutex); - - return rc; -} - -static u32 error_type; -static u32 error_flags; -static u64 error_param1; -static u64 error_param2; -static u64 error_param3; -static u64 error_param4; -static struct dentry *einj_debug_dir; -static struct { u32 mask; const char *str; } const einj_error_type_string[] = { - { BIT(0), "Processor Correctable" }, - { BIT(1), "Processor Uncorrectable non-fatal" }, - { BIT(2), "Processor Uncorrectable fatal" }, - { BIT(3), "Memory Correctable" }, - { BIT(4), "Memory Uncorrectable non-fatal" }, - { BIT(5), "Memory Uncorrectable fatal" }, - { BIT(6), "PCI Express Correctable" }, - { BIT(7), "PCI Express Uncorrectable non-fatal" }, - { BIT(8), "PCI Express Uncorrectable fatal" }, - { BIT(9), "Platform Correctable" }, - { BIT(10), "Platform Uncorrectable non-fatal" }, - { BIT(11), "Platform Uncorrectable fatal"}, - { BIT(12), "CXL.cache Protocol Correctable" }, - { BIT(13), "CXL.cache Protocol Uncorrectable non-fatal" }, - { BIT(14), "CXL.cache Protocol Uncorrectable fatal" }, - { BIT(15), "CXL.mem Protocol Correctable" }, - { BIT(16), "CXL.mem Protocol Uncorrectable non-fatal" }, - { BIT(17), "CXL.mem Protocol Uncorrectable fatal" }, - { BIT(31), "Vendor Defined Error Types" }, -}; - -static int available_error_type_show(struct seq_file *m, void *v) -{ - int rc; - u32 error_type = 0; - - rc = einj_get_available_error_type(&error_type); - if (rc) - return rc; - for (int pos = 0; pos < ARRAY_SIZE(einj_error_type_string); pos++) - if (error_type & einj_error_type_string[pos].mask) - seq_printf(m, "0x%08x\t%s\n", einj_error_type_string[pos].mask, - einj_error_type_string[pos].str); - - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(available_error_type); - -static int error_type_get(void *data, u64 *val) -{ - *val = error_type; - - return 0; -} - -static int error_type_set(void *data, u64 val) -{ - int rc; - u32 available_error_type = 0; - u32 tval, vendor; - - /* Only low 32 bits for error type are valid */ - if (val & GENMASK_ULL(63, 32)) - return -EINVAL; - - /* - * Vendor defined types have 0x80000000 bit set, and - * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE - */ - vendor = val & ACPI5_VENDOR_BIT; - tval = val & 0x7fffffff; - - /* Only one error type can be specified */ - if (tval & (tval - 1)) - return -EINVAL; - if (!vendor) { - rc = einj_get_available_error_type(&available_error_type); - if (rc) - return rc; - if (!(val & available_error_type)) - return -EINVAL; - } - error_type = val; - - return 0; -} - -DEFINE_DEBUGFS_ATTRIBUTE(error_type_fops, error_type_get, error_type_set, - "0x%llx\n"); - -static int error_inject_set(void *data, u64 val) -{ - if (!error_type) - return -EINVAL; - - return einj_error_inject(error_type, error_flags, error_param1, error_param2, - error_param3, error_param4); -} - -DEFINE_DEBUGFS_ATTRIBUTE(error_inject_fops, NULL, error_inject_set, "%llu\n"); - -static int einj_check_table(struct acpi_table_einj *einj_tab) -{ - if ((einj_tab->header_length != - (sizeof(struct acpi_table_einj) - sizeof(einj_tab->header))) - && (einj_tab->header_length != sizeof(struct acpi_table_einj))) - return -EINVAL; - if (einj_tab->header.length < sizeof(struct acpi_table_einj)) - return -EINVAL; - if (einj_tab->entries != - (einj_tab->header.length - sizeof(struct acpi_table_einj)) / - sizeof(struct acpi_einj_entry)) - return -EINVAL; - - return 0; -} - -static int __init einj_probe(struct platform_device *pdev) -{ - int rc; - acpi_status status; - struct apei_exec_context ctx; - - if (acpi_disabled) { - pr_debug("ACPI disabled.\n"); - return -ENODEV; - } - - status = acpi_get_table(ACPI_SIG_EINJ, 0, - (struct acpi_table_header **)&einj_tab); - if (status == AE_NOT_FOUND) { - pr_debug("EINJ table not found.\n"); - return -ENODEV; - } else if (ACPI_FAILURE(status)) { - pr_err("Failed to get EINJ table: %s\n", - acpi_format_exception(status)); - return -EINVAL; - } - - rc = einj_check_table(einj_tab); - if (rc) { - pr_warn(FW_BUG "Invalid EINJ table.\n"); - goto err_put_table; - } - - rc = -ENOMEM; - einj_debug_dir = debugfs_create_dir("einj", apei_get_debugfs_dir()); - - debugfs_create_file("available_error_type", S_IRUSR, einj_debug_dir, - NULL, &available_error_type_fops); - debugfs_create_file_unsafe("error_type", 0600, einj_debug_dir, - NULL, &error_type_fops); - debugfs_create_file_unsafe("error_inject", 0200, einj_debug_dir, - NULL, &error_inject_fops); - - apei_resources_init(&einj_resources); - einj_exec_ctx_init(&ctx); - rc = apei_exec_collect_resources(&ctx, &einj_resources); - if (rc) { - pr_err("Error collecting EINJ resources.\n"); - goto err_fini; - } - - rc = apei_resources_request(&einj_resources, "APEI EINJ"); - if (rc) { - pr_err("Error requesting memory/port resources.\n"); - goto err_fini; - } - - rc = apei_exec_pre_map_gars(&ctx); - if (rc) { - pr_err("Error pre-mapping GARs.\n"); - goto err_release; - } - - einj_param = einj_get_parameter_address(); - if ((param_extension || acpi5) && einj_param) { - debugfs_create_x32("flags", S_IRUSR | S_IWUSR, einj_debug_dir, - &error_flags); - debugfs_create_x64("param1", S_IRUSR | S_IWUSR, einj_debug_dir, - &error_param1); - debugfs_create_x64("param2", S_IRUSR | S_IWUSR, einj_debug_dir, - &error_param2); - debugfs_create_x64("param3", S_IRUSR | S_IWUSR, einj_debug_dir, - &error_param3); - debugfs_create_x64("param4", S_IRUSR | S_IWUSR, einj_debug_dir, - &error_param4); - debugfs_create_x32("notrigger", S_IRUSR | S_IWUSR, - einj_debug_dir, ¬rigger); - } - - if (vendor_dev[0]) { - vendor_blob.data = vendor_dev; - vendor_blob.size = strlen(vendor_dev); - debugfs_create_blob("vendor", S_IRUSR, einj_debug_dir, - &vendor_blob); - debugfs_create_x32("vendor_flags", S_IRUSR | S_IWUSR, - einj_debug_dir, &vendor_flags); - } - - if (vendor_errors.size) - debugfs_create_blob("oem_error", 0600, einj_debug_dir, - &vendor_errors); - - pr_info("Error INJection is initialized.\n"); - - return 0; - -err_release: - apei_resources_release(&einj_resources); -err_fini: - apei_resources_fini(&einj_resources); - debugfs_remove_recursive(einj_debug_dir); -err_put_table: - acpi_put_table((struct acpi_table_header *)einj_tab); - - return rc; -} - -static void __exit einj_remove(struct platform_device *pdev) -{ - struct apei_exec_context ctx; - - if (einj_param) { - acpi_size size = (acpi5) ? - sizeof(struct set_error_type_with_address) : - sizeof(struct einj_parameter); - - acpi_os_unmap_iomem(einj_param, size); - if (vendor_errors.size) - acpi_os_unmap_memory(vendor_errors.data, vendor_errors.size); - } - einj_exec_ctx_init(&ctx); - apei_exec_post_unmap_gars(&ctx); - apei_resources_release(&einj_resources); - apei_resources_fini(&einj_resources); - debugfs_remove_recursive(einj_debug_dir); - acpi_put_table((struct acpi_table_header *)einj_tab); -} - -static struct platform_device *einj_dev; -static struct platform_driver einj_driver = { - .remove_new = einj_remove, - .driver = { - .name = "acpi-einj", - }, -}; - -static int __init einj_init(void) -{ - struct platform_device_info einj_dev_info = { - .name = "acpi-einj", - .id = -1, - }; - int rc; - - einj_dev = platform_device_register_full(&einj_dev_info); - if (IS_ERR(einj_dev)) - return PTR_ERR(einj_dev); - - rc = platform_driver_probe(&einj_driver, einj_probe); - einj_initialized = rc == 0; - - return 0; -} - -static void __exit einj_exit(void) -{ - if (einj_initialized) - platform_driver_unregister(&einj_driver); - - platform_device_del(einj_dev); -} - -module_init(einj_init); -module_exit(einj_exit); - -MODULE_AUTHOR("Huang Ying"); -MODULE_DESCRIPTION("APEI Error INJection support"); -MODULE_LICENSE("GPL"); diff --git a/include/linux/einj-cxl.h b/include/linux/einj-cxl.h new file mode 100644 index 000000000000..624ff6ff41f9 --- /dev/null +++ b/include/linux/einj-cxl.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * CXL protocol Error INJection support. + * + * Copyright (c) 2023 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Ben Cheatham + */ +#ifndef EINJ_CXL_H +#define EINJ_CXL_H + +#include +#include + +struct pci_dev; +struct seq_file; + +#if IS_ENABLED(CONFIG_ACPI_APEI_EINJ_CXL) +int einj_cxl_available_error_type_show(struct seq_file *m, void *v); +int einj_cxl_inject_error(struct pci_dev *dport_dev, u64 type); +int einj_cxl_inject_rch_error(u64 rcrb, u64 type); +bool einj_cxl_is_initialized(void); +#else /* !IS_ENABLED(CONFIG_ACPI_APEI_EINJ_CXL) */ +static inline int einj_cxl_available_error_type_show(struct seq_file *m, + void *v) +{ + return -ENXIO; +} + +static inline int einj_cxl_inject_error(struct pci_dev *dport_dev, u64 type) +{ + return -ENXIO; +} + +static inline int einj_cxl_inject_rch_error(u64 rcrb, u64 type) +{ + return -ENXIO; +} + +static inline bool einj_cxl_is_initialized(void) { return false; } +#endif /* CONFIG_ACPI_APEI_EINJ_CXL */ + +#endif /* EINJ_CXL_H */ -- cgit v1.2.3 From c6c3187d66bc4e87086036266def4170742d7214 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Sat, 17 Feb 2024 22:39:46 +0100 Subject: lib/firmware_table: Provide buffer length argument to cdat_table_parse() There exist card implementations with a CDAT table using a fixed size buffer, but with entries filled in that do not fill the whole table length size. Then, the last entry in the CDAT table may not mark the end of the CDAT table buffer specified by the length field in the CDAT header. It can be shorter with trailing unused (zero'ed) data. The actual table length is determined while reading all CDAT entries of the table with DOE. If the table is greater than expected (containing zero'ed trailing data), the CDAT parser fails with: [ 48.691717] Malformed DSMAS table length: (24:0) [ 48.702084] [CDAT:0x00] Invalid zero length [ 48.711460] cxl_port endpoint1: Failed to parse CDAT: -22 In addition, a check of the table buffer length is missing to prevent an out-of-bound access then parsing the CDAT table. Hardening code against device returning borked table. Fix that by providing an optional buffer length argument to acpi_parse_entries_array() that can be used by cdat_table_parse() to propagate the buffer size down to its users to check the buffer length. This also prevents a possible out-of-bound access mentioned. Add a check to warn about a malformed CDAT table length. Cc: Rafael J. Wysocki Cc: Len Brown Reviewed-by: Dave Jiang Signed-off-by: Robert Richter Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/ZdEnopFO0Tl3t2O1@rric.localdomain Signed-off-by: Dan Williams --- drivers/acpi/tables.c | 2 +- drivers/cxl/core/cdat.c | 6 +++--- drivers/cxl/core/pci.c | 8 +++++++- include/linux/fw_table.h | 4 +++- lib/fw_table.c | 15 ++++++++++----- 5 files changed, 24 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index b07f7d091d13..b976e5fc3fbc 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -253,7 +253,7 @@ int __init_or_acpilib acpi_table_parse_entries_array( count = acpi_parse_entries_array(id, table_size, (union fw_table_header *)table_header, - proc, proc_num, max_entries); + 0, proc, proc_num, max_entries); acpi_put_table(table_header); return count; diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 08fd0baea7a0..c8737e480789 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -149,13 +149,13 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port, int rc; rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, - dsmas_xa, port->cdat.table); + dsmas_xa, port->cdat.table, port->cdat.length); rc = cdat_table_parse_output(rc); if (rc) return rc; rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler, - dsmas_xa, port->cdat.table); + dsmas_xa, port->cdat.table, port->cdat.length); return cdat_table_parse_output(rc); } @@ -477,7 +477,7 @@ void cxl_switch_parse_cdat(struct cxl_port *port) return; rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler, - port, port->cdat.table); + port, port->cdat.table, port->cdat.length); rc = cdat_table_parse_output(rc); if (rc) dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc); diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 184deff887d0..0df09bd79408 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -623,7 +623,7 @@ void read_cdat_data(struct cxl_port *port) struct pci_dev *pdev = NULL; struct cxl_memdev *cxlmd; struct cdat_doe_rsp *buf; - size_t length; + size_t table_length, length; int rc; if (is_cxl_memdev(uport)) { @@ -662,10 +662,16 @@ void read_cdat_data(struct cxl_port *port) if (!buf) goto err; + table_length = length; + rc = cxl_cdat_read_table(dev, doe_mb, buf, &length); if (rc) goto err; + if (table_length != length) + dev_warn(dev, "Malformed CDAT table length (%zu:%zu), discarding trailing data\n", + table_length, length); + if (cdat_checksum(buf->data, length)) goto err; diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h index 95421860397a..3ff4c277296f 100644 --- a/include/linux/fw_table.h +++ b/include/linux/fw_table.h @@ -40,12 +40,14 @@ union acpi_subtable_headers { int acpi_parse_entries_array(char *id, unsigned long table_size, union fw_table_header *table_header, + unsigned long max_length, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries); int cdat_table_parse(enum acpi_cdat_type type, acpi_tbl_entry_handler_arg handler_arg, void *arg, - struct acpi_table_cdat *table_header); + struct acpi_table_cdat *table_header, + unsigned long length); /* CXL is the only non-ACPI consumer of the FIRMWARE_TABLE library */ #if IS_ENABLED(CONFIG_ACPI) && !IS_ENABLED(CONFIG_CXL_BUS) diff --git a/lib/fw_table.c b/lib/fw_table.c index c3569d2ba503..16291814450e 100644 --- a/lib/fw_table.c +++ b/lib/fw_table.c @@ -127,6 +127,7 @@ static __init_or_fwtbl_lib int call_handler(struct acpi_subtable_proc *proc, * * @id: table id (for debugging purposes) * @table_size: size of the root table + * @max_length: maximum size of the table (ignore if 0) * @table_header: where does the table start? * @proc: array of acpi_subtable_proc struct containing entry id * and associated handler with it @@ -148,18 +149,21 @@ static __init_or_fwtbl_lib int call_handler(struct acpi_subtable_proc *proc, int __init_or_fwtbl_lib acpi_parse_entries_array(char *id, unsigned long table_size, union fw_table_header *table_header, + unsigned long max_length, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries) { - unsigned long table_end, subtable_len, entry_len; + unsigned long table_len, table_end, subtable_len, entry_len; struct acpi_subtable_entry entry; enum acpi_subtable_type type; int count = 0; int i; type = acpi_get_subtable_type(id); - table_end = (unsigned long)table_header + - acpi_table_get_length(type, table_header); + table_len = acpi_table_get_length(type, table_header); + if (max_length && max_length < table_len) + table_len = max_length; + table_end = (unsigned long)table_header + table_len; /* Parse all entries looking for a match. */ @@ -208,7 +212,8 @@ int __init_or_fwtbl_lib cdat_table_parse(enum acpi_cdat_type type, acpi_tbl_entry_handler_arg handler_arg, void *arg, - struct acpi_table_cdat *table_header) + struct acpi_table_cdat *table_header, + unsigned long length) { struct acpi_subtable_proc proc = { .id = type, @@ -222,6 +227,6 @@ cdat_table_parse(enum acpi_cdat_type type, return acpi_parse_entries_array(ACPI_SIG_CDAT, sizeof(struct acpi_table_cdat), (union fw_table_header *)table_header, - &proc, 1, 0); + length, &proc, 1, 0); } EXPORT_SYMBOL_FWTBL_LIB(cdat_table_parse); -- cgit v1.2.3 From 9d9539db8638cfe053fcd1f441746f0e2c8c2d32 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Tue, 12 Mar 2024 10:39:44 +0100 Subject: pidfs: remove config option As Linus suggested this enables pidfs unconditionally. A key property to retain is the ability to compare pidfds by inode number (cf. [1]). That's extremely helpful just as comparing namespace file descriptors by inode number is. They are used in a variety of scenarios where they need to be compared, e.g., when receiving a pidfd via SO_PEERPIDFD from a socket to trivially authenticate a the sender and various other use-cases. For 64bit systems this is pretty trivial to do. For 32bit it's slightly more annoying as we discussed but we simply add a dumb ida based allocator that gets used on 32bit. This gives the same guarantees about inode numbers on 64bit without any overflow risk. Practically, we'll never run into overflow issues because we're constrained by the number of processes that can exist on 32bit and by the number of open files that can exist on a 32bit system. On 64bit none of this matters and things are very simple. If 32bit also needs the uniqueness guarantee they can simply parse the contents of /proc//fd/. The uniqueness guarantees have a variety of use-cases. One of the most obvious ones is that they will make pidfiles (or "pidfdfiles", I guess) reliable as the unique identifier can be placed into there that won't be reycled. Also a frequent request. Note, I took the chance and simplified path_from_stashed() even further. Instead of passing the inode number explicitly to path_from_stashed() we let the filesystem handle that internally. So path_from_stashed() ends up even simpler than it is now. This is also a good solution allowing the cleanup code to be clean and consistent between 32bit and 64bit. The cleanup path in prepare_anon_dentry() is also switched around so we put the inode before the dentry allocation. This means we only have to call the cleanup handler for the filesystem's inode data once and can rely ->evict_inode() otherwise. Aside from having to have a bit of extra code for 32bit it actually ends up a nice cleanup for path_from_stashed() imho. Tested on both 32 and 64bit including error injection. Link: https://github.com/systemd/systemd/pull/31713 [1] Link: https://lore.kernel.org/r/20240312-dingo-sehnlich-b3ecc35c6de7@brauner Signed-off-by: Christian Brauner Signed-off-by: Linus Torvalds --- fs/Kconfig | 7 ---- fs/internal.h | 6 +-- fs/libfs.c | 33 +++++++++-------- fs/nsfs.c | 11 ++++-- fs/pidfs.c | 101 +++++++++++++++++++++++--------------------------- include/linux/pid.h | 6 +-- include/linux/pidfs.h | 1 - kernel/pid.c | 6 --- 8 files changed, 78 insertions(+), 93 deletions(-) (limited to 'include/linux') diff --git a/fs/Kconfig b/fs/Kconfig index 4bc7dd420874..ea2f77446080 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -173,13 +173,6 @@ source "fs/proc/Kconfig" source "fs/kernfs/Kconfig" source "fs/sysfs/Kconfig" -config FS_PID - bool "Pseudo filesystem for process file descriptors" - depends on 64BIT - default y - help - Pidfs implements advanced features for process file descriptors. - config TMPFS bool "Tmpfs virtual memory file system support (former shm fs)" depends on SHMEM diff --git a/fs/internal.h b/fs/internal.h index 49c1fcfee4b3..7ca738904e34 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -313,8 +313,8 @@ struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap); void mnt_idmap_put(struct mnt_idmap *idmap); struct stashed_operations { void (*put_data)(void *data); - void (*init_inode)(struct inode *inode, void *data); + int (*init_inode)(struct inode *inode, void *data); }; -int path_from_stashed(struct dentry **stashed, unsigned long ino, - struct vfsmount *mnt, void *data, struct path *path); +int path_from_stashed(struct dentry **stashed, struct vfsmount *mnt, void *data, + struct path *path); void stashed_dentry_prune(struct dentry *dentry); diff --git a/fs/libfs.c b/fs/libfs.c index 0d14ae808fcf..3a6f2cb364f8 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -2001,34 +2001,40 @@ static inline struct dentry *get_stashed_dentry(struct dentry *stashed) } static struct dentry *prepare_anon_dentry(struct dentry **stashed, - unsigned long ino, struct super_block *sb, void *data) { struct dentry *dentry; struct inode *inode; const struct stashed_operations *sops = sb->s_fs_info; - - dentry = d_alloc_anon(sb); - if (!dentry) - return ERR_PTR(-ENOMEM); + int ret; inode = new_inode_pseudo(sb); if (!inode) { - dput(dentry); + sops->put_data(data); return ERR_PTR(-ENOMEM); } - inode->i_ino = ino; inode->i_flags |= S_IMMUTABLE; inode->i_mode = S_IFREG; simple_inode_init_ts(inode); - sops->init_inode(inode, data); + + ret = sops->init_inode(inode, data); + if (ret < 0) { + iput(inode); + return ERR_PTR(ret); + } /* Notice when this is changed. */ WARN_ON_ONCE(!S_ISREG(inode->i_mode)); WARN_ON_ONCE(!IS_IMMUTABLE(inode)); + dentry = d_alloc_anon(sb); + if (!dentry) { + iput(inode); + return ERR_PTR(-ENOMEM); + } + /* Store address of location where dentry's supposed to be stashed. */ dentry->d_fsdata = stashed; @@ -2062,7 +2068,6 @@ static struct dentry *stash_dentry(struct dentry **stashed, /** * path_from_stashed - create path from stashed or new dentry * @stashed: where to retrieve or stash dentry - * @ino: inode number to use * @mnt: mnt of the filesystems to use * @data: data to store in inode->i_private * @path: path to create @@ -2077,8 +2082,8 @@ static struct dentry *stash_dentry(struct dentry **stashed, * * Return: On success zero and on failure a negative error is returned. */ -int path_from_stashed(struct dentry **stashed, unsigned long ino, - struct vfsmount *mnt, void *data, struct path *path) +int path_from_stashed(struct dentry **stashed, struct vfsmount *mnt, void *data, + struct path *path) { struct dentry *dentry; const struct stashed_operations *sops = mnt->mnt_sb->s_fs_info; @@ -2091,11 +2096,9 @@ int path_from_stashed(struct dentry **stashed, unsigned long ino, } /* Allocate a new dentry. */ - dentry = prepare_anon_dentry(stashed, ino, mnt->mnt_sb, data); - if (IS_ERR(dentry)) { - sops->put_data(data); + dentry = prepare_anon_dentry(stashed, mnt->mnt_sb, data); + if (IS_ERR(dentry)) return PTR_ERR(dentry); - } /* Added a new dentry. @data is now owned by the filesystem. */ path->dentry = stash_dentry(stashed, dentry); diff --git a/fs/nsfs.c b/fs/nsfs.c index 7aaafb5cb9fc..07e22a15ef02 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -56,7 +56,7 @@ int ns_get_path_cb(struct path *path, ns_get_path_helper_t *ns_get_cb, if (!ns) return -ENOENT; - return path_from_stashed(&ns->stashed, ns->inum, nsfs_mnt, ns, path); + return path_from_stashed(&ns->stashed, nsfs_mnt, ns, path); } struct ns_get_path_task_args { @@ -101,8 +101,7 @@ int open_related_ns(struct ns_common *ns, return PTR_ERR(relative); } - err = path_from_stashed(&relative->stashed, relative->inum, nsfs_mnt, - relative, &path); + err = path_from_stashed(&relative->stashed, nsfs_mnt, relative, &path); if (err < 0) { put_unused_fd(fd); return err; @@ -199,11 +198,15 @@ static const struct super_operations nsfs_ops = { .show_path = nsfs_show_path, }; -static void nsfs_init_inode(struct inode *inode, void *data) +static int nsfs_init_inode(struct inode *inode, void *data) { + struct ns_common *ns = data; + inode->i_private = data; inode->i_mode |= S_IRUGO; inode->i_fop = &ns_file_operations; + inode->i_ino = ns->inum; + return 0; } static void nsfs_put_data(void *data) diff --git a/fs/pidfs.c b/fs/pidfs.c index 8fd71a00be9c..a63d5d24aa02 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -16,17 +16,6 @@ #include "internal.h" -static int pidfd_release(struct inode *inode, struct file *file) -{ -#ifndef CONFIG_FS_PID - struct pid *pid = file->private_data; - - file->private_data = NULL; - put_pid(pid); -#endif - return 0; -} - #ifdef CONFIG_PROC_FS /** * pidfd_show_fdinfo - print information about a pidfd @@ -120,7 +109,6 @@ static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts) } static const struct file_operations pidfs_file_operations = { - .release = pidfd_release, .poll = pidfd_poll, #ifdef CONFIG_PROC_FS .show_fdinfo = pidfd_show_fdinfo, @@ -131,16 +119,45 @@ struct pid *pidfd_pid(const struct file *file) { if (file->f_op != &pidfs_file_operations) return ERR_PTR(-EBADF); -#ifdef CONFIG_FS_PID return file_inode(file)->i_private; -#else - return file->private_data; -#endif } -#ifdef CONFIG_FS_PID static struct vfsmount *pidfs_mnt __ro_after_init; +#if BITS_PER_LONG == 32 +/* + * Provide a fallback mechanism for 32-bit systems so processes remain + * reliably comparable by inode number even on those systems. + */ +static DEFINE_IDA(pidfd_inum_ida); + +static int pidfs_inum(struct pid *pid, unsigned long *ino) +{ + int ret; + + ret = ida_alloc_range(&pidfd_inum_ida, RESERVED_PIDS + 1, + UINT_MAX, GFP_ATOMIC); + if (ret < 0) + return -ENOSPC; + + *ino = ret; + return 0; +} + +static inline void pidfs_free_inum(unsigned long ino) +{ + if (ino > 0) + ida_free(&pidfd_inum_ida, ino); +} +#else +static inline int pidfs_inum(struct pid *pid, unsigned long *ino) +{ + *ino = pid->ino; + return 0; +} +#define pidfs_free_inum(ino) ((void)(ino)) +#endif + /* * The vfs falls back to simple_setattr() if i_op->setattr() isn't * implemented. Let's reject it completely until we have a clean @@ -173,6 +190,7 @@ static void pidfs_evict_inode(struct inode *inode) clear_inode(inode); put_pid(pid); + pidfs_free_inum(inode->i_ino); } static const struct super_operations pidfs_sops = { @@ -183,8 +201,10 @@ static const struct super_operations pidfs_sops = { static char *pidfs_dname(struct dentry *dentry, char *buffer, int buflen) { - return dynamic_dname(buffer, buflen, "pidfd:[%lu]", - d_inode(dentry)->i_ino); + struct inode *inode = d_inode(dentry); + struct pid *pid = inode->i_private; + + return dynamic_dname(buffer, buflen, "pidfd:[%llu]", pid->ino); } static const struct dentry_operations pidfs_dentry_operations = { @@ -193,13 +213,19 @@ static const struct dentry_operations pidfs_dentry_operations = { .d_prune = stashed_dentry_prune, }; -static void pidfs_init_inode(struct inode *inode, void *data) +static int pidfs_init_inode(struct inode *inode, void *data) { inode->i_private = data; inode->i_flags |= S_PRIVATE; inode->i_mode |= S_IRWXU; inode->i_op = &pidfs_inode_operations; inode->i_fop = &pidfs_file_operations; + /* + * Inode numbering for pidfs start at RESERVED_PIDS + 1. This + * avoids collisions with the root inode which is 1 for pseudo + * filesystems. + */ + return pidfs_inum(data, &inode->i_ino); } static void pidfs_put_data(void *data) @@ -240,13 +266,7 @@ struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags) struct path path; int ret; - /* - * Inode numbering for pidfs start at RESERVED_PIDS + 1. - * This avoids collisions with the root inode which is 1 - * for pseudo filesystems. - */ - ret = path_from_stashed(&pid->stashed, pid->ino, pidfs_mnt, - get_pid(pid), &path); + ret = path_from_stashed(&pid->stashed, pidfs_mnt, get_pid(pid), &path); if (ret < 0) return ERR_PTR(ret); @@ -261,30 +281,3 @@ void __init pidfs_init(void) if (IS_ERR(pidfs_mnt)) panic("Failed to mount pidfs pseudo filesystem"); } - -bool is_pidfs_sb(const struct super_block *sb) -{ - return sb == pidfs_mnt->mnt_sb; -} - -#else /* !CONFIG_FS_PID */ - -struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags) -{ - struct file *pidfd_file; - - pidfd_file = anon_inode_getfile("[pidfd]", &pidfs_file_operations, pid, - flags | O_RDWR); - if (IS_ERR(pidfd_file)) - return pidfd_file; - - get_pid(pid); - return pidfd_file; -} - -void __init pidfs_init(void) { } -bool is_pidfs_sb(const struct super_block *sb) -{ - return false; -} -#endif diff --git a/include/linux/pid.h b/include/linux/pid.h index c79a0efd0258..a3aad9b4074c 100644 --- a/include/linux/pid.h +++ b/include/linux/pid.h @@ -45,6 +45,8 @@ * find_pid_ns() using the int nr and struct pid_namespace *ns. */ +#define RESERVED_PIDS 300 + struct upid { int nr; struct pid_namespace *ns; @@ -55,10 +57,8 @@ struct pid refcount_t count; unsigned int level; spinlock_t lock; -#ifdef CONFIG_FS_PID struct dentry *stashed; - unsigned long ino; -#endif + u64 ino; /* lists of tasks that use this pid */ struct hlist_head tasks[PIDTYPE_MAX]; struct hlist_head inodes; diff --git a/include/linux/pidfs.h b/include/linux/pidfs.h index 40dd325a32a6..75bdf9807802 100644 --- a/include/linux/pidfs.h +++ b/include/linux/pidfs.h @@ -4,6 +4,5 @@ struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags); void __init pidfs_init(void); -bool is_pidfs_sb(const struct super_block *sb); #endif /* _LINUX_PID_FS_H */ diff --git a/kernel/pid.c b/kernel/pid.c index 99a0c5eb24b8..da76ed1873f7 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -62,17 +62,13 @@ struct pid init_struct_pid = { int pid_max = PID_MAX_DEFAULT; -#define RESERVED_PIDS 300 - int pid_max_min = RESERVED_PIDS + 1; int pid_max_max = PID_MAX_LIMIT; -#ifdef CONFIG_FS_PID /* * Pseudo filesystems start inode numbering after one. We use Reserved * PIDs as a natural offset. */ static u64 pidfs_ino = RESERVED_PIDS; -#endif /* * PID-map pages start out as NULL, they get allocated upon @@ -280,10 +276,8 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid, spin_lock_irq(&pidmap_lock); if (!(ns->pid_allocated & PIDNS_ADDING)) goto out_unlock; -#ifdef CONFIG_FS_PID pid->stashed = NULL; pid->ino = ++pidfs_ino; -#endif for ( ; upid >= pid->numbers; --upid) { /* Make the PID visible to find_pid_ns. */ idr_replace(&upid->ns->idr, pid, upid->nr); -- cgit v1.2.3 From 0225bdfafd818f895fa4a4512f124a1614e011e2 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 1 Feb 2024 06:28:41 -0500 Subject: mempool: kvmalloc pool Add mempool_init_kvmalloc_pool() and mempool_create_kvmalloc_pool(), which wrap kvmalloc() instead of kmalloc() - kmalloc() with a vmalloc() fallback. This is part of a bcachefs cleanup - dropping an internal kvpmalloc() helper (which predates kvmalloc()) along with mempool helpers; this replaces the bcachefs-private kvpmalloc_pool. Signed-off-by: Kent Overstreet Cc: linux-mm@kvack.org --- include/linux/mempool.h | 13 +++++++++++++ mm/mempool.c | 13 +++++++++++++ 2 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mempool.h b/include/linux/mempool.h index 7be1e32e6d42..16c5cc807ff6 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -95,6 +95,19 @@ static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size) (void *) size); } +void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data); +void mempool_kvfree(void *element, void *pool_data); + +static inline int mempool_init_kvmalloc_pool(mempool_t *pool, int min_nr, size_t size) +{ + return mempool_init(pool, min_nr, mempool_kvmalloc, mempool_kvfree, (void *) size); +} + +static inline mempool_t *mempool_create_kvmalloc_pool(int min_nr, size_t size) +{ + return mempool_create(min_nr, mempool_kvmalloc, mempool_kvfree, (void *) size); +} + /* * A mempool_alloc_t and mempool_free_t for a simple page allocator that * allocates pages of the order specified by pool_data diff --git a/mm/mempool.c b/mm/mempool.c index dbbf0e9fb424..076c736f5f1f 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -590,6 +590,19 @@ void mempool_kfree(void *element, void *pool_data) } EXPORT_SYMBOL(mempool_kfree); +void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data) +{ + size_t size = (size_t)pool_data; + return kvmalloc(size, gfp_mask); +} +EXPORT_SYMBOL(mempool_kvmalloc); + +void mempool_kvfree(void *element, void *pool_data) +{ + kvfree(element); +} +EXPORT_SYMBOL(mempool_kvfree); + /* * A simple mempool-backed page allocator that allocates pages * of the order specified by pool_data. -- cgit v1.2.3 From 9448e55d032d99af8e23487f51a542d51b2f1a48 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 25 Feb 2024 14:27:11 +0000 Subject: of: Add cleanup.h based auto release via __free(device_node) markings The recent addition of scope based cleanup support to the kernel provides a convenient tool to reduce the chances of leaking reference counts where of_node_put() should have been called in an error path. This enables struct device_node *child __free(device_node) = NULL; for_each_child_of_node(np, child) { if (test) return test; } with no need for a manual call of of_node_put(). A following patch will reduce the scope of the child variable to the for loop, to avoid an issues with ordering of autocleanup, and make it obvious when this assigned a non NULL value. In this simple example the gains are small but there are some very complex error handling cases buried in these loops that will be greatly simplified by enabling early returns with out the need for this manual of_node_put() call. Note that there are coccinelle checks in scripts/coccinelle/iterators/for_each_child.cocci to detect a failure to call of_node_put(). This new approach does not cause false positives. Longer term we may want to add scripting to check this new approach is done correctly with no double of_node_put() calls being introduced due to the auto cleanup. It may also be useful to script finding places this new approach is useful. Signed-off-by: Jonathan Cameron Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240225142714.286440-2-jic23@kernel.org Signed-off-by: Rob Herring --- include/linux/of.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/of.h b/include/linux/of.h index d5e7acdc8c8e..2992e24cd72b 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -13,6 +13,7 @@ */ #include #include +#include #include #include #include @@ -134,6 +135,7 @@ static inline struct device_node *of_node_get(struct device_node *node) } static inline void of_node_put(struct device_node *node) { } #endif /* !CONFIG_OF_DYNAMIC */ +DEFINE_FREE(device_node, struct device_node *, if (_T) of_node_put(_T)) /* Pointer for first entry in chain of all nodes. */ extern struct device_node *of_root; -- cgit v1.2.3 From 34af4554fb0ce164e2c4876683619eb1e23848d4 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 25 Feb 2024 14:27:12 +0000 Subject: of: Introduce for_each_*_child_of_node_scoped() to automate of_node_put() handling To avoid issues with out of order cleanup, or ambiguity about when the auto freed data is first instantiated, do it within the for loop definition. The disadvantage is that the struct device_node *child variable creation is not immediately obvious where this is used. However, in many cases, if there is another definition of struct device_node *child; the compiler / static analysers will notify us that it is unused, or uninitialized. Note that, in the vast majority of cases, the _available_ form should be used and as code is converted to these scoped handers, we should confirm that any cases that do not check for available have a good reason not to. Signed-off-by: Jonathan Cameron Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240225142714.286440-3-jic23@kernel.org Signed-off-by: Rob Herring --- include/linux/of.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/of.h b/include/linux/of.h index 2992e24cd72b..4677e50d52b7 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -1420,10 +1420,23 @@ static inline int of_property_read_s32(const struct device_node *np, #define for_each_child_of_node(parent, child) \ for (child = of_get_next_child(parent, NULL); child != NULL; \ child = of_get_next_child(parent, child)) + +#define for_each_child_of_node_scoped(parent, child) \ + for (struct device_node *child __free(device_node) = \ + of_get_next_child(parent, NULL); \ + child != NULL; \ + child = of_get_next_child(parent, child)) + #define for_each_available_child_of_node(parent, child) \ for (child = of_get_next_available_child(parent, NULL); child != NULL; \ child = of_get_next_available_child(parent, child)) +#define for_each_available_child_of_node_scoped(parent, child) \ + for (struct device_node *child __free(device_node) = \ + of_get_next_available_child(parent, NULL); \ + child != NULL; \ + child = of_get_next_available_child(parent, child)) + #define for_each_of_cpu_node(cpu) \ for (cpu = of_get_next_cpu_node(NULL); cpu != NULL; \ cpu = of_get_next_cpu_node(cpu)) -- cgit v1.2.3 From 66a67c860cce3643248f7e80ee095b946829a342 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 28 Feb 2024 18:28:48 -0500 Subject: fs: file_remove_privs_flags() Rename and export __file_remove_privs(); for a buffered write path that doesn't take the inode lock we need to be able to check if the operation needs to do work first. Signed-off-by: Kent Overstreet Cc: Alexander Viro Cc: Christian Brauner --- fs/inode.c | 7 ++++--- include/linux/fs.h | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index 91048c4c9c9e..b465afdbfcef 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2031,7 +2031,7 @@ static int __remove_privs(struct mnt_idmap *idmap, return notify_change(idmap, dentry, &newattrs, NULL); } -static int __file_remove_privs(struct file *file, unsigned int flags) +int file_remove_privs_flags(struct file *file, unsigned int flags) { struct dentry *dentry = file_dentry(file); struct inode *inode = file_inode(file); @@ -2056,6 +2056,7 @@ static int __file_remove_privs(struct file *file, unsigned int flags) inode_has_no_xattr(inode); return error; } +EXPORT_SYMBOL_GPL(file_remove_privs_flags); /** * file_remove_privs - remove special file privileges (suid, capabilities) @@ -2068,7 +2069,7 @@ static int __file_remove_privs(struct file *file, unsigned int flags) */ int file_remove_privs(struct file *file) { - return __file_remove_privs(file, 0); + return file_remove_privs_flags(file, 0); } EXPORT_SYMBOL(file_remove_privs); @@ -2161,7 +2162,7 @@ static int file_modified_flags(struct file *file, int flags) * Clear the security bits if the process is not being run by root. * This keeps people from modifying setuid and setgid binaries. */ - ret = __file_remove_privs(file, flags); + ret = file_remove_privs_flags(file, flags); if (ret) return ret; diff --git a/include/linux/fs.h b/include/linux/fs.h index 1fbc72c5f112..14ea66b62823 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3004,6 +3004,7 @@ extern struct inode *new_inode_pseudo(struct super_block *sb); extern struct inode *new_inode(struct super_block *sb); extern void free_inode_nonrcu(struct inode *inode); extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *); +extern int file_remove_privs_flags(struct file *file, unsigned int flags); extern int file_remove_privs(struct file *); int setattr_should_drop_sgid(struct mnt_idmap *idmap, const struct inode *inode); -- cgit v1.2.3 From 3a319a2476d27e0b6c3cac3ebf6e3d0b665a06e5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 7 Mar 2024 22:32:06 -0500 Subject: lib/generic-radix-tree.c: Make nodes more reasonably sized this code originally used the page allocator directly, but most code shouldn't do that - PAGE_SIZE varies with architecture, and slab is faster. 4k is also on the large side for typical usage, 512 bytes is a better choice for typical usage that might be somewhat sparse. Signed-off-by: Kent Overstreet --- include/linux/generic-radix-tree.h | 29 ++++++++++++++++------------- lib/generic-radix-tree.c | 35 ++++++++++++----------------------- 2 files changed, 28 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h index 847413164738..f3512fddf3d7 100644 --- a/include/linux/generic-radix-tree.h +++ b/include/linux/generic-radix-tree.h @@ -5,7 +5,7 @@ * DOC: Generic radix trees/sparse arrays * * Very simple and minimalistic, supporting arbitrary size entries up to - * PAGE_SIZE. + * GENRADIX_NODE_SIZE. * * A genradix is defined with the type it will store, like so: * @@ -45,12 +45,15 @@ struct genradix_root; +#define GENRADIX_NODE_SHIFT 9 +#define GENRADIX_NODE_SIZE (1U << GENRADIX_NODE_SHIFT) + struct __genradix { struct genradix_root *root; }; /* - * NOTE: currently, sizeof(_type) must not be larger than PAGE_SIZE: + * NOTE: currently, sizeof(_type) must not be larger than GENRADIX_NODE_SIZE: */ #define __GENRADIX_INITIALIZER \ @@ -101,14 +104,14 @@ void __genradix_free(struct __genradix *); static inline size_t __idx_to_offset(size_t idx, size_t obj_size) { if (__builtin_constant_p(obj_size)) - BUILD_BUG_ON(obj_size > PAGE_SIZE); + BUILD_BUG_ON(obj_size > GENRADIX_NODE_SIZE); else - BUG_ON(obj_size > PAGE_SIZE); + BUG_ON(obj_size > GENRADIX_NODE_SIZE); if (!is_power_of_2(obj_size)) { - size_t objs_per_page = PAGE_SIZE / obj_size; + size_t objs_per_page = GENRADIX_NODE_SIZE / obj_size; - return (idx / objs_per_page) * PAGE_SIZE + + return (idx / objs_per_page) * GENRADIX_NODE_SIZE + (idx % objs_per_page) * obj_size; } else { return idx * obj_size; @@ -118,9 +121,9 @@ static inline size_t __idx_to_offset(size_t idx, size_t obj_size) #define __genradix_cast(_radix) (typeof((_radix)->type[0]) *) #define __genradix_obj_size(_radix) sizeof((_radix)->type[0]) #define __genradix_objs_per_page(_radix) \ - (PAGE_SIZE / sizeof((_radix)->type[0])) + (GENRADIX_NODE_SIZE / sizeof((_radix)->type[0])) #define __genradix_page_remainder(_radix) \ - (PAGE_SIZE % sizeof((_radix)->type[0])) + (GENRADIX_NODE_SIZE % sizeof((_radix)->type[0])) #define __genradix_idx_to_offset(_radix, _idx) \ __idx_to_offset(_idx, __genradix_obj_size(_radix)) @@ -217,8 +220,8 @@ static inline void __genradix_iter_advance(struct genradix_iter *iter, iter->offset += obj_size; if (!is_power_of_2(obj_size) && - (iter->offset & (PAGE_SIZE - 1)) + obj_size > PAGE_SIZE) - iter->offset = round_up(iter->offset, PAGE_SIZE); + (iter->offset & (GENRADIX_NODE_SIZE - 1)) + obj_size > GENRADIX_NODE_SIZE) + iter->offset = round_up(iter->offset, GENRADIX_NODE_SIZE); iter->pos++; } @@ -235,8 +238,8 @@ static inline void __genradix_iter_rewind(struct genradix_iter *iter, return; } - if ((iter->offset & (PAGE_SIZE - 1)) == 0) - iter->offset -= PAGE_SIZE % obj_size; + if ((iter->offset & (GENRADIX_NODE_SIZE - 1)) == 0) + iter->offset -= GENRADIX_NODE_SIZE % obj_size; iter->offset -= obj_size; iter->pos--; @@ -263,7 +266,7 @@ static inline void __genradix_iter_rewind(struct genradix_iter *iter, genradix_for_each_from(_radix, _iter, _p, 0) #define genradix_last_pos(_radix) \ - (SIZE_MAX / PAGE_SIZE * __genradix_objs_per_page(_radix) - 1) + (SIZE_MAX / GENRADIX_NODE_SIZE * __genradix_objs_per_page(_radix) - 1) /** * genradix_for_each_reverse - iterate over entry in a genradix, reverse order diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c index 41f1bcdc4488..aaefb9b678c8 100644 --- a/lib/generic-radix-tree.c +++ b/lib/generic-radix-tree.c @@ -5,7 +5,7 @@ #include #include -#define GENRADIX_ARY (PAGE_SIZE / sizeof(struct genradix_node *)) +#define GENRADIX_ARY (GENRADIX_NODE_SIZE / sizeof(struct genradix_node *)) #define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY) struct genradix_node { @@ -14,13 +14,13 @@ struct genradix_node { struct genradix_node *children[GENRADIX_ARY]; /* Leaf: */ - u8 data[PAGE_SIZE]; + u8 data[GENRADIX_NODE_SIZE]; }; }; static inline int genradix_depth_shift(unsigned depth) { - return PAGE_SHIFT + GENRADIX_ARY_SHIFT * depth; + return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth; } /* @@ -33,7 +33,7 @@ static inline size_t genradix_depth_size(unsigned depth) /* depth that's needed for a genradix that can address up to ULONG_MAX: */ #define GENRADIX_MAX_DEPTH \ - DIV_ROUND_UP(BITS_PER_LONG - PAGE_SHIFT, GENRADIX_ARY_SHIFT) + DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT) #define GENRADIX_DEPTH_MASK \ ((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1)) @@ -79,23 +79,12 @@ EXPORT_SYMBOL(__genradix_ptr); static inline struct genradix_node *genradix_alloc_node(gfp_t gfp_mask) { - struct genradix_node *node; - - node = (struct genradix_node *)__get_free_page(gfp_mask|__GFP_ZERO); - - /* - * We're using pages (not slab allocations) directly for kernel data - * structures, so we need to explicitly inform kmemleak of them in order - * to avoid false positive memory leak reports. - */ - kmemleak_alloc(node, PAGE_SIZE, 1, gfp_mask); - return node; + return kzalloc(GENRADIX_NODE_SIZE, gfp_mask); } static inline void genradix_free_node(struct genradix_node *node) { - kmemleak_free(node); - free_page((unsigned long)node); + kfree(node); } /* @@ -200,7 +189,7 @@ restart: i++; iter->offset = round_down(iter->offset + objs_per_ptr, objs_per_ptr); - iter->pos = (iter->offset >> PAGE_SHIFT) * + iter->pos = (iter->offset >> GENRADIX_NODE_SHIFT) * objs_per_page; if (i == GENRADIX_ARY) goto restart; @@ -209,7 +198,7 @@ restart: n = n->children[i]; } - return &n->data[iter->offset & (PAGE_SIZE - 1)]; + return &n->data[iter->offset & (GENRADIX_NODE_SIZE - 1)]; } EXPORT_SYMBOL(__genradix_iter_peek); @@ -235,7 +224,7 @@ restart: if (ilog2(iter->offset) >= genradix_depth_shift(level)) { iter->offset = genradix_depth_size(level); - iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page; + iter->pos = (iter->offset >> GENRADIX_NODE_SHIFT) * objs_per_page; iter->offset -= obj_size_plus_page_remainder; iter->pos--; @@ -251,7 +240,7 @@ restart: size_t objs_per_ptr = genradix_depth_size(level); iter->offset = round_down(iter->offset, objs_per_ptr); - iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page; + iter->pos = (iter->offset >> GENRADIX_NODE_SHIFT) * objs_per_page; if (!iter->offset) return NULL; @@ -267,7 +256,7 @@ restart: n = n->children[i]; } - return &n->data[iter->offset & (PAGE_SIZE - 1)]; + return &n->data[iter->offset & (GENRADIX_NODE_SIZE - 1)]; } EXPORT_SYMBOL(__genradix_iter_peek_prev); @@ -289,7 +278,7 @@ int __genradix_prealloc(struct __genradix *radix, size_t size, { size_t offset; - for (offset = 0; offset < size; offset += PAGE_SIZE) + for (offset = 0; offset < size; offset += GENRADIX_NODE_SIZE) if (!__genradix_ptr_alloc(radix, offset, gfp_mask)) return -ENOMEM; -- cgit v1.2.3 From a5a858f622a0aff5cdb5e271442cd01b2a01467f Mon Sep 17 00:00:00 2001 From: Casey Schaufler Date: Thu, 14 Mar 2024 11:31:26 -0400 Subject: lsm: use 32-bit compatible data types in LSM syscalls Change the size parameters in lsm_list_modules(), lsm_set_self_attr() and lsm_get_self_attr() from size_t to u32. This avoids the need to have different interfaces for 32 and 64 bit systems. Cc: stable@vger.kernel.org Fixes: a04a1198088a ("LSM: syscalls for current process attributes") Fixes: ad4aff9ec25f ("LSM: Create lsm_list_modules system call") Signed-off-by: Casey Schaufler Reported-and-reviewed-by: Dmitry V. Levin [PM: subject and metadata tweaks, syscall.h fixes] Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 4 ++-- include/linux/security.h | 8 ++++---- include/linux/syscalls.h | 6 +++--- security/apparmor/lsm.c | 4 ++-- security/lsm_syscalls.c | 10 +++++----- security/security.c | 12 ++++++------ security/selinux/hooks.c | 4 ++-- security/smack/smack_lsm.c | 4 ++-- tools/testing/selftests/lsm/common.h | 6 +++--- tools/testing/selftests/lsm/lsm_get_self_attr_test.c | 10 +++++----- tools/testing/selftests/lsm/lsm_list_modules_test.c | 8 ++++---- tools/testing/selftests/lsm/lsm_set_self_attr_test.c | 6 +++--- 12 files changed, 41 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index a8057a3f8de6..334e00efbde4 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -280,9 +280,9 @@ LSM_HOOK(int, 0, netlink_send, struct sock *sk, struct sk_buff *skb) LSM_HOOK(void, LSM_RET_VOID, d_instantiate, struct dentry *dentry, struct inode *inode) LSM_HOOK(int, -EOPNOTSUPP, getselfattr, unsigned int attr, - struct lsm_ctx __user *ctx, size_t *size, u32 flags) + struct lsm_ctx __user *ctx, u32 *size, u32 flags) LSM_HOOK(int, -EOPNOTSUPP, setselfattr, unsigned int attr, - struct lsm_ctx *ctx, size_t size, u32 flags) + struct lsm_ctx *ctx, u32 size, u32 flags) LSM_HOOK(int, -EINVAL, getprocattr, struct task_struct *p, const char *name, char **value) LSM_HOOK(int, -EINVAL, setprocattr, const char *name, void *value, size_t size) diff --git a/include/linux/security.h b/include/linux/security.h index f249f5b9a9d7..41a8f667bdfa 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -491,9 +491,9 @@ int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops, unsigned nsops, int alter); void security_d_instantiate(struct dentry *dentry, struct inode *inode); int security_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx, - size_t __user *size, u32 flags); + u32 __user *size, u32 flags); int security_setselfattr(unsigned int attr, struct lsm_ctx __user *ctx, - size_t size, u32 flags); + u32 size, u32 flags); int security_getprocattr(struct task_struct *p, int lsmid, const char *name, char **value); int security_setprocattr(int lsmid, const char *name, void *value, size_t size); @@ -507,7 +507,7 @@ int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen); int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen); int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen); int security_locked_down(enum lockdown_reason what); -int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, size_t *uctx_len, +int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, u32 *uctx_len, void *val, size_t val_len, u64 id, u64 flags); #else /* CONFIG_SECURITY */ @@ -1478,7 +1478,7 @@ static inline int security_locked_down(enum lockdown_reason what) return 0; } static inline int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, - size_t *uctx_len, void *val, size_t val_len, + u32 *uctx_len, void *val, size_t val_len, u64 id, u64 flags) { return -EOPNOTSUPP; diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 77eb9b0e7685..e619ac10cd23 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -960,10 +960,10 @@ asmlinkage long sys_cachestat(unsigned int fd, struct cachestat __user *cstat, unsigned int flags); asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags); asmlinkage long sys_lsm_get_self_attr(unsigned int attr, struct lsm_ctx *ctx, - size_t *size, __u32 flags); + u32 *size, u32 flags); asmlinkage long sys_lsm_set_self_attr(unsigned int attr, struct lsm_ctx *ctx, - size_t size, __u32 flags); -asmlinkage long sys_lsm_list_modules(u64 *ids, size_t *size, u32 flags); + u32 size, u32 flags); +asmlinkage long sys_lsm_list_modules(u64 *ids, u32 *size, u32 flags); /* * Architecture-specific system calls diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 9a3dcaafb5b1..cef8c466af80 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -779,7 +779,7 @@ static int apparmor_sb_pivotroot(const struct path *old_path, } static int apparmor_getselfattr(unsigned int attr, struct lsm_ctx __user *lx, - size_t *size, u32 flags) + u32 *size, u32 flags) { int error = -ENOENT; struct aa_task_ctx *ctx = task_ctx(current); @@ -924,7 +924,7 @@ fail: } static int apparmor_setselfattr(unsigned int attr, struct lsm_ctx *ctx, - size_t size, u32 flags) + u32 size, u32 flags) { int rc; diff --git a/security/lsm_syscalls.c b/security/lsm_syscalls.c index 5d391b1f7e69..8440948a690c 100644 --- a/security/lsm_syscalls.c +++ b/security/lsm_syscalls.c @@ -53,7 +53,7 @@ u64 lsm_name_to_attr(const char *name) * value indicating the reason for the error is returned. */ SYSCALL_DEFINE4(lsm_set_self_attr, unsigned int, attr, struct lsm_ctx __user *, - ctx, size_t, size, u32, flags) + ctx, u32, size, u32, flags) { return security_setselfattr(attr, ctx, size, flags); } @@ -75,7 +75,7 @@ SYSCALL_DEFINE4(lsm_set_self_attr, unsigned int, attr, struct lsm_ctx __user *, * a negative value indicating the error is returned. */ SYSCALL_DEFINE4(lsm_get_self_attr, unsigned int, attr, struct lsm_ctx __user *, - ctx, size_t __user *, size, u32, flags) + ctx, u32 __user *, size, u32, flags) { return security_getselfattr(attr, ctx, size, flags); } @@ -93,11 +93,11 @@ SYSCALL_DEFINE4(lsm_get_self_attr, unsigned int, attr, struct lsm_ctx __user *, * required size. In all other cases a negative value indicating the * error is returned. */ -SYSCALL_DEFINE3(lsm_list_modules, u64 __user *, ids, size_t __user *, size, +SYSCALL_DEFINE3(lsm_list_modules, u64 __user *, ids, u32 __user *, size, u32, flags) { - size_t total_size = lsm_active_cnt * sizeof(*ids); - size_t usize; + u32 total_size = lsm_active_cnt * sizeof(*ids); + u32 usize; int i; if (flags) diff --git a/security/security.c b/security/security.c index a07f43c58d65..5b2e0a15377d 100644 --- a/security/security.c +++ b/security/security.c @@ -785,7 +785,7 @@ static int lsm_superblock_alloc(struct super_block *sb) * Returns 0 on success, -E2BIG if userspace buffer is not large enough, * -EFAULT on a copyout error, -ENOMEM if memory can't be allocated. */ -int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, size_t *uctx_len, +int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, u32 *uctx_len, void *val, size_t val_len, u64 id, u64 flags) { @@ -3931,14 +3931,14 @@ EXPORT_SYMBOL(security_d_instantiate); * If @size is insufficient to contain the data -E2BIG is returned. */ int security_getselfattr(unsigned int attr, struct lsm_ctx __user *uctx, - size_t __user *size, u32 flags) + u32 __user *size, u32 flags) { struct security_hook_list *hp; struct lsm_ctx lctx = { .id = LSM_ID_UNDEF, }; u8 __user *base = (u8 __user *)uctx; - size_t total = 0; - size_t entrysize; - size_t left; + u32 entrysize; + u32 total = 0; + u32 left; bool toobig = false; bool single = false; int count = 0; @@ -4024,7 +4024,7 @@ int security_getselfattr(unsigned int attr, struct lsm_ctx __user *uctx, * LSM specific failure. */ int security_setselfattr(unsigned int attr, struct lsm_ctx __user *uctx, - size_t size, u32 flags) + u32 size, u32 flags) { struct security_hook_list *hp; struct lsm_ctx *lctx; diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 8db4875164ab..3448454c82d0 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6559,7 +6559,7 @@ abort_change: * There will only ever be one attribute. */ static int selinux_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx, - size_t *size, u32 flags) + u32 *size, u32 flags) { int rc; char *val = NULL; @@ -6574,7 +6574,7 @@ static int selinux_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx, } static int selinux_setselfattr(unsigned int attr, struct lsm_ctx *ctx, - size_t size, u32 flags) + u32 size, u32 flags) { int rc; diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 28be26712396..146667937811 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -3653,7 +3653,7 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode) * There will only ever be one attribute. */ static int smack_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx, - size_t *size, u32 flags) + u32 *size, u32 flags) { int rc; struct smack_known *skp; @@ -3774,7 +3774,7 @@ static int do_setattr(u64 attr, void *value, size_t size) * Returns 0 on success, an error code otherwise. */ static int smack_setselfattr(unsigned int attr, struct lsm_ctx *ctx, - size_t size, u32 flags) + u32 size, u32 flags) { int rc; diff --git a/tools/testing/selftests/lsm/common.h b/tools/testing/selftests/lsm/common.h index d404329e5eeb..06d12110d241 100644 --- a/tools/testing/selftests/lsm/common.h +++ b/tools/testing/selftests/lsm/common.h @@ -7,7 +7,7 @@ #ifndef lsm_get_self_attr static inline int lsm_get_self_attr(unsigned int attr, struct lsm_ctx *ctx, - size_t *size, __u32 flags) + __u32 *size, __u32 flags) { return syscall(__NR_lsm_get_self_attr, attr, ctx, size, flags); } @@ -15,14 +15,14 @@ static inline int lsm_get_self_attr(unsigned int attr, struct lsm_ctx *ctx, #ifndef lsm_set_self_attr static inline int lsm_set_self_attr(unsigned int attr, struct lsm_ctx *ctx, - size_t size, __u32 flags) + __u32 size, __u32 flags) { return syscall(__NR_lsm_set_self_attr, attr, ctx, size, flags); } #endif #ifndef lsm_list_modules -static inline int lsm_list_modules(__u64 *ids, size_t *size, __u32 flags) +static inline int lsm_list_modules(__u64 *ids, __u32 *size, __u32 flags) { return syscall(__NR_lsm_list_modules, ids, size, flags); } diff --git a/tools/testing/selftests/lsm/lsm_get_self_attr_test.c b/tools/testing/selftests/lsm/lsm_get_self_attr_test.c index e0e313d9047a..df215e4aa63f 100644 --- a/tools/testing/selftests/lsm/lsm_get_self_attr_test.c +++ b/tools/testing/selftests/lsm/lsm_get_self_attr_test.c @@ -40,7 +40,7 @@ TEST(size_null_lsm_get_self_attr) TEST(ctx_null_lsm_get_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); - size_t size = page_size; + __u32 size = page_size; int rc; rc = lsm_get_self_attr(LSM_ATTR_CURRENT, NULL, &size, 0); @@ -57,7 +57,7 @@ TEST(size_too_small_lsm_get_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); struct lsm_ctx *ctx = calloc(page_size, 1); - size_t size = 1; + __u32 size = 1; ASSERT_NE(NULL, ctx); errno = 0; @@ -77,7 +77,7 @@ TEST(flags_zero_lsm_get_self_attr) const long page_size = sysconf(_SC_PAGESIZE); struct lsm_ctx *ctx = calloc(page_size, 1); __u64 *syscall_lsms = calloc(page_size, 1); - size_t size; + __u32 size; int lsmcount; int i; @@ -117,7 +117,7 @@ TEST(flags_overset_lsm_get_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); struct lsm_ctx *ctx = calloc(page_size, 1); - size_t size; + __u32 size; ASSERT_NE(NULL, ctx); @@ -140,7 +140,7 @@ TEST(flags_overset_lsm_get_self_attr) TEST(basic_lsm_get_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); - size_t size = page_size; + __u32 size = page_size; struct lsm_ctx *ctx = calloc(page_size, 1); struct lsm_ctx *tctx = NULL; __u64 *syscall_lsms = calloc(page_size, 1); diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 4d5d4cee2586..06d24d4679a6 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -31,7 +31,7 @@ TEST(size_null_lsm_list_modules) TEST(ids_null_lsm_list_modules) { const long page_size = sysconf(_SC_PAGESIZE); - size_t size = page_size; + __u32 size = page_size; errno = 0; ASSERT_EQ(-1, lsm_list_modules(NULL, &size, 0)); @@ -43,7 +43,7 @@ TEST(size_too_small_lsm_list_modules) { const long page_size = sysconf(_SC_PAGESIZE); __u64 *syscall_lsms = calloc(page_size, 1); - size_t size = 1; + __u32 size = 1; ASSERT_NE(NULL, syscall_lsms); errno = 0; @@ -58,7 +58,7 @@ TEST(flags_set_lsm_list_modules) { const long page_size = sysconf(_SC_PAGESIZE); __u64 *syscall_lsms = calloc(page_size, 1); - size_t size = page_size; + __u32 size = page_size; ASSERT_NE(NULL, syscall_lsms); errno = 0; @@ -72,7 +72,7 @@ TEST(flags_set_lsm_list_modules) TEST(correct_lsm_list_modules) { const long page_size = sysconf(_SC_PAGESIZE); - size_t size = page_size; + __u32 size = page_size; __u64 *syscall_lsms = calloc(page_size, 1); char *sysfs_lsms = calloc(page_size, 1); char *name; diff --git a/tools/testing/selftests/lsm/lsm_set_self_attr_test.c b/tools/testing/selftests/lsm/lsm_set_self_attr_test.c index e9712c6cf596..66dec47e3ca3 100644 --- a/tools/testing/selftests/lsm/lsm_set_self_attr_test.c +++ b/tools/testing/selftests/lsm/lsm_set_self_attr_test.c @@ -25,7 +25,7 @@ TEST(size_too_small_lsm_set_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); struct lsm_ctx *ctx = calloc(page_size, 1); - size_t size = page_size; + __u32 size = page_size; ASSERT_NE(NULL, ctx); if (attr_lsm_count()) { @@ -41,7 +41,7 @@ TEST(flags_zero_lsm_set_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); struct lsm_ctx *ctx = calloc(page_size, 1); - size_t size = page_size; + __u32 size = page_size; ASSERT_NE(NULL, ctx); if (attr_lsm_count()) { @@ -57,7 +57,7 @@ TEST(flags_overset_lsm_set_self_attr) { const long page_size = sysconf(_SC_PAGESIZE); char *ctx = calloc(page_size, 1); - size_t size = page_size; + __u32 size = page_size; struct lsm_ctx *tctx = (struct lsm_ctx *)ctx; ASSERT_NE(NULL, ctx); -- cgit v1.2.3 From e54e09c05c00120cbe817bdb037088035be4bd79 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 12 Mar 2024 09:55:45 -0600 Subject: net: remove {revc,send}msg_copy_msghdr() from exports The only user of these was io_uring, and it's not using them anymore. Make them static and remove them from the socket header file. Signed-off-by: Jens Axboe Link: https://lore.kernel.org/r/1b6089d3-c1cf-464a-abd3-b0f0b6bb2523@kernel.dk Signed-off-by: Jakub Kicinski --- include/linux/socket.h | 7 ------- net/socket.c | 14 +++++++------- tools/perf/trace/beauty/include/linux/socket.h | 7 ------- 3 files changed, 7 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index cfcb7e2c3813..139c330ccf2c 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -422,13 +422,6 @@ extern long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg, struct user_msghdr __user *umsg, struct sockaddr __user *uaddr, unsigned int flags); -extern int sendmsg_copy_msghdr(struct msghdr *msg, - struct user_msghdr __user *umsg, unsigned flags, - struct iovec **iov); -extern int recvmsg_copy_msghdr(struct msghdr *msg, - struct user_msghdr __user *umsg, unsigned flags, - struct sockaddr __user **uaddr, - struct iovec **iov); extern int __copy_msghdr(struct msghdr *kmsg, struct user_msghdr *umsg, struct sockaddr __user **save_addr); diff --git a/net/socket.c b/net/socket.c index 7e9c8fc9a5b4..e5f3af49a8b6 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2600,9 +2600,9 @@ out: return err; } -int sendmsg_copy_msghdr(struct msghdr *msg, - struct user_msghdr __user *umsg, unsigned flags, - struct iovec **iov) +static int sendmsg_copy_msghdr(struct msghdr *msg, + struct user_msghdr __user *umsg, unsigned flags, + struct iovec **iov) { int err; @@ -2753,10 +2753,10 @@ SYSCALL_DEFINE4(sendmmsg, int, fd, struct mmsghdr __user *, mmsg, return __sys_sendmmsg(fd, mmsg, vlen, flags, true); } -int recvmsg_copy_msghdr(struct msghdr *msg, - struct user_msghdr __user *umsg, unsigned flags, - struct sockaddr __user **uaddr, - struct iovec **iov) +static int recvmsg_copy_msghdr(struct msghdr *msg, + struct user_msghdr __user *umsg, unsigned flags, + struct sockaddr __user **uaddr, + struct iovec **iov) { ssize_t err; diff --git a/tools/perf/trace/beauty/include/linux/socket.h b/tools/perf/trace/beauty/include/linux/socket.h index cfcb7e2c3813..139c330ccf2c 100644 --- a/tools/perf/trace/beauty/include/linux/socket.h +++ b/tools/perf/trace/beauty/include/linux/socket.h @@ -422,13 +422,6 @@ extern long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg, struct user_msghdr __user *umsg, struct sockaddr __user *uaddr, unsigned int flags); -extern int sendmsg_copy_msghdr(struct msghdr *msg, - struct user_msghdr __user *umsg, unsigned flags, - struct iovec **iov); -extern int recvmsg_copy_msghdr(struct msghdr *msg, - struct user_msghdr __user *umsg, unsigned flags, - struct sockaddr __user **uaddr, - struct iovec **iov); extern int __copy_msghdr(struct msghdr *kmsg, struct user_msghdr *umsg, struct sockaddr __user **save_addr); -- cgit v1.2.3 From 152609795dbf02f004c86049b75c23f4e68071d8 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Sat, 16 Mar 2024 01:10:21 +0100 Subject: fbcon: Increase maximum font width x height to 64 x 128 By using bitmaps we actually support whatever size we would want, but the console currently limits fonts to 64x128 (which gives 60x16 text on 4k screens), so we don't need more for now, and we can easily increase later. Signed-off-by: Samuel Thibault Signed-off-by: Helge Deller --- drivers/firmware/efi/earlycon.c | 2 +- drivers/video/fbdev/arkfb.c | 15 +++++++++++---- drivers/video/fbdev/core/fbcon.c | 16 +++++++++------- drivers/video/fbdev/core/fbmem.c | 12 ++++++------ drivers/video/fbdev/core/svgalib.c | 15 +++++++++++---- drivers/video/fbdev/s3fb.c | 15 +++++++++++---- drivers/video/fbdev/vga16fb.c | 6 +++++- drivers/video/fbdev/vt8623fb.c | 15 +++++++++++---- drivers/video/sticore.c | 2 +- include/linux/fb.h | 18 ++++++++++++------ include/linux/font.h | 3 ++- lib/fonts/fonts.c | 15 +++++++++------ 12 files changed, 89 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/earlycon.c b/drivers/firmware/efi/earlycon.c index f80a9af3d16e..d18a1a5de144 100644 --- a/drivers/firmware/efi/earlycon.c +++ b/drivers/firmware/efi/earlycon.c @@ -252,7 +252,7 @@ static int __init efi_earlycon_setup(struct earlycon_device *device, if (si->lfb_depth != 32) return -ENODEV; - font = get_default_font(xres, yres, -1, -1); + font = get_default_font(xres, yres, NULL, NULL); if (!font) return -ENODEV; diff --git a/drivers/video/fbdev/arkfb.c b/drivers/video/fbdev/arkfb.c index dca9c0325b3f..082501feceb9 100644 --- a/drivers/video/fbdev/arkfb.c +++ b/drivers/video/fbdev/arkfb.c @@ -622,8 +622,13 @@ static int arkfb_set_par(struct fb_info *info) info->tileops = NULL; /* in 4bpp supports 8p wide tiles only, any tiles otherwise */ - info->pixmap.blit_x = (bpp == 4) ? (1 << (8 - 1)) : (~(u32)0); - info->pixmap.blit_y = ~(u32)0; + if (bpp == 4) { + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + } else { + bitmap_fill(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + } + bitmap_fill(info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); offset_value = (info->var.xres_virtual * bpp) / 64; screen_size = info->var.yres_virtual * info->fix.line_length; @@ -635,8 +640,10 @@ static int arkfb_set_par(struct fb_info *info) info->tileops = &arkfb_tile_ops; /* supports 8x16 tiles only */ - info->pixmap.blit_x = 1 << (8 - 1); - info->pixmap.blit_y = 1 << (16 - 1); + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + bitmap_zero(info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); + set_bit(16 - 1, info->pixmap.blit_y); offset_value = info->var.xres_virtual / 16; screen_size = (info->var.xres_virtual * info->var.yres_virtual) / 64; diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 1183e7a871f8..4b67b32fdbc7 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2485,12 +2485,12 @@ static int fbcon_set_font(struct vc_data *vc, struct console_font *font, h > FBCON_SWAP(info->var.rotate, info->var.yres, info->var.xres)) return -EINVAL; - if (font->width > 32 || font->height > 32) + if (font->width > FB_MAX_BLIT_WIDTH || font->height > FB_MAX_BLIT_HEIGHT) return -EINVAL; /* Make sure drawing engine can handle the font */ - if (!(info->pixmap.blit_x & BIT(font->width - 1)) || - !(info->pixmap.blit_y & BIT(font->height - 1))) + if (!test_bit(font->width - 1, info->pixmap.blit_x) || + !test_bit(font->height - 1, info->pixmap.blit_y)) return -EINVAL; /* Make sure driver can handle the font length */ @@ -3084,8 +3084,8 @@ void fbcon_get_requirement(struct fb_info *info, vc = vc_cons[i].d; if (vc && vc->vc_mode == KD_TEXT && info->node == con2fb_map[i]) { - caps->x |= 1 << (vc->vc_font.width - 1); - caps->y |= 1 << (vc->vc_font.height - 1); + set_bit(vc->vc_font.width - 1, caps->x); + set_bit(vc->vc_font.height - 1, caps->y); charcnt = vc->vc_font.charcount; if (caps->len < charcnt) caps->len = charcnt; @@ -3096,8 +3096,10 @@ void fbcon_get_requirement(struct fb_info *info, if (vc && vc->vc_mode == KD_TEXT && info->node == con2fb_map[fg_console]) { - caps->x = 1 << (vc->vc_font.width - 1); - caps->y = 1 << (vc->vc_font.height - 1); + bitmap_zero(caps->x, FB_MAX_BLIT_WIDTH); + set_bit(vc->vc_font.width - 1, caps->x); + bitmap_zero(caps->y, FB_MAX_BLIT_HEIGHT); + set_bit(vc->vc_font.height - 1, caps->y); caps->len = vc->vc_font.charcount; } } diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c index fc206755f5f6..5ca18bfe11f6 100644 --- a/drivers/video/fbdev/core/fbmem.c +++ b/drivers/video/fbdev/core/fbmem.c @@ -212,8 +212,8 @@ static int fb_check_caps(struct fb_info *info, struct fb_var_screeninfo *var, fbcon_get_requirement(info, &caps); info->fbops->fb_get_caps(info, &fbcaps, var); - if (((fbcaps.x ^ caps.x) & caps.x) || - ((fbcaps.y ^ caps.y) & caps.y) || + if (!bitmap_subset(caps.x, fbcaps.x, FB_MAX_BLIT_WIDTH) || + !bitmap_subset(caps.y, fbcaps.y, FB_MAX_BLIT_HEIGHT) || (fbcaps.len < caps.len)) err = -EINVAL; @@ -420,11 +420,11 @@ static int do_register_framebuffer(struct fb_info *fb_info) } fb_info->pixmap.offset = 0; - if (!fb_info->pixmap.blit_x) - fb_info->pixmap.blit_x = ~(u32)0; + if (bitmap_empty(fb_info->pixmap.blit_x, FB_MAX_BLIT_WIDTH)) + bitmap_fill(fb_info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); - if (!fb_info->pixmap.blit_y) - fb_info->pixmap.blit_y = ~(u32)0; + if (bitmap_empty(fb_info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT)) + bitmap_fill(fb_info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); if (!fb_info->modelist.prev || !fb_info->modelist.next) INIT_LIST_HEAD(&fb_info->modelist); diff --git a/drivers/video/fbdev/core/svgalib.c b/drivers/video/fbdev/core/svgalib.c index 2cba158888ea..821b89a0a645 100644 --- a/drivers/video/fbdev/core/svgalib.c +++ b/drivers/video/fbdev/core/svgalib.c @@ -354,12 +354,19 @@ void svga_get_caps(struct fb_info *info, struct fb_blit_caps *caps, { if (var->bits_per_pixel == 0) { /* can only support 256 8x16 bitmap */ - caps->x = 1 << (8 - 1); - caps->y = 1 << (16 - 1); + bitmap_zero(caps->x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, caps->x); + bitmap_zero(caps->y, FB_MAX_BLIT_HEIGHT); + set_bit(16 - 1, caps->y); caps->len = 256; } else { - caps->x = (var->bits_per_pixel == 4) ? 1 << (8 - 1) : ~(u32)0; - caps->y = ~(u32)0; + if (var->bits_per_pixel == 4) { + bitmap_zero(caps->x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, caps->x); + } else { + bitmap_fill(caps->x, FB_MAX_BLIT_WIDTH); + } + bitmap_fill(caps->y, FB_MAX_BLIT_HEIGHT); caps->len = ~(u32)0; } } diff --git a/drivers/video/fbdev/s3fb.c b/drivers/video/fbdev/s3fb.c index 07722a5ea8ef..ff84106ecf1c 100644 --- a/drivers/video/fbdev/s3fb.c +++ b/drivers/video/fbdev/s3fb.c @@ -617,8 +617,13 @@ static int s3fb_set_par(struct fb_info *info) info->tileops = NULL; /* in 4bpp supports 8p wide tiles only, any tiles otherwise */ - info->pixmap.blit_x = (bpp == 4) ? (1 << (8 - 1)) : (~(u32)0); - info->pixmap.blit_y = ~(u32)0; + if (bpp == 4) { + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + } else { + bitmap_fill(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + } + bitmap_fill(info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); offset_value = (info->var.xres_virtual * bpp) / 64; screen_size = info->var.yres_virtual * info->fix.line_length; @@ -630,8 +635,10 @@ static int s3fb_set_par(struct fb_info *info) info->tileops = fasttext ? &s3fb_fast_tile_ops : &s3fb_tile_ops; /* supports 8x16 tiles only */ - info->pixmap.blit_x = 1 << (8 - 1); - info->pixmap.blit_y = 1 << (16 - 1); + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + bitmap_zero(info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); + set_bit(16 - 1, info->pixmap.blit_y); offset_value = info->var.xres_virtual / 16; screen_size = (info->var.xres_virtual * info->var.yres_virtual) / 64; diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c index b485e9198201..a87bafbb119c 100644 --- a/drivers/video/fbdev/vga16fb.c +++ b/drivers/video/fbdev/vga16fb.c @@ -1353,7 +1353,11 @@ static int vga16fb_probe(struct platform_device *dev) info->var = vga16fb_defined; info->fix = vga16fb_fix; /* supports rectangles with widths of multiples of 8 */ - info->pixmap.blit_x = 1 << 7 | 1 << 15 | 1 << 23 | 1 << 31; + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + set_bit(16 - 1, info->pixmap.blit_x); + set_bit(24 - 1, info->pixmap.blit_x); + set_bit(32 - 1, info->pixmap.blit_x); info->flags = FBINFO_HWACCEL_YPAN; i = (info->var.bits_per_pixel == 8) ? 256 : 16; diff --git a/drivers/video/fbdev/vt8623fb.c b/drivers/video/fbdev/vt8623fb.c index f8d022cb61e8..df984f3a7ff6 100644 --- a/drivers/video/fbdev/vt8623fb.c +++ b/drivers/video/fbdev/vt8623fb.c @@ -390,8 +390,13 @@ static int vt8623fb_set_par(struct fb_info *info) info->tileops = NULL; /* in 4bpp supports 8p wide tiles only, any tiles otherwise */ - info->pixmap.blit_x = (bpp == 4) ? (1 << (8 - 1)) : (~(u32)0); - info->pixmap.blit_y = ~(u32)0; + if (bpp == 4) { + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + } else { + bitmap_fill(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + } + bitmap_fill(info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); offset_value = (info->var.xres_virtual * bpp) / 64; fetch_value = ((info->var.xres * bpp) / 128) + 4; @@ -408,8 +413,10 @@ static int vt8623fb_set_par(struct fb_info *info) info->tileops = &vt8623fb_tile_ops; /* supports 8x16 tiles only */ - info->pixmap.blit_x = 1 << (8 - 1); - info->pixmap.blit_y = 1 << (16 - 1); + bitmap_zero(info->pixmap.blit_x, FB_MAX_BLIT_WIDTH); + set_bit(8 - 1, info->pixmap.blit_x); + bitmap_zero(info->pixmap.blit_y, FB_MAX_BLIT_HEIGHT); + set_bit(16 - 1, info->pixmap.blit_y); offset_value = info->var.xres_virtual / 16; fetch_value = (info->var.xres / 8) + 8; diff --git a/drivers/video/sticore.c b/drivers/video/sticore.c index 7115b325817f..88a1758616e0 100644 --- a/drivers/video/sticore.c +++ b/drivers/video/sticore.c @@ -529,7 +529,7 @@ sti_select_fbfont(struct sti_cooked_rom *cooked_rom, const char *fbfont_name) if (fbfont_name && strlen(fbfont_name)) fbfont = find_font(fbfont_name); if (!fbfont) - fbfont = get_default_font(1024,768, ~(u32)0, ~(u32)0); + fbfont = get_default_font(1024, 768, NULL, NULL); if (!fbfont) return NULL; diff --git a/include/linux/fb.h b/include/linux/fb.h index 05dc9624897d..7d7c7791fd26 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -143,9 +143,13 @@ struct fb_event { void *data; }; +/* Enough for the VT console needs, see its max_font_width/height */ +#define FB_MAX_BLIT_WIDTH 64 +#define FB_MAX_BLIT_HEIGHT 128 + struct fb_blit_caps { - u32 x; - u32 y; + DECLARE_BITMAP(x, FB_MAX_BLIT_WIDTH); + DECLARE_BITMAP(y, FB_MAX_BLIT_HEIGHT); u32 len; u32 flags; }; @@ -192,10 +196,12 @@ struct fb_pixmap { u32 scan_align; /* alignment per scanline */ u32 access_align; /* alignment per read/write (bits) */ u32 flags; /* see FB_PIXMAP_* */ - u32 blit_x; /* supported bit block dimensions (1-32)*/ - u32 blit_y; /* Format: blit_x = 1 << (width - 1) */ - /* blit_y = 1 << (height - 1) */ - /* if 0, will be set to 0xffffffff (all)*/ + /* supported bit block dimensions */ + /* Format: test_bit(width - 1, blit_x) */ + /* test_bit(height - 1, blit_y) */ + /* if zero, will be set to full (all) */ + DECLARE_BITMAP(blit_x, FB_MAX_BLIT_WIDTH); + DECLARE_BITMAP(blit_y, FB_MAX_BLIT_HEIGHT); /* access methods */ void (*writeio)(struct fb_info *info, void __iomem *dst, void *src, unsigned int size); void (*readio) (struct fb_info *info, void *dst, void __iomem *src, unsigned int size); diff --git a/include/linux/font.h b/include/linux/font.h index abf1442ce719..81caffd51bb4 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -57,7 +57,8 @@ extern const struct font_desc *find_font(const char *name); /* Get the default font for a specific screen size */ extern const struct font_desc *get_default_font(int xres, int yres, - u32 font_w, u32 font_h); + unsigned long *font_w, + unsigned long *font_h); /* Max. length for the name of a predefined font */ #define MAX_FONT_NAME 32 diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index 973866438608..47e34950b665 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -96,18 +96,21 @@ EXPORT_SYMBOL(find_font); * get_default_font - get default font * @xres: screen size of X * @yres: screen size of Y - * @font_w: bit array of supported widths (1 - 32) - * @font_h: bit array of supported heights (1 - 32) + * @font_w: bit array of supported widths (1 - FB_MAX_BLIT_WIDTH) + * @font_h: bit array of supported heights (1 - FB_MAX_BLIT_HEIGHT) * * Get the default font for a specified screen size. * Dimensions are in pixels. * + * font_w or font_h being NULL means all values are supported. + * * Returns %NULL if no font is found, or a pointer to the * chosen font. * */ -const struct font_desc *get_default_font(int xres, int yres, u32 font_w, - u32 font_h) +const struct font_desc *get_default_font(int xres, int yres, + unsigned long *font_w, + unsigned long *font_h) { int i, c, cc, res; const struct font_desc *f, *g; @@ -135,8 +138,8 @@ const struct font_desc *get_default_font(int xres, int yres, u32 font_w, if (res > 20) c += 20 - res; - if ((font_w & (1U << (f->width - 1))) && - (font_h & (1U << (f->height - 1)))) + if ((!font_w || test_bit(f->width - 1, font_w)) && + (!font_h || test_bit(f->height - 1, font_h))) c += 1000; if (c > cc) { -- cgit v1.2.3 From 56a34d799bfa53064e7b8bd354aacd176aeaecc8 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Mon, 26 Feb 2024 16:00:08 +0530 Subject: kexec/kdump: make struct crash_mem available without CONFIG_CRASH_DUMP struct crash_mem defined under include/linux/crash_core.h represents a list of memory ranges. While it is used to represent memory ranges for kdump kernel, it can also be used for other kind of memory ranges. In fact, KEXEC_FILE_LOAD syscall in powerpc uses this structure to represent reserved memory ranges and exclude memory ranges needed to find the right memory regions to load kexec kernel. So, make the definition of crash_mem structure available for !CONFIG_CRASH_DUMP case too. Signed-off-by: Hari Bathini Acked-by: Baoquan He Signed-off-by: Michael Ellerman Link: https://msgid.link/20240226103010.589537-2-hbathini@linux.ibm.com --- include/linux/crash_core.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index 23270b16e1db..d33352c2e386 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -8,6 +8,12 @@ struct kimage; +struct crash_mem { + unsigned int max_nr_ranges; + unsigned int nr_ranges; + struct range ranges[] __counted_by(max_nr_ranges); +}; + #ifdef CONFIG_CRASH_DUMP int crash_shrink_memory(unsigned long new_size); @@ -51,12 +57,6 @@ static inline unsigned int crash_get_elfcorehdr_size(void) { return 0; } /* Alignment required for elf header segment */ #define ELF_CORE_HEADER_ALIGN 4096 -struct crash_mem { - unsigned int max_nr_ranges; - unsigned int nr_ranges; - struct range ranges[] __counted_by(max_nr_ranges); -}; - extern int crash_exclude_mem_range(struct crash_mem *mem, unsigned long long mstart, unsigned long long mend); -- cgit v1.2.3 From 59a55a63c24624c7ad268f12c8f82d142ef6a6d4 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 14 Mar 2024 15:24:13 +0100 Subject: fs,block: get holder during claim Now that we open block devices as files we need to deal with the realities that closing is a deferred operation. An operation on the block device such as e.g., freeze, thaw, or removal that runs concurrently with umount, tries to acquire a stable reference on the holder. The holder might already be gone though. Make that reliable by grabbing a passive reference to the holder during bdev_open() and releasing it during bdev_release(). Fixes: f3a608827d1f ("bdev: open block device as files") # mainline only Reported-by: Christoph Hellwig Link: https://lore.kernel.org/r/ZfEQQ9jZZVes0WCZ@infradead.org Reviewed-by: Jan Kara Reviewed-by: Christoph Hellwig Tested-by: Yi Zhang Reported-by: https://lore.kernel.org/r/CAHj4cs8tbDwKRwfS1=DmooP73ysM__xAb2PQc6XsAmWR+VuYmg@mail.gmail.com Link: https://lore.kernel.org/r/20240315-freibad-annehmbar-ca68c375af91@brauner Signed-off-by: Christian Brauner --- block/bdev.c | 7 +++++++ fs/super.c | 18 ++++++++++++++++++ include/linux/blkdev.h | 10 ++++++++++ 3 files changed, 35 insertions(+) (limited to 'include/linux') diff --git a/block/bdev.c b/block/bdev.c index e7adaaf1c219..7a5f611c3d2e 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -583,6 +583,9 @@ static void bd_finish_claiming(struct block_device *bdev, void *holder, mutex_unlock(&bdev->bd_holder_lock); bd_clear_claiming(whole, holder); mutex_unlock(&bdev_lock); + + if (hops && hops->get_holder) + hops->get_holder(holder); } /** @@ -605,6 +608,7 @@ EXPORT_SYMBOL(bd_abort_claiming); static void bd_end_claim(struct block_device *bdev, void *holder) { struct block_device *whole = bdev_whole(bdev); + const struct blk_holder_ops *hops = bdev->bd_holder_ops; bool unblock = false; /* @@ -627,6 +631,9 @@ static void bd_end_claim(struct block_device *bdev, void *holder) whole->bd_holder = NULL; mutex_unlock(&bdev_lock); + if (hops && hops->put_holder) + hops->put_holder(holder); + /* * If this was the last claim, remove holder link and unblock evpoll if * it was a write holder. diff --git a/fs/super.c b/fs/super.c index ee05ab6b37e7..71d9779c42b1 100644 --- a/fs/super.c +++ b/fs/super.c @@ -1515,11 +1515,29 @@ static int fs_bdev_thaw(struct block_device *bdev) return error; } +static void fs_bdev_super_get(void *data) +{ + struct super_block *sb = data; + + spin_lock(&sb_lock); + sb->s_count++; + spin_unlock(&sb_lock); +} + +static void fs_bdev_super_put(void *data) +{ + struct super_block *sb = data; + + put_super(sb); +} + const struct blk_holder_ops fs_holder_ops = { .mark_dead = fs_bdev_mark_dead, .sync = fs_bdev_sync, .freeze = fs_bdev_freeze, .thaw = fs_bdev_thaw, + .get_holder = fs_bdev_super_get, + .put_holder = fs_bdev_super_put, }; EXPORT_SYMBOL_GPL(fs_holder_ops); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f9b87c39cab0..c3e8f7cf96be 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1505,6 +1505,16 @@ struct blk_holder_ops { * Thaw the file system mounted on the block device. */ int (*thaw)(struct block_device *bdev); + + /* + * If needed, get a reference to the holder. + */ + void (*get_holder)(void *holder); + + /* + * Release the holder. + */ + void (*put_holder)(void *holder); }; /* -- cgit v1.2.3 From c3198822c6cb9fb588e446540485669cc81c5d34 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Fri, 8 Mar 2024 17:26:00 +0200 Subject: net: esp: fix bad handling of pages from page_pool When the skb is reorganized during esp_output (!esp->inline), the pages coming from the original skb fragments are supposed to be released back to the system through put_page. But if the skb fragment pages are originating from a page_pool, calling put_page on them will trigger a page_pool leak which will eventually result in a crash. This leak can be easily observed when using CONFIG_DEBUG_VM and doing ipsec + gre (non offloaded) forwarding: BUG: Bad page state in process ksoftirqd/16 pfn:1451b6 page:00000000de2b8d32 refcount:0 mapcount:0 mapping:0000000000000000 index:0x1451b6000 pfn:0x1451b6 flags: 0x200000000000000(node=0|zone=2) page_type: 0xffffffff() raw: 0200000000000000 dead000000000040 ffff88810d23c000 0000000000000000 raw: 00000001451b6000 0000000000000001 00000000ffffffff 0000000000000000 page dumped because: page_pool leak Modules linked in: ip_gre gre mlx5_ib mlx5_core xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink iptable_nat nf_nat xt_addrtype br_netfilter rpcrdma rdma_ucm ib_iser libiscsi scsi_transport_iscsi ib_umad rdma_cm ib_ipoib iw_cm ib_cm ib_uverbs ib_core overlay zram zsmalloc fuse [last unloaded: mlx5_core] CPU: 16 PID: 96 Comm: ksoftirqd/16 Not tainted 6.8.0-rc4+ #22 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014 Call Trace: dump_stack_lvl+0x36/0x50 bad_page+0x70/0xf0 free_unref_page_prepare+0x27a/0x460 free_unref_page+0x38/0x120 esp_ssg_unref.isra.0+0x15f/0x200 esp_output_tail+0x66d/0x780 esp_xmit+0x2c5/0x360 validate_xmit_xfrm+0x313/0x370 ? validate_xmit_skb+0x1d/0x330 validate_xmit_skb_list+0x4c/0x70 sch_direct_xmit+0x23e/0x350 __dev_queue_xmit+0x337/0xba0 ? nf_hook_slow+0x3f/0xd0 ip_finish_output2+0x25e/0x580 iptunnel_xmit+0x19b/0x240 ip_tunnel_xmit+0x5fb/0xb60 ipgre_xmit+0x14d/0x280 [ip_gre] dev_hard_start_xmit+0xc3/0x1c0 __dev_queue_xmit+0x208/0xba0 ? nf_hook_slow+0x3f/0xd0 ip_finish_output2+0x1ca/0x580 ip_sublist_rcv_finish+0x32/0x40 ip_sublist_rcv+0x1b2/0x1f0 ? ip_rcv_finish_core.constprop.0+0x460/0x460 ip_list_rcv+0x103/0x130 __netif_receive_skb_list_core+0x181/0x1e0 netif_receive_skb_list_internal+0x1b3/0x2c0 napi_gro_receive+0xc8/0x200 gro_cell_poll+0x52/0x90 __napi_poll+0x25/0x1a0 net_rx_action+0x28e/0x300 __do_softirq+0xc3/0x276 ? sort_range+0x20/0x20 run_ksoftirqd+0x1e/0x30 smpboot_thread_fn+0xa6/0x130 kthread+0xcd/0x100 ? kthread_complete_and_exit+0x20/0x20 ret_from_fork+0x31/0x50 ? kthread_complete_and_exit+0x20/0x20 ret_from_fork_asm+0x11/0x20 The suggested fix is to introduce a new wrapper (skb_page_unref) that covers page refcounting for page_pool pages as well. Cc: stable@vger.kernel.org Fixes: 6a5bcd84e886 ("page_pool: Allow drivers to hint on SKB recycling") Reported-and-tested-by: Anatoli N.Chechelnickiy Reported-by: Ian Kumlien Link: https://lore.kernel.org/netdev/CAA85sZvvHtrpTQRqdaOx6gd55zPAVsqMYk_Lwh4Md5knTq7AyA@mail.gmail.com Signed-off-by: Dragos Tatulea Reviewed-by: Mina Almasry Reviewed-by: Jakub Kicinski Acked-by: Ilias Apalodimas Signed-off-by: Steffen Klassert --- include/linux/skbuff.h | 10 ++++++++++ net/ipv4/esp4.c | 8 ++++---- net/ipv6/esp6.c | 8 ++++---- 3 files changed, 18 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 3023bc2be6a1..b49a7d6591e8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3523,6 +3523,16 @@ int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, struct bpf_prog *prog); bool napi_pp_put_page(struct page *page, bool napi_safe); +static inline void +skb_page_unref(const struct sk_buff *skb, struct page *page, bool napi_safe) +{ +#ifdef CONFIG_PAGE_POOL + if (skb->pp_recycle && napi_pp_put_page(page, napi_safe)) + return; +#endif + put_page(page); +} + static inline void napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe) { diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 4dd9e5040672..d33d12421814 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -95,7 +95,7 @@ static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, __alignof__(struct scatterlist)); } -static void esp_ssg_unref(struct xfrm_state *x, void *tmp) +static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb) { struct crypto_aead *aead = x->data; int extralen = 0; @@ -114,7 +114,7 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp) */ if (req->src != req->dst) for (sg = sg_next(req->src); sg; sg = sg_next(sg)) - put_page(sg_page(sg)); + skb_page_unref(skb, sg_page(sg), false); } #ifdef CONFIG_INET_ESPINTCP @@ -260,7 +260,7 @@ static void esp_output_done(void *data, int err) } tmp = ESP_SKB_CB(skb)->tmp; - esp_ssg_unref(x, tmp); + esp_ssg_unref(x, tmp, skb); kfree(tmp); if (xo && (xo->flags & XFRM_DEV_RESUME)) { @@ -639,7 +639,7 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * } if (sg != dsg) - esp_ssg_unref(x, tmp); + esp_ssg_unref(x, tmp, skb); if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) err = esp_output_tail_tcp(x, skb); diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 6e6efe026cdc..7371886d4f9f 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -112,7 +112,7 @@ static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, __alignof__(struct scatterlist)); } -static void esp_ssg_unref(struct xfrm_state *x, void *tmp) +static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb) { struct crypto_aead *aead = x->data; int extralen = 0; @@ -131,7 +131,7 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp) */ if (req->src != req->dst) for (sg = sg_next(req->src); sg; sg = sg_next(sg)) - put_page(sg_page(sg)); + skb_page_unref(skb, sg_page(sg), false); } #ifdef CONFIG_INET6_ESPINTCP @@ -294,7 +294,7 @@ static void esp_output_done(void *data, int err) } tmp = ESP_SKB_CB(skb)->tmp; - esp_ssg_unref(x, tmp); + esp_ssg_unref(x, tmp, skb); kfree(tmp); esp_output_encap_csum(skb); @@ -677,7 +677,7 @@ int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info } if (sg != dsg) - esp_ssg_unref(x, tmp); + esp_ssg_unref(x, tmp, skb); if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) err = esp_output_tail_tcp(x, skb); -- cgit v1.2.3 From 35c3e27917568192927c785fc380f139255468b4 Mon Sep 17 00:00:00 2001 From: Abhishek Chauhan Date: Thu, 14 Mar 2024 12:24:04 -0700 Subject: Revert "net: Re-use and set mono_delivery_time bit for userspace tstamp packets" This reverts commit 885c36e59f46375c138de18ff1692f18eff67b7f. The patch currently broke the bpf selftest test_tc_dtime because uapi field __sk_buff->tstamp_type depends on skb->mono_delivery_time which does not necessarily mean mono with the original fix as the bit was re-used for userspace timestamp as well to avoid tstamp reset in the forwarding path. To solve this we need to keep mono_delivery_time as is and introduce another bit called user_delivery_time and fall back to the initial proposal of setting the user_delivery_time bit based on sk_clockid set from userspace. Fixes: 885c36e59f46 ("net: Re-use and set mono_delivery_time bit for userspace tstamp packets") Link: https://lore.kernel.org/netdev/bc037db4-58bb-4861-ac31-a361a93841d3@linux.dev/ Signed-off-by: Abhishek Chauhan Acked-by: Daniel Borkmann Acked-by: Martin KaFai Lau Signed-off-by: David S. Miller --- include/linux/skbuff.h | 6 +++--- net/ipv4/ip_output.c | 1 - net/ipv4/raw.c | 1 - net/ipv6/ip6_output.c | 2 +- net/ipv6/raw.c | 2 +- net/packet/af_packet.c | 4 +--- 6 files changed, 6 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 3023bc2be6a1..7d56ce195120 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -822,9 +822,9 @@ typedef unsigned char *sk_buff_data_t; * @decrypted: Decrypted SKB * @slow_gro: state present at GRO time, slower prepare step required * @mono_delivery_time: When set, skb->tstamp has the - * delivery_time in mono clock base (i.e., EDT) or a clock base chosen - * by SO_TXTIME. If zero, skb->tstamp has the (rcv) timestamp at - * ingress. + * delivery_time in mono clock base (i.e. EDT). Otherwise, the + * skb->tstamp has the (rcv) timestamp at ingress and + * delivery_time at egress. * @napi_id: id of the NAPI struct this skb came from * @sender_cpu: (aka @napi_id) source CPU in XPS * @alloc_cpu: CPU which did the skb allocation. diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 33f93dc730a3..1fe794967211 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1458,7 +1458,6 @@ struct sk_buff *__ip_make_skb(struct sock *sk, skb->priority = (cork->tos != -1) ? cork->priority: READ_ONCE(sk->sk_priority); skb->mark = cork->mark; skb->tstamp = cork->transmit_time; - skb->mono_delivery_time = !!skb->tstamp; /* * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec * on dst refcount diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index 42ac434cfcfa..12b3740393ba 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -360,7 +360,6 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4, skb->priority = READ_ONCE(sk->sk_priority); skb->mark = sockc->mark; skb->tstamp = sockc->transmit_time; - skb->mono_delivery_time = !!skb->tstamp; skb_dst_set(skb, &rt->dst); *rtp = NULL; diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 02eeca5492cd..b9dd3a66e423 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1925,7 +1925,7 @@ struct sk_buff *__ip6_make_skb(struct sock *sk, skb->priority = READ_ONCE(sk->sk_priority); skb->mark = cork->base.mark; skb->tstamp = cork->base.transmit_time; - skb->mono_delivery_time = !!skb->tstamp; + ip6_cork_steal_dst(skb, cork); IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS); if (proto == IPPROTO_ICMPV6) { diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index ca49e6617afa..0d896ca7b589 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -622,7 +622,7 @@ static int rawv6_send_hdrinc(struct sock *sk, struct msghdr *msg, int length, skb->priority = READ_ONCE(sk->sk_priority); skb->mark = sockc->mark; skb->tstamp = sockc->transmit_time; - skb->mono_delivery_time = !!skb->tstamp; + skb_put(skb, length); skb_reset_network_header(skb); iph = ipv6_hdr(skb); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 7cfc7d301508..18f616f487ea 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2057,7 +2057,7 @@ retry: skb->priority = READ_ONCE(sk->sk_priority); skb->mark = READ_ONCE(sk->sk_mark); skb->tstamp = sockc.transmit_time; - skb->mono_delivery_time = !!skb->tstamp; + skb_setup_tx_timestamp(skb, sockc.tsflags); if (unlikely(extra_len == 4)) @@ -2586,7 +2586,6 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, skb->priority = READ_ONCE(po->sk.sk_priority); skb->mark = READ_ONCE(po->sk.sk_mark); skb->tstamp = sockc->transmit_time; - skb->mono_delivery_time = !!skb->tstamp; skb_setup_tx_timestamp(skb, sockc->tsflags); skb_zcopy_set_nouarg(skb, ph.raw); @@ -3065,7 +3064,6 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) skb->priority = READ_ONCE(sk->sk_priority); skb->mark = sockc.mark; skb->tstamp = sockc.transmit_time; - skb->mono_delivery_time = !!skb->tstamp; if (unlikely(extra_len == 4)) skb->no_fcs = 1; -- cgit v1.2.3 From 70a6ed553f7d3504febac467cb4a0bae621ba3c6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 22 Feb 2024 16:14:19 -0500 Subject: tracing: Use EVENT_NULL_STR macro instead of open coding "(null)" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TRACE_EVENT macros has some dependency if a __string() field is NULL, where it will save "(null)" as the string. This string is also used by __assign_str(). It's better to create a single macro instead of having something that will not be caught by the compiler if there is an unfortunate typo. Link: https://lore.kernel.org/linux-trace-kernel/20240222211443.106216915@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Andrew Morton Cc: Ville Syrjälä Cc: Rodrigo Vivi Cc: Chuck Lever Suggested-by: Mathieu Desnoyers Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_events.h | 3 +++ include/trace/events/sunrpc.h | 12 ++++++------ include/trace/stages/stage5_get_offsets.h | 4 ++-- include/trace/stages/stage6_event_callback.h | 8 ++++---- 4 files changed, 15 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index fc6d0af56bb1..6f9bdfb09d1d 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -17,6 +17,9 @@ struct dentry; struct bpf_prog; union bpf_attr; +/* Used for event string fields when they are NULL */ +#define EVENT_NULL_STR "(null)" + const char *trace_print_flags_seq(struct trace_seq *p, const char *delim, unsigned long flags, const struct trace_print_flags *flag_array); diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index cdd3a45e6003..ce6a85b82afa 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -1327,18 +1327,18 @@ TRACE_EVENT(xs_stream_read_data, __field(ssize_t, err) __field(size_t, total) __string(addr, xprt ? xprt->address_strings[RPC_DISPLAY_ADDR] : - "(null)") + EVENT_NULL_STR) __string(port, xprt ? xprt->address_strings[RPC_DISPLAY_PORT] : - "(null)") + EVENT_NULL_STR) ), TP_fast_assign( __entry->err = err; __entry->total = total; __assign_str(addr, xprt ? - xprt->address_strings[RPC_DISPLAY_ADDR] : "(null)"); + xprt->address_strings[RPC_DISPLAY_ADDR] : EVENT_NULL_STR); __assign_str(port, xprt ? - xprt->address_strings[RPC_DISPLAY_PORT] : "(null)"); + xprt->address_strings[RPC_DISPLAY_PORT] : EVENT_NULL_STR); ), TP_printk("peer=[%s]:%s err=%zd total=%zu", __get_str(addr), @@ -1783,7 +1783,7 @@ TRACE_EVENT(svc_process, __string(service, name) __string(procedure, svc_proc_name(rqst)) __string(addr, rqst->rq_xprt ? - rqst->rq_xprt->xpt_remotebuf : "(null)") + rqst->rq_xprt->xpt_remotebuf : EVENT_NULL_STR) ), TP_fast_assign( @@ -1793,7 +1793,7 @@ TRACE_EVENT(svc_process, __assign_str(service, name); __assign_str(procedure, svc_proc_name(rqst)); __assign_str(addr, rqst->rq_xprt ? - rqst->rq_xprt->xpt_remotebuf : "(null)"); + rqst->rq_xprt->xpt_remotebuf : EVENT_NULL_STR); ), TP_printk("addr=%s xid=0x%08x service=%s vers=%u proc=%s", diff --git a/include/trace/stages/stage5_get_offsets.h b/include/trace/stages/stage5_get_offsets.h index 20b801ed3fd4..e6b96608f452 100644 --- a/include/trace/stages/stage5_get_offsets.h +++ b/include/trace/stages/stage5_get_offsets.h @@ -47,7 +47,7 @@ #undef __string #define __string(item, src) __dynamic_array(char, item, \ - strlen((const char *)(src) ? : "(null)") + 1) \ + strlen((const char *)(src) ? : EVENT_NULL_STR) + 1) \ __data_offsets->item##_ptr_ = src; #undef __string_len @@ -70,7 +70,7 @@ #undef __rel_string #define __rel_string(item, src) __rel_dynamic_array(char, item, \ - strlen((const char *)(src) ? : "(null)") + 1) \ + strlen((const char *)(src) ? : EVENT_NULL_STR) + 1) \ __data_offsets->item##_ptr_ = src; #undef __rel_string_len diff --git a/include/trace/stages/stage6_event_callback.h b/include/trace/stages/stage6_event_callback.h index 38732855eadb..2bfd49713b42 100644 --- a/include/trace/stages/stage6_event_callback.h +++ b/include/trace/stages/stage6_event_callback.h @@ -32,14 +32,14 @@ #undef __assign_str #define __assign_str(dst, src) \ - memcpy(__get_str(dst), __data_offsets.dst##_ptr_ ? : "(null)", \ + memcpy(__get_str(dst), __data_offsets.dst##_ptr_ ? : EVENT_NULL_STR, \ __get_dynamic_array_len(dst)) #undef __assign_str_len #define __assign_str_len(dst, src, len) \ do { \ memcpy(__get_str(dst), \ - __data_offsets.dst##_ptr_ ? : "(null)", len); \ + __data_offsets.dst##_ptr_ ? : EVENT_NULL_STR, len); \ __get_str(dst)[len] = '\0'; \ } while(0) @@ -94,14 +94,14 @@ #undef __assign_rel_str #define __assign_rel_str(dst, src) \ - memcpy(__get_rel_str(dst), __data_offsets.dst##_ptr_ ? : "(null)", \ + memcpy(__get_rel_str(dst), __data_offsets.dst##_ptr_ ? : EVENT_NULL_STR, \ __get_rel_dynamic_array_len(dst)) #undef __assign_rel_str_len #define __assign_rel_str_len(dst, src, len) \ do { \ memcpy(__get_rel_str(dst), \ - __data_offsets.dst##_ptr_ ? : "(null)", len); \ + __data_offsets.dst##_ptr_ ? : EVENT_NULL_STR, len); \ __get_rel_str(dst)[len] = '\0'; \ } while (0) -- cgit v1.2.3 From 19f0423fd55c301c8edaea286e568ec657f42750 Mon Sep 17 00:00:00 2001 From: Huang Yiwei Date: Fri, 23 Feb 2024 16:31:26 +0800 Subject: tracing: Support to dump instance traces by ftrace_dump_on_oops Currently ftrace only dumps the global trace buffer on an OOPs. For debugging a production usecase, instance trace will be helpful to check specific problems since global trace buffer may be used for other purposes. This patch extend the ftrace_dump_on_oops parameter to dump a specific or multiple trace instances: - ftrace_dump_on_oops=0: as before -- don't dump - ftrace_dump_on_oops[=1]: as before -- dump the global trace buffer on all CPUs - ftrace_dump_on_oops=2 or =orig_cpu: as before -- dump the global trace buffer on CPU that triggered the oops - ftrace_dump_on_oops=: new behavior -- dump the tracing instance matching - ftrace_dump_on_oops[=2/orig_cpu],[=2/orig_cpu], [=2/orig_cpu]: new behavior -- dump the global trace buffer and multiple instance buffer on all CPUs, or only dump on CPU that triggered the oops if =2 or =orig_cpu is given Also, the sysctl node can handle the input accordingly. Link: https://lore.kernel.org/linux-trace-kernel/20240223083126.1817731-1-quic_hyiwei@quicinc.com Cc: Ross Zwisler Cc: Cc: Cc: Cc: Cc: Cc: Cc: Signed-off-by: Huang Yiwei Signed-off-by: Steven Rostedt (Google) --- Documentation/admin-guide/kernel-parameters.txt | 26 +++- Documentation/admin-guide/sysctl/kernel.rst | 30 ++++- include/linux/ftrace.h | 4 +- include/linux/kernel.h | 1 + kernel/sysctl.c | 4 +- kernel/trace/trace.c | 156 ++++++++++++++++++------ kernel/trace/trace_selftest.c | 2 +- 7 files changed, 168 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 479ff1737c2f..fa871d53641c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1572,12 +1572,28 @@ The above will cause the "foo" tracing instance to trigger a snapshot at the end of boot up. - ftrace_dump_on_oops[=orig_cpu] + ftrace_dump_on_oops[=2(orig_cpu) | =][, | + ,=2(orig_cpu)] [FTRACE] will dump the trace buffers on oops. - If no parameter is passed, ftrace will dump - buffers of all CPUs, but if you pass orig_cpu, it will - dump only the buffer of the CPU that triggered the - oops. + If no parameter is passed, ftrace will dump global + buffers of all CPUs, if you pass 2 or orig_cpu, it + will dump only the buffer of the CPU that triggered + the oops, or the specific instance will be dumped if + its name is passed. Multiple instance dump is also + supported, and instances are separated by commas. Each + instance supports only dump on CPU that triggered the + oops by passing 2 or orig_cpu to it. + + ftrace_dump_on_oops=foo=orig_cpu + + The above will dump only the buffer of "foo" instance + on CPU that triggered the oops. + + ftrace_dump_on_oops,foo,bar=orig_cpu + + The above will dump global buffer on all CPUs, the + buffer of "foo" instance on all CPUs and the buffer + of "bar" instance on CPU that triggered the oops. ftrace_filter=[function-list] [FTRACE] Limit the functions traced by the function diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index 6584a1f9bfe3..ea8e5f152edc 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -296,12 +296,30 @@ kernel panic). This will output the contents of the ftrace buffers to the console. This is very useful for capturing traces that lead to crashes and outputting them to a serial console. -= =================================================== -0 Disabled (default). -1 Dump buffers of all CPUs. -2 Dump the buffer of the CPU that triggered the oops. -= =================================================== - +======================= =========================================== +0 Disabled (default). +1 Dump buffers of all CPUs. +2(orig_cpu) Dump the buffer of the CPU that triggered the + oops. + Dump the specific instance buffer on all CPUs. +=2(orig_cpu) Dump the specific instance buffer on the CPU + that triggered the oops. +======================= =========================================== + +Multiple instance dump is also supported, and instances are separated +by commas. If global buffer also needs to be dumped, please specify +the dump mode (1/2/orig_cpu) first for global buffer. + +So for example to dump "foo" and "bar" instance buffer on all CPUs, +user can:: + + echo "foo,bar" > /proc/sys/kernel/ftrace_dump_on_oops + +To dump global buffer and "foo" instance buffer on all +CPUs along with the "bar" instance buffer on CPU that triggered the +oops, user can:: + + echo "1,foo,bar=2" > /proc/sys/kernel/ftrace_dump_on_oops ftrace_enabled, stack_tracer_enabled ==================================== diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index e8921871ef9a..54d53f345d14 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1151,7 +1151,9 @@ static inline void unpause_graph_tracing(void) { } #ifdef CONFIG_TRACING enum ftrace_dump_mode; -extern enum ftrace_dump_mode ftrace_dump_on_oops; +#define MAX_TRACER_SIZE 100 +extern char ftrace_dump_on_oops[]; +extern int ftrace_dump_on_oops_enabled(void); extern int tracepoint_printk; extern void disable_trace_on_warning(void); diff --git a/include/linux/kernel.h b/include/linux/kernel.h index d718fbec72dd..be2e8c0a187e 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -215,6 +215,7 @@ enum ftrace_dump_mode { DUMP_NONE, DUMP_ALL, DUMP_ORIG, + DUMP_PARAM, }; #ifdef CONFIG_TRACING diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 157f7ce2942d..81cc974913bb 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1710,9 +1710,9 @@ static struct ctl_table kern_table[] = { { .procname = "ftrace_dump_on_oops", .data = &ftrace_dump_on_oops, - .maxlen = sizeof(int), + .maxlen = MAX_TRACER_SIZE, .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dostring, }, { .procname = "traceoff_on_warning", diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 1e1fd377a1cf..233d1af39fff 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -130,9 +130,12 @@ cpumask_var_t __read_mostly tracing_buffer_mask; * /proc/sys/kernel/ftrace_dump_on_oops * Set 1 if you want to dump buffers of all CPUs * Set 2 if you want to dump the buffer of the CPU that triggered oops + * Set instance name if you want to dump the specific trace instance + * Multiple instance dump is also supported, and instances are seperated + * by commas. */ - -enum ftrace_dump_mode ftrace_dump_on_oops; +/* Set to string format zero to disable by default */ +char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0"; /* When set, tracing will stop when a WARN*() is hit */ int __disable_trace_on_warning; @@ -178,7 +181,6 @@ static void ftrace_trace_userstack(struct trace_array *tr, struct trace_buffer *buffer, unsigned int trace_ctx); -#define MAX_TRACER_SIZE 100 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; static char *default_bootup_tracer; @@ -201,19 +203,33 @@ static int __init set_cmdline_ftrace(char *str) } __setup("ftrace=", set_cmdline_ftrace); +int ftrace_dump_on_oops_enabled(void) +{ + if (!strcmp("0", ftrace_dump_on_oops)) + return 0; + else + return 1; +} + static int __init set_ftrace_dump_on_oops(char *str) { - if (*str++ != '=' || !*str || !strcmp("1", str)) { - ftrace_dump_on_oops = DUMP_ALL; + if (!*str) { + strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE); return 1; } - if (!strcmp("orig_cpu", str) || !strcmp("2", str)) { - ftrace_dump_on_oops = DUMP_ORIG; - return 1; - } + if (*str == ',') { + strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE); + strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1); + return 1; + } + + if (*str++ == '=') { + strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE); + return 1; + } - return 0; + return 0; } __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); @@ -9832,14 +9848,14 @@ static struct notifier_block trace_die_notifier = { static int trace_die_panic_handler(struct notifier_block *self, unsigned long ev, void *unused) { - if (!ftrace_dump_on_oops) + if (!ftrace_dump_on_oops_enabled()) return NOTIFY_DONE; /* The die notifier requires DIE_OOPS to trigger */ if (self == &trace_die_notifier && ev != DIE_OOPS) return NOTIFY_DONE; - ftrace_dump(ftrace_dump_on_oops); + ftrace_dump(DUMP_PARAM); return NOTIFY_DONE; } @@ -9880,12 +9896,12 @@ trace_printk_seq(struct trace_seq *s) trace_seq_init(s); } -void trace_init_global_iter(struct trace_iterator *iter) +static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr) { - iter->tr = &global_trace; + iter->tr = tr; iter->trace = iter->tr->current_trace; iter->cpu_file = RING_BUFFER_ALL_CPUS; - iter->array_buffer = &global_trace.array_buffer; + iter->array_buffer = &tr->array_buffer; if (iter->trace && iter->trace->open) iter->trace->open(iter); @@ -9905,22 +9921,19 @@ void trace_init_global_iter(struct trace_iterator *iter) iter->fmt_size = STATIC_FMT_BUF_SIZE; } -void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) +void trace_init_global_iter(struct trace_iterator *iter) +{ + trace_init_iter(iter, &global_trace); +} + +static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode) { /* use static because iter can be a bit big for the stack */ static struct trace_iterator iter; - static atomic_t dump_running; - struct trace_array *tr = &global_trace; unsigned int old_userobj; unsigned long flags; int cnt = 0, cpu; - /* Only allow one dump user at a time. */ - if (atomic_inc_return(&dump_running) != 1) { - atomic_dec(&dump_running); - return; - } - /* * Always turn off tracing when we dump. * We don't need to show trace output of what happens @@ -9929,12 +9942,12 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) * If the user does a sysrq-z, then they can re-enable * tracing with echo 1 > tracing_on. */ - tracing_off(); + tracer_tracing_off(tr); local_irq_save(flags); /* Simulate the iterator */ - trace_init_global_iter(&iter); + trace_init_iter(&iter, tr); for_each_tracing_cpu(cpu) { atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); @@ -9945,21 +9958,15 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) /* don't look at user memory in panic mode */ tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; - switch (oops_dump_mode) { - case DUMP_ALL: - iter.cpu_file = RING_BUFFER_ALL_CPUS; - break; - case DUMP_ORIG: + if (dump_mode == DUMP_ORIG) iter.cpu_file = raw_smp_processor_id(); - break; - case DUMP_NONE: - goto out_enable; - default: - printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); + else iter.cpu_file = RING_BUFFER_ALL_CPUS; - } - printk(KERN_TRACE "Dumping ftrace buffer:\n"); + if (tr == &global_trace) + printk(KERN_TRACE "Dumping ftrace buffer:\n"); + else + printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name); /* Did function tracer already get disabled? */ if (ftrace_is_dead()) { @@ -10001,15 +10008,84 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) else printk(KERN_TRACE "---------------------------------\n"); - out_enable: tr->trace_flags |= old_userobj; for_each_tracing_cpu(cpu) { atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); } - atomic_dec(&dump_running); local_irq_restore(flags); } + +static void ftrace_dump_by_param(void) +{ + bool first_param = true; + char dump_param[MAX_TRACER_SIZE]; + char *buf, *token, *inst_name; + struct trace_array *tr; + + strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE); + buf = dump_param; + + while ((token = strsep(&buf, ",")) != NULL) { + if (first_param) { + first_param = false; + if (!strcmp("0", token)) + continue; + else if (!strcmp("1", token)) { + ftrace_dump_one(&global_trace, DUMP_ALL); + continue; + } + else if (!strcmp("2", token) || + !strcmp("orig_cpu", token)) { + ftrace_dump_one(&global_trace, DUMP_ORIG); + continue; + } + } + + inst_name = strsep(&token, "="); + tr = trace_array_find(inst_name); + if (!tr) { + printk(KERN_TRACE "Instance %s not found\n", inst_name); + continue; + } + + if (token && (!strcmp("2", token) || + !strcmp("orig_cpu", token))) + ftrace_dump_one(tr, DUMP_ORIG); + else + ftrace_dump_one(tr, DUMP_ALL); + } +} + +void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) +{ + static atomic_t dump_running; + + /* Only allow one dump user at a time. */ + if (atomic_inc_return(&dump_running) != 1) { + atomic_dec(&dump_running); + return; + } + + switch (oops_dump_mode) { + case DUMP_ALL: + ftrace_dump_one(&global_trace, DUMP_ALL); + break; + case DUMP_ORIG: + ftrace_dump_one(&global_trace, DUMP_ORIG); + break; + case DUMP_PARAM: + ftrace_dump_by_param(); + break; + case DUMP_NONE: + break; + default: + printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); + ftrace_dump_one(&global_trace, DUMP_ALL); + } + + atomic_dec(&dump_running); +} EXPORT_SYMBOL_GPL(ftrace_dump); #define WRITE_BUFSIZE 4096 diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index 529590499b1f..e9c5058a8efd 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -768,7 +768,7 @@ static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace) if (unlikely(++graph_hang_thresh > GRAPH_MAX_FUNC_TEST)) { ftrace_graph_stop(); printk(KERN_WARNING "BUG: Function graph tracer hang!\n"); - if (ftrace_dump_on_oops) { + if (ftrace_dump_on_oops_enabled()) { ftrace_dump(DUMP_ALL); /* ftrace_dump() disables tracing */ tracing_on(); -- cgit v1.2.3 From 1b273124107cc8b9dd52228eba701efa516a3d92 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 28 Feb 2024 13:31:12 -0500 Subject: tracepoints: Use WARN() and not WARN_ON() for warnings There are two WARN_ON*() warnings in tracepoint.h that deal with RCU usage. But when they trigger, especially from using a TRACE_EVENT() macro, the information is not very helpful and is confusing: ------------[ cut here ]------------ WARNING: CPU: 0 PID: 0 at include/trace/events/lock.h:24 lock_acquire+0x2b2/0x2d0 Where the above warning takes you to: TRACE_EVENT(lock_acquire, <<<--- line 24 in lock.h TP_PROTO(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, struct lockdep_map *next_lock, unsigned long ip), [..] Change the WARN_ON_ONCE() to WARN_ONCE() and add a string that allows someone to search for exactly where the bug happened. Link: https://lore.kernel.org/linux-trace-kernel/20240228133112.0d64fb1b@gandalf.local.home Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Thomas Gleixner Reported-by: Borislav Petkov Tested-by: Borislav Petkov (AMD) Reviewed-by: Paul E. McKenney Signed-off-by: Steven Rostedt (Google) --- include/linux/tracepoint.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 88c0ba623ee6..689b6d71590e 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -199,7 +199,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) if (!(cond)) \ return; \ \ - if (WARN_ON_ONCE(RCUIDLE_COND(rcuidle))) \ + if (WARN_ONCE(RCUIDLE_COND(rcuidle), \ + "Bad RCU usage for tracepoint")) \ return; \ \ /* keep srcu and sched-rcu usage consistent */ \ @@ -259,7 +260,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) TP_ARGS(args), \ TP_CONDITION(cond), 0); \ if (IS_ENABLED(CONFIG_LOCKDEP) && (cond)) { \ - WARN_ON_ONCE(!rcu_is_watching()); \ + WARN_ONCE(!rcu_is_watching(), \ + "RCU not watching for tracepoint"); \ } \ } \ __DECLARE_TRACE_RCU(name, PARAMS(proto), PARAMS(args), \ -- cgit v1.2.3 From 0a926fc972532788719fd03c4a44724ec23c1875 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sat, 3 Feb 2024 00:38:57 +0800 Subject: vDPA: introduce get_vq_size to vdpa_config_ops This commit introduces a new interface get_vq_size to vDPA config ops, this new interface intends to report the size of a specific virtqueue Signed-off-by: Zhu Lingshan Message-Id: <20240202163905.8834-3-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 8 ++++++++ include/linux/vdpa.h | 5 +++++ 2 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index aef92a7c57f3..ba52d128aeb7 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -687,6 +687,14 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, if (!ops->set_group_asid) return -EOPNOTSUPP; return ops->set_group_asid(vdpa, idx, s.num); + case VHOST_VDPA_GET_VRING_SIZE: + if (!ops->get_vq_size) + return -EOPNOTSUPP; + s.index = idx; + s.num = ops->get_vq_size(vdpa, idx); + if (copy_to_user(argp, &s, sizeof(s))) + return -EFAULT; + return 0; case VHOST_GET_VRING_BASE: r = ops->get_vq_state(v->vdpa, idx, &vq_state); if (r) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index db15ac07f8a6..4097e8e92860 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -195,6 +195,10 @@ struct vdpa_map_file { * @idx: virtqueue index * Returns int: irq number of a virtqueue, * negative number if no irq assigned. + * @get_vq_size: Get the size of a specific virtqueue (optional) + * @vdev: vdpa device + * @idx: virtqueue index + * Return u16: the size of the virtqueue * @get_vq_align: Get the virtqueue align requirement * for the device * @vdev: vdpa device @@ -386,6 +390,7 @@ struct vdpa_config_ops { (*get_vq_notification)(struct vdpa_device *vdev, u16 idx); /* vq irq is not expected to be changed once DRIVER_OK is set */ int (*get_vq_irq)(struct vdpa_device *vdev, u16 idx); + u16 (*get_vq_size)(struct vdpa_device *vdev, u16 idx); /* Device ops */ u32 (*get_vq_align)(struct vdpa_device *vdev); -- cgit v1.2.3 From c2475a9a789721bfdcc1b16aaf61ccfecb891914 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Mon, 19 Feb 2024 02:55:57 +0800 Subject: vDPA: report virtio-block capacity to user space This commit allows userspace to query capacity of a virtio-block device. Signed-off-by: Zhu Lingshan Message-Id: <20240218185606.13509-2-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 35 +++++++++++++++++++++++++++++++++++ include/linux/vdpa.h | 1 + include/uapi/linux/vdpa.h | 2 ++ 3 files changed, 38 insertions(+) (limited to 'include/linux') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index c73c06102e7e..613bec31f59e 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -944,6 +944,38 @@ static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *ms return vdpa_dev_net_mq_config_fill(msg, features_device, &config); } +static int +vdpa_dev_blk_capacity_config_fill(struct sk_buff *msg, + const struct virtio_blk_config *config) +{ + u64 val_u64; + + val_u64 = __virtio64_to_cpu(true, config->capacity); + + return nla_put_u64_64bit(msg, VDPA_ATTR_DEV_BLK_CFG_CAPACITY, + val_u64, VDPA_ATTR_PAD); +} + +static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev, + struct sk_buff *msg) +{ + struct virtio_blk_config config = {}; + u64 features_device; + + vdev->config->get_config(vdev, 0, &config, sizeof(config)); + + features_device = vdev->config->get_device_features(vdev); + + if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_FEATURES, features_device, + VDPA_ATTR_PAD)) + return -EMSGSIZE; + + if (vdpa_dev_blk_capacity_config_fill(msg, &config)) + return -EMSGSIZE; + + return 0; +} + static int vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq, int flags, struct netlink_ext_ack *extack) @@ -988,6 +1020,9 @@ vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, case VIRTIO_ID_NET: err = vdpa_dev_net_config_fill(vdev, msg); break; + case VIRTIO_ID_BLOCK: + err = vdpa_dev_blk_config_fill(vdev, msg); + break; default: err = -EOPNOTSUPP; break; diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 4097e8e92860..7977ca03ac7a 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -7,6 +7,7 @@ #include #include #include +#include #include /** diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 54b649ab0f22..1bf69226cb96 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -56,6 +56,8 @@ enum vdpa_attr { /* virtio features that are provisioned to the vDPA device */ VDPA_ATTR_DEV_FEATURES, /* u64 */ + VDPA_ATTR_DEV_BLK_CFG_CAPACITY, /* u64 */ + /* new attributes must be added above here */ VDPA_ATTR_MAX, }; -- cgit v1.2.3 From f6e0a4984c2e7244689ea87b62b433bed9d07e94 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 14 Mar 2024 20:08:45 +0000 Subject: net: move dev->state into net_device_read_txrx group dev->state can be read in rx and tx fast paths. netif_running() which needs dev->state is called from - enqueue_to_backlog() [RX path] - __dev_direct_xmit() [TX path] Fixes: 43a71cd66b9c ("net-device: reorganize net_device fast path variables") Signed-off-by: Eric Dumazet Cc: Coco Li Reviewed-by: Jiri Pirko Link: https://lore.kernel.org/r/20240314200845.3050179-1-edumazet@google.com Signed-off-by: Paolo Abeni --- Documentation/networking/net_cachelines/net_device.rst | 2 +- include/linux/netdevice.h | 2 +- net/core/dev.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index dceb49d56a91..70c4fb9d4e5c 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -13,7 +13,7 @@ struct_dev_ifalias* ifalias unsigned_long mem_end unsigned_long mem_start unsigned_long base_addr -unsigned_long state +unsigned_long state read_mostly read_mostly netif_running(dev) struct_list_head dev_list struct_list_head napi_list struct_list_head unreg_list diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c6f6ac779b34..cb37817d6382 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2072,6 +2072,7 @@ struct net_device { struct pcpu_sw_netstats __percpu *tstats; struct pcpu_dstats __percpu *dstats; }; + unsigned long state; unsigned int flags; unsigned short hard_header_len; netdev_features_t features; @@ -2117,7 +2118,6 @@ struct net_device { * part of the usual set specified in Space.c. */ - unsigned long state; struct list_head dev_list; struct list_head napi_list; diff --git a/net/core/dev.c b/net/core/dev.c index 722787c32755..303a6ff46e4e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -11665,11 +11665,12 @@ static void __init net_dev_struct_check(void) /* TXRX read-mostly hotpath */ CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr); - CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 38); + CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46); /* RX read-mostly hotpath */ CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific); -- cgit v1.2.3 From 2d9d9f256c8c85049306df3131ec7c81f9d8317c Mon Sep 17 00:00:00 2001 From: Herve Codina Date: Thu, 14 Mar 2024 13:00:06 +0100 Subject: lib/bitmap: Fix bitmap_scatter() and bitmap_gather() kernel doc The make htmldoc command failed with the following error ... include/linux/bitmap.h:524: ERROR: Unexpected indentation. ... include/linux/bitmap.h:524: CRITICAL: Unexpected section title or transition. Move the visual representation to a literal block. Fixes: de5f84338970 ("lib/bitmap: Introduce bitmap_scatter() and bitmap_gather() helpers") Reported-by: Stephen Rothwell Closes: https://lore.kernel.org/linux-kernel/20240312153059.3ffde1b7@canb.auug.org.au/ Signed-off-by: Herve Codina Reviewed-by: Andy Shevchenko Reviewed-by: Bagas Sanjaya Acked-by: Yury Norov Link: https://lore.kernel.org/r/20240314120006.458580-1-herve.codina@bootlin.com Signed-off-by: Jakub Kicinski --- include/linux/bitmap.h | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index fb3a9c93ac86..aa4096126553 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -522,17 +522,18 @@ static inline void bitmap_replace(unsigned long *dst, * * (Bits 0, 1, 2, 3, 4, 5 are copied to the bits 0, 1, 4, 8, 9, 12) * - * A more 'visual' description of the operation: - * src: 0000000001011010 - * |||||| - * +------+||||| - * | +----+|||| - * | |+----+||| - * | || +-+|| - * | || | || - * mask: ...v..vv...v..vv - * ...0..11...0..10 - * dst: 0000001100000010 + * A more 'visual' description of the operation:: + * + * src: 0000000001011010 + * |||||| + * +------+||||| + * | +----+|||| + * | |+----+||| + * | || +-+|| + * | || | || + * mask: ...v..vv...v..vv + * ...0..11...0..10 + * dst: 0000001100000010 * * A relationship exists between bitmap_scatter() and bitmap_gather(). * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation. @@ -568,16 +569,17 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, * * (Bits 0, 1, 4, 8, 9, 12 are copied to the bits 0, 1, 2, 3, 4, 5) * - * A more 'visual' description of the operation: - * mask: ...v..vv...v..vv - * src: 0000001100000010 - * ^ ^^ ^ 0 - * | || | 10 - * | || > 010 - * | |+--> 1010 - * | +--> 11010 - * +----> 011010 - * dst: 0000000000011010 + * A more 'visual' description of the operation:: + * + * mask: ...v..vv...v..vv + * src: 0000001100000010 + * ^ ^^ ^ 0 + * | || | 10 + * | || > 010 + * | |+--> 1010 + * | +--> 11010 + * +----> 011010 + * dst: 0000000000011010 * * A relationship exists between bitmap_gather() and bitmap_scatter(). See * bitmap_scatter() for the bitmap scatter detailed operations. -- cgit v1.2.3 From 1a77557d48cff187a169c2aec01c0dd78a5e7e50 Mon Sep 17 00:00:00 2001 From: Yan Zhai Date: Tue, 19 Mar 2024 13:44:34 -0700 Subject: rcu: add a helper to report consolidated flavor QS When under heavy load, network processing can run CPU-bound for many tens of seconds. Even in preemptible kernels (non-RT kernel), this can block RCU Tasks grace periods, which can cause trace-event removal to take more than a minute, which is unacceptably long. This commit therefore creates a new helper function that passes through both RCU and RCU-Tasks quiescent states every 100 milliseconds. This hard-coded value suffices for current workloads. Suggested-by: Paul E. McKenney Reviewed-by: Jesper Dangaard Brouer Signed-off-by: Yan Zhai Reviewed-by: Paul E. McKenney Acked-by: Jesper Dangaard Brouer Link: https://lore.kernel.org/r/90431d46ee112d2b0af04dbfe936faaca11810a5.1710877680.git.yan@cloudflare.com Signed-off-by: Jakub Kicinski --- include/linux/rcupdate.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 16f519914415..17d7ed5f3ae6 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -247,6 +247,37 @@ do { \ cond_resched(); \ } while (0) +/** + * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states + * @old_ts: jiffies at start of processing. + * + * This helper is for long-running softirq handlers, such as NAPI threads in + * networking. The caller should initialize the variable passed in as @old_ts + * at the beginning of the softirq handler. When invoked frequently, this macro + * will invoke rcu_softirq_qs() every 100 milliseconds thereafter, which will + * provide both RCU and RCU-Tasks quiescent states. Note that this macro + * modifies its old_ts argument. + * + * Because regions of code that have disabled softirq act as RCU read-side + * critical sections, this macro should be invoked with softirq (and + * preemption) enabled. + * + * The macro is not needed when CONFIG_PREEMPT_RT is defined. RT kernels would + * have more chance to invoke schedule() calls and provide necessary quiescent + * states. As a contrast, calling cond_resched() only won't achieve the same + * effect because cond_resched() does not provide RCU-Tasks quiescent states. + */ +#define rcu_softirq_qs_periodic(old_ts) \ +do { \ + if (!IS_ENABLED(CONFIG_PREEMPT_RT) && \ + time_after(jiffies, (old_ts) + HZ / 10)) { \ + preempt_disable(); \ + rcu_softirq_qs(); \ + preempt_enable(); \ + (old_ts) = jiffies; \ + } \ +} while (0) + /* * Infrastructure to implement the synchronize_() primitives in * TREE_RCU and rcu_barrier_() primitives in TINY_RCU. -- cgit v1.2.3 From d8e45f2929b94099913eb66c3ebb18b5063e9421 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 6 Mar 2024 15:51:36 -0800 Subject: overflow: Change DEFINE_FLEX to take __counted_by member The norm should be flexible array structures with __counted_by annotations, so DEFINE_FLEX() is updated to expect that. Rename the non-annotated version to DEFINE_RAW_FLEX(), and update the few existing users. Additionally add selftests for the macros. Reviewed-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20240306235128.it.933-kees@kernel.org Reviewed-by: Przemek Kitszel Signed-off-by: Kees Cook --- drivers/net/ethernet/intel/ice/ice_base.c | 4 ++-- drivers/net/ethernet/intel/ice/ice_common.c | 4 ++-- drivers/net/ethernet/intel/ice/ice_ddp.c | 8 ++++---- drivers/net/ethernet/intel/ice/ice_lag.c | 6 +++--- drivers/net/ethernet/intel/ice/ice_sched.c | 4 ++-- drivers/net/ethernet/intel/ice/ice_switch.c | 10 +++++----- include/linux/overflow.h | 25 +++++++++++++++++++++---- lib/overflow_kunit.c | 19 +++++++++++++++++++ 8 files changed, 58 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index d2fd315556a3..a545a7917e4f 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -956,7 +956,7 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_tx_ring **tx_rings, u16 q_idx) { - DEFINE_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); + DEFINE_RAW_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); if (q_idx >= vsi->alloc_txq || !tx_rings || !tx_rings[q_idx]) return -EINVAL; @@ -978,7 +978,7 @@ int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_tx_ring **tx_rings, static int ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_tx_ring **rings, u16 count) { - DEFINE_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); + DEFINE_RAW_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); int err = 0; u16 q_idx; diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 4d8111aeb0ff..db4b2844e1f7 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -4695,7 +4695,7 @@ ice_dis_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u8 num_queues, enum ice_disq_rst_src rst_src, u16 vmvf_num, struct ice_sq_cd *cd) { - DEFINE_FLEX(struct ice_aqc_dis_txq_item, qg_list, q_id, 1); + DEFINE_RAW_FLEX(struct ice_aqc_dis_txq_item, qg_list, q_id, 1); u16 i, buf_size = __struct_size(qg_list); struct ice_q_ctx *q_ctx; int status = -ENOENT; @@ -4917,7 +4917,7 @@ int ice_dis_vsi_rdma_qset(struct ice_port_info *pi, u16 count, u32 *qset_teid, u16 *q_id) { - DEFINE_FLEX(struct ice_aqc_dis_txq_item, qg_list, q_id, 1); + DEFINE_RAW_FLEX(struct ice_aqc_dis_txq_item, qg_list, q_id, 1); u16 qg_size = __struct_size(qg_list); struct ice_hw *hw; int status = 0; diff --git a/drivers/net/ethernet/intel/ice/ice_ddp.c b/drivers/net/ethernet/intel/ice/ice_ddp.c index 7532d11ad7f3..fc91c4d41186 100644 --- a/drivers/net/ethernet/intel/ice/ice_ddp.c +++ b/drivers/net/ethernet/intel/ice/ice_ddp.c @@ -1938,8 +1938,8 @@ static enum ice_ddp_state ice_init_pkg_info(struct ice_hw *hw, */ static enum ice_ddp_state ice_get_pkg_info(struct ice_hw *hw) { - DEFINE_FLEX(struct ice_aqc_get_pkg_info_resp, pkg_info, pkg_info, - ICE_PKG_CNT); + DEFINE_RAW_FLEX(struct ice_aqc_get_pkg_info_resp, pkg_info, pkg_info, + ICE_PKG_CNT); u16 size = __struct_size(pkg_info); u32 i; @@ -1990,8 +1990,8 @@ static enum ice_ddp_state ice_chk_pkg_compat(struct ice_hw *hw, struct ice_pkg_hdr *ospkg, struct ice_seg **seg) { - DEFINE_FLEX(struct ice_aqc_get_pkg_info_resp, pkg, pkg_info, - ICE_PKG_CNT); + DEFINE_RAW_FLEX(struct ice_aqc_get_pkg_info_resp, pkg, pkg_info, + ICE_PKG_CNT); u16 size = __struct_size(pkg); enum ice_ddp_state state; u32 i; diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index 467372d541d2..f97128b69f87 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -491,7 +491,7 @@ static void ice_lag_move_vf_node_tc(struct ice_lag *lag, u8 oldport, u8 newport, u16 vsi_num, u8 tc) { - DEFINE_FLEX(struct ice_aqc_move_elem, buf, teid, 1); + DEFINE_RAW_FLEX(struct ice_aqc_move_elem, buf, teid, 1); struct device *dev = ice_pf_to_dev(lag->pf); u16 numq, valq, num_moved, qbuf_size; u16 buf_size = __struct_size(buf); @@ -849,7 +849,7 @@ static void ice_lag_reclaim_vf_tc(struct ice_lag *lag, struct ice_hw *src_hw, u16 vsi_num, u8 tc) { - DEFINE_FLEX(struct ice_aqc_move_elem, buf, teid, 1); + DEFINE_RAW_FLEX(struct ice_aqc_move_elem, buf, teid, 1); struct device *dev = ice_pf_to_dev(lag->pf); u16 numq, valq, num_moved, qbuf_size; u16 buf_size = __struct_size(buf); @@ -1873,7 +1873,7 @@ static void ice_lag_move_vf_nodes_tc_sync(struct ice_lag *lag, struct ice_hw *dest_hw, u16 vsi_num, u8 tc) { - DEFINE_FLEX(struct ice_aqc_move_elem, buf, teid, 1); + DEFINE_RAW_FLEX(struct ice_aqc_move_elem, buf, teid, 1); struct device *dev = ice_pf_to_dev(lag->pf); u16 numq, valq, num_moved, qbuf_size; u16 buf_size = __struct_size(buf); diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index d174a4eeb899..a1525992d14b 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -237,7 +237,7 @@ static int ice_sched_remove_elems(struct ice_hw *hw, struct ice_sched_node *parent, u32 node_teid) { - DEFINE_FLEX(struct ice_aqc_delete_elem, buf, teid, 1); + DEFINE_RAW_FLEX(struct ice_aqc_delete_elem, buf, teid, 1); u16 buf_size = __struct_size(buf); u16 num_groups_removed = 0; int status; @@ -2219,7 +2219,7 @@ int ice_sched_move_nodes(struct ice_port_info *pi, struct ice_sched_node *parent, u16 num_items, u32 *list) { - DEFINE_FLEX(struct ice_aqc_move_elem, buf, teid, 1); + DEFINE_RAW_FLEX(struct ice_aqc_move_elem, buf, teid, 1); u16 buf_len = __struct_size(buf); struct ice_sched_node *node; u16 i, grps_movd = 0; diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index f84bab80ca42..d4baae8c3b72 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -1812,7 +1812,7 @@ ice_aq_alloc_free_vsi_list(struct ice_hw *hw, u16 *vsi_list_id, enum ice_sw_lkup_type lkup_type, enum ice_adminq_opc opc) { - DEFINE_FLEX(struct ice_aqc_alloc_free_res_elem, sw_buf, elem, 1); + DEFINE_RAW_FLEX(struct ice_aqc_alloc_free_res_elem, sw_buf, elem, 1); u16 buf_len = __struct_size(sw_buf); struct ice_aqc_res_elem *vsi_ele; int status; @@ -2081,7 +2081,7 @@ ice_aq_get_recipe_to_profile(struct ice_hw *hw, u32 profile_id, u8 *r_bitmap, */ int ice_alloc_recipe(struct ice_hw *hw, u16 *rid) { - DEFINE_FLEX(struct ice_aqc_alloc_free_res_elem, sw_buf, elem, 1); + DEFINE_RAW_FLEX(struct ice_aqc_alloc_free_res_elem, sw_buf, elem, 1); u16 buf_len = __struct_size(sw_buf); int status; @@ -4418,7 +4418,7 @@ int ice_alloc_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, u16 *counter_id) { - DEFINE_FLEX(struct ice_aqc_alloc_free_res_elem, buf, elem, 1); + DEFINE_RAW_FLEX(struct ice_aqc_alloc_free_res_elem, buf, elem, 1); u16 buf_len = __struct_size(buf); int status; @@ -4446,7 +4446,7 @@ int ice_free_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, u16 counter_id) { - DEFINE_FLEX(struct ice_aqc_alloc_free_res_elem, buf, elem, 1); + DEFINE_RAW_FLEX(struct ice_aqc_alloc_free_res_elem, buf, elem, 1); u16 buf_len = __struct_size(buf); int status; @@ -4476,7 +4476,7 @@ ice_free_res_cntr(struct ice_hw *hw, u8 type, u8 alloc_shared, u16 num_items, */ int ice_share_res(struct ice_hw *hw, u16 type, u8 shared, u16 res_id) { - DEFINE_FLEX(struct ice_aqc_alloc_free_res_elem, buf, elem, 1); + DEFINE_RAW_FLEX(struct ice_aqc_alloc_free_res_elem, buf, elem, 1); u16 buf_len = __struct_size(buf); u16 res_type; int status; diff --git a/include/linux/overflow.h b/include/linux/overflow.h index aa691f2119b0..0c7e3dcfe867 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -398,7 +398,7 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) * @count: Number of elements in the array; must be compile-time const. * @initializer: initializer expression (could be empty for no init). */ -#define _DEFINE_FLEX(type, name, member, count, initializer) \ +#define _DEFINE_FLEX(type, name, member, count, initializer...) \ _Static_assert(__builtin_constant_p(count), \ "onstack flex array members require compile-time const count"); \ union { \ @@ -408,8 +408,8 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) type *name = (type *)&name##_u /** - * DEFINE_FLEX() - Define an on-stack instance of structure with a trailing - * flexible array member. + * DEFINE_RAW_FLEX() - Define an on-stack instance of structure with a trailing + * flexible array member, when it does not have a __counted_by annotation. * * @type: structure type name, including "struct" keyword. * @name: Name for a variable to define. @@ -420,7 +420,24 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) * flexible array member. * Use __struct_size(@name) to get compile-time size of it afterwards. */ -#define DEFINE_FLEX(type, name, member, count) \ +#define DEFINE_RAW_FLEX(type, name, member, count) \ _DEFINE_FLEX(type, name, member, count, = {}) +/** + * DEFINE_FLEX() - Define an on-stack instance of structure with a trailing + * flexible array member. + * + * @TYPE: structure type name, including "struct" keyword. + * @NAME: Name for a variable to define. + * @MEMBER: Name of the array member. + * @COUNTER: Name of the __counted_by member. + * @COUNT: Number of elements in the array; must be compile-time const. + * + * Define a zeroed, on-stack, instance of @TYPE structure with a trailing + * flexible array member. + * Use __struct_size(@NAME) to get compile-time size of it afterwards. + */ +#define DEFINE_FLEX(TYPE, NAME, MEMBER, COUNTER, COUNT) \ + _DEFINE_FLEX(TYPE, NAME, MEMBER, COUNT, = { .obj.COUNTER = COUNT, }) + #endif /* __LINUX_OVERFLOW_H */ diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c index 65e8a72a83bf..4ef31b0bb74d 100644 --- a/lib/overflow_kunit.c +++ b/lib/overflow_kunit.c @@ -1172,6 +1172,24 @@ static void castable_to_type_test(struct kunit *test) #undef TEST_CASTABLE_TO_TYPE } +struct foo { + int a; + u32 counter; + s16 array[] __counted_by(counter); +}; + +static void DEFINE_FLEX_test(struct kunit *test) +{ + DEFINE_RAW_FLEX(struct foo, two, array, 2); + DEFINE_FLEX(struct foo, eight, array, counter, 8); + DEFINE_FLEX(struct foo, empty, array, counter, 0); + + KUNIT_EXPECT_EQ(test, __struct_size(two), + sizeof(struct foo) + sizeof(s16) + sizeof(s16)); + KUNIT_EXPECT_EQ(test, __struct_size(eight), 24); + KUNIT_EXPECT_EQ(test, __struct_size(empty), sizeof(struct foo)); +} + static struct kunit_case overflow_test_cases[] = { KUNIT_CASE(u8_u8__u8_overflow_test), KUNIT_CASE(s8_s8__s8_overflow_test), @@ -1194,6 +1212,7 @@ static struct kunit_case overflow_test_cases[] = { KUNIT_CASE(overflows_type_test), KUNIT_CASE(same_type_test), KUNIT_CASE(castable_to_type_test), + KUNIT_CASE(DEFINE_FLEX_test), {} }; -- cgit v1.2.3