Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git

15 files changed, 659 insertions, 247 deletions

diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c
index d9fb587a62c5..cf51b9c39f4e 100644
--- a/drivers/net/thunderbolt/main.c
+++ b/drivers/net/thunderbolt/main.c
@@ -540,11 +540,12 @@ static int tbnet_alloc_rx_buffers(struct tbnet *net, unsigned int nbuffers)
 		trace_tbnet_alloc_rx_frame(index, tf->page, dma_addr,
 					   DMA_FROM_DEVICE);
 
-		tb_ring_rx(ring->ring, &tf->frame);
+		tb_ring_rx_more(ring->ring, &tf->frame);
 
 		ring->prod++;
 	}
 
+	tb_ring_notify(ring->ring);
 	return 0;
 
 err_free:
@@ -1243,7 +1244,8 @@ static netdev_tx_t tbnet_start_xmit(struct sk_buff *skb,
 		goto err_drop;
 
 	for (i = 0; i < frame_index + 1; i++)
-		tb_ring_tx(net->tx_ring.ring, &frames[i]->frame);
+		tb_ring_tx_more(net->tx_ring.ring, &frames[i]->frame);
+	tb_ring_notify(net->tx_ring.ring);
 
 	if (net->svc->prtcstns & TBNET_MATCH_FRAGS_ID)
 		atomic_inc(&net->frame_id);
diff --git a/drivers/thunderbolt/debugfs.c b/drivers/thunderbolt/debugfs.c
index 6e9080e7bcec..ccefc5dd1ded 100644
--- a/drivers/thunderbolt/debugfs.c
+++ b/drivers/thunderbolt/debugfs.c
@@ -37,6 +37,9 @@
 
 #define COUNTER_SET_LEN		3
 
+#define HW_MARGINING_DWORDS		3
+#define SW_MARGINING_DWORDS		4
+
 /*
  * USB4 spec doesn't specify dwell range, the range of 100 ms to 500 ms
  * probed to give good results.
@@ -494,7 +497,7 @@ struct tb_margining {
 	unsigned int gen;
 	bool asym_rx;
 	u32 caps[3];
-	u32 results[3];
+	u32 results[5];
 	enum usb4_margining_lane lanes;
 	unsigned int min_ber_level;
 	unsigned int max_ber_level;
@@ -612,6 +615,11 @@ supports_optional_voltage_offset_range(const struct tb_margining *margining)
 	return margining->caps[0] & USB4_MARGIN_CAP_0_OPT_VOLTAGE_SUPPORT;
 }
 
+static bool supports_extended_error_counter(const struct tb_margining *margining)
+{
+	return margining->caps[0] & USB4_MARGIN_CAP_0_EXT_ERR_COUNTER;
+}
+
 static ssize_t
 margining_ber_level_write(struct file *file, const char __user *user_buf,
 			   size_t count, loff_t *ppos)
@@ -701,6 +709,8 @@ static int margining_caps_show(struct seq_file *s, void *not_used)
 		ber_level_show(s, margining->max_ber_level);
 	} else {
 		seq_puts(s, "# hardware margining: no\n");
+		seq_printf(s, "# extended error counter: %s\n",
+			   str_yes_no(supports_extended_error_counter(margining)));
 	}
 
 	seq_printf(s, "# all lanes simultaneously: %s\n",
@@ -1150,36 +1160,91 @@ static int margining_mode_show(struct seq_file *s, void *not_used)
 }
 DEBUGFS_ATTR_RW(margining_mode);
 
+static u32 margining_get_lane_error(const u32 *results, unsigned int lane, bool extended)
+{
+	u32 error_count;
+
+	switch (lane) {
+	case USB4_MARGINING_LANE_RX0:
+		error_count = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK, results[0]);
+		break;
+	case USB4_MARGINING_LANE_RX1:
+		error_count = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK, results[0]);
+		break;
+	case USB4_MARGINING_LANE_RX2:
+		error_count = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK, results[0]);
+		break;
+	default:
+		return 0;
+	}
+
+	if (extended)
+		error_count |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK, results[1 + lane]);
+
+	return error_count;
+}
+
 static int margining_run_sw(struct tb_margining *margining,
 			    struct usb4_port_margining_params *params)
 {
+	bool extended_err = supports_extended_error_counter(margining);
 	u32 nsamples = margining->dwell_time / DWELL_SAMPLE_INTERVAL;
 	int ret, i;
+	u32 dwords;
+
+	if (params->error_counter != USB4_MARGIN_SW_ERROR_COUNTER_START)
+		goto out_stop_or_clear;
+
+	if (extended_err)
+		dwords = SW_MARGINING_DWORDS;
+	else
+		dwords = 1;
 
 	ret = usb4_port_sw_margin(margining->port, margining->target, margining->index,
 				  params, margining->results);
 	if (ret)
-		goto out_stop;
+		return ret;
 
 	for (i = 0; i <= nsamples; i++) {
 		u32 errors = 0;
 
 		ret = usb4_port_sw_margin_errors(margining->port, margining->target,
-						 margining->index, &margining->results[1]);
-		if (ret)
+						 margining->index, &margining->results[1],
+						 dwords);
+		if (ret) {
+			tb_port_warn(margining->port, "failed to read margining error counters\n");
 			break;
+		}
 
-		if (margining->lanes == USB4_MARGINING_LANE_RX0)
-			errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK,
-					   margining->results[1]);
-		else if (margining->lanes == USB4_MARGINING_LANE_RX1)
-			errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK,
-					   margining->results[1]);
-		else if (margining->lanes == USB4_MARGINING_LANE_RX2)
-			errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK,
-					   margining->results[1]);
-		else if (margining->lanes == USB4_MARGINING_LANE_ALL)
-			errors = margining->results[1];
+		if (margining->lanes == USB4_MARGINING_LANE_ALL) {
+			if (extended_err) {
+				/* Extended counters with all lanes: check each active lane */
+				int width = tb_port_get_link_width(margining->port);
+				unsigned int lane, max_lane = 2;
+
+				if (width == TB_LINK_WIDTH_ASYM_TX) {
+					max_lane = 1;
+				} else if ((width == TB_LINK_WIDTH_ASYM_RX) && margining->asym_rx) {
+					max_lane = 3;
+				} else if (width < 0) {
+					tb_port_warn(margining->port, "failed to read link width\n");
+					break;
+				}
+
+				for (lane = 0; lane < max_lane; lane++) {
+					errors = margining_get_lane_error(&margining->results[1],
+									  lane, extended_err);
+					if (errors)
+						break;
+				}
+			} else {
+				/* Standard counters: all lane errors fit in results[1]*/
+				errors = margining->results[1];
+			}
+		} else {
+			errors = margining_get_lane_error(&margining->results[1],
+							  margining->lanes, extended_err);
+		}
 
 		/* Any errors stop the test */
 		if (errors)
@@ -1187,15 +1252,15 @@ static int margining_run_sw(struct tb_margining *margining,
 
 		fsleep(DWELL_SAMPLE_INTERVAL * USEC_PER_MSEC);
 	}
+	params->error_counter = USB4_MARGIN_SW_ERROR_COUNTER_STOP;
 
-out_stop:
+out_stop_or_clear:
 	/*
-	 * Stop the counters but don't clear them to allow the
+	 * Stop the counters or clear them as per the
 	 * different error counter configurations.
 	 */
-	margining_modify_error_counter(margining, margining->lanes,
-				       USB4_MARGIN_SW_ERROR_COUNTER_STOP);
-	return ret;
+	return margining_modify_error_counter(margining, margining->lanes,
+					      params->error_counter);
 }
 
 static int validate_margining(struct tb_margining *margining)
@@ -1272,7 +1337,7 @@ static int margining_run_write(void *data, u64 val)
 
 	if (margining->software) {
 		struct usb4_port_margining_params params = {
-			.error_counter = USB4_MARGIN_SW_ERROR_COUNTER_CLEAR,
+			.error_counter = margining->error_counter,
 			.lanes = margining->lanes,
 			.time = margining->time,
 			.voltage_time_offset = margining->voltage_time_offset,
@@ -1303,7 +1368,7 @@ static int margining_run_write(void *data, u64 val)
 			    margining->lanes);
 
 		ret = usb4_port_hw_margin(port, margining->target, margining->index, &params,
-					  margining->results, ARRAY_SIZE(margining->results));
+					  margining->results, HW_MARGINING_DWORDS);
 	}
 
 	if (down_sw)
@@ -1425,7 +1490,7 @@ static int margining_results_show(struct seq_file *s, void *not_used)
 	seq_printf(s, "0x%08x\n", margining->results[0]);
 	/* Only the hardware margining has two result dwords */
 	if (!margining->software) {
-		for (int i = 1; i < ARRAY_SIZE(margining->results); i++)
+		for (int i = 1; i < HW_MARGINING_DWORDS; i++)
 			seq_printf(s, "0x%08x\n", margining->results[i]);
 
 		if (margining->lanes == USB4_MARGINING_LANE_ALL) {
@@ -1444,18 +1509,30 @@ static int margining_results_show(struct seq_file *s, void *not_used)
 		u32 lane_errors, result;
 
 		seq_printf(s, "0x%08x\n", margining->results[1]);
+		if (supports_extended_error_counter(margining)) {
+			seq_printf(s, "0x%08x\n", margining->results[2]);
+			seq_printf(s, "0x%08x\n", margining->results[3]);
+			seq_printf(s, "0x%08x\n", margining->results[4]);
+		}
+
 
 		result = FIELD_GET(USB4_MARGIN_SW_LANES_MASK, margining->results[0]);
 		if (result == USB4_MARGINING_LANE_RX0 ||
 		    result == USB4_MARGINING_LANE_ALL) {
 			lane_errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK,
 						margining->results[1]);
+			if (supports_extended_error_counter(margining))
+				lane_errors |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK,
+							  margining->results[2]);
 			seq_printf(s, "# lane 0 errors: %u\n", lane_errors);
 		}
 		if (result == USB4_MARGINING_LANE_RX1 ||
 		    result == USB4_MARGINING_LANE_ALL) {
 			lane_errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK,
 						margining->results[1]);
+			if (supports_extended_error_counter(margining))
+				lane_errors |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK,
+							  margining->results[3]);
 			seq_printf(s, "# lane 1 errors: %u\n", lane_errors);
 		}
 		if (margining->asym_rx &&
@@ -1463,6 +1540,9 @@ static int margining_results_show(struct seq_file *s, void *not_used)
 		     result == USB4_MARGINING_LANE_ALL)) {
 			lane_errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK,
 						margining->results[1]);
+			if (supports_extended_error_counter(margining))
+				lane_errors |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK,
+							  margining->results[4]);
 			seq_printf(s, "# lane 2 errors: %u\n", lane_errors);
 		}
 	}
diff --git a/drivers/thunderbolt/dma_test.c b/drivers/thunderbolt/dma_test.c
index 519c67678b08..bcecb0edcb81 100644
--- a/drivers/thunderbolt/dma_test.c
+++ b/drivers/thunderbolt/dma_test.c
@@ -215,11 +215,6 @@ static void dma_test_stop_rings(struct dma_test *dt)
 {
 	int ret;
 
-	if (dt->rx_ring)
-		tb_ring_stop(dt->rx_ring);
-	if (dt->tx_ring)
-		tb_ring_stop(dt->tx_ring);
-
 	ret = tb_xdomain_disable_paths(dt->xd, dt->tx_hopid,
 				       dt->tx_ring ? dt->tx_ring->hop : -1,
 				       dt->rx_hopid,
@@ -227,6 +222,11 @@ static void dma_test_stop_rings(struct dma_test *dt)
 	if (ret)
 		dev_warn(&dt->svc->dev, "failed to disable DMA paths\n");
 
+	if (dt->rx_ring)
+		tb_ring_stop(dt->rx_ring);
+	if (dt->tx_ring)
+		tb_ring_stop(dt->tx_ring);
+
 	dma_test_free_rings(dt);
 }
 
diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c
index 2a13fa6888ba..2acfe83928b6 100644
--- a/drivers/thunderbolt/eeprom.c
+++ b/drivers/thunderbolt/eeprom.c
@@ -324,7 +324,7 @@ int tb_drom_read_uid_only(struct tb_switch *sw, u64 *uid)
 }
 
 static int tb_drom_parse_entry_generic(struct tb_switch *sw,
-		struct tb_drom_entry_header *header)
+		const struct tb_drom_entry_header *header)
 {
 	const struct tb_drom_entry_generic *entry =
 		(const struct tb_drom_entry_generic *)header;
@@ -360,7 +360,7 @@ static int tb_drom_parse_entry_generic(struct tb_switch *sw,
 }
 
 static int tb_drom_parse_entry_port(struct tb_switch *sw,
-				    struct tb_drom_entry_header *header)
+				    const struct tb_drom_entry_header *header)
 {
 	struct tb_port *port;
 	int res;
@@ -386,11 +386,13 @@ static int tb_drom_parse_entry_port(struct tb_switch *sw,
 	type &= 0xffffff;
 
 	if (type == TB_TYPE_PORT) {
-		struct tb_drom_entry_port *entry = (void *) header;
+		const struct tb_drom_entry_port *entry =
+			(const struct tb_drom_entry_port *)header;
+
 		if (header->len != sizeof(*entry)) {
 			tb_sw_warn(sw,
 				"port entry has size %#x (expected %#zx)\n",
-				header->len, sizeof(struct tb_drom_entry_port));
+				header->len, sizeof(*entry));
 			return -EIO;
 		}
 		port->link_nr = entry->link_nr;
@@ -421,9 +423,16 @@ static int tb_drom_parse_entries(struct tb_switch *sw, size_t header_size)
 	int res;
 
 	while (pos < drom_size) {
-		struct tb_drom_entry_header *entry = (void *) (sw->drom + pos);
-		if (pos + 1 == drom_size || pos + entry->len > drom_size
-				|| !entry->len) {
+		const struct tb_drom_entry_header *entry;
+
+		if (drom_size - pos < sizeof(*entry)) {
+			tb_sw_warn(sw, "DROM buffer overrun\n");
+			return -EIO;
+		}
+
+		entry = (const struct tb_drom_entry_header *)(sw->drom + pos);
+		if (entry->len < sizeof(*entry) ||
+		    entry->len > drom_size - pos) {
 			tb_sw_warn(sw, "DROM buffer overrun\n");
 			return -EIO;
 		}
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index be18f7b6595a..e99a3fcc4a29 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -41,6 +41,7 @@ static bool host_reset = true;
 module_param(host_reset, bool, 0444);
 MODULE_PARM_DESC(host_reset, "reset USB4 host router (default: true)");
 
+/* Returns absolute bit number of the ring in the interrupt registers */
 static int ring_interrupt_index(const struct tb_ring *ring)
 {
 	int bit = ring->hop;
@@ -49,24 +50,41 @@ static int ring_interrupt_index(const struct tb_ring *ring)
 	return bit;
 }
 
-static void nhi_mask_interrupt(struct tb_nhi *nhi, int mask, int ring)
+static void nhi_mask_interrupt(struct tb_nhi *nhi, u32 mask, int reg_index)
 {
-	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT) {
-		u32 val;
+	int offset = reg_index * 4;
+	u32 val;
 
-		val = ioread32(nhi->iobase + REG_RING_INTERRUPT_BASE + ring);
-		iowrite32(val & ~mask, nhi->iobase + REG_RING_INTERRUPT_BASE + ring);
-	} else {
-		iowrite32(mask, nhi->iobase + REG_RING_INTERRUPT_MASK_CLEAR_BASE + ring);
-	}
+	/* Use shadow copy instead of reading the register */
+	val = nhi->interrupt_mask[reg_index] & ~mask;
+	nhi->interrupt_mask[reg_index] = val;
+
+	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT)
+		iowrite32(val, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
+	else
+		iowrite32(mask, nhi->iobase + REG_RING_INTERRUPT_MASK_CLEAR_BASE + offset);
 }
 
-static void nhi_clear_interrupt(struct tb_nhi *nhi, int ring)
+static void nhi_unmask_interrupt(struct tb_nhi *nhi, u32 mask, int reg_index)
 {
+	int offset = reg_index * 4;
+	u32 val;
+
+	/* Use shadow copy instead of reading the register */
+	val = nhi->interrupt_mask[reg_index] | mask;
+	nhi->interrupt_mask[reg_index] = val;
+
+	iowrite32(val, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
+}
+
+static void nhi_clear_interrupt(struct tb_nhi *nhi, int reg_index)
+{
+	int offset = reg_index * 4;
+
 	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT)
-		ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + ring);
+		ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + offset);
 	else
-		iowrite32(~0, nhi->iobase + REG_RING_INT_CLEAR + ring);
+		iowrite32(~0, nhi->iobase + REG_RING_INT_CLEAR + offset);
 }
 
 /*
@@ -76,22 +94,17 @@ static void nhi_clear_interrupt(struct tb_nhi *nhi, int ring)
  */
 static void ring_interrupt_active(struct tb_ring *ring, bool active)
 {
-	int index = ring_interrupt_index(ring) / 32 * 4;
-	int reg = REG_RING_INTERRUPT_BASE + index;
-	int interrupt_bit = ring_interrupt_index(ring) & 31;
-	int mask = 1 << interrupt_bit;
+	int interrupt_index = ring_interrupt_index(ring);
+	int reg_index = interrupt_index / 32;
+	int reg = REG_RING_INTERRUPT_BASE + reg_index * 4;
+	int interrupt_bit = interrupt_index % 32;
+	u32 mask = BIT(interrupt_bit);
 	u32 old, new;
 
 	if (ring->irq > 0) {
 		u32 step, shift, ivr, misc, itr;
 		void __iomem *ivr_base;
 		int auto_clear_bit;
-		int index;
-
-		if (ring->is_tx)
-			index = ring->hop;
-		else
-			index = ring->hop + ring->nhi->hop_count;
 
 		/*
 		 * Intel routers support a bit that isn't part of
@@ -114,8 +127,8 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 				  ring->nhi->iobase + REG_DMA_MISC);
 
 		ivr_base = ring->nhi->iobase + REG_INT_VEC_ALLOC_BASE;
-		step = index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
-		shift = index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
+		step = interrupt_index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
+		shift = interrupt_index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
 		ivr = ioread32(ivr_base + step);
 		ivr &= ~(REG_INT_VEC_ALLOC_MASK << shift);
 		if (active)
@@ -129,7 +142,7 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 			  ring->vector * 4);
 	}
 
-	old = ioread32(ring->nhi->iobase + reg);
+	old = ring->nhi->interrupt_mask[reg_index];
 	if (active)
 		new = old | mask;
 	else
@@ -139,15 +152,24 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 		"%s interrupt at register %#x bit %d (%#x -> %#x)\n",
 		active ? "enabling" : "disabling", reg, interrupt_bit, old, new);
 
-	if (new == old)
-		dev_WARN(ring->nhi->dev, "interrupt for %s %d is already %s\n",
-			 RING_TYPE(ring), ring->hop,
-			 str_enabled_disabled(active));
+	if (new == old) {
+		/*
+		 * Rings that are polled mask the interrupt while the
+		 * completions are being advanced (see __ring_interrupt())
+		 * so for those it can already be disabled by the time
+		 * the ring is stopped.
+		 */
+		if (active || !ring->start_poll)
+			dev_WARN(ring->nhi->dev,
+				 "interrupt for %s %d is already %s\n",
+				 RING_TYPE(ring), ring->hop,
+				 str_enabled_disabled(active));
+	}
 
 	if (active)
-		iowrite32(new, ring->nhi->iobase + reg);
+		nhi_unmask_interrupt(ring->nhi, mask, reg_index);
 	else
-		nhi_mask_interrupt(ring->nhi, mask, index);
+		nhi_mask_interrupt(ring->nhi, mask, reg_index);
 }
 
 /*
@@ -160,11 +182,11 @@ void nhi_disable_interrupts(struct tb_nhi *nhi)
 	int i = 0;
 	/* disable interrupts */
 	for (i = 0; i < RING_INTERRUPT_REG_COUNT(nhi); i++)
-		nhi_mask_interrupt(nhi, ~0, 4 * i);
+		nhi_mask_interrupt(nhi, ~0, i);
 
 	/* clear interrupt status bits */
 	for (i = 0; i < RING_NOTIFY_REG_COUNT(nhi); i++)
-		nhi_clear_interrupt(nhi, 4 * i);
+		nhi_clear_interrupt(nhi, i);
 }
 
 /* ring helper methods */
@@ -227,12 +249,37 @@ static bool ring_empty(struct tb_ring *ring)
 	return ring->head == ring->tail;
 }
 
+static void __ring_notify(struct tb_ring *ring)
+{
+	lockdep_assert_held(&ring->lock);
+
+	if (ring->notify_pending) {
+		/*
+		 * The doorbell carries the absolute index of the head
+		 * so a single write covers all the descriptors posted
+		 * since the previous one.
+		 */
+		if (ring->is_tx)
+			ring_iowrite_prod(ring, ring->head);
+		else
+			ring_iowrite_cons(ring, ring->head);
+	}
+	ring->notify_pending = false;
+}
+
 /*
  * ring_write_descriptors() - post frames from ring->queue to the controller
+ * @ring: Ring to post the frames to
+ * @notify: Notify the controller about the posted descriptors
+ *
+ * Unless @notify is %true the controller is not notified about the posted
+ * descriptors and the caller is expected to call tb_ring_notify() once it
+ * is done queuing frames. This allows batching of frames before
+ * updating the producer/consumer indices.
  *
  * ring->lock is held.
  */
-static void ring_write_descriptors(struct tb_ring *ring)
+static void ring_write_descriptors(struct tb_ring *ring, bool notify)
 {
 	struct ring_frame *frame, *n;
 	struct ring_desc *descriptor;
@@ -256,11 +303,11 @@ static void ring_write_descriptors(struct tb_ring *ring)
 			descriptor->sof = frame->sof;
 		}
 		ring->head = (ring->head + 1) % ring->size;
-		if (ring->is_tx)
-			ring_iowrite_prod(ring, ring->head);
-		else
-			ring_iowrite_cons(ring, ring->head);
+		ring->notify_pending = true;
 	}
+
+	if (notify)
+		__ring_notify(ring);
 }
 
 /*
@@ -305,7 +352,7 @@ static void ring_work(struct work_struct *work)
 		}
 		ring->tail = (ring->tail + 1) % ring->size;
 	}
-	ring_write_descriptors(ring);
+	ring_write_descriptors(ring, true);
 
 invoke_callback:
 	/* allow callbacks to schedule new work */
@@ -324,7 +371,7 @@ invoke_callback:
 	wake_up(&ring->wait);
 }
 
-int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
+int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame, bool more)
 {
 	unsigned long flags;
 	int ret = 0;
@@ -332,7 +379,7 @@ int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
 	spin_lock_irqsave(&ring->lock, flags);
 	if (ring->running) {
 		list_add_tail(&frame->list, &ring->queue);
-		ring_write_descriptors(ring);
+		ring_write_descriptors(ring, !more);
 	} else {
 		ret = -ESHUTDOWN;
 	}
@@ -342,6 +389,22 @@ int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
 EXPORT_SYMBOL_GPL(__tb_ring_enqueue);
 
 /**
+ * tb_ring_notify() - Notify the controller about the queued frames
+ * @ring: Ring to notify
+ *
+ * Notifies the controller about frames that were enqueued using
+ * tb_ring_tx_more() or tb_ring_rx_more(). Does nothing if there are no
+ * such frames pending.
+ */
+void tb_ring_notify(struct tb_ring *ring)
+{
+	guard(spinlock_irqsave)(&ring->lock);
+	if (ring->running)
+		__ring_notify(ring);
+}
+EXPORT_SYMBOL_GPL(tb_ring_notify);
+
+/**
  * tb_ring_poll() - Poll one completed frame from the ring
  * @ring: Ring to poll
  *
@@ -377,6 +440,13 @@ struct ring_frame *tb_ring_poll(struct tb_ring *ring)
 		}
 
 		ring->tail = (ring->tail + 1) % ring->size;
+
+		/*
+		 * There is one more slot available so we can push next
+		 * descriptor to the ring. This is needed when the ring
+		 * is created with %RING_FLAG_NO_INTERRUPT.
+		 */
+		ring_write_descriptors(ring, true);
 	}
 
 unlock:
@@ -385,19 +455,35 @@ unlock:
 }
 EXPORT_SYMBOL_GPL(tb_ring_poll);
 
+/**
+ * tb_ring_poll_pending() - Does the ring have completed frame
+ * @ring: Ring to check
+ *
+ * Can be used to check whether there is a completed frame in the ring
+ * that next call to tb_ring_poll() returns.
+ *
+ * Return: %true if a completed frame is waiting, %false otherwise.
+ */
+bool tb_ring_poll_pending(struct tb_ring *ring)
+{
+	guard(spinlock_irqsave)(&ring->lock);
+
+	if (!ring->running || ring_empty(ring))
+		return false;
+	return !!(ring->descriptors[ring->tail].flags & RING_DESC_COMPLETED);
+}
+EXPORT_SYMBOL_GPL(tb_ring_poll_pending);
+
 static void __ring_interrupt_mask(struct tb_ring *ring, bool mask)
 {
-	int idx = ring_interrupt_index(ring);
-	int reg = REG_RING_INTERRUPT_BASE + idx / 32 * 4;
-	int bit = idx % 32;
-	u32 val;
+	int interrupt_index = ring_interrupt_index(ring);
+	int reg_index = interrupt_index / 32;
+	int interrupt_bit = interrupt_index % 32;
 
-	val = ioread32(ring->nhi->iobase + reg);
 	if (mask)
-		val &= ~BIT(bit);
+		nhi_mask_interrupt(ring->nhi, BIT(interrupt_bit), reg_index);
 	else
-		val |= BIT(bit);
-	iowrite32(val, ring->nhi->iobase + reg);
+		nhi_unmask_interrupt(ring->nhi, BIT(interrupt_bit), reg_index);
 }
 
 /* Both @nhi->lock and @ring->lock should be held */
@@ -788,6 +874,7 @@ void tb_ring_stop(struct tb_ring *ring)
 	ring_iowrite32desc(ring, 0, 12);
 	ring->head = 0;
 	ring->tail = 0;
+	ring->notify_pending = false;
 	ring->running = false;
 
 err:
@@ -1182,24 +1269,40 @@ static void nhi_reset(struct tb_nhi *nhi)
 
 static struct tb *nhi_select_cm(struct tb_nhi *nhi)
 {
+	bool linked = false;
 	struct tb *tb;
 
 	/*
 	 * USB4 case is simple. If we got control of any of the
 	 * capabilities, we use software CM.
 	 */
-	if (tb_acpi_is_native())
-		return tb_probe(nhi);
+	if (!tb_acpi_is_native()) {
+		/*
+		 * Either firmware based CM is running (we did not get
+		 * control from the firmware) or this is pre-USB4 PC so
+		 * try first firmware CM and then fallback to software CM.
+		 */
+		tb = icm_probe(nhi);
+		if (tb)
+			return tb;
+	}
 
-	/*
-	 * Either firmware based CM is running (we did not get control
-	 * from the firmware) or this is pre-USB4 PC so try first
-	 * firmware CM and then fallback to software CM.
-	 */
-	tb = icm_probe(nhi);
+	tb = tb_probe(nhi);
 	if (!tb)
-		tb = tb_probe(nhi);
+		return NULL;
 
+	if (nhi->ops->add_links)
+		linked = nhi->ops->add_links(nhi);
+	if (!linked)
+		linked = tb_acpi_add_links(nhi);
+	/*
+	 * Device links are needed to make sure we establish tunnels
+	 * before protocol stacks are resumed so complain here if we
+	 * found them missing.
+	 */
+	if (!linked)
+		dev_warn(nhi->dev,
+			 "device links to tunneled native ports are missing!\n");
 	return tb;
 }
 
@@ -1222,7 +1325,10 @@ int nhi_probe(struct tb_nhi *nhi)
 				     sizeof(*nhi->tx_rings), GFP_KERNEL);
 	nhi->rx_rings = devm_kcalloc(dev, nhi->hop_count,
 				     sizeof(*nhi->rx_rings), GFP_KERNEL);
-	if (!nhi->tx_rings || !nhi->rx_rings)
+	nhi->interrupt_mask = devm_kcalloc(dev, RING_INTERRUPT_REG_COUNT(nhi),
+					   sizeof(*nhi->interrupt_mask),
+					   GFP_KERNEL);
+	if (!nhi->tx_rings || !nhi->rx_rings || !nhi->interrupt_mask)
 		return -ENOMEM;
 
 	nhi_reset(nhi);
diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h
index d488eadadfce..88af9ab9fde7 100644
--- a/drivers/thunderbolt/nhi.h
+++ b/drivers/thunderbolt/nhi.h
@@ -41,6 +41,7 @@ extern const struct dev_pm_ops nhi_pm_ops;
 /**
  * struct tb_nhi_ops - NHI specific optional operations
  * @init: NHI specific initialization
+ * @add_links: Setup device links for tunneling native protocols
  * @suspend_noirq: NHI specific suspend_noirq hook
  * @resume_noirq: NHI specific resume_noirq hook
  * @runtime_suspend: NHI specific runtime_suspend hook
@@ -55,6 +56,7 @@ extern const struct dev_pm_ops nhi_pm_ops;
  */
 struct tb_nhi_ops {
 	int (*init)(struct tb_nhi *nhi);
+	bool (*add_links)(struct tb_nhi *nhi);
 	int (*suspend_noirq)(struct tb_nhi *nhi, bool wakeup);
 	int (*resume_noirq)(struct tb_nhi *nhi);
 	int (*runtime_suspend)(struct tb_nhi *nhi);
diff --git a/drivers/thunderbolt/path.c b/drivers/thunderbolt/path.c
index b2c322e76b8a..05249eed3f64 100644
--- a/drivers/thunderbolt/path.c
+++ b/drivers/thunderbolt/path.c
@@ -398,7 +398,7 @@ static int __tb_path_deactivate_hop(struct tb_port *port, int hop_index,
 		return ret;
 
 	/* Wait until it is drained */
-	timeout = ktime_add_ms(ktime_get(), 500);
+	timeout = ktime_add_ms(ktime_get(), port->pp_timeout_msec);
 	do {
 		ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
 		if (ret)
diff --git a/drivers/thunderbolt/pci.c b/drivers/thunderbolt/pci.c
index 8462ccb59b7e..4408229d8ac7 100644
--- a/drivers/thunderbolt/pci.c
+++ b/drivers/thunderbolt/pci.c
@@ -18,6 +18,7 @@
 #include <linux/property.h>
 #include <linux/string_helpers.h>
 #include <linux/suspend.h>
+#include <linux/platform_data/x86/apple.h>
 
 #include "nhi.h"
 #include "nhi_regs.h"
@@ -108,6 +109,82 @@ static void nhi_pci_check_iommu(struct tb_nhi_pci *nhi_pci)
 		str_enabled_disabled(port_ok));
 }
 
+static bool add_link(struct tb_nhi *nhi, struct pci_dev *pdev)
+{
+	const struct device_link *link;
+
+	link = device_link_add(&pdev->dev, nhi->dev,
+			       DL_FLAG_AUTOREMOVE_SUPPLIER |
+			       DL_FLAG_PM_RUNTIME);
+	if (!link) {
+		dev_warn(nhi->dev, "device link creation from %s failed\n",
+			 dev_name(&pdev->dev));
+		return false;
+	}
+
+	dev_dbg(nhi->dev, "created link from %s\n", dev_name(&pdev->dev));
+	return true;
+}
+
+/*
+ * During suspend the Thunderbolt controller is reset and all PCIe
+ * tunnels are lost. The NHI driver will try to reestablish all tunnels
+ * during resume. This adds device links between the tunneled PCIe
+ * downstream ports and the NHI so that the device core will make sure
+ * NHI is resumed first before the rest.
+ */
+static bool nhi_pci_add_links(struct tb_nhi *nhi)
+{
+	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
+	struct pci_dev *upstream, *pdev;
+	bool ret;
+
+	if (!x86_apple_machine)
+		return false;
+
+	switch (nhi_pdev->device) {
+	case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
+	case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
+	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
+	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
+	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
+	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
+		break;
+	default:
+		return false;
+	}
+
+	upstream = pci_upstream_bridge(nhi_pdev);
+	while (upstream) {
+		if (!pci_is_pcie(upstream))
+			return false;
+		if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
+			break;
+		upstream = pci_upstream_bridge(upstream);
+	}
+
+	if (!upstream)
+		return false;
+
+	/*
+	 * For each hotplug downstream port, create add device link back
+	 * to NHI so that PCIe tunnels can be re-established after
+	 * sleep.
+	 */
+	ret = false;
+	for_each_pci_bridge(pdev, upstream->subordinate) {
+		if (!pci_is_pcie(pdev))
+			continue;
+		if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||
+		    !pdev->is_pciehp)
+			continue;
+
+		ret |= add_link(nhi, pdev);
+	}
+
+	return ret;
+}
+
 static int nhi_pci_init_msi(struct tb_nhi *nhi)
 {
 	struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi);
@@ -251,6 +328,7 @@ static bool nhi_pci_is_present(struct tb_nhi *nhi)
 }
 
 static const struct tb_nhi_ops pci_nhi_default_ops = {
+	.add_links = nhi_pci_add_links,
 	.pre_nvm_auth = nhi_pci_start_dma_port,
 	.post_nvm_auth = nhi_pci_complete_dma_port,
 	.request_ring_irq = nhi_pci_ring_request_msix,
@@ -421,6 +499,40 @@ static int icl_nhi_resume(struct tb_nhi *nhi)
 	return 0;
 }
 
+static bool icl_nhi_add_links(struct tb_nhi *nhi)
+{
+	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
+	struct pci_bus *bus = nhi_pdev->bus;
+	struct pci_dev *pdev;
+	bool ret;
+
+	if (!x86_apple_machine)
+		return false;
+
+	/*
+	 * On integrated controllers the tunneled PCIe root ports are
+	 * directly under the host bridge.
+	 */
+	if (!pci_is_root_bus(bus))
+		return false;
+
+	ret = false;
+	for_each_pci_bridge(pdev, bus) {
+		switch (nhi_pdev->device) {
+		case PCI_DEVICE_ID_INTEL_ICL_NHI0:
+			if (pdev->device == 0x8a1d || pdev->device == 0x8a1f)
+				ret |= add_link(nhi, pdev);
+			break;
+		case PCI_DEVICE_ID_INTEL_ICL_NHI1:
+			if (pdev->device == 0x8a21 || pdev->device == 0x8a23)
+				ret |= add_link(nhi, pdev);
+			break;
+		}
+	}
+
+	return ret;
+}
+
 static void icl_nhi_shutdown(struct tb_nhi *nhi)
 {
 	nhi_pci_release_irq(nhi);
@@ -430,6 +542,7 @@ static void icl_nhi_shutdown(struct tb_nhi *nhi)
 
 static const struct tb_nhi_ops icl_nhi_ops = {
 	.init = icl_nhi_resume,
+	.add_links = icl_nhi_add_links,
 	.suspend_noirq = icl_nhi_suspend_noirq,
 	.resume_noirq = icl_nhi_resume,
 	.runtime_suspend = icl_nhi_suspend,
diff --git a/drivers/thunderbolt/quirks.c b/drivers/thunderbolt/quirks.c
index 96b7fe0386e3..2722918e0e09 100644
--- a/drivers/thunderbolt/quirks.c
+++ b/drivers/thunderbolt/quirks.c
@@ -52,6 +52,19 @@ static void quirk_block_rpm_in_redrive(struct tb_switch *sw)
 	tb_sw_dbg(sw, "preventing runtime PM in DP redrive mode\n");
 }
 
+static void quirk_stuck_pending(struct tb_switch *sw)
+{
+	struct tb_port *port;
+
+	tb_switch_for_each_port(sw, port) {
+		if (!tb_port_is_nhi(port))
+			continue;
+
+		port->pp_timeout_msec = 0;
+		tb_port_dbg(port, "pending bit does not clear, reading it once\n");
+	}
+}
+
 struct tb_quirk {
 	u16 hw_vendor_id;
 	u16 hw_device_id;
@@ -117,6 +130,11 @@ static const struct tb_quirk tb_quirks[] = {
 	{ 0x0438, 0x0209, 0x0000, 0x0000, quirk_clx_disable },
 	{ 0x0438, 0x020a, 0x0000, 0x0000, quirk_clx_disable },
 	{ 0x0438, 0x020b, 0x0000, 0x0000, quirk_clx_disable },
+	/*
+	 * ASMedia ASM4242 never clears the Pending Packets bit of its host
+	 * interface adapter.
+	 */
+	{ 0x174c, 0x2428, 0x0000, 0x0000, quirk_stuck_pending },
 };
 
 /**
diff --git a/drivers/thunderbolt/sb_regs.h b/drivers/thunderbolt/sb_regs.h
index 5391502a4b87..b801f3a694be 100644
--- a/drivers/thunderbolt/sb_regs.h
+++ b/drivers/thunderbolt/sb_regs.h
@@ -59,6 +59,7 @@ enum usb4_sb_opcode {
 #define USB4_MARGIN_CAP_0_MAX_VOLTAGE_OFFSET_MASK GENMASK(18, 13)
 #define USB4_MARGIN_CAP_0_OPT_VOLTAGE_SUPPORT	BIT(19)
 #define USB4_MARGIN_CAP_0_VOLT_STEPS_OPT_MASK	GENMASK(26, 20)
+#define USB4_MARGIN_CAP_0_EXT_ERR_COUNTER	BIT(28)
 #define USB4_MARGIN_CAP_1_MAX_VOLT_OFS_OPT_MASK GENMASK(7, 0)
 #define USB4_MARGIN_CAP_1_TIME_DESTR		BIT(8)
 #define USB4_MARGIN_CAP_1_TIME_INDP_MASK	GENMASK(10, 9)
@@ -108,5 +109,6 @@ enum usb4_sb_opcode {
 #define USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK	GENMASK(3, 0)
 #define USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK	GENMASK(7, 4)
 #define USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK	GENMASK(11, 8)
+#define USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK	GENMASK(15, 4)
 
 #endif
diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c
index 988765c98c8c..4f9a57b77bfa 100644
--- a/drivers/thunderbolt/stream.c
+++ b/drivers/thunderbolt/stream.c
@@ -131,14 +131,20 @@ struct tbstream_ring {
  * @ring_size: Size of the rings
  * @throttling: Interrupt throttling rate in ns
  * @busy_poll: Instead of interrupts, busy poll the rings
+ * @rx_pending: Receive ring has completions that need to be advanced
  * @users: Number of times @cdev has been opened
- * @closed: CLOSE packet was received
+ * @close_received: CLOSE packet appeared in the RX ring
+ * @closed: CLOSE packet was handled in the read side
  * @removed: Userspace removed the ConfigFS group underneath.
  * @wait: Waitqueue for open, read and write
  * @lock: Lock protecting this structure
  * @tx_ring: Transmit ring
  * @rx_ring: Receive ring
  * @list: Stream devices are linked through this
+ *
+ * @close_received is used on write side to notify the writer that the
+ * other side sent CLOSE. @closed on the other hand is used on the read
+ * side to make read(2) return EOF to the caller.
  */
 struct tbstream_dev {
 	struct config_group group;
@@ -151,7 +157,9 @@ struct tbstream_dev {
 	unsigned int ring_size;
 	unsigned int throttling;
 	bool busy_poll;
+	bool rx_pending;
 	int users;
+	bool close_received;
 	bool closed;
 	bool removed;
 	wait_queue_head_t wait;
@@ -278,6 +286,14 @@ static inline bool tbstream_ring_available(const struct tbstream_ring *ring)
 	return ring->prod > ring->cons;
 }
 
+static void tbstream_ring_poll(struct tbstream_ring *ring)
+{
+	struct ring_frame *frame;
+
+	while ((frame = tb_ring_poll(ring->ring)))
+		frame->callback(ring->ring, frame, false);
+}
+
 static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev)
 {
 	if (sdev->stream)
@@ -338,9 +354,19 @@ static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev)
 	return sdev->removed;
 }
 
+static inline bool tbstream_dev_close_received(const struct tbstream_dev *sdev)
+{
+	return READ_ONCE(sdev->close_received);
+}
+
 static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev)
 {
-	return sdev->closed;
+	return READ_ONCE(sdev->closed);
+}
+
+static inline bool tbstream_dev_rx_pending(const struct tbstream_dev *sdev)
+{
+	return READ_ONCE(sdev->rx_pending);
 }
 
 static void
@@ -349,6 +375,7 @@ tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
 {
 	struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);
 	struct tbstream_dev *sdev = sf->sdev;
+	__poll_t mask;
 
 	if (canceled)
 		return;
@@ -356,12 +383,17 @@ tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
 	sf->completed = true;
 	sdev->rx_ring.prod++;
 
-	if (sf->frame.flags & RING_DESC_CRC_ERROR)
-		pr_warn("RX CRC error\n");
-	else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN)
-		pr_warn("RX buffer overrun\n");
-	else
-		wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
+	mask = EPOLLIN | EPOLLRDNORM;
+	/*
+	 * The CLOSE packet does not have a payload so the flags do not
+	 * matter. The read_iter() deals with the flags.
+	 */
+	if (sf->frame.eof == TBSTREAM_CLOSE) {
+		WRITE_ONCE(sdev->close_received, true);
+		mask |= EPOLLHUP;
+	}
+
+	wake_up_interruptible_poll(&sdev->wait, mask);
 }
 
 static struct tbstream_frame *
@@ -538,38 +570,27 @@ tbstream_dev_send_data(struct tbstream_dev *sdev, struct iov_iter *from,
 	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
 }
 
-static void
-tbstream_dev_poll_ring(struct tbstream_dev *sdev, struct tbstream_ring *ring)
-{
-	struct ring_frame *frame;
-
-	if (!sdev->busy_poll)
-		return;
-
-	while ((frame = tb_ring_poll(ring->ring)))
-		frame->callback(ring->ring, frame, false);
-}
-
 static int tbstream_dev_send_close(struct tbstream_dev *sdev)
 {
 	struct tbstream_frame *sf;
+	ktime_t timeout;
 
-	if (sdev->busy_poll) {
+	/*
+	 * Wait for the ring to have available slots before we send the
+	 * CLOSE packet.
+	 */
+	timeout = ktime_add_ms(ktime_get(), 500);
+	do {
+		if (tbstream_ring_available(&sdev->tx_ring))
+			break;
 		/*
-		 * When busy polling it's the write(2) path that
-		 * advances the completions so it is possible that the
-		 * ring is full at this point. Advance the ring here so
-		 * that there is room for the CLOSE packet to be sent.
+		 * For busy polling we need to advance the ring here as
+		 * well to make the slots available.
 		 */
-		ktime_t timeout = ktime_add_ms(ktime_get(), 500);
-
-		do {
-			if (tbstream_ring_available(&sdev->tx_ring))
-				break;
-			tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
-			fsleep(15);
-		} while (ktime_before(ktime_get(), timeout));
-	}
+		if (sdev->busy_poll)
+			tbstream_ring_poll(&sdev->tx_ring);
+		fsleep(15);
+	} while (ktime_before(ktime_get(), timeout));
 
 	sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256);
 	if (IS_ERR(sf))
@@ -577,16 +598,45 @@ static int tbstream_dev_send_close(struct tbstream_dev *sdev)
 	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
 }
 
+static void tbstream_dev_start_poll(void *data)
+{
+	struct tbstream_dev *sdev = data;
+
+	WRITE_ONCE(sdev->rx_pending, true);
+	wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
+}
+
+/* sdev->lock must be held */
+static void tbstream_dev_advance_rx(struct tbstream_dev *sdev)
+{
+	/*
+	 * Clear before running the completions so that an interrupt
+	 * that arrives while we are doing that is not missed.
+	 */
+	WRITE_ONCE(sdev->rx_pending, false);
+	tbstream_ring_poll(&sdev->rx_ring);
+}
+
+/* sdev->lock must be held */
+static void tbstream_dev_complete_rx(struct tbstream_dev *sdev)
+{
+	if (!sdev->busy_poll)
+		tb_ring_poll_complete(sdev->rx_ring.ring);
+}
+
 static int tbstream_dev_start(struct tbstream_dev *sdev)
 {
 	struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
 	unsigned int flags = RING_FLAG_FRAME | RING_FLAG_E2E;
+	void (*start_poll)(void *) = NULL;
 	u16 sof_mask, eof_mask;
 	struct tb_ring *ring;
 	int ret, e2e_tx_hop;
 
 	if (sdev->busy_poll)
 		flags |= RING_FLAG_NO_INTERRUPT;
+	else
+		start_poll = tbstream_dev_start_poll;
 
 	ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size, flags);
 	if (!ring)
@@ -602,7 +652,8 @@ static int tbstream_dev_start(struct tbstream_dev *sdev)
 	eof_mask = BIT(TBSTREAM_DATA) | BIT(TBSTREAM_CLOSE);
 
 	ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size, flags,
-				e2e_tx_hop, sof_mask, eof_mask, NULL, NULL);
+				e2e_tx_hop, sof_mask, eof_mask, start_poll,
+				sdev);
 	if (!ring) {
 		ret = -ENOMEM;
 		goto err_free_tx_buffers;
@@ -619,6 +670,8 @@ static int tbstream_dev_start(struct tbstream_dev *sdev)
 	tb_ring_throttling(sdev->tx_ring.ring, sdev->throttling);
 	tb_ring_throttling(sdev->rx_ring.ring, sdev->throttling);
 
+	sdev->rx_pending = false;
+
 	tb_ring_start(sdev->tx_ring.ring);
 	tb_ring_start(sdev->rx_ring.ring);
 
@@ -665,19 +718,16 @@ static void tbstream_dev_stop(struct tbstream_dev *sdev)
 		do {
 			if (tbstream_dev_tx_drained(sdev))
 				break;
-			tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
+			tbstream_ring_poll(&sdev->tx_ring);
 			fsleep(15);
 		} while (ktime_before(ktime_get(), timeout));
-
-		tb_ring_stop(sdev->tx_ring.ring);
-		tb_ring_stop(sdev->rx_ring.ring);
 	} else {
 		tb_ring_flush(sdev->tx_ring.ring, 500);
-		tb_ring_stop(sdev->tx_ring.ring);
-		tb_ring_flush(sdev->rx_ring.ring, 500);
-		tb_ring_stop(sdev->rx_ring.ring);
 	}
 
+	tb_ring_stop(sdev->tx_ring.ring);
+	tb_ring_stop(sdev->rx_ring.ring);
+
 	xd = tbstream_dev_xdomain(sdev);
 	if (xd) {
 		tb_xdomain_disable_paths(xd, sdev->out_hopid,
@@ -707,6 +757,45 @@ static int tbstream_dev_lock(struct tbstream_dev *sdev, bool nowait)
 	return 0;
 }
 
+/* Must not be called with @sdev->lock held */
+static int tbstream_dev_busy_poll_wait(struct tbstream_dev *sdev,
+				       struct tbstream_ring *ring)
+{
+	for (;;) {
+		if (signal_pending(current))
+			return -ERESTARTSYS;
+		if (tb_ring_poll_pending(ring->ring))
+			return 0;
+		/*
+		 * For TX ring we need to check the RX side too because
+		 * it might have received CLOSE packet.
+		 */
+		if (ring == &sdev->tx_ring &&
+		    tb_ring_poll_pending(sdev->rx_ring.ring))
+			return 0;
+		if (tbstream_dev_valid(sdev) != 0 ||
+		    tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev))
+			return 0;
+		cond_resched();
+	}
+}
+
+static bool
+tbstream_dev_has_event(struct tbstream_dev *sdev, struct tbstream_ring *ring)
+{
+	if (tbstream_dev_valid(sdev) != 0)
+		return true;
+	if (tbstream_dev_closed(sdev))
+		return true;
+	if (tbstream_dev_removed(sdev))
+		return true;
+	if (tbstream_dev_rx_pending(sdev))
+		return true;
+	if (tbstream_ring_available(ring))
+		return true;
+	return tb_ring_poll_pending(sdev->rx_ring.ring);
+}
+
 static ssize_t
 tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 {
@@ -725,8 +814,8 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 		return ret;
 
 	for (;;) {
-		/* When busy polling, advance any completions manually */
-		tbstream_dev_poll_ring(sdev, &sdev->rx_ring);
+		/* Advance RX completions */
+		tbstream_dev_advance_rx(sdev);
 
 		ret = tbstream_dev_valid(sdev);
 		if (ret) {
@@ -742,21 +831,20 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 		if (tbstream_ring_available(&sdev->rx_ring))
 			break;
 
+		/* Polled all we could. Re-enable the interrupt now. */
+		tbstream_dev_complete_rx(sdev);
 		mutex_unlock(&sdev->lock);
 
 		if (nowait)
 			return -EAGAIN;
 
 		if (sdev->busy_poll) {
-			if (signal_pending(current))
-				return -ERESTARTSYS;
-			cond_resched();
+			ret = tbstream_dev_busy_poll_wait(sdev, &sdev->rx_ring);
+			if (ret)
+				return ret;
 		} else {
 			ret = wait_event_interruptible(sdev->wait,
-					tbstream_ring_available(&sdev->rx_ring) ||
-					tbstream_dev_valid(sdev) != 0 ||
-					tbstream_dev_closed(sdev) ||
-					tbstream_dev_removed(sdev));
+				tbstream_dev_has_event(sdev, &sdev->rx_ring));
 			if (ret)
 				return ret;
 		}
@@ -783,7 +871,20 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 		if (sf->frame.eof == TBSTREAM_CLOSE) {
 			if (!nbytes) {
 				tbstream_dev_consume_rx(sdev);
-				sdev->closed = true;
+				WRITE_ONCE(sdev->closed, true);
+			}
+			break;
+		} else if (sf->frame.flags &
+			   (RING_DESC_CRC_ERROR | RING_DESC_BUFFER_OVERRUN)) {
+			/*
+			 * If something was already read return that now
+			 * and next read will report the error.
+			 */
+			if (!nbytes) {
+				pr_warn("corrupted frame received, flags %#x\n",
+					sf->frame.flags);
+				tbstream_dev_consume_rx(sdev);
+				ret = -EIO;
 			}
 			break;
 		}
@@ -819,6 +920,25 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 	return nbytes;
 }
 
+static void tbstream_dev_advance_both(struct tbstream_dev *sdev)
+{
+	/*
+	 * When busy polling, advance TX completions manually.
+	 *
+	 * We also need to advance the RX side in both modes to be able
+	 * to receive CLOSE packet from the other peer if there is no
+	 * reader.
+	 */
+	if (sdev->busy_poll) {
+		tbstream_ring_poll(&sdev->tx_ring);
+		tbstream_dev_advance_rx(sdev);
+	} else if (tbstream_dev_rx_pending(sdev) ||
+		   tb_ring_poll_pending(sdev->rx_ring.ring)) {
+		tbstream_dev_advance_rx(sdev);
+		tbstream_dev_complete_rx(sdev);
+	}
+}
+
 static ssize_t
 tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 {
@@ -837,7 +957,8 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 		return ret;
 
 	for (;;) {
-		tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
+		/* Advance TX (and RX) completions */
+		tbstream_dev_advance_both(sdev);
 
 		ret = tbstream_dev_valid(sdev);
 		if (ret) {
@@ -845,7 +966,12 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 			return ret;
 		}
 
-		if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) {
+		if (tbstream_dev_close_received(sdev)) {
+			mutex_unlock(&sdev->lock);
+			return -EPIPE;
+		}
+
+		if (tbstream_dev_removed(sdev)) {
 			mutex_unlock(&sdev->lock);
 			return -ENXIO;
 		}
@@ -859,15 +985,13 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 			return -EAGAIN;
 
 		if (sdev->busy_poll) {
-			if (signal_pending(current))
-				return -ERESTARTSYS;
-			cond_resched();
+			ret = tbstream_dev_busy_poll_wait(sdev, &sdev->tx_ring);
+			if (ret)
+				return ret;
 		} else {
 			ret = wait_event_interruptible(sdev->wait,
-					tbstream_ring_available(&sdev->tx_ring) ||
-					tbstream_dev_valid(sdev) != 0 ||
-					tbstream_dev_closed(sdev) ||
-					tbstream_dev_removed(sdev));
+				tbstream_dev_has_event(sdev, &sdev->tx_ring) ||
+				tbstream_dev_close_received(sdev));
 			if (ret)
 				return ret;
 		}
@@ -917,14 +1041,28 @@ tbstream_dev_fops_poll(struct file *file, struct poll_table_struct *wait)
 
 	poll_wait(file, &sdev->wait, wait);
 	guard(mutex)(&sdev->lock);
-	if (tbstream_dev_valid(sdev) != 0) {
-		mask |= EPOLLHUP | EPOLLERR;
+	if (tbstream_dev_valid(sdev) != 0)
+		return EPOLLHUP | EPOLLERR;
+
+	/*
+	 * The RX completions are only advanced from here and from
+	 * read(2) so do that now, otherwise we would never report
+	 * anything to be available.
+	 */
+	tbstream_dev_advance_rx(sdev);
+
+	if (tbstream_dev_close_received(sdev))
+		mask |= EPOLLHUP;
+	if (tbstream_ring_available(&sdev->tx_ring))
+		mask |= EPOLLOUT | EPOLLWRNORM;
+	if (tbstream_ring_available(&sdev->rx_ring)) {
+		mask |= EPOLLIN | EPOLLRDNORM;
 	} else {
-		if (tbstream_ring_available(&sdev->tx_ring))
-			mask |= EPOLLOUT | EPOLLWRNORM;
-		if (tbstream_ring_available(&sdev->rx_ring))
+		tbstream_dev_complete_rx(sdev);
+		if (tb_ring_poll_pending(sdev->rx_ring.ring))
 			mask |= EPOLLIN | EPOLLRDNORM;
 	}
+
 	return mask;
 }
 
@@ -974,6 +1112,8 @@ static int tbstream_dev_fops_open(struct inode *inode, struct file *file)
 			sdev->users--;
 			goto err_unlock;
 		}
+
+		sdev->close_received = false;
 		sdev->closed = false;
 	}
 
@@ -998,11 +1138,18 @@ static int tbstream_dev_fops_release(struct inode *inode, struct file *file)
 	mutex_lock(&sdev->lock);
 	if (--sdev->users == 0) {
 		/*
+		 * Advance now in case there is CLOSE waiting in the RX
+		 * ring.
+		 */
+		tbstream_dev_advance_both(sdev);
+		/*
 		 * Send CLOSE tunneled packet to notify the other end
-		 * that we are closing the file. We do this twice if the
-		 * first one fails.
+		 * that we are closing the file, if it is not closed
+		 * already.
 		 */
-		tbstream_dev_send_close(sdev);
+		if (!tbstream_dev_close_received(sdev) &&
+		    tbstream_dev_valid(sdev) == 0)
+			tbstream_dev_send_close(sdev);
 		tbstream_dev_stop(sdev);
 	}
 	mutex_unlock(&sdev->lock);
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index cf571a7c98d0..d22d8db8f890 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -19,6 +19,9 @@
 
 #include "tb.h"
 
+/* How long a hop is given to drain when a path is deactivated */
+#define TB_PORT_PENDING_TIMEOUT		500 /* ms */
+
 /* Switch NVM support */
 
 struct nvm_auth_status {
@@ -712,6 +715,7 @@ static int tb_init_port(struct tb_port *port)
 	int cap;
 
 	INIT_LIST_HEAD(&port->list);
+	port->pp_timeout_msec = TB_PORT_PENDING_TIMEOUT;
 
 	/* Control adapter does not have configuration space */
 	if (!port->port)
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index 4e5d0578fd4b..4da608ccbccb 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -10,7 +10,6 @@
 #include <linux/errno.h>
 #include <linux/delay.h>
 #include <linux/pm_runtime.h>
-#include <linux/platform_data/x86/apple.h>
 
 #include "tb.h"
 #include "tb_regs.h"
@@ -3334,75 +3333,6 @@ static const struct tb_cm_ops tb_cm_ops = {
 	.disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
 };
 
-/*
- * During suspend the Thunderbolt controller is reset and all PCIe
- * tunnels are lost. The NHI driver will try to reestablish all tunnels
- * during resume. This adds device links between the tunneled PCIe
- * downstream ports and the NHI so that the device core will make sure
- * NHI is resumed first before the rest.
- */
-static bool tb_apple_add_links(struct tb_nhi *nhi)
-{
-	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
-	struct pci_dev *upstream, *pdev;
-	bool ret;
-
-	if (!x86_apple_machine)
-		return false;
-
-	switch (nhi_pdev->device) {
-	case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
-	case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
-	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
-	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
-		break;
-	default:
-		return false;
-	}
-
-	upstream = pci_upstream_bridge(nhi_pdev);
-	while (upstream) {
-		if (!pci_is_pcie(upstream))
-			return false;
-		if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
-			break;
-		upstream = pci_upstream_bridge(upstream);
-	}
-
-	if (!upstream)
-		return false;
-
-	/*
-	 * For each hotplug downstream port, create add device link
-	 * back to NHI so that PCIe tunnels can be re-established after
-	 * sleep.
-	 */
-	ret = false;
-	for_each_pci_bridge(pdev, upstream->subordinate) {
-		const struct device_link *link;
-
-		if (!pci_is_pcie(pdev))
-			continue;
-		if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||
-		    !pdev->is_pciehp)
-			continue;
-
-		link = device_link_add(&pdev->dev, nhi->dev,
-				       DL_FLAG_AUTOREMOVE_SUPPLIER |
-				       DL_FLAG_PM_RUNTIME);
-		if (link) {
-			dev_dbg(nhi->dev, "created link from %s\n",
-				dev_name(&pdev->dev));
-			ret = true;
-		} else {
-			dev_warn(nhi->dev, "device link creation from %s failed\n",
-				 dev_name(&pdev->dev));
-		}
-	}
-
-	return ret;
-}
-
 struct tb *tb_probe(struct tb_nhi *nhi)
 {
 	struct tb_cm *tcm;
@@ -3427,13 +3357,5 @@ struct tb *tb_probe(struct tb_nhi *nhi)
 
 	tb_dbg(tb, "using software connection manager\n");
 
-	/*
-	 * Device links are needed to make sure we establish tunnels
-	 * before the PCIe/USB stack is resumed so complain here if we
-	 * found them missing.
-	 */
-	if (!tb_apple_add_links(nhi) && !tb_acpi_add_links(nhi))
-		tb_warn(tb, "device links to tunneled native ports are missing!\n");
-
 	return tb;
 }
diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h
index 4373336d9425..e86845762e22 100644
--- a/drivers/thunderbolt/tb.h
+++ b/drivers/thunderbolt/tb.h
@@ -273,6 +273,8 @@ struct tb_bandwidth_group {
  * @max_bw: Maximum possible bandwidth through this adapter if set to
  *	    non-zero.
  * @redrive: For DP IN, if true the adapter is in redrive mode.
+ * @pp_timeout_msec: How long a hop of this adapter is given to drain when a
+ *		     path is deactivated. %0 means a single read.
  *
  * In USB4 terminology this structure represents an adapter (protocol or
  * lane adapter).
@@ -302,6 +304,7 @@ struct tb_port {
 	struct list_head group_list;
 	unsigned int max_bw;
 	bool redrive;
+	unsigned int pp_timeout_msec;
 };
 
 /**
@@ -1439,7 +1442,7 @@ int usb4_port_sw_margin(struct tb_port *port, enum usb4_sb_target target,
 			u8 index, const struct usb4_port_margining_params *params,
 			u32 *results);
 int usb4_port_sw_margin_errors(struct tb_port *port, enum usb4_sb_target target,
-			       u8 index, u32 *errors);
+			       u8 index, u32 *errors, size_t dwords);
 
 int usb4_port_retimer_set_inbound_sbtx(struct tb_port *port, u8 index);
 int usb4_port_retimer_unset_inbound_sbtx(struct tb_port *port, u8 index);
diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c
index 5fd4fe070f25..c9b3931c45ed 100644
--- a/drivers/thunderbolt/usb4.c
+++ b/drivers/thunderbolt/usb4.c
@@ -1826,23 +1826,27 @@ int usb4_port_sw_margin(struct tb_port *port, enum usb4_sb_target target,
  * @target: Sideband target
  * @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
  * @errors: Error metadata is copied here.
+ * @dwords: Number of dwords to read error counter values
  *
  * This reads back the software margining error counters from the port.
  *
  * Return: %0 on success, negative errno otherwise.
  */
 int usb4_port_sw_margin_errors(struct tb_port *port, enum usb4_sb_target target,
-			       u8 index, u32 *errors)
+			       u8 index, u32 *errors, size_t dwords)
 {
 	int ret;
+	u8 reg;
 
 	ret = usb4_port_sb_op(port, target, index,
 			      USB4_SB_OPCODE_READ_SW_MARGIN_ERR, 150);
 	if (ret)
 		return ret;
 
-	return usb4_port_sb_read(port, target, index, USB4_SB_METADATA, errors,
-				 sizeof(*errors));
+	reg = (dwords > 1) ? USB4_SB_DATA : USB4_SB_METADATA;
+
+	return usb4_port_sb_read(port, target, index, reg, errors,
+				 sizeof(*errors) * dwords);
 }
 
 static inline int usb4_port_retimer_op(struct tb_port *port, u8 index,