diff options
Diffstat (limited to 'drivers/irqchip/irq-riscv-imsic-state.c')
| -rw-r--r-- | drivers/irqchip/irq-riscv-imsic-state.c | 151 |
1 files changed, 114 insertions, 37 deletions
diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c index b97e6cd89ed7..bdf5cd2037f2 100644 --- a/drivers/irqchip/irq-riscv-imsic-state.c +++ b/drivers/irqchip/irq-riscv-imsic-state.c @@ -124,10 +124,11 @@ void __imsic_eix_update(unsigned long base_id, unsigned long num_id, bool pend, } } -static void __imsic_local_sync(struct imsic_local_priv *lpriv) +static bool __imsic_local_sync(struct imsic_local_priv *lpriv) { - struct imsic_local_config *mlocal; - struct imsic_vector *vec, *mvec; + struct imsic_local_config *tlocal, *mlocal; + struct imsic_vector *vec, *tvec, *mvec; + bool ret = true; int i; lockdep_assert_held(&lpriv->lock); @@ -143,35 +144,97 @@ static void __imsic_local_sync(struct imsic_local_priv *lpriv) __imsic_id_clear_enable(i); /* - * If the ID was being moved to a new ID on some other CPU - * then we can get a MSI during the movement so check the - * ID pending bit and re-trigger the new ID on other CPU - * using MMIO write. + * Clear the previous vector pointer of the new vector only + * after the movement is complete on the old CPU. */ - mvec = READ_ONCE(vec->move); - WRITE_ONCE(vec->move, NULL); - if (mvec && mvec != vec) { - if (__imsic_id_read_clear_pending(i)) { + mvec = READ_ONCE(vec->move_prev); + if (mvec) { + /* + * If the old vector has not been updated then + * try again in the next sync-up call. + */ + if (READ_ONCE(mvec->move_next)) { + ret = false; + continue; + } + + WRITE_ONCE(vec->move_prev, NULL); + } + + /* + * If a vector was being moved to a new vector on some other + * CPU then we can get a MSI during the movement so check the + * ID pending bit and re-trigger the new ID on other CPU using + * MMIO write. + */ + mvec = READ_ONCE(vec->move_next); + if (mvec) { + /* + * Devices having non-atomic MSI update might see + * an intermediate state so check both old ID and + * new ID for pending interrupts. + * + * For details, see imsic_irq_set_affinity(). + */ + tvec = vec->local_id == mvec->local_id ? + NULL : &lpriv->vectors[mvec->local_id]; + + if (tvec && !irq_can_move_in_process_context(irq_get_irq_data(vec->irq)) && + __imsic_id_read_clear_pending(tvec->local_id)) { + /* Retrigger temporary vector if it was already in-use */ + if (READ_ONCE(tvec->enable)) { + tlocal = per_cpu_ptr(imsic->global.local, tvec->cpu); + writel_relaxed(tvec->local_id, tlocal->msi_va); + } + + mlocal = per_cpu_ptr(imsic->global.local, mvec->cpu); + writel_relaxed(mvec->local_id, mlocal->msi_va); + } + + if (__imsic_id_read_clear_pending(vec->local_id)) { mlocal = per_cpu_ptr(imsic->global.local, mvec->cpu); writel_relaxed(mvec->local_id, mlocal->msi_va); } - imsic_vector_free(&lpriv->vectors[i]); + WRITE_ONCE(vec->move_next, NULL); + imsic_vector_free(vec); } skip: bitmap_clear(lpriv->dirty_bitmap, i, 1); } + + return ret; } -void imsic_local_sync_all(void) +#ifdef CONFIG_SMP +static void __imsic_local_timer_start(struct imsic_local_priv *lpriv) +{ + lockdep_assert_held(&lpriv->lock); + + if (!timer_pending(&lpriv->timer)) { + lpriv->timer.expires = jiffies + 1; + add_timer_on(&lpriv->timer, smp_processor_id()); + } +} +#else +static inline void __imsic_local_timer_start(struct imsic_local_priv *lpriv) +{ +} +#endif + +void imsic_local_sync_all(bool force_all) { struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); unsigned long flags; raw_spin_lock_irqsave(&lpriv->lock, flags); - bitmap_fill(lpriv->dirty_bitmap, imsic->global.nr_ids + 1); - __imsic_local_sync(lpriv); + + if (force_all) + bitmap_fill(lpriv->dirty_bitmap, imsic->global.nr_ids + 1); + if (!__imsic_local_sync(lpriv)) + __imsic_local_timer_start(lpriv); + raw_spin_unlock_irqrestore(&lpriv->lock, flags); } @@ -190,12 +253,7 @@ void imsic_local_delivery(bool enable) #ifdef CONFIG_SMP static void imsic_local_timer_callback(struct timer_list *timer) { - struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); - unsigned long flags; - - raw_spin_lock_irqsave(&lpriv->lock, flags); - __imsic_local_sync(lpriv); - raw_spin_unlock_irqrestore(&lpriv->lock, flags); + imsic_local_sync_all(false); } static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu) @@ -216,14 +274,11 @@ static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu */ if (cpu_online(cpu)) { if (cpu == smp_processor_id()) { - __imsic_local_sync(lpriv); - return; + if (__imsic_local_sync(lpriv)) + return; } - if (!timer_pending(&lpriv->timer)) { - lpriv->timer.expires = jiffies + 1; - add_timer_on(&lpriv->timer, cpu); - } + __imsic_local_timer_start(lpriv); } } #else @@ -278,8 +333,26 @@ void imsic_vector_unmask(struct imsic_vector *vec) raw_spin_unlock(&lpriv->lock); } -static bool imsic_vector_move_update(struct imsic_local_priv *lpriv, struct imsic_vector *vec, - bool new_enable, struct imsic_vector *new_move) +void imsic_vector_force_move_cleanup(struct imsic_vector *vec) +{ + struct imsic_local_priv *lpriv; + struct imsic_vector *mvec; + unsigned long flags; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + raw_spin_lock_irqsave(&lpriv->lock, flags); + + mvec = READ_ONCE(vec->move_prev); + WRITE_ONCE(vec->move_prev, NULL); + if (mvec) + imsic_vector_free(mvec); + + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +} + +static bool imsic_vector_move_update(struct imsic_local_priv *lpriv, + struct imsic_vector *vec, bool is_old_vec, + bool new_enable, struct imsic_vector *move_vec) { unsigned long flags; bool enabled; @@ -289,7 +362,10 @@ static bool imsic_vector_move_update(struct imsic_local_priv *lpriv, struct imsi /* Update enable and move details */ enabled = READ_ONCE(vec->enable); WRITE_ONCE(vec->enable, new_enable); - WRITE_ONCE(vec->move, new_move); + if (is_old_vec) + WRITE_ONCE(vec->move_next, move_vec); + else + WRITE_ONCE(vec->move_prev, move_vec); /* Mark the vector as dirty and synchronize */ bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); @@ -322,8 +398,8 @@ void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_ve * interrupt on the old vector while device was being moved * to the new vector. */ - enabled = imsic_vector_move_update(old_lpriv, old_vec, false, new_vec); - imsic_vector_move_update(new_lpriv, new_vec, enabled, new_vec); + enabled = imsic_vector_move_update(old_lpriv, old_vec, true, false, new_vec); + imsic_vector_move_update(new_lpriv, new_vec, false, enabled, old_vec); } #ifdef CONFIG_GENERIC_IRQ_DEBUGFS @@ -368,7 +444,7 @@ struct imsic_vector *imsic_vector_from_local_id(unsigned int cpu, unsigned int l return &lpriv->vectors[local_id]; } -struct imsic_vector *imsic_vector_alloc(unsigned int hwirq, const struct cpumask *mask) +struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *mask) { struct imsic_vector *vec = NULL; struct imsic_local_priv *lpriv; @@ -384,9 +460,10 @@ struct imsic_vector *imsic_vector_alloc(unsigned int hwirq, const struct cpumask lpriv = per_cpu_ptr(imsic->lpriv, cpu); vec = &lpriv->vectors[local_id]; - vec->hwirq = hwirq; + vec->irq = irq; vec->enable = false; - vec->move = NULL; + vec->move_next = NULL; + vec->move_prev = NULL; return vec; } @@ -396,7 +473,7 @@ void imsic_vector_free(struct imsic_vector *vec) unsigned long flags; raw_spin_lock_irqsave(&imsic->matrix_lock, flags); - vec->hwirq = UINT_MAX; + vec->irq = 0; irq_matrix_free(imsic->matrix, vec->cpu, vec->local_id, false); raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); } @@ -455,7 +532,7 @@ static int __init imsic_local_init(void) vec = &lpriv->vectors[i]; vec->cpu = cpu; vec->local_id = i; - vec->hwirq = UINT_MAX; + vec->irq = 0; } } |