diff options
Diffstat (limited to 'kernel/trace/ring_buffer.c')
| -rw-r--r-- | kernel/trace/ring_buffer.c | 551 |
1 files changed, 342 insertions, 209 deletions
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 56a328e94395..04bb94c29f58 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -270,7 +270,8 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event) if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX) return length; length -= RB_EVNT_HDR_SIZE; - if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) + if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]) || + RB_FORCE_8BYTE_ALIGNMENT) length -= sizeof(event->array[0]); return length; } @@ -329,6 +330,14 @@ struct buffer_data_read_page { struct buffer_data_page *data; /* actual data, stored in this page */ }; +static __always_inline unsigned int rb_read_page_capacity(struct buffer_data_read_page *rpage) +{ + return (PAGE_SIZE << rpage->order) - BUF_PAGE_HDR_SIZE; +} + +/* The number of bits for static buffer ids */ +#define RB_STATIC_BITS 30 + /* * Note, the buffer_page list must be first. The buffer pages * are allocated in cache lines, which means that each buffer @@ -344,7 +353,7 @@ struct buffer_page { local_t entries; /* entries on this page */ unsigned long real_end; /* real end of data */ unsigned order; /* order of the page */ - u32 id:30; /* ID for external mapping */ + u32 id:RB_STATIC_BITS; /* ID for external mapping */ u32 range:1; /* Mapped via a range */ struct buffer_data_page *page; /* Actual data page */ }; @@ -390,6 +399,17 @@ static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) return rb_data_page_size(bpage->page); } +/** + * rb_page_capacity - Get the capacity of a buffer page + * @bpage: The buffer page + * + * Return: The maximum size available for events in the given buffer page. + */ +static __always_inline unsigned int rb_page_capacity(struct buffer_page *bpage) +{ + return (PAGE_SIZE << bpage->order) - BUF_PAGE_HDR_SIZE; +} + static void free_buffer_page(struct buffer_page *bpage) { /* Range pages are not to be freed */ @@ -404,11 +424,11 @@ static void free_buffer_page(struct buffer_page *bpage) */ #define alloc_cpu_buffer(cpu) (struct ring_buffer_per_cpu *) \ kzalloc_node(ALIGN(sizeof(struct ring_buffer_per_cpu), \ - cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); + cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)) #define alloc_cpu_page(cpu) (struct buffer_page *) \ kzalloc_node(ALIGN(sizeof(struct buffer_page), \ - cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); + cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)) static struct buffer_data_page *alloc_cpu_data(int cpu, int order) { @@ -502,11 +522,11 @@ struct ring_buffer_per_cpu { int cpu; atomic_t record_disabled; atomic_t resize_disabled; - struct trace_buffer *buffer; + struct trace_buffer *buffer; raw_spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; - struct buffer_data_page *free_page; + struct buffer_data_read_page free_page; unsigned long nr_pages; unsigned int current_context; struct list_head *pages; @@ -540,7 +560,6 @@ struct ring_buffer_per_cpu { /* pages removed since last reset */ unsigned long pages_removed; - unsigned int mapped; unsigned int user_mapped; /* user space mapping */ struct mutex mapping_lock; struct buffer_page **subbuf_ids; /* ID to subbuf VA */ @@ -560,7 +579,6 @@ struct ring_buffer_per_cpu { struct trace_buffer { unsigned flags; - int cpus; atomic_t record_disabled; atomic_t resizing; cpumask_var_t cpumask; @@ -585,11 +603,72 @@ struct trace_buffer { struct ring_buffer_meta *meta; - unsigned int subbuf_size; unsigned int subbuf_order; - unsigned int max_data_size; }; +static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer) +{ + return PAGE_SIZE << buffer->subbuf_order; +} + +/** + * rb_subbuf_capacity - Get the capacity of a subbuffer + * @buffer: A trace buffer + * + * Unsafe to use without holding trace_buffer::mutex or with resizing enabled. + * Consider rb_page_capacity() instead. + * + * Return: The maximum size available for events in a trace buffer subbuffer. + */ +static __always_inline unsigned int rb_subbuf_capacity(struct trace_buffer *buffer) +{ + return rb_subbuf_size(buffer) - BUF_PAGE_HDR_SIZE; +} + +/** + * rb_subbuf_max_data_size - Get the maximum payload size of a single event + * @buffer: A trace buffer + * + * Return: The maximum data payload size that can be stored in a single event. + */ +static __always_inline unsigned int rb_subbuf_max_data_size(struct trace_buffer *buffer) +{ + struct ring_buffer_event *event; + + /* + * surely rb_subbuf_capacity() is bigger than + * RINGBUF_TYPE_DATA_TYPE_LEN_MAX (see ring_buffer_event_length). + */ + return rb_subbuf_capacity(buffer) - RB_EVNT_HDR_SIZE - sizeof(event->array[0]); +} + +/** + * rb_subbuf_start - Get the start address of a subbuffer + * @buffer: A trace buffer + * @addr: An address of an event on a subbuffer + * + * Return: The start of the subbuffer for where @addr sits + */ +static __always_inline +unsigned long rb_subbuf_start(struct trace_buffer *buffer, unsigned long addr) +{ + return addr & ~((unsigned long)(rb_subbuf_size(buffer) - 1)); +} + +static bool rb_is_static(struct ring_buffer_per_cpu *cpu_buffer) +{ + return cpu_buffer->user_mapped || cpu_buffer->remote || cpu_buffer->ring_meta; +} + +static unsigned long rb_static_max_pages(void) +{ + /* + * Static ring buffers are using bpage::id and must account for the + * reader page. + */ + return (1UL << RB_STATIC_BITS) - 1; +} + struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; @@ -629,7 +708,7 @@ int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq trace_seq_printf(s, "\tfield: char data;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), data), - (unsigned int)(buffer ? buffer->subbuf_size : + (unsigned int)(buffer ? rb_subbuf_capacity(buffer) : PAGE_SIZE - BUF_PAGE_HDR_SIZE), (unsigned int)is_signed_type(char)); @@ -825,14 +904,13 @@ static void rb_wake_up_waiters(struct irq_work *work) struct ring_buffer_per_cpu *cpu_buffer = container_of(rbwork, struct ring_buffer_per_cpu, irq_work); - /* Called from interrupt context */ - raw_spin_lock(&cpu_buffer->reader_lock); - rbwork->wakeup_full = false; - rbwork->full_waiters_pending = false; + scoped_guard(raw_spinlock_irqsave, &cpu_buffer->reader_lock) { + rbwork->wakeup_full = false; + rbwork->full_waiters_pending = false; - /* Waking up all waiters, they will reset the shortest full */ - cpu_buffer->shortest_full = 0; - raw_spin_unlock(&cpu_buffer->reader_lock); + /* Waking up all waiters, they will reset the shortest full */ + cpu_buffer->shortest_full = 0; + } wake_up_all(&rbwork->full_waiters); } @@ -1607,7 +1685,7 @@ out_locked: * This is used to help find the next per cpu subbuffer within a mapped range. */ static unsigned long -rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) +rb_range_align_subbuf(unsigned long addr, unsigned int subbuf_size, unsigned long nr_subbufs) { addr += sizeof(struct ring_buffer_cpu_meta) + sizeof(int) * nr_subbufs; @@ -1617,13 +1695,12 @@ rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) /* * Return the ring_buffer_meta for a given @cpu. */ -static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) +static void *rb_range_meta(struct trace_buffer *buffer, unsigned long nr_pages, int cpu) { - int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + unsigned int subbuf_size = rb_subbuf_size(buffer); struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; - unsigned long ptr; - int nr_subbufs; + unsigned long ptr, nr_subbufs; bmeta = buffer->meta; if (!bmeta) @@ -1669,7 +1746,7 @@ static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) /* Return the start of subbufs given the meta pointer */ static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta) { - int subbuf_size = meta->subbuf_size; + unsigned int subbuf_size = meta->subbuf_size; unsigned long ptr; ptr = (unsigned long)meta; @@ -1681,11 +1758,11 @@ static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta) /* * Return a specific sub-buffer for a given @cpu defined by @idx. */ -static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx) +static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, unsigned long idx) { struct ring_buffer_cpu_meta *meta; + unsigned int subbuf_size; unsigned long ptr; - int subbuf_size; meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu); if (!meta) @@ -1701,7 +1778,7 @@ static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx) ptr = (unsigned long)rb_subbufs_from_meta(meta); - ptr += subbuf_size * idx; + ptr += (unsigned long)subbuf_size * idx; if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end) return NULL; @@ -1778,13 +1855,12 @@ static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size) * must be the same. */ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, - struct trace_buffer *buffer, int nr_pages, + struct trace_buffer *buffer, unsigned long nr_pages, unsigned long *subbuf_mask) { - int subbuf_size = PAGE_SIZE; unsigned long buffers_start; unsigned long buffers_end; - int i; + unsigned long i; if (!subbuf_mask) return false; @@ -1794,8 +1870,13 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } + if (meta->nr_subbufs != nr_pages + 1) { + pr_info("Ring buffer boot meta [%d] invalid nr_subbufs\n", cpu); + return false; + } + buffers_start = meta->first_buffer; - buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); + buffers_end = meta->first_buffer + (meta->nr_subbufs * PAGE_SIZE); /* Is the head and commit buffers within the range of buffers? */ if (meta->head_buffer < buffers_start || @@ -2033,8 +2114,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) struct buffer_page *head_page, *orig_head, *orig_reader; struct rb_validation_state state = { 0 }; bool skip = false; + unsigned long i; int ret; - int i; if (!meta || !meta->head_buffer) return; @@ -2085,7 +2166,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) rb_validate_buffer(head_page, cpu_buffer, meta, &state, 0, state.ts); } if (i) - pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); + pr_info("Ring buffer [%d] rewound %lu pages\n", cpu_buffer->cpu, i); /* The last rewound page must be skipped. */ if (head_page != orig_head) @@ -2169,7 +2250,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) } } -static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size) +static void rb_range_meta_init(struct trace_buffer *buffer, unsigned long nr_pages, + int scratch_size) { struct ring_buffer_cpu_meta *meta; unsigned long *subbuf_mask; @@ -2269,8 +2351,8 @@ static int rbm_show(struct seq_file *m, void *v) rb_meta_subbuf_idx(meta, (void *)meta->head_buffer)); seq_printf(m, "commit_buffer: %d\n", rb_meta_subbuf_idx(meta, (void *)meta->commit_buffer)); - seq_printf(m, "subbuf_size: %d\n", meta->subbuf_size); - seq_printf(m, "nr_subbufs: %d\n", meta->nr_subbufs); + seq_printf(m, "subbuf_size: %u\n", meta->subbuf_size); + seq_printf(m, "nr_subbufs: %u\n", meta->nr_subbufs); return 0; } @@ -2329,10 +2411,7 @@ static struct ring_buffer_desc *ring_buffer_desc(struct trace_buffer_desc *trace size_t len; int i; - if (!trace_desc) - return NULL; - - if (cpu >= trace_desc->nr_cpus) + if (!trace_desc || !trace_desc->nr_cpus) return NULL; end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len); @@ -2358,7 +2437,7 @@ static void *ring_buffer_desc_page(struct ring_buffer_desc *desc, unsigned int p } static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, - long nr_pages, struct list_head *pages) + unsigned long nr_pages, struct list_head *pages) { struct trace_buffer *buffer = cpu_buffer->buffer; struct ring_buffer_cpu_meta *meta = NULL; @@ -2434,8 +2513,8 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, bpage->id = i + 1; cpu_buffer->subbuf_ids[i + 1] = bpage; } else { - int order = cpu_buffer->buffer->subbuf_order; - bpage->page = alloc_cpu_data(cpu_buffer->cpu, order); + bpage->page = alloc_cpu_data(cpu_buffer->cpu, + cpu_buffer->buffer->subbuf_order); if (!bpage->page) goto free_pages; } @@ -2486,7 +2565,7 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, } static struct ring_buffer_per_cpu * -rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) +rb_allocate_cpu_buffer(struct trace_buffer *buffer, unsigned long nr_pages, int cpu) { struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = alloc_cpu_buffer(cpu); @@ -2512,6 +2591,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) bpage = alloc_cpu_page(cpu); if (!bpage) return NULL; + bpage->order = cpu_buffer->buffer->subbuf_order; rb_check_bpage(cpu_buffer, bpage); @@ -2522,7 +2602,6 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) * Range mapped buffers have the same restrictions as memory * mapped ones do. */ - cpu_buffer->mapped = 1; cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu); bpage->page = rb_range_buffer(cpu_buffer, 0); if (!bpage->page) @@ -2530,6 +2609,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) if (cpu_buffer->ring_meta->head_buffer) rb_meta_buffer_update(cpu_buffer, bpage); bpage->range = 1; + + atomic_inc(&cpu_buffer->resize_disabled); } else if (buffer->remote) { struct ring_buffer_desc *desc = ring_buffer_desc(buffer->remote->desc, cpu); @@ -2539,8 +2620,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) cpu_buffer->remote = buffer->remote; cpu_buffer->meta_page = (struct trace_buffer_meta *)(void *)desc->meta_va; cpu_buffer->nr_pages = nr_pages; - cpu_buffer->subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, - sizeof(*cpu_buffer->subbuf_ids), GFP_KERNEL); + cpu_buffer->subbuf_ids = kzalloc_objs(*cpu_buffer->subbuf_ids, + cpu_buffer->nr_pages + 1); if (!cpu_buffer->subbuf_ids) goto fail_free_reader; @@ -2555,8 +2636,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) bpage->range = 1; cpu_buffer->subbuf_ids[0] = bpage; } else { - int order = cpu_buffer->buffer->subbuf_order; - bpage->page = alloc_cpu_data(cpu, order); + bpage->page = alloc_cpu_data(cpu, bpage->order); if (!bpage->page) goto fail_free_reader; } @@ -2601,6 +2681,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) return_ptr(cpu_buffer); fail_free_reader: + kfree(cpu_buffer->subbuf_ids); free_buffer_page(cpu_buffer->reader_page); return NULL; @@ -2629,7 +2710,7 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) free_buffer_page(bpage); } - free_page((unsigned long)cpu_buffer->free_page); + free_pages((unsigned long)cpu_buffer->free_page.data, cpu_buffer->free_page.order); kfree(cpu_buffer); } @@ -2641,8 +2722,8 @@ static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) struct ring_buffer_cpu_meta *meta; struct buffer_data_page *dpage; unsigned long entry_bytes = 0; + unsigned int subbuf_size; unsigned long ptr; - int subbuf_size; int invalid = 0; int cpu; int i; @@ -2712,8 +2793,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, struct ring_buffer_remote *remote) { struct trace_buffer *buffer __free(kfree) = NULL; - long nr_pages; - int subbuf_size; + unsigned int subbuf_size; + unsigned long nr_pages; int bsize; int cpu; int ret; @@ -2729,10 +2810,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, buffer->subbuf_order = order; subbuf_size = (PAGE_SIZE << order); - buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; - - /* Max payload is buffer page size - header (8bytes) */ - buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); buffer->flags = flags; buffer->clock = trace_clock_local; @@ -2741,7 +2818,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); init_waitqueue_head(&buffer->irq_work.waiters); - buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), @@ -2772,6 +2848,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, size = end - buffers_start; size = size / nr_cpu_ids; + if (size < sizeof(struct ring_buffer_cpu_meta)) + goto fail_free_buffers; /* * The number of sub-buffers (nr_pages) is determined by the * total size allocated minus the meta data size. @@ -2781,6 +2859,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, */ nr_pages = (size - sizeof(struct ring_buffer_cpu_meta)) / (subbuf_size + sizeof(int)); + + if (nr_pages > rb_static_max_pages()) + goto fail_free_buffers; + /* Need at least two pages plus the reader page */ if (nr_pages < 3) goto fail_free_buffers; @@ -2813,12 +2895,15 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, /* The writer is remote. This ring-buffer is read-only */ atomic_inc(&buffer->record_disabled); nr_pages = desc->nr_page_va - 1; + + if (nr_pages > rb_static_max_pages()) + goto fail_free_buffers; + if (nr_pages < 2) goto fail_free_buffers; } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); if (nr_pages < 2) nr_pages = 2; } @@ -3201,7 +3286,7 @@ static void update_pages_handler(struct work_struct *work) * @size: the new size. * @cpu_id: the cpu buffer to resize * - * Minimum size is 2 * buffer->subbuf_size. + * Minimum size is 2 * rb_subbuf_capacity(buffer). * * Returns 0 on success and < 0 on failure. */ @@ -3223,12 +3308,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, !cpumask_test_cpu(cpu_id, buffer->cpumask)) return 0; - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; - /* * Keep CPUs from coming online while resizing to synchronize * with new per CPU buffers being created. @@ -3239,6 +3318,12 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + + /* we need a minimum of two pages */ + if (nr_pages < 2) + nr_pages = 2; + if (cpu_id == RING_BUFFER_ALL_CPUS) { /* * Don't succeed if resizing is disabled, as a reader might be @@ -3511,7 +3596,7 @@ rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event { unsigned long addr = (unsigned long)event; - addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; + addr &= (unsigned long)rb_subbuf_size(cpu_buffer->buffer) - 1; return addr - BUF_PAGE_HDR_SIZE; } @@ -3753,8 +3838,8 @@ static inline void rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, unsigned long tail, struct rb_event_info *info) { - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); struct buffer_page *tail_page = info->tail_page; + unsigned long bsize = rb_page_capacity(tail_page); struct ring_buffer_event *event; unsigned long length = info->length; @@ -4099,8 +4184,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, new_index = rb_event_index(cpu_buffer, event); old_index = new_index + rb_event_ts_length(event); - addr = (unsigned long)event; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, (unsigned long)event); bpage = READ_ONCE(cpu_buffer->tail_page); @@ -4765,7 +4849,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, tail = write - info->length; /* See if we shot pass the end of this buffer page */ - if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { + if (unlikely(write > rb_page_capacity(tail_page))) { check_buffer(cpu_buffer, info, CHECK_FULL_PAGE); return rb_move_tail(cpu_buffer, tail, info); } @@ -4901,7 +4985,7 @@ rb_reserve_next_event(struct trace_buffer *buffer, if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { add_ts_default = RB_ADD_STAMP_ABSOLUTE; info.length += RB_LEN_TIME_EXTEND; - if (info.length > cpu_buffer->buffer->max_data_size) + if (info.length > rb_subbuf_max_data_size(cpu_buffer->buffer)) goto out_fail; } else { add_ts_default = RB_ADD_STAMP_NONE; @@ -4976,7 +5060,7 @@ ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) if (unlikely(atomic_read(&cpu_buffer->record_disabled))) goto out; - if (unlikely(length > buffer->max_data_size)) + if (unlikely(length > rb_subbuf_max_data_size(buffer))) goto out; if (unlikely(trace_recursive_lock(cpu_buffer))) @@ -5010,7 +5094,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage = cpu_buffer->commit_page; struct buffer_page *start; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, addr); /* Do the likely case first */ if (likely(bpage->page == (void *)addr)) { @@ -5123,7 +5207,7 @@ int ring_buffer_write(struct trace_buffer *buffer, if (atomic_read(&cpu_buffer->record_disabled)) return -EBUSY; - if (length > buffer->max_data_size) + if (length > rb_subbuf_max_data_size(buffer)) return -EBUSY; if (unlikely(trace_recursive_lock(cpu_buffer))) @@ -5751,8 +5835,11 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id]; - WARN_ON_ONCE(cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu, - cpu_buffer->remote->priv)); + if (cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu, + cpu_buffer->remote->priv)) { + pr_warn_ratelimited("Remote reader page swap failed\n"); + return NULL; + } /* nr_pages doesn't include the reader page */ if (WARN_ON_ONCE(cpu_buffer->meta_page->reader.id > cpu_buffer->nr_pages)) return NULL; @@ -5785,6 +5872,7 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) cpu_buffer->head_page = new_head; cpu_buffer->reader_page = new_reader; + cpu_buffer->reader_page->read = 0; cpu_buffer->pages = &new_head->list; cpu_buffer->read_stamp = new_reader->page->time_stamp; cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events; @@ -5795,13 +5883,12 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) static struct buffer_page * __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { - int max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); + unsigned long max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; struct buffer_page *reader = NULL; + unsigned long nr_loops = 0; unsigned long overwrite; unsigned long flags; int missed_events = 0; - int nr_loops = 0; bool ret; local_irq_save(flags); @@ -5944,7 +6031,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) #define USECS_WAIT 1000000 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) { /* If the write is past the end of page, a writer is still updating it */ - if (likely(!reader || rb_page_write(reader) <= bsize)) + if (likely(!reader || rb_page_write(reader) <= rb_page_capacity(reader))) break; udelay(1); @@ -6119,8 +6206,8 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) struct trace_buffer *buffer; struct ring_buffer_per_cpu *cpu_buffer; struct ring_buffer_event *event; - int nr_loops = 0; - int max_loops; + unsigned long nr_loops = 0; + unsigned long max_loops; if (ts) *ts = 0; @@ -6377,36 +6464,44 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume); struct ring_buffer_iter * ring_buffer_read_start(struct trace_buffer *buffer, int cpu, gfp_t flags) { + struct ring_buffer_iter *iter __free(kfree) = kzalloc_obj(*iter, flags); struct ring_buffer_per_cpu *cpu_buffer; - struct ring_buffer_iter *iter; - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return NULL; - - iter = kzalloc_obj(*iter, flags); if (!iter) return NULL; - /* Holds the entire event: data and meta data */ - iter->event_size = buffer->subbuf_size; - iter->event = kmalloc(iter->event_size, flags); - if (!iter->event) { - kfree(iter); + if (!cpumask_test_cpu(cpu, buffer->cpumask)) return NULL; - } cpu_buffer = buffer->buffers[cpu]; - iter->cpu_buffer = cpu_buffer; + /* + * Only KDB is using GFP_ATOMIC, for the others, lock the buffer to + * prevent concurrent resizing. + */ + if (gfpflags_allow_blocking(flags)) + mutex_lock(&buffer->mutex); atomic_inc(&cpu_buffer->resize_disabled); + if (gfpflags_allow_blocking(flags)) + mutex_unlock(&buffer->mutex); + + /* Holds the entire event: data and meta data. */ + iter->event_size = rb_page_capacity(READ_ONCE(cpu_buffer->reader_page)); + iter->event = kmalloc(iter->event_size, flags); + if (!iter->event) { + atomic_dec(&cpu_buffer->resize_disabled); + return NULL; + } + iter->cpu_buffer = cpu_buffer; + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); arch_spin_lock(&cpu_buffer->lock); rb_iter_reset(iter); arch_spin_unlock(&cpu_buffer->lock); - return iter; + return_ptr(iter); } EXPORT_SYMBOL_GPL(ring_buffer_read_start); @@ -6460,7 +6555,7 @@ unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 0; - return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; + return rb_subbuf_capacity(buffer) * buffer->buffers[cpu]->nr_pages; } EXPORT_SYMBOL_GPL(ring_buffer_size); @@ -6474,8 +6569,9 @@ unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer) { /* If abs timestamp is requested, events have a timestamp too */ if (ring_buffer_time_stamp_abs(buffer)) - return buffer->max_data_size - RB_LEN_TIME_EXTEND; - return buffer->max_data_size; + return rb_subbuf_max_data_size(buffer) - RB_LEN_TIME_EXTEND; + + return rb_subbuf_max_data_size(buffer); } EXPORT_SYMBOL_GPL(ring_buffer_max_event_size); @@ -6607,12 +6703,11 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) rb_head_page_activate(cpu_buffer); cpu_buffer->pages_removed = 0; - if (cpu_buffer->mapped) { - rb_update_meta_page(cpu_buffer); - if (cpu_buffer->ring_meta) { - struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; - meta->commit_buffer = meta->head_buffer; - } + rb_update_meta_page(cpu_buffer); + if (cpu_buffer->ring_meta) { + struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; + + meta->commit_buffer = meta->head_buffer; } } @@ -6852,7 +6947,7 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, { struct ring_buffer_per_cpu *cpu_buffer_a; struct ring_buffer_per_cpu *cpu_buffer_b; - int ret = -EINVAL; + int ret = -EBUSY; if (!cpumask_test_cpu(cpu, buffer_a->cpumask) || !cpumask_test_cpu(cpu, buffer_b->cpumask)) @@ -6861,8 +6956,8 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, cpu_buffer_a = buffer_a->buffers[cpu]; cpu_buffer_b = buffer_b->buffers[cpu]; - /* It's up to the callers to not try to swap mapped buffers */ - if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped)) + /* It's up to the callers to not try to swap static buffers */ + if (WARN_ON_ONCE(rb_is_static(cpu_buffer_a) || rb_is_static(cpu_buffer_b))) return -EBUSY; /* At least make sure the two buffers are somewhat the same */ @@ -6893,10 +6988,10 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, atomic_inc(&cpu_buffer_a->record_disabled); atomic_inc(&cpu_buffer_b->record_disabled); - ret = -EBUSY; - if (local_read(&cpu_buffer_a->committing)) + /* Do not swap if either buffer is in the process of writing */ + if (cpu_buffer_a->current_context) goto out_dec; - if (local_read(&cpu_buffer_b->committing)) + if (cpu_buffer_b->current_context) goto out_dec; /* @@ -6928,56 +7023,78 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); * ring_buffer_alloc_read_page - allocate a page to read from buffer * @buffer: the buffer to allocate for. * @cpu: the cpu buffer to allocate. + * @rpage: pointer to pass in an already allocated page (can be NULL) + * and returns the allocated page. * - * This function is used in conjunction with ring_buffer_read_page. + * This function is used in conjunction with ring_buffer_read_page(). * When reading a full page from the ring buffer, these functions * can be used to speed up the process. The calling function should * allocate a few pages first with this function. Then when it * needs to get pages from the ring buffer, it passes the result - * of this function into ring_buffer_read_page, which will swap + * of this function into ring_buffer_read_page(), which will swap * the page that was allocated, with the read page of the buffer. * + * If @rpage is provided, and it has a different order than the current + * subbuffer order, its payload will be freed and re-allocated. If it + * already matches the order, it is simply returned. + * * Returns: - * The page allocated, or ERR_PTR + * 0 on success, < 0 on error */ -struct buffer_data_read_page * -ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) +int ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu, + struct buffer_data_read_page **rpage) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_read_page *bpage = NULL; unsigned long flags; + unsigned int order; if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return ERR_PTR(-ENODEV); + return -ENODEV; - bpage = kzalloc_obj(*bpage); - if (!bpage) - return ERR_PTR(-ENOMEM); + if (!rpage) + return -EINVAL; - bpage->order = buffer->subbuf_order; + order = READ_ONCE(buffer->subbuf_order); + + if (*rpage) { + if ((*rpage)->order == order) + return 0; + + /* We can reuse rpage, but we discard the payload */ + free_pages((unsigned long)(*rpage)->data, (*rpage)->order); + (*rpage)->data = NULL; + } else { + *rpage = kzalloc_obj(**rpage); + if (!*rpage) + return -ENOMEM; + } + + (*rpage)->order = order; cpu_buffer = buffer->buffers[cpu]; + local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (cpu_buffer->free_page) { - bpage->data = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + if (cpu_buffer->free_page.data) { + **rpage = cpu_buffer->free_page; + cpu_buffer->free_page.data = NULL; } arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); - if (bpage->data) { - rb_init_data_page(bpage->data); + if ((*rpage)->data) { + rb_init_data_page((*rpage)->data); } else { - bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order); - if (!bpage->data) { - kfree(bpage); - return ERR_PTR(-ENOMEM); + (*rpage)->data = alloc_cpu_data(cpu, (*rpage)->order); + if (!(*rpage)->data) { + kfree(*rpage); + *rpage = NULL; + return -ENOMEM; } } - return bpage; + return 0; } EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); @@ -6985,21 +7102,30 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); * ring_buffer_free_read_page - free an allocated read page * @buffer: the buffer the page was allocate for * @cpu: the cpu buffer the page came from - * @data_page: the page to free + * @rpage: the buffer_data_read_page to free * * Free a page allocated from ring_buffer_alloc_read_page. */ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, - struct buffer_data_read_page *data_page) + struct buffer_data_read_page *rpage) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_page *dpage = data_page->data; - struct page *page = virt_to_page(dpage); + struct buffer_data_page *dpage; unsigned long flags; + struct page *page; if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) return; + if (!rpage) + return; + + dpage = rpage->data; + if (!dpage) + goto out; + + page = virt_to_page(dpage); + cpu_buffer = buffer->buffers[cpu]; /* @@ -7007,14 +7133,14 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, * is different from the subbuffer order of the buffer - * we can't reuse it */ - if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) + if (page_ref_count(page) > 1 || rpage->order != READ_ONCE(buffer->subbuf_order)) goto out; local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (!cpu_buffer->free_page) { - cpu_buffer->free_page = dpage; + if (!cpu_buffer->free_page.data) { + cpu_buffer->free_page = *rpage; dpage = NULL; } @@ -7022,8 +7148,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, local_irq_restore(flags); out: - free_pages((unsigned long)dpage, data_page->order); - kfree(data_page); + free_pages((unsigned long)dpage, rpage->order); + kfree(rpage); } EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); @@ -7074,7 +7200,6 @@ int ring_buffer_read_page(struct trace_buffer *buffer, unsigned int size; unsigned int read; u64 save_timestamp; - bool force_memcpy; if (!cpumask_test_cpu(cpu, buffer->cpumask)) return -1; @@ -7091,13 +7216,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (!data_page || !data_page->data) return -1; - if (data_page->order != buffer->subbuf_order) - return -1; - dpage = data_page->data; if (!dpage) return -1; + len = min_t(size_t, len, rb_read_page_capacity(data_page)); + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); reader = rb_get_reader_page(cpu_buffer); @@ -7113,33 +7237,26 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; - force_memcpy = cpu_buffer->mapped || cpu_buffer->remote; - - /* - * If this page has been partially read or - * if len is not big enough to read the rest of the page or - * a writer is still on the page, then - * we must copy the data from the page to the buffer. - * Otherwise, we can simply swap the page with the one passed in. - */ + /* + * It is not possible to swap the reader page if: + * - It has been partially read + * - len is not big enough to read it entirely + * - A writer is still on it + * - The ring buffer is static + * - The order doesn't match + */ if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || - force_memcpy) { + rb_is_static(cpu_buffer) || + data_page->order != reader->order) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; unsigned int event_size; unsigned int flags = 0; - /* - * If a full page is expected, this can still be returned - * if there's been a previous partial read and the - * rest of the page can be read and the commit page is off - * the reader page. - */ - if (full && - (!read || (len < (size - read)) || - cpu_buffer->reader_page == cpu_buffer->commit_page)) + /* If a full page is requested, it cannot be the commit page */ + if (full && cpu_buffer->reader_page == cpu_buffer->commit_page) return -1; if (len > (size - read)) @@ -7174,7 +7291,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, rpos = reader->read; pos += event_size; - if (rpos >= event_size) + if (rpos >= size) break; event = rb_reader_event(cpu_buffer); @@ -7225,7 +7342,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (missed_events > 0 && - buffer->subbuf_size - size >= sizeof(missed_events)) { + rb_read_page_capacity(data_page) - size >= sizeof(missed_events)) { memcpy(&dpage->data[size], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7245,8 +7362,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* * This page may be off to user land. Zero it out here. */ - if (size < buffer->subbuf_size) - memset(&dpage->data[size], 0, buffer->subbuf_size - size); + if (size < rb_read_page_capacity(data_page)) + memset(&dpage->data[size], 0, rb_read_page_capacity(data_page) - size); return read; } @@ -7265,6 +7382,18 @@ void *ring_buffer_read_page_data(struct buffer_data_read_page *page) EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); /** + * ring_buffer_read_page_size - get size of the read page. + * @rpage: the page to get the size from + * + * Returns size of the page in bytes. + */ +unsigned int ring_buffer_read_page_size(struct buffer_data_read_page *rpage) +{ + return rpage ? PAGE_SIZE << rpage->order : 0; +} +EXPORT_SYMBOL_GPL(ring_buffer_read_page_size); + +/** * ring_buffer_subbuf_size_get - get size of the sub buffer. * @buffer: the buffer to get the sub buffer size from * @@ -7272,7 +7401,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); */ int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) { - return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + return rb_subbuf_size(buffer); } EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get); @@ -7317,8 +7446,9 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) { struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage, *tmp; - int old_order, old_size; - int nr_pages; + unsigned int old_capacity; + unsigned long nr_pages; + int old_order; int psize; int err; int cpu; @@ -7326,9 +7456,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (!buffer || order < 0) return -EINVAL; - if (buffer->subbuf_order == order) - return 0; - psize = (1 << order) * PAGE_SIZE; if (psize <= BUF_PAGE_HDR_SIZE) return -EINVAL; @@ -7337,18 +7464,29 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; - old_order = buffer->subbuf_order; - old_size = buffer->subbuf_size; - /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); + + old_order = buffer->subbuf_order; + if (old_order == order) + return 0; + + old_capacity = rb_subbuf_capacity(buffer); + + /* The mmap fast path reads subbuf_order without buffer->mutex. */ + for_each_buffer_cpu(buffer, cpu) { + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + continue; + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return -EBUSY; + } + atomic_inc(&buffer->record_disabled); /* Make sure all commits have finished */ synchronize_rcu(); - buffer->subbuf_order = order; - buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; + WRITE_ONCE(buffer->subbuf_order, order); /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { @@ -7358,14 +7496,14 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer = buffer->buffers[cpu]; - if (cpu_buffer->mapped) { + if (atomic_read(&cpu_buffer->resize_disabled)) { err = -EBUSY; goto error; } /* Update the number of pages to match the new size */ - nr_pages = old_size * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); + nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; + nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); /* we need a minimum of two pages */ if (nr_pages < 2) @@ -7387,7 +7525,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } for_each_buffer_cpu(buffer, cpu) { - struct buffer_data_page *old_free_data_page; + struct buffer_data_read_page old_free_data_page; struct list_head old_pages; unsigned long flags; @@ -7428,8 +7566,10 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update; cpu_buffer->nr_pages_to_update = 0; + arch_spin_lock(&cpu_buffer->lock); old_free_data_page = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + cpu_buffer->free_page.data = NULL; + arch_spin_unlock(&cpu_buffer->lock); rb_head_page_activate(cpu_buffer); @@ -7440,7 +7580,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) list_del_init(&bpage->list); free_buffer_page(bpage); } - free_pages((unsigned long)old_free_data_page, old_order); + free_pages((unsigned long)old_free_data_page.data, old_free_data_page.order); rb_check_pages(cpu_buffer); } @@ -7450,8 +7590,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) return 0; error: - buffer->subbuf_order = old_order; - buffer->subbuf_size = old_size; + WRITE_ONCE(buffer->subbuf_order, old_order); atomic_dec(&buffer->record_disabled); @@ -7499,10 +7638,10 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page **subbuf_ids) { struct trace_buffer_meta *meta = cpu_buffer->meta_page; - unsigned int nr_subbufs = cpu_buffer->nr_pages + 1; + unsigned long nr_subbufs = cpu_buffer->nr_pages + 1; struct buffer_page *first_subbuf, *subbuf; - int cnt = 0; - int id = 0; + unsigned int cnt = 0; + unsigned int id = 0; id = rb_page_id(cpu_buffer, cpu_buffer->reader_page, id); subbuf_ids[id++] = cpu_buffer->reader_page; @@ -7529,7 +7668,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, meta->meta_struct_len = sizeof(*meta); meta->nr_subbufs = nr_subbufs; - meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + meta->subbuf_size = rb_subbuf_size(cpu_buffer->buffer); meta->meta_page_size = meta->subbuf_size; rb_update_meta_page(cpu_buffer); @@ -7571,11 +7710,7 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, lockdep_assert_held(&cpu_buffer->mapping_lock); - /* mapped is always greater or equal to user_mapped */ - if (WARN_ON(cpu_buffer->mapped < cpu_buffer->user_mapped)) - return -EINVAL; - - if (inc && cpu_buffer->mapped == UINT_MAX) + if (inc && cpu_buffer->user_mapped == UINT_MAX) return -EBUSY; if (WARN_ON(!inc && cpu_buffer->user_mapped == 0)) @@ -7584,13 +7719,10 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, mutex_lock(&cpu_buffer->buffer->mutex); raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); - if (inc) { + if (inc) cpu_buffer->user_mapped++; - cpu_buffer->mapped++; - } else { + else cpu_buffer->user_mapped--; - cpu_buffer->mapped--; - } raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); mutex_unlock(&cpu_buffer->buffer->mutex); @@ -7613,7 +7745,8 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, struct vm_area_struct *vma) { - unsigned long nr_subbufs, nr_pages, nr_vma_pages, pgoff = vma->vm_pgoff; + unsigned long nr_subbufs, nr_pages, nr_vma_pages; + pgoff_t pgoff = vma_start_pgoff(vma); unsigned int subbuf_pages, subbuf_order; struct page **pages __free(kfree) = NULL; int p = 0, s = 0; @@ -7736,6 +7869,9 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, /* prevent another thread from changing buffer/sub-buffer sizes */ guard(mutex)(&buffer->mutex); + if (cpu_buffer->nr_pages > rb_static_max_pages()) + return -E2BIG; + err = rb_alloc_meta_page(cpu_buffer); if (err) return err; @@ -7762,7 +7898,6 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, if (!err) { raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); /* This is the first time it is mapped by user */ - cpu_buffer->mapped++; cpu_buffer->user_mapped = 1; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); } else { @@ -7819,8 +7954,6 @@ int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); /* This is the last user space mapping */ - if (!WARN_ON_ONCE(cpu_buffer->mapped < cpu_buffer->user_mapped)) - cpu_buffer->mapped--; cpu_buffer->user_mapped = 0; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); @@ -7891,7 +8024,7 @@ consume: * missed events, then record it there. */ commit = rb_page_size(reader); - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + if (rb_page_capacity(reader) - commit >= sizeof(missed_events)) { memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7923,7 +8056,7 @@ consume: out: /* Some archs do not have data cache coherency between kernel and user-space */ flush_kernel_vmap_range(cpu_buffer->reader_page->page, - buffer->subbuf_size + BUF_PAGE_HDR_SIZE); + rb_subbuf_size(buffer)); rb_update_meta_page(cpu_buffer); @@ -8214,7 +8347,7 @@ static __init int test_ringbuffer(void) out_free: for_each_online_cpu(cpu) { - if (!rb_threads[cpu]) + if (IS_ERR_OR_NULL(rb_threads[cpu])) break; kthread_stop(rb_threads[cpu]); } |