mirror of
https://github.com/tbsdtv/linux_media.git
synced 2025-07-23 04:33:26 +02:00
4a0fef6278
task_work bits of io_uring_task are split into two cache lines causing extra cache bouncing, place them into a separate cache line. Also move the most used submission path fields closer together, so there are hot. Cc: stable@vger.kernel.org # 5.15+ Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
59 lines
1.5 KiB
C
59 lines
1.5 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
* Arbitrary limit, can be raised if need be
|
|
*/
|
|
#define IO_RINGFD_REG_MAX 16
|
|
|
|
struct io_uring_task {
|
|
/* submission side */
|
|
int cached_refs;
|
|
const struct io_ring_ctx *last;
|
|
struct io_wq *io_wq;
|
|
struct file *registered_rings[IO_RINGFD_REG_MAX];
|
|
|
|
struct xarray xa;
|
|
struct wait_queue_head wait;
|
|
atomic_t in_idle;
|
|
atomic_t inflight_tracked;
|
|
struct percpu_counter inflight;
|
|
|
|
struct { /* task_work */
|
|
spinlock_t task_lock;
|
|
bool task_running;
|
|
struct io_wq_work_list task_list;
|
|
struct io_wq_work_list prio_task_list;
|
|
struct callback_head task_work;
|
|
} ____cacheline_aligned_in_smp;
|
|
};
|
|
|
|
struct io_tctx_node {
|
|
struct list_head ctx_node;
|
|
struct task_struct *task;
|
|
struct io_ring_ctx *ctx;
|
|
};
|
|
|
|
int io_uring_alloc_task_context(struct task_struct *task,
|
|
struct io_ring_ctx *ctx);
|
|
void io_uring_del_tctx_node(unsigned long index);
|
|
int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter);
|
|
void io_uring_clean_tctx(struct io_uring_task *tctx);
|
|
|
|
void io_uring_unreg_ringfd(void);
|
|
int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg,
|
|
unsigned nr_args);
|
|
int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg,
|
|
unsigned nr_args);
|
|
|
|
/*
|
|
* Note that this task has used io_uring. We use it for cancelation purposes.
|
|
*/
|
|
static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
|
|
{
|
|
struct io_uring_task *tctx = current->io_uring;
|
|
|
|
if (likely(tctx && tctx->last == ctx))
|
|
return 0;
|
|
return __io_uring_add_tctx_node(ctx, true);
|
|
}
|