Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fs/aio.c
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/eventfd.h>
#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/compat.h>
#include <linux/migrate.h>
#include <linux/ramfs.h>
Expand Down
2 changes: 1 addition & 1 deletion fs/fs-writeback.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#include <linux/pagemap.h>
#include <linux/kthread.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/backing-dev.h>
#include <linux/tracepoint.h>
#include <linux/device.h>
Expand Down
95 changes: 95 additions & 0 deletions include/linux/blk_plug.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BLK_PLUG_H
#define _LINUX_BLK_PLUG_H

#include <linux/sched.h>

struct blk_plug_cb;
typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *cb, bool from_schedule);

struct rq_list {
struct request *head;
struct request *tail;
};

#ifdef CONFIG_BLOCK
/*
* blk_plug permits building a queue of related requests by holding the I/O
* fragments for a short period. This allows merging of sequential requests
* into single larger request. As the requests are moved from a per-task list to
* the device's request_queue in a batch, this results in improved scalability
* as the lock contention for request_queue lock is reduced.
*
* It is ok not to disable preemption when adding the request to the plug list
* or when attempting a merge. For details, please see schedule() where
* blk_flush_plug() is called.
*/
struct blk_plug {
struct rq_list mq_list; /* blk-mq requests */

/* if ios_left is > 1, we can batch tag/rq allocations */
struct rq_list cached_rqs;
u64 cur_ktime;
unsigned short nr_ios;

unsigned short rq_count;

bool multiple_queues;
bool has_elevator;

struct list_head cb_list; /* md requires an unplug callback */
};

void blk_start_plug(struct blk_plug *);
void blk_start_plug_nr_ios(struct blk_plug *, unsigned short);
void blk_finish_plug(struct blk_plug *);

void __blk_flush_plug(struct blk_plug *plug, bool from_schedule);
static inline void blk_flush_plug(struct blk_plug *plug, bool async)
{
if (plug)
__blk_flush_plug(plug, async);
}

static __always_inline void blk_plug_invalidate_ts(void)
{
if (unlikely(current->flags & PF_BLOCK_TS)) {
current->plug->cur_ktime = 0;
current->flags &= ~PF_BLOCK_TS;
}
}

struct blk_plug_cb {
struct list_head list;
blk_plug_cb_fn callback;
void *data;
};

struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
int size);
#else /* CONFIG_BLOCK */
struct blk_plug {
};

static inline void blk_start_plug(struct blk_plug *plug)
{
}

static inline void blk_start_plug_nr_ios(struct blk_plug *plug,
unsigned short nr_ios)
{
}

static inline void blk_finish_plug(struct blk_plug *plug)
{
}

static inline void blk_flush_plug(struct blk_plug *plug, bool async)
{
}

static inline void blk_plug_invalidate_ts(void)
{
}
#endif /* CONFIG_BLOCK */
#endif /* _LINUX_BLK_PLUG_H */
86 changes: 1 addition & 85 deletions include/linux/blkdev.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#include <linux/types.h>
#include <linux/blk_types.h>
#include <linux/blk_plug.h>
#include <linux/device.h>
#include <linux/list.h>
#include <linux/llist.h>
Expand All @@ -21,7 +22,6 @@
#include <linux/rcupdate.h>
#include <linux/percpu-refcount.h>
#include <linux/blkzoned.h>
#include <linux/sched.h>
#include <linux/sbitmap.h>
#include <linux/uuid.h>
#include <linux/xarray.h>
Expand Down Expand Up @@ -1169,94 +1169,10 @@ extern void blk_put_queue(struct request_queue *);

void blk_mark_disk_dead(struct gendisk *disk);

struct rq_list {
struct request *head;
struct request *tail;
};

#ifdef CONFIG_BLOCK
/*
* blk_plug permits building a queue of related requests by holding the I/O
* fragments for a short period. This allows merging of sequential requests
* into single larger request. As the requests are moved from a per-task list to
* the device's request_queue in a batch, this results in improved scalability
* as the lock contention for request_queue lock is reduced.
*
* It is ok not to disable preemption when adding the request to the plug list
* or when attempting a merge. For details, please see schedule() where
* blk_flush_plug() is called.
*/
struct blk_plug {
struct rq_list mq_list; /* blk-mq requests */

/* if ios_left is > 1, we can batch tag/rq allocations */
struct rq_list cached_rqs;
u64 cur_ktime;
unsigned short nr_ios;

unsigned short rq_count;

bool multiple_queues;
bool has_elevator;

struct list_head cb_list; /* md requires an unplug callback */
};

struct blk_plug_cb;
typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
struct blk_plug_cb {
struct list_head list;
blk_plug_cb_fn callback;
void *data;
};
extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
void *data, int size);
extern void blk_start_plug(struct blk_plug *);
extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short);
extern void blk_finish_plug(struct blk_plug *);

void __blk_flush_plug(struct blk_plug *plug, bool from_schedule);
static inline void blk_flush_plug(struct blk_plug *plug, bool async)
{
if (plug)
__blk_flush_plug(plug, async);
}

static __always_inline void blk_plug_invalidate_ts(void)
{
if (unlikely(current->flags & PF_BLOCK_TS)) {
current->plug->cur_ktime = 0;
current->flags &= ~PF_BLOCK_TS;
}
}

int blkdev_issue_flush(struct block_device *bdev);
long nr_blockdev_pages(void);
#else /* CONFIG_BLOCK */
struct blk_plug {
};

static inline void blk_start_plug_nr_ios(struct blk_plug *plug,
unsigned short nr_ios)
{
}

static inline void blk_start_plug(struct blk_plug *plug)
{
}

static inline void blk_finish_plug(struct blk_plug *plug)
{
}

static inline void blk_flush_plug(struct blk_plug *plug, bool async)
{
}

static inline void blk_plug_invalidate_ts(void)
{
}

static inline int blkdev_issue_flush(struct block_device *bdev)
{
return 0;
Expand Down
2 changes: 1 addition & 1 deletion include/linux/io_uring_types.h
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#ifndef IO_URING_TYPES_H
#define IO_URING_TYPES_H

#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/hashtable.h>
#include <linux/task_work.h>
#include <linux/bitmap.h>
Expand Down
1 change: 1 addition & 0 deletions io_uring/io_uring.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#define IOU_CORE_H

#include <linux/errno.h>
#include <linux/file.h>
#include <linux/lockdep.h>
#include <linux/resume_user_mode.h>
#include <linux/poll.h>
Expand Down
1 change: 1 addition & 0 deletions io_uring/kbuf.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/poll.h>
#include <linux/uio.h>
#include <linux/vmalloc.h>
#include <linux/io_uring.h>

Expand Down
2 changes: 2 additions & 0 deletions io_uring/rsrc.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
#ifndef IOU_RSRC_H
#define IOU_RSRC_H

#include <linux/bvec.h>
#include <linux/io_uring_types.h>
#include <linux/lockdep.h>
#include <linux/uio.h>

#define IO_VEC_CACHE_SOFT_CAP 256

Expand Down
1 change: 1 addition & 0 deletions io_uring/rw.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include <linux/io_uring_types.h>
#include <linux/pagemap.h>
#include <linux/uio.h>

struct io_meta_state {
u32 seed;
Expand Down
1 change: 0 additions & 1 deletion kernel/exit.c
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
#include <linux/audit.h> /* for audit_free() */
#include <linux/resource.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/blkdev.h>
#include <linux/task_work.h>
#include <linux/fs_struct.h>
#include <linux/init_task.h>
Expand Down
1 change: 0 additions & 1 deletion kernel/sched/core.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
#include <linux/sched/rseq_api.h>
#include <linux/sched/rt.h>

#include <linux/blkdev.h>
#include <linux/context_tracking.h>
#include <linux/cpuset.h>
#include <linux/delayacct.h>
Expand Down
2 changes: 1 addition & 1 deletion mm/madvise.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#include <linux/ksm.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/backing-dev.h>
#include <linux/pagewalk.h>
#include <linux/swap.h>
Expand Down
1 change: 0 additions & 1 deletion mm/page-writeback.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#include <linux/init.h>
#include <linux/backing-dev.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/blkdev.h>
#include <linux/mpage.h>
#include <linux/rmap.h>
#include <linux/percpu.h>
Expand Down
2 changes: 1 addition & 1 deletion mm/readahead.c
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
* ->read_folio() which may be less efficient.
*/

#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/kernel.h>
#include <linux/dax.h>
#include <linux/gfp.h>
Expand Down
2 changes: 1 addition & 1 deletion mm/swap_state.c
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
#include <linux/pagemap.h>
#include <linux/folio_batch.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/migrate.h>
#include <linux/vmalloc.h>
#include <linux/huge_mm.h>
Expand Down
2 changes: 1 addition & 1 deletion mm/vmscan.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#include <linux/vmstat.h>
#include <linux/file.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/blk_plug.h>
#include <linux/buffer_head.h> /* for buffer_heads_over_limit */
#include <linux/mm_inline.h>
#include <linux/backing-dev.h>
Expand Down