-
Notifications
You must be signed in to change notification settings - Fork 4
ROX-30299: Track io uring operations #1331
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5b90a55
a28ab9a
807044f
bc7c395
2859434
013c0e1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| FROM quay.io/centos/centos:stream9 AS builder | ||
|
|
||
| RUN dnf install -y --enablerepo=crb gcc glibc-static | ||
|
|
||
| WORKDIR /build | ||
| COPY io_uring_write_raw.c . | ||
| RUN cc -static -o io_uring_write_raw io_uring_write_raw.c | ||
|
|
||
| FROM quay.io/centos/centos:stream9-minimal | ||
|
|
||
| COPY --from=builder /build/io_uring_write_raw /usr/local/bin/ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| /* | ||
| * Helper that uses raw io_uring syscalls to write content to a file. | ||
| * No liburing dependency — only uses io_uring_setup/io_uring_enter | ||
| * syscalls directly, so it can be statically linked. | ||
| * | ||
| * Usage: io_uring_write_raw <file> <content> | ||
| * | ||
| * Exit codes: | ||
| * 0 - success | ||
| * 1 - usage error | ||
| * 2 - io_uring not available | ||
| * 3 - I/O error | ||
| */ | ||
| #include <errno.h> | ||
| #include <fcntl.h> | ||
| #include <linux/io_uring.h> | ||
| #include <stdio.h> | ||
| #include <string.h> | ||
| #include <sys/mman.h> | ||
| #include <sys/syscall.h> | ||
| #include <unistd.h> | ||
|
|
||
| struct ring { | ||
| int fd; | ||
| struct io_uring_sqe *sqes; | ||
| unsigned *sq_tail; | ||
| unsigned *sq_mask; | ||
| unsigned *sq_array; | ||
| struct io_uring_cqe *cqes; | ||
| unsigned *cq_head; | ||
| unsigned *cq_tail; | ||
| unsigned *cq_mask; | ||
| }; | ||
|
|
||
| static int ring_init(struct ring *r, unsigned entries) | ||
| { | ||
| struct io_uring_params p; | ||
|
|
||
| memset(&p, 0, sizeof(p)); | ||
|
|
||
| int fd = syscall(__NR_io_uring_setup, entries, &p); | ||
| if (fd < 0) | ||
| return -1; | ||
|
|
||
| size_t sq_sz = p.sq_off.array + p.sq_entries * sizeof(unsigned); | ||
| size_t cq_sz = p.cq_off.cqes + | ||
| p.cq_entries * sizeof(struct io_uring_cqe); | ||
| size_t sqe_sz = p.sq_entries * sizeof(struct io_uring_sqe); | ||
|
|
||
| void *sq = mmap(NULL, sq_sz, PROT_READ | PROT_WRITE, | ||
| MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING); | ||
| void *cq = mmap(NULL, cq_sz, PROT_READ | PROT_WRITE, | ||
| MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING); | ||
| void *sqes = mmap(NULL, sqe_sz, PROT_READ | PROT_WRITE, | ||
| MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES); | ||
|
|
||
| if (sq == MAP_FAILED || cq == MAP_FAILED || sqes == MAP_FAILED) { | ||
| close(fd); | ||
| return -1; | ||
| } | ||
|
|
||
| r->fd = fd; | ||
| r->sqes = sqes; | ||
| r->sq_tail = sq + p.sq_off.tail; | ||
| r->sq_mask = sq + p.sq_off.ring_mask; | ||
| r->sq_array = sq + p.sq_off.array; | ||
| r->cqes = cq + p.cq_off.cqes; | ||
| r->cq_head = cq + p.cq_off.head; | ||
| r->cq_tail = cq + p.cq_off.tail; | ||
| r->cq_mask = cq + p.cq_off.ring_mask; | ||
|
|
||
| return 0; | ||
| } | ||
|
|
||
| static struct io_uring_sqe *get_sqe(struct ring *r) | ||
| { | ||
| unsigned tail = __atomic_load_n(r->sq_tail, __ATOMIC_RELAXED); | ||
| unsigned idx = tail & *r->sq_mask; | ||
| struct io_uring_sqe *sqe = &r->sqes[idx]; | ||
|
|
||
| memset(sqe, 0, sizeof(*sqe)); | ||
| return sqe; | ||
| } | ||
|
|
||
| static int submit_and_wait(struct ring *r, struct io_uring_cqe **cqe) | ||
| { | ||
| unsigned tail = __atomic_load_n(r->sq_tail, __ATOMIC_RELAXED); | ||
| unsigned idx = tail & *r->sq_mask; | ||
|
|
||
| r->sq_array[idx] = idx; | ||
| __atomic_store_n(r->sq_tail, tail + 1, __ATOMIC_RELEASE); | ||
|
|
||
| int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, | ||
| IORING_ENTER_GETEVENTS, NULL, 0); | ||
| if (ret < 0) | ||
| return -1; | ||
|
|
||
| unsigned head = __atomic_load_n(r->cq_head, __ATOMIC_RELAXED); | ||
|
|
||
| *cqe = &r->cqes[head & *r->cq_mask]; | ||
| return 0; | ||
| } | ||
|
|
||
| static void cqe_advance(struct ring *r) | ||
| { | ||
| unsigned head = __atomic_load_n(r->cq_head, __ATOMIC_RELAXED); | ||
|
|
||
| __atomic_store_n(r->cq_head, head + 1, __ATOMIC_RELEASE); | ||
| } | ||
|
|
||
| int main(int argc, char *argv[]) | ||
| { | ||
| struct ring r; | ||
| struct io_uring_sqe *sqe; | ||
| struct io_uring_cqe *cqe; | ||
|
|
||
| if (argc != 3) { | ||
| fprintf(stderr, "Usage: %s <file> <content>\n", argv[0]); | ||
| return 1; | ||
| } | ||
|
|
||
| if (ring_init(&r, 4) < 0) { | ||
| fprintf(stderr, "io_uring_setup: %s\n", strerror(errno)); | ||
| return 2; | ||
| } | ||
|
|
||
| /* Open file via io_uring */ | ||
| sqe = get_sqe(&r); | ||
| sqe->opcode = IORING_OP_OPENAT; | ||
| sqe->fd = AT_FDCWD; | ||
| sqe->addr = (unsigned long)argv[1]; | ||
| sqe->open_flags = O_WRONLY | O_TRUNC; | ||
| if (submit_and_wait(&r, &cqe) < 0 || cqe->res < 0) { | ||
| fprintf(stderr, "openat: %s\n", | ||
| strerror(-(cqe ? cqe->res : errno))); | ||
| return 3; | ||
| } | ||
| int fd = cqe->res; | ||
| cqe_advance(&r); | ||
|
|
||
| /* Write content via io_uring */ | ||
| sqe = get_sqe(&r); | ||
| sqe->opcode = IORING_OP_WRITE; | ||
| sqe->fd = fd; | ||
| sqe->addr = (unsigned long)argv[2]; | ||
| sqe->len = strlen(argv[2]); | ||
| if (submit_and_wait(&r, &cqe) < 0 || cqe->res < 0) { | ||
| fprintf(stderr, "write: %s\n", | ||
| strerror(-(cqe ? cqe->res : errno))); | ||
| return 3; | ||
| } | ||
| cqe_advance(&r); | ||
|
|
||
| /* Close file via io_uring */ | ||
| sqe = get_sqe(&r); | ||
| sqe->opcode = IORING_OP_CLOSE; | ||
| sqe->fd = fd; | ||
| if (submit_and_wait(&r, &cqe) < 0 || cqe->res < 0) { | ||
| fprintf(stderr, "close: %s\n", | ||
| strerror(-(cqe ? cqe->res : errno))); | ||
| return 3; | ||
| } | ||
| cqe_advance(&r); | ||
|
|
||
| close(r.fd); | ||
| return 0; | ||
| } |
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test seems to only pass if the write done via
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is failing locally for me, because it sees the open event. Now that I build the binary in the image, the CI tests are failing for the same reason. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import os | ||
|
|
||
| import docker | ||
| import docker.models.containers | ||
| import docker.models.images | ||
| import pytest | ||
|
|
||
| from event import Event, EventType, Process | ||
| from server import EventServer | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def io_uring_image(docker_client: docker.DockerClient): | ||
| image, _ = docker_client.images.build( | ||
| path='containers/io-uring', | ||
| tag='io-uring:latest', | ||
| dockerfile='Containerfile', | ||
| ) | ||
| return image | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def get_io_uring_container( | ||
| io_uring_image: docker.models.images.Image, | ||
| docker_client: docker.DockerClient, | ||
| monitored_dir: str, | ||
| ): | ||
| container = docker_client.containers.run( | ||
| io_uring_image.tags[0], | ||
| detach=True, | ||
| tty=True, | ||
| name='io-uring', | ||
| security_opt=['seccomp=unconfined'], | ||
| volumes={ | ||
| monitored_dir: { | ||
| 'bind': '/data', | ||
| 'mode': 'z', | ||
| }, | ||
| }, | ||
| ) | ||
|
|
||
| yield container | ||
|
|
||
| container.stop(timeout=1) | ||
| container.remove() | ||
|
|
||
|
|
||
| def test_io_uring_write( | ||
| monitored_dir: str, | ||
| server: EventServer, | ||
| get_io_uring_container: docker.models.containers.Container, | ||
| ): | ||
| """ | ||
| Verifies that io_uring write operations modify files but are not | ||
| currently tracked by fact. | ||
|
|
||
| Creates a file with 'hi', modifies it to 'bye' via io_uring | ||
| (open, write, and close all go through io_uring, bypassing the | ||
| normal syscall path), then verifies the content changed and that | ||
| only the expected creation events are captured. | ||
| """ | ||
| fut = os.path.join(monitored_dir, 'io_uring_test.txt') | ||
| process = Process.from_proc() | ||
|
|
||
| # Create file with initial content via normal I/O. | ||
| with open(fut, 'w') as f: | ||
| f.write('hi') | ||
|
|
||
| creation = Event( | ||
| process=process, | ||
| event_type=EventType.CREATION, | ||
| file=fut, | ||
| host_path=fut, | ||
| ) | ||
| server.wait_events([creation]) | ||
|
|
||
| # Modify the file using io_uring inside the container. | ||
| exit_code, output = get_io_uring_container.exec_run( | ||
| ['io_uring_write_raw', '/data/io_uring_test.txt', 'bye'], | ||
| ) | ||
| if exit_code == 2: | ||
| pytest.skip(f'io_uring not supported: {output.decode()}') | ||
| assert exit_code == 0, f'io_uring write failed: {output.decode()}' | ||
|
|
||
| # Create a sentinel file via normal I/O to verify event ordering. | ||
| # With strict=True (the default), any unexpected event appearing | ||
| # before the sentinel would cause the test to fail. | ||
| sentinel = os.path.join(monitored_dir, 'sentinel.txt') | ||
| with open(sentinel, 'w') as f: | ||
| f.write('sentinel') | ||
|
|
||
| sentinel_event = Event( | ||
| process=process, | ||
| event_type=EventType.CREATION, | ||
| file=sentinel, | ||
| host_path=sentinel, | ||
| ) | ||
| server.wait_events([sentinel_event]) | ||
|
|
||
| # Verify the file was actually modified by io_uring. | ||
| with open(fut) as f: | ||
| assert f.read() == 'bye' |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this needed because some systems might not have
liburing+ devel installed? If so, as my original comment stated, we probably want to containerize the binary and just run that.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think the greatest advantage is that it gives greater control and transparency into which syscalls are made. With
liburingversion if the library changes, the syscalls could change and potentially break the test. I have removedio_uring_write.cand will only useio_uring_write_raw.c.