Skip to content

Commit 3799508

Browse files
authored
Merge pull request #2378 from vchamarthi/asv-benchmarks
Add ASV benchmark suite for dpctl
2 parents 8890726 + 2110684 commit 3799508

13 files changed

Lines changed: 928 additions & 0 deletions

benchmarks/README.md

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# dpctl ASV Benchmarks
2+
3+
Runtime-overhead benchmarks for [dpctl](https://github.com/IntelPython/dpctl)
4+
using [ASV](https://asv.readthedocs.io/en/stable/): object construction,
5+
queue caching, USM allocation, device enumeration, kernel bundle
6+
compilation, data movement, kernel submission. No compute throughput.
7+
8+
## Coverage
9+
10+
| File | API |
11+
|------|-----|
12+
| `bench_construct.py` | `SyclDevice`/`SyclContext`/`SyclQueue`/`SyclPlatform` construction, device/queue attribute reads |
13+
| `bench_queue_cache.py` | `get_device_cached_queue` for each key kind, vs. an uncached baseline |
14+
| `bench_usm.py` | `MemoryUSM{Device,Host,Shared}` alloc/free, aligned alloc, first touch, USM pointer queries |
15+
| `bench_enumerate.py` | `get_devices`, `get_num_devices`, `get_platforms`, `select_*_device`, `has_*_devices`, `select_device_with_aspects` |
16+
| `bench_compile.py` | Kernel bundles from SPIR-V / OpenCL C source / SYCL source, kernel lookup, availability probes |
17+
| `bench_copy.py` | `SyclQueue.memcpy`/`memcpy_async`/`fill`/`memset`, `_Memory.copy_to_host`/`copy_from_host`/`copy_from_device` |
18+
| `bench_submit.py` | `submit`, `submit_async`, batched submission, `submit_barrier`, idle `wait` |
19+
20+
## Device axis
21+
22+
Benchmarks parameterize over the `cpu`/`gpu` filter selectors and skip when a
23+
selector has no matching device, so the same benchmark names run on any
24+
node. Sizes over 25% of a device's `global_mem_size` skip the same way.
25+
26+
## Compilation caching
27+
28+
`benchmarks/__init__.py` disables the persistent JIT cache
29+
(`SYCL_CACHE_PERSISTENT=0`). `time_bundle_from_source_cold` mints a unique
30+
kernel name per call to defeat the in-memory cache too; `_warm` reuses one
31+
name to measure the cache-hit path.
32+
33+
`create_kernel_bundle_from_source` runs on the OpenCL backend only.
34+
`create_kernel_bundle_from_sycl_source` needs a device where
35+
`can_compile("sycl")` is true.
36+
37+
## Running
38+
39+
```bash
40+
pip install ".[benchmark]"
41+
cd benchmarks && asv machine --yes && asv run --python=same --quick HEAD^!
42+
```
43+
44+
One module: `asv run --python=same --quick --bench bench_compile HEAD^!`
45+
46+
Compare commits: `asv continuous --python=same HEAD~1 HEAD`
47+
48+
View results: `asv publish && asv preview`

benchmarks/asv.conf.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"version": 1,
3+
"project": "dpctl",
4+
"project_url": "https://github.com/IntelPython/dpctl",
5+
"show_commit_url": "https://github.com/IntelPython/dpctl/commit/",
6+
"repo": "..",
7+
"branches": [
8+
"master",
9+
"dev-milestone"
10+
],
11+
"environment_type": "conda",
12+
"conda_channels": [
13+
"https://software.repos.intel.com/python/conda/",
14+
"conda-forge"
15+
],
16+
"benchmark_dir": "benchmarks",
17+
"env_dir": ".asv/env",
18+
"results_dir": ".asv/results",
19+
"html_dir": ".asv/html",
20+
"build_cache_size": 2,
21+
"default_benchmark_timeout": 900,
22+
"regressions_thresholds": {
23+
".*": 0.2
24+
}
25+
}

benchmarks/benchmarks/__init__.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Data Parallel Control (dpctl)
2+
#
3+
# Copyright 2026 Intel Corporation
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""ASV benchmarks for dpctl."""
18+
19+
import os
20+
21+
# Disable the persistent JIT cache before dpctl is imported, so cold-compile
22+
# benchmarks in bench_compile.py measure a real compile, not a cache hit.
23+
os.environ.setdefault("SYCL_CACHE_PERSISTENT", "0")

benchmarks/benchmarks/_utils.py

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
# Data Parallel Control (dpctl)
2+
#
3+
# Copyright 2026 Intel Corporation
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Shared utilities for dpctl benchmarks.
18+
19+
Every benchmark that needs a device goes through :func:`queue_for` or
20+
:func:`device_for` so that benchmark names stay identical on every node in
21+
the pool: a node without a given device reports the parameter as skipped
22+
rather than failing the whole suite.
23+
"""
24+
25+
import os
26+
27+
from asv_runner.benchmarks.mark import SkipNotImplemented
28+
29+
import dpctl
30+
31+
# Device selectors
32+
_SELECTORS = ["opencl:cpu", "level_zero:gpu"]
33+
34+
# Allocation and transfer sizes, in bytes.
35+
_SIZES = [4 * 1024, 1024**2, 16 * 1024**2, 256 * 1024**2]
36+
37+
# USM kinds.
38+
_USM_TYPES = ["device", "host", "shared"]
39+
40+
# Max fraction of device memory a single allocation may claim.
41+
_MEM_BUDGET = 0.25
42+
43+
_queues = {}
44+
_devices = {}
45+
_spirv = None
46+
47+
48+
def queue_for(selector):
49+
"""Return a memoized queue for *selector*, or skip when unavailable."""
50+
if selector not in _queues:
51+
try:
52+
_queues[selector] = dpctl.SyclQueue(selector)
53+
except dpctl.SyclQueueCreationError:
54+
_queues[selector] = None
55+
q = _queues[selector]
56+
if q is None:
57+
raise SkipNotImplemented(f"no {selector} device available")
58+
return q
59+
60+
61+
def device_for(selector):
62+
"""Return a memoized device for *selector*, or skip when unavailable."""
63+
if selector not in _devices:
64+
try:
65+
_devices[selector] = dpctl.SyclDevice(selector)
66+
except dpctl.SyclDeviceCreationError:
67+
_devices[selector] = None
68+
d = _devices[selector]
69+
if d is None:
70+
raise SkipNotImplemented(f"no {selector} device available")
71+
return d
72+
73+
74+
def usm_class(usm_type):
75+
"""Return the dpctl.memory class allocating *usm_type* memory."""
76+
import dpctl.memory as dpm
77+
78+
return {
79+
"device": dpm.MemoryUSMDevice,
80+
"host": dpm.MemoryUSMHost,
81+
"shared": dpm.MemoryUSMShared,
82+
}[usm_type]
83+
84+
85+
def skip_unless_fits(queue, nbytes):
86+
"""Skip when *nbytes* exceeds this device's allocation budget."""
87+
budget = _MEM_BUDGET * queue.sycl_device.global_mem_size
88+
if nbytes > budget:
89+
raise SkipNotImplemented(
90+
f"{nbytes} bytes exceeds the device memory budget"
91+
)
92+
93+
94+
def opencl_queue_or_skip():
95+
"""Return a memoized OpenCL queue, or skip.
96+
97+
``create_kernel_bundle_from_source`` only supports the OpenCL backend.
98+
"""
99+
return queue_for("opencl")
100+
101+
102+
def sycl_source_queue_or_skip(selector):
103+
"""Return a queue whose device can compile SYCL source, or skip."""
104+
try:
105+
import dpctl.compiler as dpc
106+
except ImportError:
107+
raise SkipNotImplemented("dpctl.compiler is not available")
108+
q = queue_for(selector)
109+
if not dpc.is_sycl_source_compilation_available():
110+
raise SkipNotImplemented("SYCL source compilation extension absent")
111+
if not q.sycl_device.can_compile("sycl"):
112+
raise SkipNotImplemented("device cannot compile SYCL source")
113+
return q
114+
115+
116+
def spirv_bytes():
117+
"""Return the SPIR-V module shipped with the installed dpctl, or skip.
118+
119+
Defines ``add(int*, int*, int*)`` and ``axpy(int*, int*, int*, int)``.
120+
"""
121+
global _spirv
122+
if _spirv is None:
123+
path = os.path.join(
124+
os.path.dirname(os.path.abspath(dpctl.__file__)),
125+
"tests",
126+
"input_files",
127+
"multi_kernel.spv",
128+
)
129+
if not os.path.exists(path):
130+
raise SkipNotImplemented(f"SPIR-V module not found at {path}")
131+
with open(path, "rb") as fh:
132+
_spirv = fh.read()
133+
return _spirv
134+
135+
136+
def ocl_axpy_source(kernel_name="axpy"):
137+
"""Return OpenCL C source for an axpy kernel called *kernel_name*."""
138+
return (
139+
f"kernel void {kernel_name}("
140+
" global int *a, global int *b, global int *c, int d) {"
141+
" size_t index = get_global_id(0);"
142+
" c[index] = d * a[index] + b[index];"
143+
"}"
144+
)
145+
146+
147+
def sycl_axpy_source(kernel_name="axpy"):
148+
"""Return SYCL source for an axpy function called *kernel_name*."""
149+
return f"""
150+
#include <sycl/sycl.hpp>
151+
152+
extern "C" SYCL_EXTERNAL
153+
void {kernel_name}(int* a, int* b, int* c, int d, size_t i) {{
154+
c[i] = d * a[i] + b[i];
155+
}}
156+
"""
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Data Parallel Control (dpctl)
2+
#
3+
# Copyright 2026 Intel Corporation
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Benchmarks for kernel bundle creation.
18+
19+
Cold and warm compilation are separate benchmarks. Cold gives every call a
20+
kernel name no compiler has seen, so neither the in-memory nor the
21+
persistent cache (disabled in ``benchmarks/__init__.py``) can serve it; warm
22+
re-submits identical source to measure the cache-hit path instead.
23+
"""
24+
25+
import itertools
26+
27+
from asv_runner.benchmarks.mark import SkipNotImplemented
28+
29+
try:
30+
import dpctl.compiler as dpc
31+
except ImportError:
32+
dpc = None
33+
34+
from ._utils import (
35+
_SELECTORS,
36+
ocl_axpy_source,
37+
opencl_queue_or_skip,
38+
queue_for,
39+
spirv_bytes,
40+
sycl_axpy_source,
41+
sycl_source_queue_or_skip,
42+
)
43+
44+
45+
class BundleFromSPIRV:
46+
"""Kernel bundle from a pre-compiled SPIR-V module."""
47+
48+
params = [_SELECTORS]
49+
param_names = ["selector"]
50+
51+
def setup(self, selector):
52+
if dpc is None:
53+
raise SkipNotImplemented("dpctl.compiler is not available")
54+
self.queue = queue_for(selector)
55+
self.spirv = spirv_bytes()
56+
self.bundle = dpc.create_kernel_bundle_from_spirv(
57+
self.queue, self.spirv
58+
)
59+
60+
def time_bundle_from_spirv(self, selector):
61+
dpc.create_kernel_bundle_from_spirv(self.queue, self.spirv)
62+
63+
def time_get_sycl_kernel(self, selector):
64+
self.bundle.get_sycl_kernel("axpy")
65+
66+
def time_has_sycl_kernel(self, selector):
67+
self.bundle.has_sycl_kernel("axpy")
68+
69+
70+
class BundleFromOpenCLSource:
71+
"""Kernel bundle built from OpenCL C source (OpenCL backend only)."""
72+
73+
timeout = 300
74+
number = 1
75+
repeat = 3
76+
warmup_time = 0
77+
78+
def setup(self):
79+
if dpc is None:
80+
raise SkipNotImplemented("dpctl.compiler is not available")
81+
self.queue = opencl_queue_or_skip()
82+
self.counter = itertools.count()
83+
self.warm_source = ocl_axpy_source()
84+
dpc.create_kernel_bundle_from_source(self.queue, self.warm_source)
85+
86+
def time_bundle_from_source_cold(self):
87+
name = f"axpy_{next(self.counter)}"
88+
dpc.create_kernel_bundle_from_source(self.queue, ocl_axpy_source(name))
89+
90+
def time_bundle_from_source_warm(self):
91+
dpc.create_kernel_bundle_from_source(self.queue, self.warm_source)
92+
93+
94+
class BundleFromSYCLSource:
95+
"""Kernel bundle built from SYCL source via the kernel_compiler
96+
extension.
97+
98+
Skipped unless the extension is present and the device reports it can
99+
compile SYCL source.
100+
"""
101+
102+
params = [_SELECTORS]
103+
param_names = ["selector"]
104+
timeout = 600
105+
number = 1
106+
repeat = 2
107+
warmup_time = 0
108+
109+
def setup(self, selector):
110+
if dpc is None:
111+
raise SkipNotImplemented("dpctl.compiler is not available")
112+
self.queue = sycl_source_queue_or_skip(selector)
113+
self.counter = itertools.count()
114+
self.warm_source = sycl_axpy_source()
115+
dpc.create_kernel_bundle_from_sycl_source(self.queue, self.warm_source)
116+
117+
def time_bundle_from_sycl_source_cold(self, selector):
118+
name = f"axpy_{next(self.counter)}"
119+
dpc.create_kernel_bundle_from_sycl_source(
120+
self.queue, sycl_axpy_source(name)
121+
)
122+
123+
def time_bundle_from_sycl_source_warm(self, selector):
124+
dpc.create_kernel_bundle_from_sycl_source(self.queue, self.warm_source)
125+
126+
127+
class SourceCompilationProbe:
128+
"""Cost of the availability probes themselves.
129+
130+
Called by consumers before every compilation attempt.
131+
"""
132+
133+
params = [_SELECTORS]
134+
param_names = ["selector"]
135+
136+
def setup(self, selector):
137+
self.device = queue_for(selector).sycl_device
138+
139+
def time_can_compile(self, selector):
140+
self.device.can_compile("sycl")

0 commit comments

Comments
 (0)