diff --git a/master/custom/worker_downtime.py b/master/custom/worker_downtime.py new file mode 100644 index 00000000..3f73936b --- /dev/null +++ b/master/custom/worker_downtime.py @@ -0,0 +1,49 @@ +from datetime import datetime, timedelta + + +def no_builds_between(start, end, *, day_of_week=None, tz=None): + """A function for "builder.canStartBuild" that implements planned downtime + + Avoid a build to be started between start and end time and delay such + builds until end time. + """ + + start = datetime.strptime(start, "%H:%M").time() + end = datetime.strptime(end, "%H:%M").time() + def canStartBuild(builder, wfb, request): + now_dt = datetime.now(tz=tz) + if day_of_week is not None and now_dt.weekday() != day_of_week: + return True + now = now_dt.time() + if is_within_time_range(now, start, end): + delay = get_delay(now, end) + # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions + wfb.worker.quarantine_timeout = delay + wfb.worker.putInQuarantine() + # This does not take the worker out of quarantine, it only resets + # the timeout value to default (restarting the default + # exponential backoff) + wfb.worker.resetQuarantine() + return False + # Schedule the build now + return True + return canStartBuild + + +def is_within_time_range(now, start, end): + if start <= end: + return start <= now <= end + else: + return now >= start or now <= end + + +def get_delay(now, end): + today = datetime.today() + now = datetime.combine(today, now) + end = datetime.combine(today, end) + + if now > end: + end += timedelta(days=1) + + difference = end - now + return difference.total_seconds() diff --git a/master/custom/workers.py b/master/custom/workers.py index ebd7dd9b..b1b4ab07 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -3,10 +3,12 @@ # vim:set ts=8 sw=4 sts=4 et: from functools import partial +from zoneinfo import ZoneInfo from buildbot.plugins import worker as _worker from custom.factories import MAIN_BRANCH_NAME +from custom.worker_downtime import no_builds_between # By default, the buildmaster sends a simple, non-blocking message to each @@ -32,6 +34,7 @@ def __init__( parallel_tests=None, timeout_factor=1, exclude_test_resources=None, + downtime=None ): self.name = name self.tags = tags or set() @@ -41,6 +44,7 @@ def __init__( self.parallel_tests = parallel_tests self.timeout_factor = timeout_factor self.exclude_test_resources = exclude_test_resources or [] + self.downtime = downtime worker_settings = settings.workers[name] owner = name.split("-")[0] @@ -55,6 +59,12 @@ def __init__( notify_on_missing=emails, keepalive_interval=KEEPALIVE) +# Some of Itamar's workers are reprovisioned every Wednesday at 9am PT. +# Builds scheduled between 8am - 10am PT on Wednesdays will be delayed to +# 10am PT. +itamaro_downtime = no_builds_between( + "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") +) def get_workers(settings): cpw = partial(CPythonWorker, settings) @@ -160,6 +170,10 @@ def get_workers(settings): tags=['linux', 'unix', 'ubuntu', 'arm', 'arm64', 'aarch64', 'bigmem'], not_branches=['3.10', '3.11', '3.12', '3.13', '3.14'], parallel_tests=8, + # This worker runs pyperformance at 12am UTC. + # If a build is scheduled between 10pm UTC and 2am UTC, + # it will be delayed to 2am UTC. + downtime=no_builds_between("22:00", "2:00") ), cpw( name="cstratak-fedora-rawhide-s390x", @@ -294,6 +308,12 @@ def get_workers(settings): tags=['windows', 'win11', 'amd64', 'x86-64', 'bigmem'], not_branches=['3.10', '3.11', '3.12', '3.13', '3.14'], parallel_tests=4, + # This worker restarts every day at 9am UTC to work around issues + # stemming from failing bigmem tests trashing disk space and + # fragmenting RAM. + # Builds scheduled between 07:20am - 9:20am UTC will be delayed + # to 9:20am UTC. + downtime=no_builds_between("7:20", "9:20") ), cpw( name="itamaro-centos-aws", @@ -301,6 +321,7 @@ def get_workers(settings): not_branches=['3.10', '3.11', '3.12'], parallel_tests=10, parallel_builders=2, + downtime=itamaro_downtime, ), cpw( name="itamaro-win64-srv-22-aws", @@ -308,6 +329,7 @@ def get_workers(settings): not_branches=['3.10', '3.11', '3.12'], parallel_tests=10, parallel_builders=2, + downtime=itamaro_downtime, ), cpw( name="itamaro-macos-intel-aws", diff --git a/master/master.cfg b/master/master.cfg index 174a903f..3c03e74b 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -14,7 +14,7 @@ import os import subprocess import sys -from datetime import datetime, timedelta +from datetime import timedelta from functools import partial from zoneinfo import ZoneInfo @@ -52,6 +52,7 @@ from custom.builders import ( # noqa: E402 STABLE, ONLY_MAIN_BRANCH, ) +from custom.worker_downtime import no_builds_between def set_up_sentry(): @@ -194,57 +195,13 @@ def is_important_change(change): return any(is_important_file(filename) for filename in change.files) -def is_within_time_range(now, start, end): - if start <= end: - return start <= now <= end - else: - return now >= start or now <= end - - -def get_delay(now, end): - today = datetime.today() - now = datetime.combine(today, now) - end = datetime.combine(today, end) - - if now > end: - end += timedelta(days=1) - - difference = end - now - return difference.total_seconds() - - -# Avoid a build to be started between start and end time and delay such build -# at end time -def no_builds_between(start, end, *, day_of_week=None, tz=None): - start = datetime.strptime(start, "%H:%M").time() - end = datetime.strptime(end, "%H:%M").time() - def canStartBuild(builder, wfb, request): - now_dt = datetime.now(tz=tz) - if day_of_week is not None and now_dt.weekday() != day_of_week: - return True - now = now_dt.time() - if is_within_time_range(now, start, end): - delay = get_delay(now, end) - # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions - wfb.worker.quarantine_timeout = delay - wfb.worker.putInQuarantine() - # This does not take the worker out of quarantine, it only resets - # the timeout value to default (restarting the default - # exponential backoff) - wfb.worker.resetQuarantine() - return False - # Schedule the build now - return True - return canStartBuild - - github_status_builders = [] release_status_builders = [] mail_status_builders = [] # Regular builders -for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches): +for git_url, branchname, git_branch in git_branches: buildernames = [] refleakbuildernames = [] for name, worker, buildfactory, stability, tier in BUILDERS: @@ -308,24 +265,8 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches): locks=[cpulock.access("counting")], ) - # This worker runs pyperformance at 12am UTC. If a build is scheduled between - # 10pm UTC and 2am UTC, it will be delayed to 2am UTC. - if worker.name == "diegorusso-aarch64-bigmem": - builder.canStartBuild = no_builds_between("22:00", "2:00") - - # This worker restarts every day at 9am UTC to work around issues stemming from - # failing bigmem tests trashing disk space and fragmenting RAM. Builds scheduled - # between 07:20am - 9:20am UTC will be delayed to 9:20am UTC. - if worker.name == "ambv-bb-win11": - builder.canStartBuild = no_builds_between("7:20", "9:20") - - # These workers are reprovisioned every Wednesday at 9am PT. Builds - # scheduled between 8am - 10am PT on Wednesdays will be delayed to - # 10am PT. - if worker.name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"): - builder.canStartBuild = no_builds_between( - "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") - ) + if worker.downtime: + builder.canStartBuild = worker.downtime c["builders"].append(builder) @@ -397,18 +338,8 @@ for name, worker, buildfactory, stability, tier in BUILDERS: locks=[cpulock.access("counting")], ) - # This worker runs pyperformance at 12am. If a build is scheduled between - # 10pm and 2am, it will be delayed at 2am. - if worker.name == "diegorusso-aarch64-bigmem": - builder.canStartBuild = no_builds_between("22:00", "2:00") - - # These workers are reprovisioned every Wednesday at 9am PT. Builds - # scheduled between 8am - 10am PT on Wednesdays will be delayed to - # 10am PT. - if worker.name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"): - builder.canStartBuild = no_builds_between( - "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") - ) + if worker.downtime: + builder.canStartBuild = worker.downtime c["builders"].append(builder)