Implement run_shell in Starlark

shell/BUILD

@@ -1,29 +1,55 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
 # A runtime toolchain for shell scripts.
 #
-# Use `sh_toolchain` to register a toolchain for this type.
+# Used at runtime by sh_binary and sh_test targets.
+#
+# Register `sh_toolchain` targets for this type.
 #
 # Every toolchain registered for this type has the following attributes:
 # - `path`: The path to the shell interpreter for the target platform.
 #
-# Other attribute may be present but are considered implementation details of
+# Other attributes may be present but are considered implementation details of
 # Bazel's sh_* rules.
 #
 # Toolchains registered for this type should have target constraints.
-
-load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-
 toolchain_type(
     name = "toolchain_type",
     visibility = ["//visibility:public"],
 )
 
+# An exec toolchain for shell commands.
+#
+# Used in rule implementations that use `run_shell` to run shell commands at build time.
+#
+# Register `sh_exec_toolchain` targets for this type.
+#
+# Every toolchain registered for this type has the following attributes:
+# - `path`: The path to the shell interpreter for the execution platform.
+# - `max_command_length`: The maximum command length before spilling to a helper script.
+#
+# Other attributes may be present but are considered implementation details of
+# the `run_shell` implementation.
+#
+# Toolchains registered for this type should have exec constraints.
+toolchain_type(
+    name = "exec_toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
 bzl_library(
     name = "rules_bzl",
     srcs = [
+        "run_shell.bzl",
         "sh_binary.bzl",
+        "sh_binary_info.bzl",
+        "sh_info.bzl",
         "sh_library.bzl",
         "sh_test.bzl",
     ],
     visibility = ["//visibility:public"],
-    deps = ["//shell/private:private_bzl"],
+    deps = [
+        "//shell/private:private_bzl",
+        "//shell/toolchains:toolchains_bzl",
+    ],
 )

shell/private/BUILD

@@ -4,6 +4,7 @@ bzl_library(
     name = "private_bzl",
     srcs = [
         "providers.bzl",
+        "run_shell.bzl",
         "sh_binary.bzl",
         "sh_executable.bzl",
         "sh_library.bzl",

shell/private/repositories/sh_config.bzl

@@ -23,31 +23,52 @@ _DEFAULT_SHELL_PATHS = {
     "openbsd": "/usr/local/bin/bash",
 }
 
-_UNIX_SH_TOOLCHAIN_TEMPLATE = """
+_UNIX_SH_TOOLCHAINS_TEMPLATE = """
 sh_toolchain(
     name = "{os}_sh",
     path = {sh_path},
 )
+
+sh_exec_toolchain(
+    name = "{os}_sh_exec",
+    path = {sh_path},
+    max_command_length = 64000,
+) if {sh_path} else None
 """
 
-_WINDOWS_SH_TOOLCHAIN_TEMPLATE = """
+_WINDOWS_SH_TOOLCHAINS_TEMPLATE = """
 sh_toolchain(
     name = "{os}_sh",
     path = {sh_path},
     launcher = "@bazel_tools//tools/launcher",
     launcher_maker = "@bazel_tools//tools/launcher:launcher_maker",
 )
+
+sh_exec_toolchain(
+    name = "{os}_sh_exec",
+    path = {sh_path},
+    max_command_length = 8000,
+) if {sh_path} else None
 """
 
-_TOOLCHAIN_TEMPLATE = """
+_TOOLCHAINS_TEMPLATE = """
 toolchain(
     name = "{os}_sh_toolchain",
     toolchain = ":{os}_sh",
-    toolchain_type = "@rules_shell//shell:toolchain_type",
+    toolchain_type = SH_TOOLCHAIN_TYPE,
     target_compatible_with = [
         "@platforms//os:{os}",
     ],
 )
+
+toolchain(
+    name = "{os}_sh_exec_toolchain",
+    toolchain = ":{os}_sh_exec",
+    toolchain_type = SH_EXEC_TOOLCHAIN_TYPE,
+    exec_compatible_with = [
+        "@platforms//os:{os}",
+    ],
+) if {sh_path} else None
 """
 
 def _sh_config_impl(repository_ctx):
@@ -66,23 +87,26 @@ def _sh_config_impl(repository_ctx):
         if is_host:
             # This toolchain was first added before optional toolchains were
             # available, so instead of not registering a toolchain if we
-            # couldn't find the shell, we register a toolchain with an empty
-            # path.
+            # couldn't find the shell, we register a runtime toolchain with an
+            # empty path. The exec toolchain is new and not registered if no
+            # shell is found.
             sh_path = _detect_local_shell_path(repository_ctx) or ""
         else:
             sh_path = default_shell_path
 
-        sh_toolchain_template = _WINDOWS_SH_TOOLCHAIN_TEMPLATE if os == "windows" else _UNIX_SH_TOOLCHAIN_TEMPLATE
+        sh_toolchain_template = _WINDOWS_SH_TOOLCHAINS_TEMPLATE if os == "windows" else _UNIX_SH_TOOLCHAINS_TEMPLATE
         toolchains.append(sh_toolchain_template.format(
             os = os,
             sh_path = repr(sh_path),
         ))
-        toolchains.append(_TOOLCHAIN_TEMPLATE.format(
+        toolchains.append(_TOOLCHAINS_TEMPLATE.format(
             os = os,
+            sh_path = repr(sh_path),
         ))
 
     repository_ctx.file("BUILD", """
-load("@rules_shell//shell/toolchains:sh_toolchain.bzl", "sh_toolchain")
+load("@rules_shell//shell/toolchains:sh_toolchain.bzl", "sh_toolchain", "SH_TOOLCHAIN_TYPE")
+load("@rules_shell//shell/toolchains:sh_exec_toolchain.bzl", "sh_exec_toolchain", "SH_EXEC_TOOLCHAIN_TYPE")
 """ + "\n".join(toolchains))
 
 sh_config = repository_rule(

shell/private/run_shell.bzl

@@ -0,0 +1,134 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A wrapper around `ctx.actions.run` that runs a shell command.
+
+This mirrors the behavior of the native `ctx.actions.run_shell` function.
+
+Rules calling `run_shell` must depend on the shell exec toolchain via
+`toolchains = [SH_EXEC_TOOLCHAIN_TYPE]` (loaded from
+`@rules_shell//shell/toolchains:sh_exec_toolchain.bzl`).
+"""
+
+load("@rules_shell//shell/toolchains:sh_exec_toolchain.bzl", "SH_EXEC_TOOLCHAIN_TYPE")
+
+visibility("public")
+
+def run_shell(
+        ctx,
+        *,
+        command,
+        outputs,
+        inputs = [],
+        tools = [],
+        arguments = [],
+        mnemonic = None,
+        progress_message = None,
+        use_default_shell_env = False,
+        env = None,
+        execution_requirements = None,
+        exec_group = None,
+        shadowed_action = None,
+        resource_set = None):
+    """Creates an action that runs a shell command.
+
+    Args:
+        ctx: The rule context. The calling rule must depend on the shell exec
+            toolchain type via `toolchains = [SH_EXEC_TOOLCHAIN_TYPE]` (loaded
+            from `@rules_shell//shell/toolchains:sh_exec_toolchain.bzl`).
+        command: Shell command to execute. Unlike the native
+            `ctx.actions.run_shell`, only a string is accepted; passing a
+            sequence of strings is deprecated and rejected. The command is
+            executed as `sh -c <command> "" <arguments>`, which makes the
+            `arguments` available as `$1`, `$2`, etc. If an `Args` object of
+            unknown size is passed as part of `arguments`, then the strings
+            will be at unknown indices; in this case the `$@` shell
+            substitution (retrieve all arguments) may be useful.
+        outputs: List of the output files of the action.
+        inputs: List or depset of the input files of the action.
+        tools: List or depset of any tools needed by the action. Tools are
+            executable inputs that may have their own runfiles which are
+            automatically made available to the action.
+        arguments: Command line arguments of the action. Must be a list of
+            strings or `actions.args()` objects. Bazel passes the elements in
+            this attribute as arguments to the command. The command can access
+            these arguments using shell variable substitutions such as `$1`,
+            `$2`, etc. Note that since `Args` objects are flattened before
+            indexing, if there is an `Args` object of unknown size then all
+            subsequent strings will be at unpredictable indices.
+        mnemonic: A one-word description of the action, for example,
+            CppCompile or GoLink.
+        progress_message: Progress message to show to the user during the
+            build, for example, "Compiling foo.cc to create foo.o". The message
+            may contain `%{label}`, `%{input}`, or `%{output}` patterns, which
+            are substituted with label string, first input, or output's path,
+            respectively. Prefer to use patterns instead of static strings,
+            because the former are more efficient.
+        use_default_shell_env: Whether the action should use the default shell
+            environment, which consists of a few OS-dependent variables as well
+            as variables set via `--action_env`. If both `use_default_shell_env`
+            and `env` are set, values set in `env` will overwrite the default
+            shell environment.
+        env: Sets the dictionary of environment variables.
+        execution_requirements: Information for scheduling the action.
+        exec_group: The execution group for the action. The shell exec toolchain
+            will be obtained from this group.
+        shadowed_action: An action whose inputs and environment are made
+            available to this action in addition to its own.
+        resource_set: A callback returning a dictionary of resource estimates
+            (e.g. memory, CPU) for scheduling the action locally.
+    """
+    if type(command) != type(""):
+        fail("'command' must be of type string, got %s" % type(command))
+
+    toolchains = ctx.exec_groups[exec_group] if exec_group else ctx.toolchains
+    sh_exec_toolchain = toolchains[SH_EXEC_TOOLCHAIN_TYPE].sh_exec_toolchain
+
+    interpreter_args = ctx.actions.args()
+    if len(command) <= sh_exec_toolchain.max_command_length:
+        interpreter_args.add("-c")
+        interpreter_args.add(command)
+
+        # Preserve the long-standing behavior of `ctx.actions.run_shell` passing
+        # an empty string as $0 (if any args are passed).
+        if arguments:
+            interpreter_args.add("")
+    else:
+        # Spill the command into a helper script and reference it instead. The
+        # script is executed directly rather than via `<interpeter> -c` so that
+        # the given interpreter is used without the need to synthesize a
+        # shebang.
+        interpreter_args.use_param_file("%s", use_always = True)
+        interpreter_args.set_param_file_format("multiline")
+        interpreter_args.add(command)
+
+    # shadowed_action doesn't allow an explicit None value.
+    run_kwargs = {"shadowed_action": shadowed_action} if shadowed_action else {}
+
+    ctx.actions.run(
+        executable = sh_exec_toolchain.interpreter,
+        arguments = [interpreter_args] + arguments,
+        outputs = outputs,
+        inputs = inputs,
+        tools = tools,
+        mnemonic = mnemonic,
+        progress_message = progress_message,
+        use_default_shell_env = use_default_shell_env,
+        env = env,
+        execution_requirements = execution_requirements,
+        toolchain = SH_EXEC_TOOLCHAIN_TYPE,
+        exec_group = exec_group,
+        resource_set = resource_set,
+        **run_kwargs
+    )

shell/private/sh_executable.bzl

@@ -14,12 +14,11 @@
 
 """Common code for sh_binary and sh_test rules."""
 
+load("//shell/toolchains:sh_toolchain.bzl", "SH_TOOLCHAIN_TYPE")
 load(":providers.bzl", "ShBinaryInfo", "ShInfo")
 
 visibility(["//shell"])
 
-_SH_TOOLCHAIN_TYPE = Label("//shell:toolchain_type")
-
 def _to_rlocation_path(ctx, file):
     if file.short_path.startswith("../"):
         return file.short_path[3:]
@@ -47,7 +46,7 @@ def _sh_executable_impl(ctx):
             # error.
             shebang = ""
         else:
-            shell = ctx.toolchains[_SH_TOOLCHAIN_TYPE].path
+            shell = ctx.toolchains[SH_TOOLCHAIN_TYPE].path
             shebang = "#!{}".format(shell)
         ctx.actions.write(
             entrypoint,
@@ -159,7 +158,7 @@ def _create_windows_exe_launcher(ctx, sh_toolchain, primary_output):
         outputs = [bash_launcher],
         arguments = [launcher_artifact.path, launch_info, bash_launcher.path],
         use_default_shell_env = True,
-        toolchain = _SH_TOOLCHAIN_TYPE,
+        toolchain = SH_TOOLCHAIN_TYPE,
     )
     return bash_launcher
 
@@ -171,14 +170,14 @@ def _launcher_for_windows(ctx, primary_output, main_file):
             fail("Source file is a Windows executable file, target name extension should match source file extension")
 
     # bazel_tools should always registers a toolchain for Windows, but it may have an empty path.
-    sh_toolchain = ctx.toolchains[_SH_TOOLCHAIN_TYPE]
+    sh_toolchain = ctx.toolchains[SH_TOOLCHAIN_TYPE]
     if not sh_toolchain or not sh_toolchain.path:
         # Let fail print the toolchain type with an apparent repo name.
         fail(
             """No suitable shell toolchain found:
 * if you are running Bazel on Windows, set the BAZEL_SH environment variable to the path of bash.exe
 * if you are running Bazel on a non-Windows platform but are targeting Windows, register an sh_toolchain for the""",
-            _SH_TOOLCHAIN_TYPE,
+            SH_TOOLCHAIN_TYPE,
             "toolchain type",
         )
 
@@ -232,7 +231,7 @@ most build rules</a>.
             ),
         } | extra_attrs,
         toolchains = [
-            config_common.toolchain_type(_SH_TOOLCHAIN_TYPE, mandatory = False),
+            config_common.toolchain_type(SH_TOOLCHAIN_TYPE, mandatory = False),
         ],
         provides = [ShBinaryInfo],
         **kwargs

shell/run_shell.bzl

@@ -0,0 +1,21 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A wrapper around `ctx.actions.run` that runs a shell command."""
+
+load("//shell/private:run_shell.bzl", _run_shell = "run_shell")
+
+visibility("public")
+
+run_shell = _run_shell

shell/toolchains/BUILD

@@ -0,0 +1,10 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+bzl_library(
+    name = "toolchains_bzl",
+    srcs = [
+        "sh_exec_toolchain.bzl",
+        "sh_toolchain.bzl",
+    ],
+    visibility = ["//visibility:public"],
+)