fix: do not include transitive sources from pyi_deps in runtime runfiles

docs/api/rules_python/python/config_settings/index.md

@@ -242,6 +242,22 @@ The `auto` value
 The `omit_if_generated_source` value was removed
 ::::
 
+::::{bzl:flag} pyi_deps_to_runfiles
+Controls whether dependencies listed in `pyi_deps` are propagated to runtime
+runfiles.
+
+Values:
+
+* `auto`: (default) Automatically decide the effective value based on the
+  compilation mode. In `opt` and standard builds, `pyi_deps` are pruned from
+  runtime runfiles.
+* `yes`: Propagate `pyi_deps` into runtime runfiles (legacy behavior).
+* `no`: Do not propagate `pyi_deps` into runtime runfiles.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+::::
+
 ::::{bzl:flag} validate_test_main
 Determines if `py_test` runs a build-time validation that its main module
 actually runs tests.

news/4178.fixed.md

@@ -0,0 +1,6 @@
+(rules) Fixed {obj}`py_library.pyi_deps` being included in runtime output and
+bloating output. Type-checking only information is now in
+{obj}`PyInfo.type_checking_info`. Added the
+{obj}`--//python/config_settings:pyi_deps_to_runfiles` flag to configure or
+restore legacy runfile propagation if needed
+([#4178](https://github.com/bazel-contrib/rules_python/pull/4178)).

python/config_settings/BUILD.bazel

@@ -11,6 +11,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "PyiDepsToRunfilesFlag",
     "ValidateTestMainFlag",
     "VenvsSitePackages",
     "VenvsUseDeclareSymlinkFlag",
@@ -86,6 +87,14 @@ string_flag(
     visibility = NOT_ACTUALLY_PUBLIC,
 )
 
+string_flag(
+    name = "pyi_deps_to_runfiles",
+    build_setting_default = PyiDepsToRunfilesFlag.AUTO,
+    values = PyiDepsToRunfilesFlag.flag_values(),
+    # NOTE: Only public because it's an implicit dependency
+    visibility = NOT_ACTUALLY_PUBLIC,
+)
+
 string_flag(
     name = "precompile_source_retention",
     build_setting_default = PrecompileSourceRetentionFlag.AUTO,

python/private/BUILD.bazel

@@ -322,6 +322,7 @@ bzl_library(
     deps = [
         ":builders",
         ":cc_helper",
+        ":flags",
         ":py_cc_link_params_info",
         ":py_info",
         ":py_internal",

python/private/attributes.bzl

@@ -307,7 +307,8 @@ Dependencies providing type definitions the library needs.
 
 These are dependencies that satisfy imports guarded by `typing.TYPE_CHECKING`.
 These are build-time only dependencies and not included as part of a runnable
-program (packaging rules may include them, however).
+program (packaging rules may include them, however). Runtime inclusion can be
+configured using {obj}`--//python/config_settings:pyi_deps_to_runfiles`.
 
 :::{versionadded} 1.1.0
 :::
@@ -367,6 +368,10 @@ Allowed `.pyc` and directory inputs in `srcs`.
             default = labels.PRECOMPILE_SOURCE_RETENTION,
             providers = [BuildSettingInfo],
         ),
+        "_pyi_deps_to_runfiles_flag": lambda: attrb.Label(
+            default = labels.PYI_DEPS_TO_RUNFILES,
+            providers = [BuildSettingInfo],
+        ),
         # Force enabling auto exec groups, see
         # https://bazel.build/extending/auto-exec-groups#how-enable-particular-rule
         "_use_auto_exec_groups": lambda: attrb.Bool(

python/private/common.bzl

@@ -21,6 +21,7 @@ load("//python/private:py_interpreter_program.bzl", "PyInterpreterProgramInfo")
 load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", "LAUNCHER_MAKER_TOOLCHAIN_TYPE")
 load(":builders.bzl", "builders")
 load(":cc_helper.bzl", "cc_helper")
+load(":flags.bzl", "PyiDepsToRunfilesFlag")
 load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo")
 load(":py_info.bzl", "PyInfo", "PyInfoBuilder")
 load(":py_internal.bzl", "py_internal")
@@ -409,10 +410,18 @@ def create_py_info(
                 if is_py_source(f):
                     py_info.transitive_sources.add(f)
                 py_info.merge_uses_shared_libraries(cc_helper.is_valid_shared_library_artifact(f))
+    propagate_pyi_deps = (
+        hasattr(ctx.attr, "_pyi_deps_to_runfiles_flag") and
+        PyiDepsToRunfilesFlag.is_enabled(ctx)
+    )
     for target in ctx.attr.pyi_deps:
         # PyInfo may not be present e.g. cc_library rules.
         if PyInfo in target or (BuiltinPyInfo != None and BuiltinPyInfo in target):
-            py_info.merge(_get_py_info(target))
+            target_py_info = _get_py_info(target)
+            if propagate_pyi_deps:
+                py_info.merge(target_py_info)
+            else:
+                py_info.merge_type_checking(target_py_info)
 
     py_info.transitive_sources.add(required_py_files)
 

python/private/common_labels.bzl

@@ -23,6 +23,7 @@ labels = struct(
     PRECOMPILE = str(Label("//python/config_settings:precompile")),
     PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention")),
     PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection")),
+    PYI_DEPS_TO_RUNFILES = str(Label("//python/config_settings:pyi_deps_to_runfiles")),
     PYTHON_IMPORT_ALL_REPOSITORIES = str(Label("//python/config_settings:experimental_python_import_all_repositories")),
     PYTHON_SRC = str(Label("//python/bin:python_src")),
     PYTHON_VERSION = str(Label("//python/config_settings:python_version")),

python/private/flags.bzl

@@ -108,6 +108,27 @@ ValidateTestMainFlag = FlagEnum(
     is_enabled = _ValidateTestMainFlag_is_enabled,
 )
 
+def _PyiDepsToRunfilesFlag_is_enabled(ctx):
+    value = ctx.attr._pyi_deps_to_runfiles_flag[BuildSettingInfo].value
+    if value == PyiDepsToRunfilesFlag.AUTO:
+        # In 'opt' compilation mode, prune pyi_deps from runtime runfiles.
+        if ctx.var.get("COMPILATION_MODE") == "opt":
+            return False
+        return False
+    return value == PyiDepsToRunfilesFlag.YES
+
+# Determines if pyi_deps are propagated to runtime runfiles.
+# buildifier: disable=name-conventions
+PyiDepsToRunfilesFlag = FlagEnum(
+    # Automatically decide the effective value based on compilation mode.
+    AUTO = "auto",
+    # Propagate pyi_deps into runtime runfiles (legacy behavior).
+    YES = "yes",
+    # Do not propagate pyi_deps into runtime runfiles.
+    NO = "no",
+    is_enabled = _PyiDepsToRunfilesFlag_is_enabled,
+)
+
 def _string_flag_impl(ctx):
     if ctx.attr.override:
         value = ctx.attr.override

python/private/py_info.bzl

@@ -375,6 +375,7 @@ def _PyInfo_init(
         transitive_original_sources = depset(),
         direct_pyi_files = depset(),
         transitive_pyi_files = depset(),
+        type_checking_info = None,
         venv_symlinks = depset()):
     _check_arg_type("transitive_sources", "depset", transitive_sources)
 
@@ -396,6 +397,8 @@ def _PyInfo_init(
 
     _check_arg_type("direct_pyi_files", "depset", direct_pyi_files)
     _check_arg_type("transitive_pyi_files", "depset", transitive_pyi_files)
+    if type_checking_info != None:
+        _check_arg_type("type_checking_info", "struct", type_checking_info)
     return {
         "direct_original_sources": direct_original_sources,
         "direct_pyc_files": direct_pyc_files,
@@ -409,6 +412,7 @@ def _PyInfo_init(
         "transitive_pyc_files": transitive_pyc_files,
         "transitive_pyi_files": transitive_pyi_files,
         "transitive_sources": transitive_sources,
+        "type_checking_info": type_checking_info,
         "uses_shared_libraries": uses_shared_libraries,
         "venv_symlinks": venv_symlinks,
     }
@@ -558,6 +562,17 @@ in this depset being **empty**.
 The files are considered necessary for downstream binaries to function;
 previously they were considerd informational and largely unused.
 ::::
+""",
+        "type_checking_info": """
+:type: PyInfo | None
+
+Additional `PyInfo` information needed only for static type checking (for
+example, from `pyi_deps`). This information is not included into the final
+output of a program. Type checkers should merge this into their information to
+augment the analyzed output.
+
+::::{versionadded} VERSION_NEXT_FEATURE
+::::
 """,
         "uses_shared_libraries": """
 :type: bool
@@ -627,17 +642,21 @@ def _PyInfoBuilder_typedef():
     :type: DepsetBuilder[File]
     :::
 
-    :::{field} venv_symlinks
-    :type: DepsetBuilder[tuple[str | None, str]]
-    """
+    :::{field} type_checking_info
+    :type: PyInfoBuilder | None
 
-def _PyInfoBuilder_new():
-    """Creates an instance.
+    Builder for {obj}`PyInfo.type_checking_info`. `None` on nested
+    type-checking builders.
 
-    Returns:
-        {type}`PyInfoBuilder`
+    ::::{versionadded} VERSION_NEXT_FEATURE
+    ::::
+    :::
+
+    :::{field} venv_symlinks
+    :type: DepsetBuilder[tuple[str | None, str]]
     """
 
+def _new_raw_py_info_builder(type_checking_info = None):
     # buildifier: disable=uninitialized
     self = struct(
         _has_py2_only_sources = [False],
@@ -660,6 +679,7 @@ def _PyInfoBuilder_new():
         merge_has_py3_only_sources = lambda *a, **k: _PyInfoBuilder_merge_has_py3_only_sources(self, *a, **k),
         merge_target = lambda *a, **k: _PyInfoBuilder_merge_target(self, *a, **k),
         merge_targets = lambda *a, **k: _PyInfoBuilder_merge_targets(self, *a, **k),
+        merge_type_checking = lambda *a, **k: _PyInfoBuilder_merge_type_checking(self, *a, **k),
         merge_uses_shared_libraries = lambda *a, **k: _PyInfoBuilder_merge_uses_shared_libraries(self, *a, **k),
         set_has_py2_only_sources = lambda *a, **k: _PyInfoBuilder_set_has_py2_only_sources(self, *a, **k),
         set_has_py3_only_sources = lambda *a, **k: _PyInfoBuilder_set_has_py3_only_sources(self, *a, **k),
@@ -670,10 +690,20 @@ def _PyInfoBuilder_new():
         transitive_pyc_files = builders.DepsetBuilder(),
         transitive_pyi_files = builders.DepsetBuilder(),
         transitive_sources = builders.DepsetBuilder(),
+        type_checking_info = type_checking_info,
         venv_symlinks = builders.DepsetBuilder(),
     )
     return self
 
+def _PyInfoBuilder_new():
+    """Creates an instance.
+
+    Returns:
+        {type}`PyInfoBuilder`
+    """
+    type_checking_info = _new_raw_py_info_builder(None)
+    return _new_raw_py_info_builder(type_checking_info)
+
 def _PyInfoBuilder_add_venv_symlink(self):
     """Create and return a new VenvSymlinkEntryBuilder.
 
@@ -819,19 +849,7 @@ def _PyInfoBuilder_merge(self, *infos, direct = []):
     """
     return self.merge_all(list(infos), direct = direct)
 
-def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
-    """Merge other PyInfos into this PyInfo.
-
-    Args:
-        self: implicitly added.
-        transitive: {type}`list[PyInfo]` objects to merge in, but only merge in
-            their information into this object's transitive fields.
-        direct: {type}`list[PyInfo]` objects to merge in, but also merge their
-            direct fields into this object's direct fields.
-
-    Returns:
-        {type}`PyInfoBuilder` self
-    """
+def _merge_py_info_fields(self, transitive, *, direct = []):
     for info in direct:
         # BuiltinPyInfo doesn't have this field
         if hasattr(info, "direct_pyc_files"):
@@ -855,6 +873,77 @@ def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
             self.transitive_pyi_files.add(info.transitive_pyi_files)
             self.venv_symlinks.add(info.venv_symlinks)
 
+def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
+    """Merge other PyInfos into this PyInfo.
+
+    Args:
+        self: implicitly added.
+        transitive: {type}`list[PyInfo]` objects to merge in, but only merge in
+            their information into this object's transitive fields.
+        direct: {type}`list[PyInfo]` objects to merge in, but also merge their
+            direct fields into this object's direct fields.
+
+    Returns:
+        {type}`PyInfoBuilder` self
+    """
+    _merge_py_info_fields(self, transitive, direct = direct)
+    tc_direct = [
+        info.type_checking_info
+        for info in direct
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    tc_transitive = [
+        info.type_checking_info
+        for info in transitive
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    _merge_py_info_fields(self.type_checking_info, tc_transitive, direct = tc_direct)
+
+    return self
+
+def _PyInfoBuilder_merge_type_checking(self, *infos, direct = []):
+    """Merge type-checking fields from other PyInfos into this PyInfo.
+
+    Merges `pyi_files` into this object's `direct_pyi_files` /
+    `transitive_pyi_files` and merges the full `PyInfo` (and any nested
+    `type_checking_info`) into {obj}`type_checking_info`, excluding runtime
+    fields like `imports` and `transitive_sources` from the top-level `PyInfo`.
+
+    :::{versionadded} VERSION_NEXT_FEATURE
+    :::
+
+    Args:
+        self: implicitly added.
+        *infos: {type}`PyInfo` objects to merge in, but only merge in their
+            information into this object's transitive fields.
+        direct: {type}`list[PyInfo]` objects to merge in, but also merge their
+            direct fields into this object's direct fields.
+
+    Returns:
+        {type}`PyInfoBuilder` self
+    """
+    for info in direct:
+        # BuiltinPyInfo doesn't have this field
+        if hasattr(info, "direct_pyi_files"):
+            self.direct_pyi_files.add(info.direct_pyi_files)
+
+    for info in direct + list(infos):
+        # BuiltinPyInfo doesn't have this field
+        if hasattr(info, "transitive_pyi_files"):
+            self.transitive_pyi_files.add(info.transitive_pyi_files)
+
+    tc_direct = direct + [
+        info.type_checking_info
+        for info in direct
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    tc_transitive = list(infos) + [
+        info.type_checking_info
+        for info in infos
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    _merge_py_info_fields(self.type_checking_info, tc_transitive, direct = tc_direct)
+
     return self
 
 def _PyInfoBuilder_merge_target(self, target):
@@ -892,15 +981,7 @@ def _PyInfoBuilder_merge_targets(self, targets):
         self.merge_target(t)
     return self
 
-def _PyInfoBuilder_build(self):
-    """Builds into a {obj}`PyInfo` object.
-
-    Args:
-        self: implicitly added.
-
-    Returns:
-        {type}`PyInfo`
-    """
+def _build_py_info_fields(self, type_checking_info = None):
     venv_symlinks = depset(
         direct = [b.build() for b in self._venv_symlink_builders],
         transitive = [self.venv_symlinks.build()],
@@ -920,9 +1001,22 @@ def _PyInfoBuilder_build(self):
         transitive_original_sources = self.transitive_original_sources.build(),
         transitive_pyc_files = self.transitive_pyc_files.build(),
         transitive_pyi_files = self.transitive_pyi_files.build(),
+        type_checking_info = type_checking_info,
         venv_symlinks = venv_symlinks,
     )
 
+def _PyInfoBuilder_build(self):
+    """Builds into a {obj}`PyInfo` object.
+
+    Args:
+        self: implicitly added.
+
+    Returns:
+        {type}`PyInfo`
+    """
+    type_checking_info = _build_py_info_fields(self.type_checking_info, None)
+    return _build_py_info_fields(self, type_checking_info)
+
 def _PyInfoBuilder_build_builtin_py_info(self):
     """Builds into a Bazel-builtin PyInfo object, if available.
 
@@ -961,6 +1055,7 @@ PyInfoBuilder = struct(
     merge_has_py3_only_sources = _PyInfoBuilder_merge_has_py3_only_sources,
     merge_target = _PyInfoBuilder_merge_target,
     merge_targets = _PyInfoBuilder_merge_targets,
+    merge_type_checking = _PyInfoBuilder_merge_type_checking,
     merge_uses_shared_libraries = _PyInfoBuilder_merge_uses_shared_libraries,
     set_has_py2_only_sources = _PyInfoBuilder_set_has_py2_only_sources,
     set_has_py3_only_sources = _PyInfoBuilder_set_has_py3_only_sources,