diff --git a/tests/rl/unit_tests/cpu/test_generator.py b/tests/rl/unit_tests/cpu/test_generator.py index 3d973912d6f..3caf039fd40 100644 --- a/tests/rl/unit_tests/cpu/test_generator.py +++ b/tests/rl/unit_tests/cpu/test_generator.py @@ -576,8 +576,8 @@ def test_qwen36_27b_perf_config(): config = rl_grpo_qwen3_6_27b_varlen_perf() assert config.hf_assets_path.endswith("Qwen3.6-27B") - assert "offset_rmsnorm" in config.trainer.model.local_compile_regions - assert "offset_rmsnorm" in config.generator.model.local_compile_regions + assert "offset_rmsnorm" in config.trainer.model.local_compile_regions.regions + assert "offset_rmsnorm" in config.generator.model.local_compile_regions.regions assert config.trainer.parallelism.data_parallel_shard_degree == 2 assert config.trainer.parallelism.tensor_parallel_degree == 2 assert config.generator.parallelism.tensor_parallel_degree == 4 diff --git a/tests/rl/unit_tests/cpu/test_policy_loss_configs.py b/tests/rl/unit_tests/cpu/test_policy_loss_configs.py index b79c8ef600b..facd59e2261 100644 --- a/tests/rl/unit_tests/cpu/test_policy_loss_configs.py +++ b/tests/rl/unit_tests/cpu/test_policy_loss_configs.py @@ -75,5 +75,5 @@ def test_batch_invariant_configs_disable_local_compile(module_name, factory_name factory = getattr(importlib.import_module(module_name), factory_name) config = factory() - assert config.trainer.model.local_compile_regions == [] - assert config.generator.model.local_compile_regions == [] + assert config.trainer.model.local_compile_regions.regions == [] + assert config.generator.model.local_compile_regions.regions == [] diff --git a/tests/rl/unit_tests/gpu/test_gdn_cudagraph.py b/tests/rl/unit_tests/gpu/test_gdn_cudagraph.py index 176cbc6c46e..3c3b7b56dfa 100644 --- a/tests/rl/unit_tests/gpu/test_gdn_cudagraph.py +++ b/tests/rl/unit_tests/gpu/test_gdn_cudagraph.py @@ -301,7 +301,7 @@ def recurrent(*args, **kwargs): model_config = build_model_config( "0.8B", seq_len=256, attn_backend="varlen", parallelism=ParallelismConfig() ) - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] registry.register_to_vllm( model_config, parallelism=registry.InferenceParallelismConfig(tensor_parallel_degree=1), diff --git a/tests/unit_tests/cpu/config/test_compile_config.py b/tests/unit_tests/cpu/config/test_compile_config.py index 4cbcf0ce629..f4c697d761f 100644 --- a/tests/unit_tests/cpu/config/test_compile_config.py +++ b/tests/unit_tests/cpu/config/test_compile_config.py @@ -4,24 +4,46 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +from collections.abc import Callable +from typing import Any + import pytest import torch +from torch.nn.attention.flex_attention import create_block_mask, flex_attention from torchtitan.components.loss import compute_logprobs, cross_entropy_loss, mse_loss -from torchtitan.distributed.local_compile import apply_local_compile, local_compile +from torchtitan.distributed.local_compile import local_compile, LocalCompileConfig +from torchtitan.models.common.attention import FlexInnerAttention +from torchtitan.models.common.attention.attention import _compiled_create_block_mask from torchtitan.models.common.moe import _microbatch_load_balance_local_stats +from torchtitan.models.common.vision_encoder import compiled_create_block_mask + + +class CustomDeviceLocalCompileConfig(LocalCompileConfig): + """Example configuration supplied by an out-of-tree backend.""" + + def _compile( + self, + reference: Callable[..., Any], + *, + inductor_options: dict[str, Any] | None = None, + **compile_kwargs: Any, + ) -> Callable[..., Any]: + del inductor_options + compile_kwargs["backend"] = "custom_device_backend" + return torch.compile(reference, **compile_kwargs) @pytest.fixture(autouse=True) def reset_local_compile(): - apply_local_compile([]) + LocalCompileConfig().apply() yield - apply_local_compile([]) + LocalCompileConfig().apply() -def test_apply_local_compile_rejects_unknown_name() -> None: +def test_local_compile_config_rejects_unknown_name() -> None: with pytest.raises(ValueError, match=r"foo.*registered values"): - apply_local_compile(["foo"]) + LocalCompileConfig(regions=["foo"]).apply() def test_local_compile_binds_once_from_existing_config(monkeypatch) -> None: @@ -40,12 +62,13 @@ def compiled(value: int) -> tuple[str, int]: return compiled monkeypatch.setattr(torch, "compile", fake_compile) - apply_local_compile(["test_local_compile"]) + LocalCompileConfig(regions=["test_local_compile"]).apply() assert fn(3) == ("compiled", 3) - assert len(compiled_calls) == 1 - assert "backend" not in compiled_calls[0][1] - assert compiled_calls[0][1]["fullgraph"] is True + named_calls = [call for call in compiled_calls if call[0] is fn.__wrapped__] + assert len(named_calls) == 1 + assert "backend" not in named_calls[0][1] + assert named_calls[0][1]["fullgraph"] is True def test_local_compile_forwards_compile_kwargs(monkeypatch) -> None: @@ -68,10 +91,11 @@ def fake_compile(reference, **kwargs): return reference monkeypatch.setattr(torch, "compile", fake_compile) - apply_local_compile(["test_compile_kwargs"]) + LocalCompileConfig(regions=["test_compile_kwargs"]).apply() assert fn(3) == 3 - assert compiled_calls[0][1] == { + named_calls = [call for call in compiled_calls if call[0] is fn.__wrapped__] + assert named_calls[0][1] == { "backend": "eager", "fullgraph": True, "dynamic": True, @@ -100,7 +124,7 @@ def fn(value: int) -> int: with pytest.raises( ValueError, match=r"test_non_batch_invariant.*local_compile_regions" ): - apply_local_compile(["test_non_batch_invariant"]) + LocalCompileConfig(regions=["test_non_batch_invariant"]).apply() def test_loss_functions_use_local_compile(monkeypatch) -> None: @@ -112,9 +136,18 @@ def fake_compile(reference, **kwargs): return reference monkeypatch.setattr(torch, "compile", fake_compile) - apply_local_compile(["loss"]) - - assert compiled_names == [ + LocalCompileConfig(regions=["loss"]).apply() + + assert [ + name + for name in compiled_names + if name + in { + cross_entropy_loss.__name__, + mse_loss.__name__, + compute_logprobs.__name__, + } + ] == [ cross_entropy_loss.__name__, mse_loss.__name__, compute_logprobs.__name__, @@ -129,11 +162,41 @@ def fake_compile(reference, **kwargs): return reference monkeypatch.setattr(torch, "compile", fake_compile) - apply_local_compile(["moe_aux_loss"]) + LocalCompileConfig(regions=["moe_aux_loss"]).apply() - assert compiled == [ + assert [ + call + for call in compiled + if call[0] is _microbatch_load_balance_local_stats.__wrapped__ + ] == [ ( _microbatch_load_balance_local_stats.__wrapped__, {"fullgraph": True}, ) ] + + +def test_custom_config_controls_all_torchtitan_compile_boundaries( + monkeypatch, +) -> None: + compiled_calls = [] + + @local_compile("test_custom_backend", batch_invariant=False) + def region() -> None: + pass + + def fake_compile(reference, **kwargs): + compiled_calls.append((reference, kwargs)) + return reference + + monkeypatch.setattr(torch, "compile", fake_compile) + CustomDeviceLocalCompileConfig(regions=["test_custom_backend"]).apply() + + assert callable(FlexInnerAttention._compiled_flex_attn) + assert callable(_compiled_create_block_mask) + assert callable(compiled_create_block_mask) + expected = {"backend": "custom_device_backend"} + assert (region.__wrapped__, {**expected, "fullgraph": True}) in compiled_calls + assert (flex_attention, expected) in compiled_calls + assert (create_block_mask, expected) in compiled_calls + assert sum(reference is create_block_mask for reference, _ in compiled_calls) == 2 diff --git a/tests/unit_tests/cpu/config/test_context_parallel_validation.py b/tests/unit_tests/cpu/config/test_context_parallel_validation.py index 0575d338eb8..0a1ced4cf88 100644 --- a/tests/unit_tests/cpu/config/test_context_parallel_validation.py +++ b/tests/unit_tests/cpu/config/test_context_parallel_validation.py @@ -186,6 +186,7 @@ class TestGptOssUlysses(unittest.TestCase): def _parallelize(self, inner_attention): from types import SimpleNamespace + from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.gpt_oss.model import GptOssModel with self.assertRaisesRegex(NotImplementedError, "Ulysses CP"): @@ -198,7 +199,7 @@ def _parallelize(self, inner_attention): parallelism_context=SimpleNamespace(cp_enabled=True), training=None, parallelism=None, - local_compile_regions=[], + local_compile_regions=LocalCompileConfig(), activation_remat=None, dump_folder="", ) diff --git a/tests/unit_tests/cpu/models/common/test_gated_rmsnorm.py b/tests/unit_tests/cpu/models/common/test_gated_rmsnorm.py index 4d1ecbae692..7fd0f24639e 100644 --- a/tests/unit_tests/cpu/models/common/test_gated_rmsnorm.py +++ b/tests/unit_tests/cpu/models/common/test_gated_rmsnorm.py @@ -9,13 +9,13 @@ import torch import torch.nn.functional as F -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common import GatedRMSNorm, Sigmoid, SiLU class TestGatedRMSNorm(unittest.TestCase): def setUp(self): - apply_local_compile([]) + LocalCompileConfig().apply() def test_configurable_activation_matches_reference(self): x = torch.randn(4, 3, 8) diff --git a/tests/unit_tests/cpu/models/common/test_hi_mid_lo_linear.py b/tests/unit_tests/cpu/models/common/test_hi_mid_lo_linear.py index c2664043867..490eee0b823 100644 --- a/tests/unit_tests/cpu/models/common/test_hi_mid_lo_linear.py +++ b/tests/unit_tests/cpu/models/common/test_hi_mid_lo_linear.py @@ -215,7 +215,7 @@ def test_lm_head_converter_swaps_only_lm_head(): # The head keeps 2 pieces: summed over the vocab, a third doesn't help. assert config.lm_head.backward_mode == "hi_mid" # It turns on the split's local_compile region. - assert "fp32_to_bf16_split" in config.local_compile_regions + assert "fp32_to_bf16_split" in config.local_compile_regions.regions # Converting an already converted head is a no-op. converted = config.lm_head LMHeadFP32OutputConverter.Config().build().convert(config) diff --git a/tests/unit_tests/cpu/models/kimi_k3/test_kimi_k3_config.py b/tests/unit_tests/cpu/models/kimi_k3/test_kimi_k3_config.py index e22edb1d94b..f8f3d452320 100644 --- a/tests/unit_tests/cpu/models/kimi_k3/test_kimi_k3_config.py +++ b/tests/unit_tests/cpu/models/kimi_k3/test_kimi_k3_config.py @@ -21,7 +21,7 @@ def test_kimi_k3_enables_attention_residual_local_compile() -> None: config = kimi_k3_debugmodel() model_config = cast(KimiK3Model.Config, config.model) - assert model_config.local_compile_regions == [ + assert model_config.local_compile_regions.regions == [ "loss", "gated_rmsnorm", "fused_binary_activation", diff --git a/tests/unit_tests/cpu/models/qwen3_5/test_offset_rmsnorm.py b/tests/unit_tests/cpu/models/qwen3_5/test_offset_rmsnorm.py index b43336cb5ad..04f095b348f 100644 --- a/tests/unit_tests/cpu/models/qwen3_5/test_offset_rmsnorm.py +++ b/tests/unit_tests/cpu/models/qwen3_5/test_offset_rmsnorm.py @@ -12,7 +12,7 @@ def test_qwen35_architecture_enables_offset_rmsnorm_local_compile() -> None: for config_factory in (qwen35_debugmodel, qwen36_debugmodel, qwen38_debugmodel): config = config_factory() - assert config.model.local_compile_regions == [ + assert config.model.local_compile_regions.regions == [ "loss", "fused_binary_activation", "gated_rmsnorm", diff --git a/tests/unit_tests/cpu/quantization/test_quantization.py b/tests/unit_tests/cpu/quantization/test_quantization.py index 61603ae7ced..f304707c13f 100644 --- a/tests/unit_tests/cpu/quantization/test_quantization.py +++ b/tests/unit_tests/cpu/quantization/test_quantization.py @@ -479,7 +479,7 @@ def test_qwen3_recipes_resolve(monkeypatch, recipe): assert isinstance(dataset.source, HuggingFaceRandomAccessSource.Config) assert dataset.source.path == "openai/gsm8k" assert config.checkpointer.initial_load_in_hf - assert config.model.local_compile_regions == [ + assert config.model.local_compile_regions.regions == [ "loss", "fused_binary_activation", "cos_sin_rope", @@ -987,7 +987,7 @@ def test_deepseek_nvfp4_recipes_preserve_quantization_and_routing( if bf16_tail_fraction == 0 else len(config.model.layers) // 2 ) - assert config.model.local_compile_regions == ["loss"] + assert config.model.local_compile_regions.regions == ["loss"] grouped = list(config.model.traverse(GroupedLinear.Config)) assert grouped quantized_cls = _get_nvfp4_grouped_linear_cls(GroupedLinear) diff --git a/tests/unit_tests/cpu/test_integration_test_definitions.py b/tests/unit_tests/cpu/test_integration_test_definitions.py index 0eb38f0db00..0235821f2b6 100644 --- a/tests/unit_tests/cpu/test_integration_test_definitions.py +++ b/tests/unit_tests/cpu/test_integration_test_definitions.py @@ -54,7 +54,7 @@ def test_spmd_typechecking_config_disables_local_compile() -> None: config = llama3_debugmodel_default() assert config.debug.spmd_typechecking - assert config.model.local_compile_regions == [] + assert config.model.local_compile_regions.regions == [] config.__post_init__() diff --git a/tests/unit_tests/cpu/test_trainer.py b/tests/unit_tests/cpu/test_trainer.py index c79f1b0798f..26b7349b019 100644 --- a/tests/unit_tests/cpu/test_trainer.py +++ b/tests/unit_tests/cpu/test_trainer.py @@ -756,7 +756,9 @@ def test_graph_training_engine_rejects_local_compile_regions() -> None: ): GraphTrainingEngine( config, - model_config=SimpleNamespace(local_compile_regions=["loss"]), + model_config=SimpleNamespace( + local_compile_regions=SimpleNamespace(regions=["loss"]) + ), max_num_documents=None, output_dir="", ) diff --git a/tests/unit_tests/gpu/models/common/test_gated_rmsnorm.py b/tests/unit_tests/gpu/models/common/test_gated_rmsnorm.py index 3d1604aa825..3badec1bc25 100644 --- a/tests/unit_tests/gpu/models/common/test_gated_rmsnorm.py +++ b/tests/unit_tests/gpu/models/common/test_gated_rmsnorm.py @@ -8,17 +8,17 @@ import torch -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common import GatedRMSNorm, Sigmoid @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestGatedRMSNormCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["gated_rmsnorm"]) + LocalCompileConfig(regions=["gated_rmsnorm"]).apply() def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() @staticmethod diff --git a/tests/unit_tests/gpu/models/common/test_hi_mid_lo_linear.py b/tests/unit_tests/gpu/models/common/test_hi_mid_lo_linear.py index 1cd17b19b6b..2821b03c20c 100644 --- a/tests/unit_tests/gpu/models/common/test_hi_mid_lo_linear.py +++ b/tests/unit_tests/gpu/models/common/test_hi_mid_lo_linear.py @@ -18,7 +18,7 @@ from torch.distributed.fsdp import fully_shard, MixedPrecisionPolicy from torch.fx.experimental.proxy_tensor import make_fx -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common import hi_mid_lo_linear from torchtitan.models.common.hi_mid_lo_linear import HiMidLoLinear @@ -177,11 +177,11 @@ def linear(input, weight): @contextlib.contextmanager def _compiled_split(): """Turn on the split's local_compile region, as the models that use HiMidLoLinear do.""" - apply_local_compile(["fp32_to_bf16_split"]) + LocalCompileConfig(regions=["fp32_to_bf16_split"]).apply() try: yield finally: - apply_local_compile([]) + LocalCompileConfig().apply() @pytest.mark.parametrize("dim", [0, 1]) diff --git a/tests/unit_tests/gpu/models/common/test_moe_aux_loss_compile.py b/tests/unit_tests/gpu/models/common/test_moe_aux_loss_compile.py index b5c8145489f..4482a1e710a 100644 --- a/tests/unit_tests/gpu/models/common/test_moe_aux_loss_compile.py +++ b/tests/unit_tests/gpu/models/common/test_moe_aux_loss_compile.py @@ -8,14 +8,14 @@ import torch -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common.moe import _microbatch_load_balance_local_stats @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestMoEAuxLossLocalCompile(unittest.TestCase): def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_match_eager(self): @@ -43,7 +43,7 @@ def test_forward_and_backward_match_eager(self): generator=generator, ) - apply_local_compile([]) + LocalCompileConfig().apply() eager_counts_E, eager_prob_sums_E = _microbatch_load_balance_local_stats( scores_TE, routing_map_TE, @@ -53,7 +53,7 @@ def test_forward_and_backward_match_eager(self): eager_prob_sums_E, scores_TE, grad_prob_sums_E )[0] - apply_local_compile(["moe_aux_loss"]) + LocalCompileConfig(regions=["moe_aux_loss"]).apply() compiled_counts_E, compiled_prob_sums_E = _microbatch_load_balance_local_stats( scores_TE, routing_map_TE, diff --git a/tests/unit_tests/gpu/models/common/test_situglu.py b/tests/unit_tests/gpu/models/common/test_situglu.py index 3bd6f42355a..a9b262c96f3 100644 --- a/tests/unit_tests/gpu/models/common/test_situglu.py +++ b/tests/unit_tests/gpu/models/common/test_situglu.py @@ -8,17 +8,17 @@ import torch -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common.activation import SiTUGLU @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestSiTUGLULocalCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["fused_binary_activation"]) + LocalCompileConfig(regions=["fused_binary_activation"]).apply() def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_emit_triton(self): diff --git a/tests/unit_tests/gpu/models/common/test_swiglu.py b/tests/unit_tests/gpu/models/common/test_swiglu.py index dde74bd4d75..85b09654cb8 100644 --- a/tests/unit_tests/gpu/models/common/test_swiglu.py +++ b/tests/unit_tests/gpu/models/common/test_swiglu.py @@ -9,17 +9,17 @@ import torch import torch.nn.functional as F -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common.activation import ClampedSwiGLU, SiTUGLU, SwiGLU @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestSwiGLULocalCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["fused_binary_activation"]) + LocalCompileConfig(regions=["fused_binary_activation"]).apply() def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_emit_triton(self): diff --git a/tests/unit_tests/gpu/models/kimi_k3/test_kimi_k3_attention_residual.py b/tests/unit_tests/gpu/models/kimi_k3/test_kimi_k3_attention_residual.py index ea9ef9c9b20..45456e29bdb 100644 --- a/tests/unit_tests/gpu/models/kimi_k3/test_kimi_k3_attention_residual.py +++ b/tests/unit_tests/gpu/models/kimi_k3/test_kimi_k3_attention_residual.py @@ -8,7 +8,7 @@ import torch -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common import Linear from torchtitan.models.common.nn_modules import RMSNorm from torchtitan.models.kimi_k3.model import AttentionResidual @@ -45,11 +45,11 @@ def _reference( @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestKimiK3AttentionResidualLocalCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["attention_residual"]) + LocalCompileConfig(regions=["attention_residual"]).apply() self.attention_residual = AttentionResidual.Config().build() def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_emit_triton(self): diff --git a/tests/unit_tests/gpu/models/qwen3_5/test_offset_rmsnorm.py b/tests/unit_tests/gpu/models/qwen3_5/test_offset_rmsnorm.py index f39c6fb5ba6..c4a0fa1e1b1 100644 --- a/tests/unit_tests/gpu/models/qwen3_5/test_offset_rmsnorm.py +++ b/tests/unit_tests/gpu/models/qwen3_5/test_offset_rmsnorm.py @@ -13,7 +13,7 @@ with_comms, ) -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from torchtitan.distributed.spmd_types import set_current_spmd_mesh, set_spmd_meshes from torchtitan.models.common.decoder_sharding import head_norm_sharding_config @@ -23,10 +23,10 @@ @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestOffsetRMSNormCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["offset_rmsnorm"]) + LocalCompileConfig(regions=["offset_rmsnorm"]).apply() def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_emit_triton(self): @@ -101,7 +101,7 @@ def world_size(self) -> int: @with_comms def test_compiled_qk_norm(self): - apply_local_compile(["offset_rmsnorm"]) + LocalCompileConfig(regions=["offset_rmsnorm"]).apply() device = self.device_type config = OffsetRMSNorm.Config( dim=128, @@ -137,7 +137,7 @@ def test_compiled_qk_norm(self): output.sum().backward() self.assertEqual(output.shape, x_local.shape) - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() diff --git a/tests/unit_tests/gpu/models/qwen3_5/test_rope_compile.py b/tests/unit_tests/gpu/models/qwen3_5/test_rope_compile.py index c7c91aa4ad8..d6fbe401791 100644 --- a/tests/unit_tests/gpu/models/qwen3_5/test_rope_compile.py +++ b/tests/unit_tests/gpu/models/qwen3_5/test_rope_compile.py @@ -8,7 +8,7 @@ import torch -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common.rope import CosSinRoPE from torchtitan.models.qwen3_5.rope import MRoPE @@ -16,10 +16,10 @@ @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestRoPELocalCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["cos_sin_rope"]) + LocalCompileConfig(regions=["cos_sin_rope"]).apply() def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_emit_triton(self): @@ -89,7 +89,7 @@ def test_forward_and_backward_match_eager(self): (grad_query_out, grad_key_out), ) - apply_local_compile([]) + LocalCompileConfig().apply() eager_rope = ( CosSinRoPE.Config( dim=128, @@ -222,7 +222,7 @@ def test_mrope_three_axis_positions_match_eager(self): (grad_query_out, grad_key_out), ) - apply_local_compile([]) + LocalCompileConfig().apply() eager_rope = config.build().cuda() eager_query = query.detach().clone().requires_grad_() eager_key = key.detach().clone().requires_grad_() diff --git a/tests/unit_tests/gpu/models/qwen3_5/test_sigmoid_gate.py b/tests/unit_tests/gpu/models/qwen3_5/test_sigmoid_gate.py index 83b3ab74fca..8448defd201 100644 --- a/tests/unit_tests/gpu/models/qwen3_5/test_sigmoid_gate.py +++ b/tests/unit_tests/gpu/models/qwen3_5/test_sigmoid_gate.py @@ -8,14 +8,14 @@ import torch -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.qwen3_5.moe import sigmoid_gate @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestSigmoidGateLocalCompile(unittest.TestCase): def setUp(self): - apply_local_compile(["shared_expert_gate"]) + LocalCompileConfig(regions=["shared_expert_gate"]).apply() # A local generator leaves the global RNG stream unchanged for later tests. generator = torch.Generator(device="cuda").manual_seed(0) @@ -29,7 +29,7 @@ def randn(*shape: int) -> torch.Tensor: self.grad_output = randn(64, 128) def tearDown(self): - apply_local_compile([]) + LocalCompileConfig().apply() torch._dynamo.reset() def test_forward_and_backward_match_reference(self): diff --git a/torchtitan/config/transform/lm_head_fp32.py b/torchtitan/config/transform/lm_head_fp32.py index a7cea8aa0f4..10f2ca1db59 100644 --- a/torchtitan/config/transform/lm_head_fp32.py +++ b/torchtitan/config/transform/lm_head_fp32.py @@ -67,6 +67,6 @@ def convert(self, model_config): "the model config. The torchtitan decoder names its output projection " f"{self._TARGET!r} (see torchtitan/models/common/decoder.py)." ) - if "fp32_to_bf16_split" not in model_config.local_compile_regions: - model_config.local_compile_regions.append("fp32_to_bf16_split") + if "fp32_to_bf16_split" not in model_config.local_compile_regions.regions: + model_config.local_compile_regions.regions.append("fp32_to_bf16_split") return model_config diff --git a/torchtitan/config/validation.py b/torchtitan/config/validation.py index 5bdaab9b22b..4e0b8352940 100644 --- a/torchtitan/config/validation.py +++ b/torchtitan/config/validation.py @@ -137,11 +137,12 @@ def validate_model_training_config( f"dispatcher: {type(dispatcher_config).__qualname__}." ) - if debug.spmd_typechecking and model.local_compile_regions: + if debug.spmd_typechecking and model.local_compile_regions.regions: # TODO: Remove this once Dynamo supports tracing SPMD typechecking. raise ValueError( "Local compilation is not supported with SPMD typechecking. " - "Set model.local_compile_regions=[] or disable debug.spmd_typechecking." + "Set model.local_compile_regions.regions=[] or disable " + "debug.spmd_typechecking." ) validate_context_parallel(model, parallelism) diff --git a/torchtitan/distributed/local_compile.py b/torchtitan/distributed/local_compile.py index 6774c21ebf6..2988d4c62d6 100644 --- a/torchtitan/distributed/local_compile.py +++ b/torchtitan/distributed/local_compile.py @@ -4,10 +4,11 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Function-scoped torch.compile registration.""" +"""Function-scoped torch.compile registration and configuration.""" import functools from collections.abc import Callable +from dataclasses import dataclass, field from typing import Any import torch @@ -15,24 +16,47 @@ from torchtitan.distributed.batch_invariant import is_in_batch_invariant_mode -def apply_local_compile(regions: list[str]) -> None: - """Bind registered functions to eager or compiled implementations. +_LOCAL_COMPILE_CALLBACKS: dict[str, list[Callable[["LocalCompileConfig"], None]]] = {} +_ALWAYS_COMPILE_CALLBACKS: list[Callable[["LocalCompileConfig"], None]] = [] - Process-wide: a later call replaces this choice for every model in the process. - """ - unknown = [name for name in regions if name not in _LOCAL_COMPILE_CALLBACKS] - if unknown: - raise ValueError( - f"Unknown local_compile_regions entries {unknown}; " - f"registered values are {sorted(_LOCAL_COMPILE_CALLBACKS)}" - ) - for callbacks in _LOCAL_COMPILE_CALLBACKS.values(): - for bind_local_compile_fn in callbacks: - bind_local_compile_fn(regions) +@dataclass(kw_only=True, slots=True) +class LocalCompileConfig: + """Configuration for TorchTitan-owned local ``torch.compile`` boundaries.""" + + regions: list[str] = field(default_factory=list) + def _compile( + self, + reference: Callable[..., Any], + *, + inductor_options: dict[str, Any] | None = None, + **compile_kwargs: Any, + ) -> Callable[..., Any]: + """Compile one boundary with Inductor. -_LOCAL_COMPILE_CALLBACKS: dict[str, list[Callable[[list[str]], None]]] = {} + Out-of-tree subclasses can override this hook to select another backend. + """ + if inductor_options is not None: + compile_kwargs.setdefault("options", inductor_options) + return torch.compile(reference, **compile_kwargs) + + def apply(self) -> None: + """Bind named regions and always-compiled helpers using this config.""" + unknown = [ + name for name in self.regions if name not in _LOCAL_COMPILE_CALLBACKS + ] + if unknown: + raise ValueError( + f"Unknown local_compile_regions entries {unknown}; " + f"registered values are {sorted(_LOCAL_COMPILE_CALLBACKS)}" + ) + + for callbacks in _LOCAL_COMPILE_CALLBACKS.values(): + for bind_local_compile_fn in callbacks: + bind_local_compile_fn(self) + for bind_compile_fn in _ALWAYS_COMPILE_CALLBACKS: + bind_compile_fn(self) def local_compile( @@ -56,9 +80,9 @@ def local_compile( def decorate(reference: Callable[..., Any]) -> Callable[..., Any]: fn = reference - def bind_local_compile(regions: list[str]) -> None: + def bind_local_compile(config: LocalCompileConfig) -> None: nonlocal fn - enabled = name in regions + enabled = name in config.regions batch_invariant_mode = is_in_batch_invariant_mode() if enabled and batch_invariant_mode and not batch_invariant: raise ValueError( @@ -66,7 +90,7 @@ def bind_local_compile(regions: list[str]) -> None: "batch-invariant mode; remove it from local_compile_regions." ) if enabled: - fn = torch.compile(reference, fullgraph=True, **compile_kwargs) + fn = config._compile(reference, fullgraph=True, **compile_kwargs) else: fn = reference @@ -80,4 +104,37 @@ def wrapped(*args: Any, **kwargs: Any) -> Any: return decorate -__all__ = ["apply_local_compile", "local_compile"] +def compiled( + reference: Callable[..., Any], + *, + inductor_options: dict[str, Any] | None = None, + **compile_kwargs: Any, +) -> Callable[..., Any]: + """Create an always-compiled helper controlled by ``LocalCompileConfig``.""" + fn = LocalCompileConfig()._compile( + reference, + inductor_options=inductor_options, + **compile_kwargs, + ) + + def bind_compile(config: LocalCompileConfig) -> None: + nonlocal fn + fn = config._compile( + reference, + inductor_options=inductor_options, + **compile_kwargs, + ) + + @functools.wraps(reference) + def wrapped(*args: Any, **kwargs: Any) -> Any: + return fn(*args, **kwargs) + + _ALWAYS_COMPILE_CALLBACKS.append(bind_compile) + return wrapped + + +__all__ = [ + "compiled", + "local_compile", + "LocalCompileConfig", +] diff --git a/torchtitan/distributed/pipeline_parallel.py b/torchtitan/distributed/pipeline_parallel.py index 67959106c65..769d5503a62 100644 --- a/torchtitan/distributed/pipeline_parallel.py +++ b/torchtitan/distributed/pipeline_parallel.py @@ -32,6 +32,7 @@ from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed import ParallelismContext from torchtitan.distributed.activation_remat import ActivationRematConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.base.model import BaseModel from torchtitan.models.base.module import ModuleDict, ModuleList from torchtitan.models.common.decoder import Decoder @@ -78,7 +79,7 @@ def pipeline_llm( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, device: torch.device, diff --git a/torchtitan/distributed/utils.py b/torchtitan/distributed/utils.py index 77c4dea011e..df5e77e00d2 100644 --- a/torchtitan/distributed/utils.py +++ b/torchtitan/distributed/utils.py @@ -167,11 +167,6 @@ def set_determinism( # no options) so the regional-inductor scoop configs are preserved. FlexInnerAttention.inductor_configs["max_autotune"] = False FlexInnerAttention.inductor_configs["coordinate_descent_tuning"] = False - # pyrefly: ignore [no-matching-overload] - FlexInnerAttention._compiled_flex_attn = torch.compile( - flex_attention, options=FlexInnerAttention.inductor_configs - ) - if debug_config.detect_anomaly: logger.warning( "Anomaly detection enabled. This incurs significant overhead " diff --git a/torchtitan/experiments/graph_trainer/configs.py b/torchtitan/experiments/graph_trainer/configs.py index 62e7e36314a..a3c1fd9a3ef 100644 --- a/torchtitan/experiments/graph_trainer/configs.py +++ b/torchtitan/experiments/graph_trainer/configs.py @@ -9,6 +9,7 @@ from torchtitan.components.loss import ChunkedLossWrapper from torchtitan.distributed.activation_remat import DefaultActivationRematConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.experiments.graph_trainer.chunked_loss import ( ChunkedLossWrapperWithParamGrads, ) @@ -353,7 +354,7 @@ def to_graph_trainer_config( } ) # GraphTrainer compiles the whole step (config.compile), so it drops the model's local compile regions. - graph_model.local_compile_regions = [] + graph_model.local_compile_regions = LocalCompileConfig() disable_inplace_wgrad_accum(graph_model) d["model"] = graph_model diff --git a/torchtitan/experiments/graph_trainer/trainer.py b/torchtitan/experiments/graph_trainer/trainer.py index 1b7c6c8a91e..bd8133f22f3 100644 --- a/torchtitan/experiments/graph_trainer/trainer.py +++ b/torchtitan/experiments/graph_trainer/trainer.py @@ -66,11 +66,11 @@ def __init__( ) -> None: if config.optim.enable_cuda_graph: raise ValueError("Optim CUDA graphs are not supported with GraphTrainer.") - if model_config.local_compile_regions: + if model_config.local_compile_regions.regions: raise ValueError( "GraphTrainer traces the whole step into one graph; set " - "model.local_compile_regions = [] " - f"(got {model_config.local_compile_regions})." + "model.local_compile_regions.regions = [] " + f"(got {model_config.local_compile_regions.regions})." ) if config.parallelism.fsdp_symm_mem_scope is not None: raise ValueError( @@ -359,7 +359,10 @@ class Config(Trainer.Config): compile: GraphTrainerCompileConfig = field( default_factory=GraphTrainerCompileConfig ) - """Whole-step compile. GraphTrainer requires ``model.local_compile_regions`` to be empty.""" + """Whole-step compile. + + GraphTrainer requires ``model.local_compile_regions.regions`` to be empty. + """ def __post_init__(self) -> None: Trainer.Config.__post_init__(self) diff --git a/torchtitan/experiments/transformers_modeling_backend/model.py b/torchtitan/experiments/transformers_modeling_backend/model.py index 6ec6c1080d2..6f135923a46 100644 --- a/torchtitan/experiments/transformers_modeling_backend/model.py +++ b/torchtitan/experiments/transformers_modeling_backend/model.py @@ -26,6 +26,7 @@ from torchtitan.config import TORCH_DTYPE_MAP, TrainingConfig from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.batch_invariant import is_in_batch_invariant_mode +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from torchtitan.models.base.model import BaseModel from torchtitan.models.base.module import Module, ModuleDict @@ -285,7 +286,9 @@ def __init__( ) self.sharding_config = None # HF modules run eager; the loss and the swapped-in TorchTitan MoE (SwiGLU) can compile. - self.local_compile_regions = ["loss", "fused_binary_activation"] + self.local_compile_regions = LocalCompileConfig( + regions=["loss", "fused_binary_activation"] + ) assert model_config is not None, "model_config is required" diff --git a/torchtitan/experiments/transformers_modeling_backend/parallelize.py b/torchtitan/experiments/transformers_modeling_backend/parallelize.py index 46be051bc05..a21cbe2bdee 100644 --- a/torchtitan/experiments/transformers_modeling_backend/parallelize.py +++ b/torchtitan/experiments/transformers_modeling_backend/parallelize.py @@ -27,7 +27,7 @@ enable_fsdp_symm_mem, get_fsdp_reshard_after_forward_policy, ) -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig logger = logging.getLogger(__name__) @@ -87,7 +87,7 @@ def parallelize_hf_transformers( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, **kwargs: Any, @@ -102,7 +102,7 @@ def parallelize_hf_transformers( 5. Apply AC and FSDP """ # Bind local implementations early; torch.compile traces on first use. - apply_local_compile(local_compile_regions) + local_compile_regions.apply() # Flex attention supports FSDP, TP, CP, and PP (in any combination). Under CP # the flex kernel's local SPMD boundary redistributes # k/v from seq-sharded to CP-Replicate (all-gather); see _attach_flex_kernel diff --git a/torchtitan/experiments/transformers_modeling_backend/pipeline.py b/torchtitan/experiments/transformers_modeling_backend/pipeline.py index f439c413c35..8833d566bbf 100644 --- a/torchtitan/experiments/transformers_modeling_backend/pipeline.py +++ b/torchtitan/experiments/transformers_modeling_backend/pipeline.py @@ -25,6 +25,7 @@ from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed import ParallelismContext from torchtitan.distributed.activation_remat import ActivationRematConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.pipeline_parallel import ( _build_get_mesh_callback, _build_pipeline_schedule, @@ -296,7 +297,7 @@ def pipeline_hf_transformers( *, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, device: torch.device, diff --git a/torchtitan/models/base/model.py b/torchtitan/models/base/model.py index 52760ed6a96..2b211ef18a8 100644 --- a/torchtitan/models/base/model.py +++ b/torchtitan/models/base/model.py @@ -14,7 +14,7 @@ from torchtitan.config import TrainingConfig from torchtitan.config.parallelism import ParallelismConfig -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from .module import Module @@ -86,10 +86,12 @@ class Config(Module.Config): Subclasses define model-specific hyperparameters. """ - local_compile_regions: list[str] = field(default_factory=list) - """``@local_compile`` regions this model compiles; ``[]`` runs them all eager. + local_compile_regions: LocalCompileConfig = field( + default_factory=LocalCompileConfig + ) + """Configuration for named regions, FlexAttention, and block-mask helpers. - FlexAttention manages its own compilation and is not controlled by this list. + An empty ``regions`` list runs named regions eagerly. """ @abstractmethod @@ -132,13 +134,13 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> Self: """Apply the ordered model-level parallelization lifecycle.""" # Bind local implementations early; torch.compile traces on first use. - apply_local_compile(local_compile_regions) + local_compile_regions.apply() with parallelism_context.activate_spmd(): self._parallelize(parallelism_context) if activation_remat is not None: diff --git a/torchtitan/models/common/attention/attention.py b/torchtitan/models/common/attention/attention.py index 0258daedfc1..395434af279 100644 --- a/torchtitan/models/common/attention/attention.py +++ b/torchtitan/models/common/attention/attention.py @@ -39,6 +39,7 @@ ) from torchtitan.distributed.batch_invariant import is_in_batch_invariant_mode +from torchtitan.distributed.local_compile import compiled from torchtitan.distributed.parallelism_context import MeshAxisName from torchtitan.models.base.module import Module from torchtitan.models.common.linear import Linear @@ -344,10 +345,9 @@ def build_attention_metadata( "triton.cudagraphs": False, } - # pyrefly: ignore[no-matching-overload] - _compiled_flex_attn: ClassVar[Callable] = torch.compile( + _compiled_flex_attn: ClassVar[Callable] = compiled( flex_attention, - options=inductor_configs, + inductor_options=inductor_configs, ) def __init__(self, config: Config) -> None: @@ -642,7 +642,7 @@ def sliding_window_mod( return sliding_window_mod -_compiled_create_block_mask = torch.compile(create_block_mask) +_compiled_create_block_mask = compiled(create_block_mask) def create_attention_mask(*args, **kwargs): diff --git a/torchtitan/models/common/multimodal.py b/torchtitan/models/common/multimodal.py index 26ce302f79a..9307c9a0f7e 100644 --- a/torchtitan/models/common/multimodal.py +++ b/torchtitan/models/common/multimodal.py @@ -20,7 +20,7 @@ from torchtitan.config import TORCH_DTYPE_MAP, TrainingConfig from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.activation_remat import ActivationRematConfig -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from .decoder import Decoder @@ -37,12 +37,12 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> Self: # Bind local implementations early; torch.compile traces on first use. - apply_local_compile(local_compile_regions) + local_compile_regions.apply() with parallelism_context.activate_spmd(): self._parallelize(parallelism_context) encoders = [ diff --git a/torchtitan/models/common/vision_encoder.py b/torchtitan/models/common/vision_encoder.py index a1d4b8c1fa3..ca3221a0077 100644 --- a/torchtitan/models/common/vision_encoder.py +++ b/torchtitan/models/common/vision_encoder.py @@ -28,6 +28,7 @@ import torch_remat as remat from torch.nn.attention.flex_attention import BlockMask, create_block_mask +from torchtitan.distributed.local_compile import compiled from torchtitan.distributed.parallelism_context import MeshAxisName from torchtitan.distributed.spmd_types import spmd_mesh_group from torchtitan.models.base.module import Module @@ -35,7 +36,7 @@ from torchtitan.models.common.linear import Linear from torchtitan.models.common.nn_modules import GELU, LayerNorm, RMSNorm -compiled_create_block_mask = torch.compile(create_block_mask) +compiled_create_block_mask = compiled(create_block_mask) # Applies rotary position embedding: (query, key, rope_cache) -> (query, key). RopeApply = Callable[ diff --git a/torchtitan/models/deepseek_v3/model.py b/torchtitan/models/deepseek_v3/model.py index c1ba8bc60ee..a4c1d20f7f2 100644 --- a/torchtitan/models/deepseek_v3/model.py +++ b/torchtitan/models/deepseek_v3/model.py @@ -13,6 +13,7 @@ import torch_remat as remat from torch import nn +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.base.module import Module from torchtitan.models.common.attention import ( BaseAttention, @@ -272,13 +273,15 @@ class DeepSeekV3Model(MTPDecoder): class Config(MTPDecoder.Config): dim: int = 2048 vocab_size: int = 102400 - local_compile_regions: list[str] = field( - default_factory=lambda: [ - "loss", - "fused_binary_activation", - "fp32_to_bf16_split", - "moe_aux_loss", - ] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=[ + "loss", + "fused_binary_activation", + "fp32_to_bf16_split", + "moe_aux_loss", + ] + ) ) def get_nparams_and_flops( diff --git a/torchtitan/models/deepseek_v4/model.py b/torchtitan/models/deepseek_v4/model.py index 1da25fda7a4..06ed95b7586 100644 --- a/torchtitan/models/deepseek_v4/model.py +++ b/torchtitan/models/deepseek_v4/model.py @@ -14,6 +14,7 @@ from torchtitan.config import TORCH_DTYPE_MAP, TrainingConfig from torchtitan.config.parallelism import ParallelismConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from torchtitan.models.base.module import ModuleList from torchtitan.models.common.attention import ( @@ -179,12 +180,14 @@ def _apply_fsdp( class Config(Decoder.Config): dim: int vocab_size: int - local_compile_regions: list[str] = field( - default_factory=lambda: [ - "loss", - "fused_binary_activation", - "fp32_to_bf16_split", - ] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=[ + "loss", + "fused_binary_activation", + "fp32_to_bf16_split", + ] + ) ) hc_mult: int = 4 n_mtp_layers: int = 0 diff --git a/torchtitan/models/flux/README.md b/torchtitan/models/flux/README.md index bb331f575d2..abba84856a1 100644 --- a/torchtitan/models/flux/README.md +++ b/torchtitan/models/flux/README.md @@ -51,7 +51,7 @@ MODULE=torchtitan_recipes.tests.models.flux CONFIG=flux_debugmodel_inference \ The loss function can be compiled independently with: ```python -config.model.local_compile_regions = ["loss"] +config.model.local_compile_regions.regions = ["loss"] ``` The model itself is not compiled by the standard Trainer. diff --git a/torchtitan/models/flux/model.py b/torchtitan/models/flux/model.py index 39c6480c931..46bec80d43f 100644 --- a/torchtitan/models/flux/model.py +++ b/torchtitan/models/flux/model.py @@ -18,7 +18,7 @@ from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed import context_parallel from torchtitan.distributed.activation_remat import ActivationRematConfig -from torchtitan.distributed.local_compile import apply_local_compile +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import MeshAxisName, ParallelismContext from torchtitan.distributed.spmd_types import ( annotate_replicated_parameters, @@ -103,7 +103,9 @@ class FluxModel(BaseModel): class Config(BaseModel.Config): img_in: Linear.Config txt_in: Linear.Config - local_compile_regions: list[str] = field(default_factory=lambda: ["loss"]) + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig(regions=["loss"]) + ) in_channels: int = 64 out_channels: int = 64 vec_in_dim: int = 768 @@ -235,13 +237,13 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> Self: """Apply Flux's AC-before-SPMD parallelization lifecycle.""" # Bind local implementations early; torch.compile traces on first use. - apply_local_compile(local_compile_regions) + local_compile_regions.apply() with parallelism_context.activate_spmd(): if activation_remat is not None: # Flux saves no regions under any policy: each block is diff --git a/torchtitan/models/gpt_oss/model.py b/torchtitan/models/gpt_oss/model.py index 35bd95b26f0..af275f76343 100644 --- a/torchtitan/models/gpt_oss/model.py +++ b/torchtitan/models/gpt_oss/model.py @@ -17,6 +17,7 @@ from torchtitan.config import TrainingConfig from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.activation_remat import ActivationRematConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from torchtitan.models.base.module import Module from torchtitan.models.common.attention import ( @@ -216,13 +217,15 @@ def _register_optimizer_hooks( class Config(Decoder.Config): dim: int = 2880 vocab_size: int = 201088 - local_compile_regions: list[str] = field( - default_factory=lambda: [ - "loss", - "fused_binary_activation", - "cos_sin_rope", - "fp32_to_bf16_split", - ] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=[ + "loss", + "fused_binary_activation", + "cos_sin_rope", + "fp32_to_bf16_split", + ] + ) ) def get_nparams_and_flops( @@ -250,7 +253,7 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> GptOssModel: diff --git a/torchtitan/models/kimi_k2_7/model.py b/torchtitan/models/kimi_k2_7/model.py index 6d1dd05790b..a621676c1bb 100644 --- a/torchtitan/models/kimi_k2_7/model.py +++ b/torchtitan/models/kimi_k2_7/model.py @@ -20,6 +20,7 @@ from torchtitan.config import TrainingConfig from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.activation_remat import ActivationRematConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from torchtitan.distributed.spmd_types import ( annotate_input_spmd_types, @@ -117,7 +118,7 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> KimiK25Model: diff --git a/torchtitan/models/kimi_k3/model.py b/torchtitan/models/kimi_k3/model.py index f1313a458a1..889b9276e2a 100644 --- a/torchtitan/models/kimi_k3/model.py +++ b/torchtitan/models/kimi_k3/model.py @@ -19,7 +19,7 @@ from torchtitan.config.function import Function from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.activation_remat import ActivationRematConfig -from torchtitan.distributed.local_compile import apply_local_compile, local_compile +from torchtitan.distributed.local_compile import local_compile, LocalCompileConfig from torchtitan.distributed.parallelism_context import MeshAxisName, ParallelismContext from torchtitan.distributed.spmd_types import ( annotate_input_spmd_types, @@ -391,14 +391,16 @@ class Config(Decoder.Config): attention_residual: AttentionResidual.Config = field( default_factory=AttentionResidual.Config ) - local_compile_regions: list[str] = field( - default_factory=lambda: [ - "loss", - "gated_rmsnorm", - "fused_binary_activation", - "fp32_to_bf16_split", - "attention_residual", - ] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=[ + "loss", + "gated_rmsnorm", + "fused_binary_activation", + "fp32_to_bf16_split", + "attention_residual", + ] + ) ) def get_nparams_and_flops( @@ -458,12 +460,12 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> KimiK3Model: # Bind local implementations early; torch.compile traces on first use. - apply_local_compile(local_compile_regions) + local_compile_regions.apply() with parallelism_context.activate_spmd(): annotate_replicated_parameters(self, parallelism_context) self._parallelize(parallelism_context) diff --git a/torchtitan/models/llama3/model.py b/torchtitan/models/llama3/model.py index b0335209d90..54bd7302f80 100644 --- a/torchtitan/models/llama3/model.py +++ b/torchtitan/models/llama3/model.py @@ -12,6 +12,7 @@ import torch_remat as remat from torch import nn +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common.attention import ( FlexAttentionMetadata, VarlenAttentionMetadata, @@ -83,8 +84,10 @@ class Llama3Model(Decoder): class Config(Decoder.Config): dim: int = 4096 vocab_size: int = 128256 - local_compile_regions: list[str] = field( - default_factory=lambda: ["loss", "fused_binary_activation"] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=["loss", "fused_binary_activation"] + ) ) def get_nparams_and_flops( diff --git a/torchtitan/models/muse_glimmer/model.py b/torchtitan/models/muse_glimmer/model.py index 72938708b72..0402160c20a 100644 --- a/torchtitan/models/muse_glimmer/model.py +++ b/torchtitan/models/muse_glimmer/model.py @@ -18,6 +18,7 @@ from torchtitan.config import TrainingConfig from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.activation_remat import ActivationRematConfig +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.distributed.parallelism_context import ParallelismContext from torchtitan.distributed.spmd_types import ( annotate_input_spmd_types, @@ -289,8 +290,10 @@ class MuseGlimmerModel(MultimodalModel): class Config(Decoder.Config): dim: int = 6656 vocab_size: int = 202048 - local_compile_regions: list[str] = field( - default_factory=lambda: ["loss", "fused_binary_activation"] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=["loss", "fused_binary_activation"] + ) ) # Narrows the base Decoder.Config.tok_embeddings (Embedding.Config) to the # bundled embedding+norm unit whose .embedding/.norm children carry the sharding. @@ -369,7 +372,7 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> MuseGlimmerModel: diff --git a/torchtitan/models/qwen3/model.py b/torchtitan/models/qwen3/model.py index 0ec1c454b65..bcabbf086f9 100644 --- a/torchtitan/models/qwen3/model.py +++ b/torchtitan/models/qwen3/model.py @@ -12,6 +12,7 @@ import torch.nn as nn import torch_remat as remat +from torchtitan.distributed.local_compile import LocalCompileConfig from torchtitan.models.common.attention import ( FlexAttentionMetadata, VarlenAttentionMetadata, @@ -108,13 +109,15 @@ def _register_optimizer_hooks( class Config(Decoder.Config): dim: int = 1024 vocab_size: int = 151936 - local_compile_regions: list[str] = field( - default_factory=lambda: [ - "loss", - "fused_binary_activation", - "cos_sin_rope", - "fp32_to_bf16_split", - ] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=[ + "loss", + "fused_binary_activation", + "cos_sin_rope", + "fp32_to_bf16_split", + ] + ) ) def get_nparams_and_flops( diff --git a/torchtitan/models/qwen3_5/model.py b/torchtitan/models/qwen3_5/model.py index bd68938bfb6..368f27ef7b8 100644 --- a/torchtitan/models/qwen3_5/model.py +++ b/torchtitan/models/qwen3_5/model.py @@ -18,7 +18,7 @@ from torchtitan.config import TrainingConfig from torchtitan.config.parallelism import ParallelismConfig from torchtitan.distributed.activation_remat import ActivationRematConfig -from torchtitan.distributed.local_compile import local_compile +from torchtitan.distributed.local_compile import local_compile, LocalCompileConfig from torchtitan.distributed.parallelism_context import MeshAxisName, ParallelismContext from torchtitan.distributed.spmd_types import ( annotate_input_spmd_types, @@ -353,16 +353,18 @@ def _register_optimizer_hooks( @dataclass(kw_only=True, slots=True) class Config(Decoder.Config): vision_encoder: Qwen35VisionEncoder.Config | None = None - local_compile_regions: list[str] = field( - default_factory=lambda: [ - "loss", - "fused_binary_activation", - "gated_rmsnorm", - "offset_rmsnorm", - "partial_rope", - "shared_expert_gate", - "fp32_to_bf16_split", - ] + local_compile_regions: LocalCompileConfig = field( + default_factory=lambda: LocalCompileConfig( + regions=[ + "loss", + "fused_binary_activation", + "gated_rmsnorm", + "offset_rmsnorm", + "partial_rope", + "shared_expert_gate", + "fp32_to_bf16_split", + ] + ) ) def get_nparams_and_flops( @@ -436,7 +438,7 @@ def parallelize( parallelism_context: ParallelismContext, training: TrainingConfig, parallelism: ParallelismConfig, - local_compile_regions: list[str], + local_compile_regions: LocalCompileConfig, activation_remat: ActivationRematConfig | None, dump_folder: str, ) -> Qwen35Model: diff --git a/torchtitan_recipes/models/deepseek_v3.py b/torchtitan_recipes/models/deepseek_v3.py index 5a762cdec49..9e4e8b61636 100644 --- a/torchtitan_recipes/models/deepseek_v3.py +++ b/torchtitan_recipes/models/deepseek_v3.py @@ -268,5 +268,5 @@ def deepseek_v3_671b_nvfp4_ffn_mxfp8_attn( ) ], ) - config.model.local_compile_regions = ["loss"] + config.model.local_compile_regions.regions = ["loss"] return config diff --git a/torchtitan_recipes/rl/alphabet_sort.py b/torchtitan_recipes/rl/alphabet_sort.py index f7125220ca0..29141010e90 100644 --- a/torchtitan_recipes/rl/alphabet_sort.py +++ b/torchtitan_recipes/rl/alphabet_sort.py @@ -174,7 +174,7 @@ def rl_grpo_qwen3_0_6b_varlen(*, seq_len: int = 2048) -> Controller.Config: def rl_grpo_qwen3_0_6b_varlen_no_compile(*, seq_len: int = 2048) -> Controller.Config: config = rl_grpo_qwen3_0_6b_varlen(seq_len=seq_len) for model_config in (config.trainer.model, config.generator.model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return config @@ -299,7 +299,7 @@ def rl_grpo_qwen3_0_6b_flex_batch_invariant() -> Controller.Config: ) # Local compile regions do not support batch-invariant mode. for model_config in (config.trainer.model, config.generator.model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] block_size = config.trainer.model.layers[0].attention.inner_attention.block_size config.async_loop.batcher = dataclasses.replace( config.async_loop.batcher, per_sample_pad_multiple=block_size @@ -671,7 +671,7 @@ def rl_grpo_qwen3_0_6b_varlen_batch_invariant( ) # Local compile regions do not support batch-invariant mode. for model_config in (trainer_model, generator_model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return Controller.Config( hf_assets_path="./assets/hf/Qwen3-0.6B", num_generators=3, @@ -856,7 +856,7 @@ def rl_grpo_qwen3_5_9b_varlen_batch_invariant() -> Controller.Config: ) # Local compile regions do not support batch-invariant mode. for model_config in (config.trainer.model, config.generator.model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return config diff --git a/torchtitan_recipes/tests/__init__.py b/torchtitan_recipes/tests/__init__.py index 3792be993e9..fb45108df37 100644 --- a/torchtitan_recipes/tests/__init__.py +++ b/torchtitan_recipes/tests/__init__.py @@ -27,4 +27,4 @@ def _set_spmd_typechecking(config: Trainer.Config, *, typechecking: bool) -> Non if typechecking: # Local compile does not support global SPMD typechecking yet. # TODO: Remove this once the SPMD typechecking/Dynamo issue is fixed. - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] diff --git a/torchtitan_recipes/tests/graph_trainer/numerics.py b/torchtitan_recipes/tests/graph_trainer/numerics.py index 277344d9ce7..e8d85296cce 100644 --- a/torchtitan_recipes/tests/graph_trainer/numerics.py +++ b/torchtitan_recipes/tests/graph_trainer/numerics.py @@ -55,7 +55,7 @@ def _without_inplace_wgrad_accum(config): def llama3_eager_numerics(): config = llama3_debugmodel(seq_len=2048) # Match GraphTrainer, which captures model and loss in one eager FX graph. - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] config.parallelism.tensor_parallel_degree = 2 config.parallelism.data_parallel_shard_degree = 4 config.training.num_tokens_per_microbatch_per_dp_rank = 16384 @@ -73,7 +73,7 @@ def llama3_graph_numerics(): def llama3_eager_hsdp_gradient_accumulation_numerics(): config = llama3_debugmodel(seq_len=2048) # Match GraphTrainer, which captures model and loss in one eager FX graph. - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] config.parallelism.data_parallel_replicate_degree = 2 config.parallelism.data_parallel_shard_degree = 2 config.training.num_tokens_per_microbatch_per_dp_rank = 8192 @@ -103,7 +103,7 @@ def _llama3_hsdp_pp_numerics(config): def llama3_eager_hsdp_pp_numerics(): config = llama3_debugmodel(seq_len=2048) - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] return _without_inplace_wgrad_accum(_llama3_hsdp_pp_numerics(config)) @@ -214,7 +214,7 @@ def _deepseek_v3_dist_moe_pp_numerics( runtime_config.pp_activation_slot_policy = slot_policy for _, experts, _, _ in config.model.traverse(DistMoeRoutedExperts.Config): experts.inplace_wgrad_accum = False - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] config.parallelism.pipeline_parallel_degree = 2 config.parallelism.pipeline_parallel_schedule = "Interleaved1F1B" config.parallelism.num_pp_microbatches = 8 @@ -251,7 +251,7 @@ def deepseek_v3_dist_moe_graph_pp_stage_microbatch_numerics(): def qwen3_eager_numerics(): config = qwen3_debugmodel(seq_len=2048) # Match GraphTrainer, which captures model and loss in one eager FX graph. - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] config.parallelism.tensor_parallel_degree = 2 config.parallelism.data_parallel_shard_degree = 4 config.training.num_tokens_per_microbatch_per_dp_rank = 16384 @@ -278,7 +278,7 @@ def qwen3_moe_eager_numerics(): ) config.dataloader.max_num_documents = None # Match GraphTrainer, which captures model and loss in one eager FX graph. - config.model.local_compile_regions = [] + config.model.local_compile_regions.regions = [] config.training.disable_cuda_graphs = True config.training.num_tokens_per_microbatch_per_dp_rank = 16384 return _without_inplace_wgrad_accum(config) diff --git a/torchtitan_recipes/tests/models/deepseek_v3.py b/torchtitan_recipes/tests/models/deepseek_v3.py index 65225c382bc..bc779ebaeb1 100644 --- a/torchtitan_recipes/tests/models/deepseek_v3.py +++ b/torchtitan_recipes/tests/models/deepseek_v3.py @@ -481,7 +481,7 @@ def deepseek_v3_debugmodel_nvfp4_ffn_mxfp8_attn( ), ], ) - config.model.local_compile_regions = ["loss"] + config.model.local_compile_regions.regions = ["loss"] return config @@ -530,5 +530,5 @@ def deepseek_v3_16b_nvfp4_ffn_mxfp8_attn( ) ], ) - config.model.local_compile_regions = ["loss"] + config.model.local_compile_regions.regions = ["loss"] return config diff --git a/torchtitan_recipes/tests/rl/alphabet_sort.py b/torchtitan_recipes/tests/rl/alphabet_sort.py index ce1b21069ed..d86fbadb948 100644 --- a/torchtitan_recipes/tests/rl/alphabet_sort.py +++ b/torchtitan_recipes/tests/rl/alphabet_sort.py @@ -156,7 +156,7 @@ def rl_grpo_gpt_oss_debug_varlen_no_compile( ) -> Controller.Config: config = rl_grpo_gpt_oss_debug_varlen(seq_len=seq_len) for model_config in (config.trainer.model, config.generator.model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return config @@ -194,7 +194,7 @@ def rl_grpo_gpt_oss_debug_varlen_batch_invariant() -> Controller.Config: ) # Local compile regions do not support batch-invariant mode. for model_config in (trainer_model, generator_model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return Controller.Config( hf_assets_path="tests/assets/tokenizer", async_loop=AsyncLoopConfig( @@ -439,7 +439,7 @@ def rl_grpo_qwen3_moe_debug_varlen_batch_invariant( ) # Local compile regions do not support batch-invariant mode. for model_config in (trainer_model, generator_model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return Controller.Config( hf_assets_path="tests/assets/tokenizer", async_loop=AsyncLoopConfig( @@ -604,7 +604,7 @@ def rl_grpo_qwen3_5_debug_varlen_batch_invariant( ) # Local compile regions do not support batch-invariant mode. for model_config in (config.trainer.model, config.generator.model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return config @@ -629,7 +629,7 @@ def _kimi_k3_rl_model_configs( converters=converters, ) for model_config in (trainer_model, generator_model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] return trainer_model, generator_model @@ -819,7 +819,7 @@ def rl_grpo_moe_debug_tp4_ep4() -> Controller.Config: generator_parallelism=config.generator.parallelism, ) for model_config in (config.trainer.model, config.generator.model): - model_config.local_compile_regions = [] + model_config.local_compile_regions.regions = [] # TODO: use FULL CUDA graphs once GPT-OSS CI enables a # CUDA-graph-compatible distributed MoE token dispatcher. config.generator.cuda_graph.mode = "NONE"