diff --git a/benchmarks/dynamo/distributed.py b/benchmarks/dynamo/distributed.py index 194b3906de0..b490c48ade9 100644 --- a/benchmarks/dynamo/distributed.py +++ b/benchmarks/dynamo/distributed.py @@ -81,8 +81,8 @@ def run_model(args, model, inputs, key): if args.verbose: dynamo.config.verbose = True dynamo.config.log_level = logging.DEBUG - if args.dynamo_optimize_ddp: - dynamo.config.optimize_ddp = True + if args.dynamo_no_optimize_ddp: + dynamo.config.optimize_ddp = False if args.dynamo == "inductor" and args.fsdp: torch._inductor.config.triton.cudagraphs = False log.warn("disabling inductor cudagraphs for compatibility with FSDP") @@ -129,7 +129,7 @@ if __name__ == "__main__": parser.add_argument("--trace_file", default="profile.json", help="Run the profiler") parser.add_argument("--repeat", default=10, help="Repeats for timing run") parser.add_argument( - "--dynamo_optimize_ddp", + "--dynamo_no_optimize_ddp", action="store_true", help="Enable dynamo's ddp optimizer", ) diff --git a/torch/_dynamo/config.py b/torch/_dynamo/config.py index 26efff20538..258df9989f8 100644 --- a/torch/_dynamo/config.py +++ b/torch/_dynamo/config.py @@ -138,8 +138,11 @@ capture_scalar_outputs = False enforce_cond_guards_match = True # Automatically split model graph into pieces to match DDP bucket sizes -# to allow DDP comm/compute overlap -optimize_ddp = False +# to allow DDP comm/compute overlap. Disable to allow DDP models to +# run without graph-breaks, but also without comm/compute overlap. +# set torch._dynamo.config.log_level to INFO or DEBUG for more info +# about optimize_ddp behavior. +optimize_ddp = True # If True, raises exception if TorchDynamo is called with a context manager raise_on_ctx_manager_usage = True