diff --git a/src/diffusers/quantizers/torchao/torchao_quantizer.py b/src/diffusers/quantizers/torchao/torchao_quantizer.py
index 8e781ebb143a..e31db9faa068 100644
--- a/src/diffusers/quantizers/torchao/torchao_quantizer.py
+++ b/src/diffusers/quantizers/torchao/torchao_quantizer.py
@@ -51,7 +51,8 @@
     import torch
     import torch.nn as nn
 
-if is_torchao_available():
+# `validate_environment` asks for torchao >= 0.15 before quantizing, and older versions have no `FqnToConfig`.
+if is_torchao_available() and is_torchao_version(">=", "0.15.0"):
     from torchao.quantization import FqnToConfig, quantize_
 
     if is_torchao_version(">=", "0.16.0"):