cast svd to torch_dtype

Author: Disty0

src/sdnq/loader.py

@@ -184,6 +184,9 @@ def apply_sdnq_options_to_module(model, dtype: torch.dtype = None, dequantize_fp
 
             if dtype is not None and module.sdnq_dequantizer.result_dtype not in {torch.float32, torch.float64}:
                 module.sdnq_dequantizer.result_dtype = dtype
+                if module.svd_up is not None:
+                    module.svd_up.data = module.svd_up.to(dtype=dtype)
+                    module.svd_down.data = module.svd_down.to(dtype=dtype)
 
             upcast_scale = bool(
                 dequantize_fp32
@@ -208,9 +211,6 @@ def apply_sdnq_options_to_module(model, dtype: torch.dtype = None, dequantize_fp
             module.scale.data = module.scale.to(dtype=scale_dtype)
             if module.zero_point is not None:
                 module.zero_point.data = module.zero_point.to(dtype=scale_dtype)
-            if module.svd_up is not None:
-                module.svd_up.data = module.svd_up.to(dtype=scale_dtype)
-                module.svd_down.data = module.svd_down.to(dtype=scale_dtype)
 
             if current_use_quantized_matmul is not None and current_use_quantized_matmul != module.sdnq_dequantizer.use_quantized_matmul:
                 if not module.sdnq_dequantizer.re_quantize_for_matmul and not dtype_dict[module.sdnq_dequantizer.weights_dtype]["is_packed"]:

src/sdnq/quantizer.py

@@ -309,7 +309,7 @@ def sdnq_quantize_layer_weight(weight, layer_class_name=None, weights_dtype="int
 
     if use_svd:
         try:
-            weight, svd_up, svd_down = apply_svdquant(weight, rank=svd_rank, niter=svd_steps, dtype=scale_dtype)
+            weight, svd_up, svd_down = apply_svdquant(weight, rank=svd_rank, niter=svd_steps, dtype=torch_dtype)
             if use_quantized_matmul:
                 svd_up = svd_up.t_()
                 svd_down = svd_down.t_()
@@ -422,10 +422,8 @@ def sdnq_quantize_layer_weight_dynamic(weight, layer_class_name=None, weights_dt
 
     if use_svd:
         try:
-            svd_weight, svd_up, svd_down = apply_svdquant(weight, rank=svd_rank, niter=svd_steps)
+            svd_weight, svd_up, svd_down = apply_svdquant(weight, rank=svd_rank, niter=svd_steps, dtype=torch_dtype)
             svd_up, svd_down = prepare_svd_for_matmul(svd_up, svd_down, use_quantized_matmul)
-            svd_up = svd_up.to(dtype=torch_dtype)
-            svd_down = svd_down.to(dtype=torch_dtype)
         except Exception:
             svd_up, svd_down = None, None
             svd_weight = weight