Fixed issue with heads not being added

2026-01-26 16:39:47 +00:00 · 2024-06-15 08:34:33 -06:00
parent 696f73c30d
commit ada722c9e4
1 changed files with 1 additions and 0 deletions
--- a/toolkit/models/ilora.py
+++ b/toolkit/models/ilora.py
@@ -284,6 +284,7 @@ class InstantLoRAModule(torch.nn.Module):
            hidden_size=head_dim,
            head_size=head_dim,
            num_mlp_layers=1,
+            num_heads=self.num_heads,
            output_size=self.output_size,
        )