Added stochastic rounding to adafactor. ILora adjustments

2026-04-24 00:09:26 +00:00 · 2024-03-05 07:07:09 -07:00
parent 1325613583
commit b01e8d889a
7 changed files with 153 additions and 3 deletions
--- a/toolkit/models/ilora.py
+++ b/toolkit/models/ilora.py
@@ -49,7 +49,7 @@ class InstantLoRAMidModule(torch.nn.Module):
            print(scaler.shape)
            raise e
        # apply tanh to limit values to -1 to 1
-        scaler = torch.tanh(scaler)
+        # scaler = torch.tanh(scaler)
        return x * scaler