ostris/ai-toolkit
1
0
Fork 0
mirror of https://github.com/ostris/ai-toolkit.git synced 2026-04-30 11:11:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added stochastic rounding to adafactor. ILora adjustments

Browse Source
This commit is contained in:
Jaret Burkett
2024-03-05 07:07:09 -07:00
parent 1325613583
commit b01e8d889a
7 changed files with 153 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
jobs/process/BaseSDTrainProcess.py
View File

@@ -688,6 +688,16 @@ class BaseSDTrainProcess(BaseTrainProcess):
trigger=self.trigger_word,
add_if_not_present=not is_reg,
)
if not is_reg and self.train_config.prompt_saturation_chance > 0.0:
# do random prompt saturation by expanding the prompt to hit at least 77 tokens
if random.random() < self.train_config.prompt_saturation_chance:
est_num_tokens = len(prompt.split(' '))
if est_num_tokens < 77:
num_repeats = int(77 / est_num_tokens) + 1
prompt = ', '.join([prompt] * num_repeats)
conditioned_prompts.append(prompt)
with self.timer('prepare_latents'):