diff --git a/config/examples/train_full_fine_tune_lumina.yaml b/config/examples/train_full_fine_tune_lumina.yaml new file mode 100644 index 00000000..51a61737 --- /dev/null +++ b/config/examples/train_full_fine_tune_lumina.yaml @@ -0,0 +1,99 @@ +--- +# This configuration requires 24GB of VRAM or more to operate +job: extension +config: + # this name will be the folder and filename name + name: "my_first_lumina_finetune_v1" + process: + - type: 'sd_trainer' + # root folder to save training sessions/samples/weights + training_folder: "output" + # uncomment to see performance stats in the terminal every N steps + # performance_log_every: 1000 + device: cuda:0 + # if a trigger word is specified, it will be added to captions of training data if it does not already exist + # alternatively, in your captions you can add [trigger] and it will be replaced with the trigger word + # trigger_word: "p3r5on" + save: + dtype: bf16 # precision to save + save_every: 250 # save every this many steps + max_step_saves_to_keep: 2 # how many intermittent saves to keep + save_format: 'diffusers' # 'diffusers' + datasets: + # datasets are a folder of images. captions need to be txt files with the same name as the image + # for instance image2.jpg and image2.txt. Only jpg, jpeg, and png are supported currently + # images will automatically be resized and bucketed into the resolution specified + # on windows, escape back slashes with another backslash so + # "C:\\path\\to\\images\\folder" + - folder_path: "/path/to/images/folder" + caption_ext: "txt" + caption_dropout_rate: 0.05 # will drop out the caption 5% of time + shuffle_tokens: false # shuffle caption order, split by commas + # cache_latents_to_disk: true # leave this true unless you know what you're doing + resolution: [ 512, 768, 1024 ] # lumina2 enjoys multiple resolutions + train: + batch_size: 1 + + # can be 'sigmoid', 'linear', or 'lumina2_shift' + timestep_type: 'lumina2_shift' + + steps: 2000 # total number of steps to train 500 - 4000 is a good range + gradient_accumulation: 1 + train_unet: true + train_text_encoder: false # probably won't work with lumina2 + gradient_checkpointing: true # need the on unless you have a ton of vram + noise_scheduler: "flowmatch" # for training only + optimizer: "adafactor" + lr: 3e-5 + + # Paramiter swapping can reduce vram requirements. Set factor from 1.0 to 0.0. + # 0.1 is 10% of paramiters active at easc step. Only works with adafactor + + # do_paramiter_swapping: true + # paramiter_swapping_factor: 0.9 + + # uncomment this to skip the pre training sample + # skip_first_sample: true + # uncomment to completely disable sampling + # disable_sampling: true + + # ema will smooth out learning, but could slow it down. Recommended to leave on if you have the vram + # ema_config: + # use_ema: true + # ema_decay: 0.99 + + # will probably need this if gpu supports it for lumina2, other dtypes may not work correctly + dtype: bf16 + model: + # huggingface model name or path + name_or_path: "Alpha-VLLM/Lumina-Image-2.0" + is_lumina2: true # lumina2 architecture + # you can quantize just the Gemma2 text encoder here to save vram + quantize_te: true + sample: + sampler: "flowmatch" # must match train.noise_scheduler + sample_every: 250 # sample every this many steps + width: 1024 + height: 1024 + prompts: + # you can add [trigger] to the prompts here and it will be replaced with the trigger word + # - "[trigger] holding a sign that says 'I LOVE PROMPTS!'"\ + - "woman with red hair, playing chess at the park, bomb going off in the background" + - "a woman holding a coffee cup, in a beanie, sitting at a cafe" + - "a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini" + - "a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background" + - "a bear building a log cabin in the snow covered mountains" + - "woman playing the guitar, on stage, singing a song, laser lights, punk rocker" + - "hipster man with a beard, building a chair, in a wood shop" + - "photo of a cat that is half black and half orange tabby, split down the middle. The cat has on a blue tophat. They are holding a martini glass with a pink ball of yarn in it with green knitting needles sticking out, in one paw. In the other paw, they are holding a DVD case for a movie titled, \"This is a test\" that has a golden robot on it. In the background is a busy night club with a giant mushroom man dancing with a bear." + - "a man holding a sign that says, 'this is a sign'" + - "a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle" + neg: "" + seed: 42 + walk_seed: true + guidance_scale: 4.0 + sample_steps: 25 +# you can add any additional meta info here. [name] is replaced with config name at top +meta: + name: "[name]" + version: '1.0' diff --git a/config/examples/train_lora_lumina.yaml b/config/examples/train_lora_lumina.yaml new file mode 100644 index 00000000..e5d2d756 --- /dev/null +++ b/config/examples/train_lora_lumina.yaml @@ -0,0 +1,96 @@ +--- +# This configuration requires 20GB of VRAM or more to operate +job: extension +config: + # this name will be the folder and filename name + name: "my_first_lumina_lora_v1" + process: + - type: 'sd_trainer' + # root folder to save training sessions/samples/weights + training_folder: "output" + # uncomment to see performance stats in the terminal every N steps + # performance_log_every: 1000 + device: cuda:0 + # if a trigger word is specified, it will be added to captions of training data if it does not already exist + # alternatively, in your captions you can add [trigger] and it will be replaced with the trigger word + # trigger_word: "p3r5on" + network: + type: "lora" + linear: 16 + linear_alpha: 16 + save: + dtype: bf16 # precision to save + save_every: 250 # save every this many steps + max_step_saves_to_keep: 2 # how many intermittent saves to keep + save_format: 'diffusers' # 'diffusers' + datasets: + # datasets are a folder of images. captions need to be txt files with the same name as the image + # for instance image2.jpg and image2.txt. Only jpg, jpeg, and png are supported currently + # images will automatically be resized and bucketed into the resolution specified + # on windows, escape back slashes with another backslash so + # "C:\\path\\to\\images\\folder" + - folder_path: "/path/to/images/folder" + caption_ext: "txt" + caption_dropout_rate: 0.05 # will drop out the caption 5% of time + shuffle_tokens: false # shuffle caption order, split by commas + # cache_latents_to_disk: true # leave this true unless you know what you're doing + resolution: [ 512, 768, 1024 ] # lumina2 enjoys multiple resolutions + train: + batch_size: 1 + + # can be 'sigmoid', 'linear', or 'lumina2_shift' + timestep_type: 'lumina2_shift' + + steps: 2000 # total number of steps to train 500 - 4000 is a good range + gradient_accumulation: 1 + train_unet: true + train_text_encoder: false # probably won't work with lumina2 + gradient_checkpointing: true # need the on unless you have a ton of vram + noise_scheduler: "flowmatch" # for training only + optimizer: "adamw8bit" + lr: 1e-4 + # uncomment this to skip the pre training sample + # skip_first_sample: true + # uncomment to completely disable sampling + # disable_sampling: true + + # ema will smooth out learning, but could slow it down. Recommended to leave on if you have the vram + ema_config: + use_ema: true + ema_decay: 0.99 + + # will probably need this if gpu supports it for lumina2, other dtypes may not work correctly + dtype: bf16 + model: + # huggingface model name or path + name_or_path: "Alpha-VLLM/Lumina-Image-2.0" + is_lumina2: true # lumina2 architecture + # you can quantize just the Gemma2 text encoder here to save vram + quantize_te: true + sample: + sampler: "flowmatch" # must match train.noise_scheduler + sample_every: 250 # sample every this many steps + width: 1024 + height: 1024 + prompts: + # you can add [trigger] to the prompts here and it will be replaced with the trigger word + # - "[trigger] holding a sign that says 'I LOVE PROMPTS!'"\ + - "woman with red hair, playing chess at the park, bomb going off in the background" + - "a woman holding a coffee cup, in a beanie, sitting at a cafe" + - "a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini" + - "a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background" + - "a bear building a log cabin in the snow covered mountains" + - "woman playing the guitar, on stage, singing a song, laser lights, punk rocker" + - "hipster man with a beard, building a chair, in a wood shop" + - "photo of a cat that is half black and half orange tabby, split down the middle. The cat has on a blue tophat. They are holding a martini glass with a pink ball of yarn in it with green knitting needles sticking out, in one paw. In the other paw, they are holding a DVD case for a movie titled, \"This is a test\" that has a golden robot on it. In the background is a busy night club with a giant mushroom man dancing with a bear." + - "a man holding a sign that says, 'this is a sign'" + - "a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle" + neg: "" + seed: 42 + walk_seed: true + guidance_scale: 4.0 + sample_steps: 25 +# you can add any additional meta info here. [name] is replaced with config name at top +meta: + name: "[name]" + version: '1.0'