#315 - [GH-ISSUE #362] CLIP - No progression in Training - openai/CLIP

bsakaguchi commented

2026-10-31 21:13:45 +00:00

Owner

Originally created by @TEnsorTHiru on GitHub (May 27, 2023).
Original GitHub issue: https://github.com/openai/CLIP/issues/362

I'm trying to train CLIP-ViT-B/32 in my dataset. The model is not learning anything, The validation loss doesn't reduce at all. IDK what mistake am I doing. I have attached my code here. Please do let me know If I'm doing some mistake in here

def train_epoch(epoch, model, trainloader, optim, train_image_loss, train_txt_loss):
    model.train()
    train_loss = 0.0
    pbar = tqdm(enumerate(trainloader), total=len(trainloader))
    for i, batch in pbar:
        image, caption = batch
        image = image.to(device).float()
        caption = caption.to(device).long()

        image_logits, text_logits = model(image, caption)

        gt = torch.arange(cfg.batch).to(device)

        total_train_image_loss = train_image_loss(image_logits, gt)
        total_train_text_loss = train_text_loss(text_logits, gt)
        
        total_train_loss = (total_train_image_loss + total_train_text_loss)/2
        total_train_loss.backward()

        train_loss += total_train_loss.item()
    
    train_epoch_loss = train_loss / len(trainloader)

    return train_epoch_loss


def valid_epoch(epoch, model, valloader, val_image_loss, val_txt_loss):
    model.eval()
    val_loss = 0.0
    pbar = tqdm(enumerate(valloader), total=len(valloader))
    for i, batch in pbar:
        image, caption = batch
        image = image.to(device).float()
        caption = caption.to(device).long()
        
        with torch.no_grad():
            image_logits, text_logits = model(image, caption)

        gt = torch.arange(cfg.batch).to(device)

        total_val_loss = (val_image_loss(image_logits, gt) + val_txt_loss(text_logits, gt))/2

        val_loss += total_val_loss.item()
    
    val_epoch_loss = val_loss / len(valloader)

    return val_epoch_loss


        


for i in range(FOLD_NUM):
    RUN_NAME = f"Finetune_Fold_{i}"
    

    labels = [x.strip() for x in open("labels.txt", "r").readlines()]
    images = len(glob.glob("resized2/*/*"))
    samples_per_class = [len(os.listdir(f"resized2/{x}")) for x in labels]

    model, preprocess = clip.load("ViT-B/32", device=device, jit=False)
    clip.model.convert_weights(model)
    
    for params in model.parameters():
        params.requires_grad_(True)
        
    

    train_data = CLIPDataset(data, preprocess, i, mode="train", aug=cfg.aug_mode)
    val_data = CLIPDataset(data, preprocess, i, mode="val", aug=cfg.aug_mode)

    trainloader = DataLoader(
        train_data, batch_size=cfg.batch, num_workers=16, pin_memory=True, drop_last=True
    )
    valloader = DataLoader(val_data, batch_size=cfg.batch, num_workers=16, pin_memory=True, drop_last=True)
    
    # optimizer = torch.optim.Adam(model.parameters(), lr=cfg.initial_lr, momentum=cfg.momentum, nesterov=cfg.nesterov)
    optimizer = torch.optim.Adam(model.parameters(), lr=5e-5,betas=(0.9,0.98),eps=1e-6,weight_decay=0.2) #Params used from paper, the lr is smaller, more safe for fine tuning to new dataset


    print("model, optimizer and scheduler are loaded and ready to go..........")


    train_image_loss, train_text_loss = nn.CrossEntropyLoss(), nn.CrossEntropyLoss()
    val_image_loss, val_text_loss = nn.CrossEntropyLoss(), nn.CrossEntropyLoss()

    
    with mlflow.start_run():       
        best_val_loss = float("inf")        
        no_improve_counter = 0
            
        for e in range(cfg.epochs):
            train_epoch_loss = train_epoch(e, model, trainloader, optimizer, train_image_loss, train_text_loss)
            mlflow.log_metric(key="train_loss", value=train_epoch_loss, step=e)

            val_epoch_loss = valid_epoch(e, model, valloader, val_image_loss, val_text_loss)
            mlflow.log_metric(key="val_loss", value=val_epoch_loss, step=e)

            if val_epoch_loss < best_val_loss:
                print(f"The model improved from {best_val_loss} to {val_epoch_loss}")
                best_val_loss = val_epoch_loss
                torch.save({
                    "epoch": e,
                    "model_state_dict": model.state_dict(),
                    "optimizer_state_dict": optimizer.state_dict()
                }, f"{cfg.model_dir}/clip_finetuned_fold{i}_loss{val_epoch_loss:.4f}.pth")
            

            else:
                print(f"No Improvements: {val_epoch_loss}")
                no_improve_counter += 1
                
                if no_improve_counter % cfg.counter_patience == 0:
                    break

Originally created by @TEnsorTHiru on GitHub (May 27, 2023). Original GitHub issue: https://github.com/openai/CLIP/issues/362 I'm trying to train CLIP-ViT-B/32 in my dataset. The model is not learning anything, The validation loss doesn't reduce at all. IDK what mistake am I doing. I have attached my code here. Please do let me know If I'm doing some mistake in here ``` def train_epoch(epoch, model, trainloader, optim, train_image_loss, train_txt_loss): model.train() train_loss = 0.0 pbar = tqdm(enumerate(trainloader), total=len(trainloader)) for i, batch in pbar: image, caption = batch image = image.to(device).float() caption = caption.to(device).long() image_logits, text_logits = model(image, caption) gt = torch.arange(cfg.batch).to(device) total_train_image_loss = train_image_loss(image_logits, gt) total_train_text_loss = train_text_loss(text_logits, gt) total_train_loss = (total_train_image_loss + total_train_text_loss)/2 total_train_loss.backward() train_loss += total_train_loss.item() train_epoch_loss = train_loss / len(trainloader) return train_epoch_loss def valid_epoch(epoch, model, valloader, val_image_loss, val_txt_loss): model.eval() val_loss = 0.0 pbar = tqdm(enumerate(valloader), total=len(valloader)) for i, batch in pbar: image, caption = batch image = image.to(device).float() caption = caption.to(device).long() with torch.no_grad(): image_logits, text_logits = model(image, caption) gt = torch.arange(cfg.batch).to(device) total_val_loss = (val_image_loss(image_logits, gt) + val_txt_loss(text_logits, gt))/2 val_loss += total_val_loss.item() val_epoch_loss = val_loss / len(valloader) return val_epoch_loss for i in range(FOLD_NUM): RUN_NAME = f"Finetune_Fold_{i}" labels = [x.strip() for x in open("labels.txt", "r").readlines()] images = len(glob.glob("resized2/*/*")) samples_per_class = [len(os.listdir(f"resized2/{x}")) for x in labels] model, preprocess = clip.load("ViT-B/32", device=device, jit=False) clip.model.convert_weights(model) for params in model.parameters(): params.requires_grad_(True) train_data = CLIPDataset(data, preprocess, i, mode="train", aug=cfg.aug_mode) val_data = CLIPDataset(data, preprocess, i, mode="val", aug=cfg.aug_mode) trainloader = DataLoader( train_data, batch_size=cfg.batch, num_workers=16, pin_memory=True, drop_last=True ) valloader = DataLoader(val_data, batch_size=cfg.batch, num_workers=16, pin_memory=True, drop_last=True) # optimizer = torch.optim.Adam(model.parameters(), lr=cfg.initial_lr, momentum=cfg.momentum, nesterov=cfg.nesterov) optimizer = torch.optim.Adam(model.parameters(), lr=5e-5,betas=(0.9,0.98),eps=1e-6,weight_decay=0.2) #Params used from paper, the lr is smaller, more safe for fine tuning to new dataset print("model, optimizer and scheduler are loaded and ready to go..........") train_image_loss, train_text_loss = nn.CrossEntropyLoss(), nn.CrossEntropyLoss() val_image_loss, val_text_loss = nn.CrossEntropyLoss(), nn.CrossEntropyLoss() with mlflow.start_run(): best_val_loss = float("inf") no_improve_counter = 0 for e in range(cfg.epochs): train_epoch_loss = train_epoch(e, model, trainloader, optimizer, train_image_loss, train_text_loss) mlflow.log_metric(key="train_loss", value=train_epoch_loss, step=e) val_epoch_loss = valid_epoch(e, model, valloader, val_image_loss, val_text_loss) mlflow.log_metric(key="val_loss", value=val_epoch_loss, step=e) if val_epoch_loss < best_val_loss: print(f"The model improved from {best_val_loss} to {val_epoch_loss}") best_val_loss = val_epoch_loss torch.save({ "epoch": e, "model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict() }, f"{cfg.model_dir}/clip_finetuned_fold{i}_loss{val_epoch_loss:.4f}.pth") else: print(f"No Improvements: {val_epoch_loss}") no_improve_counter += 1 if no_improve_counter % cfg.counter_patience == 0: break ```

bsakaguchi commented

2026-10-31 21:13:46 +00:00

Author

Owner

@Crawler995 commented on GitHub (Jul 20, 2023):

hello, I am trying to compress the image encoder and fine-tune it to recover the accuracy. And I face a similar problem to you that the training loss gradually reduces to 4.13 (using ~200 iterations), after it, the loss no longer reduces. This is so strange.

@Crawler995 commented on GitHub (Jul 20, 2023): hello, I am trying to compress the image encoder and fine-tune it to recover the accuracy. And I face a similar problem to you that the training loss gradually reduces to 4.13 (using ~200 iterations), after it, the loss no longer reduces. This is so strange.

bsakaguchi commented

2026-10-31 21:13:46 +00:00

Author

Owner

@TEnsorTHiru commented on GitHub (Jul 25, 2023):

hello, I am trying to compress the image encoder and fine-tune it to recover the accuracy. And I face a similar problem to you that the training loss gradually reduces to 4.13 (using ~200 iterations), after it, the loss no longer reduces. This is so strange.

If you have solved it, please LMK. I still face this issue and IDK why?

@TEnsorTHiru commented on GitHub (Jul 25, 2023): > hello, I am trying to compress the image encoder and fine-tune it to recover the accuracy. And I face a similar problem to you that the training loss gradually reduces to 4.13 (using ~200 iterations), after it, the loss no longer reduces. This is so strange. If you have solved it, please LMK. I still face this issue and IDK why?

bsakaguchi commented

2026-10-31 21:13:46 +00:00

Author

Owner

@502dxceit commented on GitHub (Oct 26, 2023):

I can't find optim.zero_grad() after loss.backward(), have you cleaned the grad after backpropagation?

@502dxceit commented on GitHub (Oct 26, 2023): I can't find optim.zero_grad() after loss.backward(), have you cleaned the grad after backpropagation?

bsakaguchi commented

2026-10-31 21:13:46 +00:00

Author

Owner

@tengshaofeng commented on GitHub (Feb 18, 2025):

@Crawler995 @502dxceit @TEnsorTHiru @longouyang @welinder this is my training code, everything gose well. https://github.com/tengshaofeng/finetune-jina-clip-v2

@tengshaofeng commented on GitHub (Feb 18, 2025): @Crawler995 @502dxceit @TEnsorTHiru @longouyang @welinder this is my training code, everything gose well. https://github.com/tengshaofeng/finetune-jina-clip-v2

bsakaguchi referenced this issue

2026-10-31 21:15:28 +00:00

[PR #315] [MERGED] Fix "AttributeError: module 'numpy' has no attribute 'float'" error in the README example #487

Rows
Columns

[GH-ISSUE #362] CLIP - No progression in Training #315