diff --git a/src/llama-build-context.cpp b/src/llama-build-context.cpp
index 0b4d1a4f..658b05bd 100644
--- a/src/llama-build-context.cpp
+++ b/src/llama-build-context.cpp
@@ -4952,8 +4952,6 @@ ggml_cgraph * llm_build_context::build_gemma2() {
 ggml_cgraph * llm_build_context::build_gemma3() {
     struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
 
-    const int64_t n_embd_head_k = hparams.n_embd_head_k;
-
     struct ggml_tensor * cur;
     struct ggml_tensor * inpL;
 
@@ -4977,6 +4975,15 @@ ggml_cgraph * llm_build_context::build_gemma3() {
     // 5 layers of local attention followed by 1 layer of global attention
     static const int sliding_window_pattern = 6;
 
+    ggml_tensor * rope_cache   = nullptr;
+    ggml_tensor * rope_cache_l = nullptr;
+    if (cparams.rope_cache && (rope_type == LLAMA_ROPE_TYPE_NEOX || rope_type == LLAMA_ROPE_TYPE_NORM)) {
+        rope_cache = ggml_rope_cache(ctx0, inp_pos, nullptr, n_rot, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+            ext_factor, attn_factor, beta_fast, beta_slow);
+        rope_cache_l = ggml_rope_cache(ctx0, inp_pos, nullptr, n_rot, n_rot, rope_type, n_ctx_orig, 10000.0f, 1.0f,
+            ext_factor, attn_factor, beta_fast, beta_slow);
+    }
+
     for (int il = 0; il < n_layer; ++il) {
         const bool is_sliding          = (il + 1) % sliding_window_pattern;
         const float freq_base_l        = is_sliding ? 10000.0f    : freq_base;
@@ -4989,24 +4996,24 @@ ggml_cgraph * llm_build_context::build_gemma3() {
 
         // self-attention
         {
-            auto [Qcur, Kcur, Vcur] = llm_build_mul_mat_qkv(gf, cur, model.layers[il].wq, nullptr,
-                    model.layers[il].wk, nullptr,
-                    model.layers[il].wv, nullptr, 0, il);
+            auto [Qcur, Kcur, Vcur] = llm_build_mul_mat_qkv(gf, cur,
+                    model.layers[il].wqkv, nullptr,
+                    model.layers[il].wqk, nullptr,
+                    model.layers[il].wq, nullptr, model.layers[il].wk, nullptr, model.layers[il].wv, nullptr,
+                    model.layers[il].attn_q_norm, model.layers[il].attn_k_norm, 0, il);
 
-            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head,    n_tokens);
-            Qcur = llm_build_norm(ctx0, Qcur, hparams, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, cb, il);
-            cb(Qcur, "Qcur_normed", il);
+            if (rope_cache) {
+                auto rcache = is_sliding ? rope_cache_l : rope_cache;
+                Qcur = ggml_rope_fast(ctx0, Qcur, rcache);
+                Kcur = ggml_rope_fast(ctx0, Kcur, rcache);
+            } else {
+                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
 
-            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
-                    ext_factor, attn_factor, beta_fast, beta_slow);
+                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+            }
             cb(Qcur, "Qcur", il);
-
-            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens);
-            Kcur = llm_build_norm(ctx0, Kcur, hparams, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, cb, il);
-            cb(Kcur, "Kcur_normed", il);
-
-            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
-                    ext_factor, attn_factor, beta_fast, beta_slow);
             cb(Kcur, "Kcur", il);
 
             cur = llm_build_kv(ctx0, lctx, kv_self, gf, model.layers[il].wo, NULL,
diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp
index 717fa9c3..04db3d90 100644
--- a/src/llama-load-tensors.cpp
+++ b/src/llama-load-tensors.cpp
@@ -28,7 +28,7 @@ struct create_tensors_helper : public create_tensors_helper_interface {
 
     virtual size_t get_ctx_size() const override { return ctx_size; }
 
-    bool merge_qkv(const LLM_TN & tn, int i, int bias);
+    bool merge_qkv(const LLM_TN & tn, int i, int bias, bool ignore_attn_scale = false);
 
     bool create_tensors() override;
 
@@ -1313,9 +1313,8 @@ bool create_tensors_helper::create_gemma_tensors(const LLM_TN & tn, int version)
 
         layer.attn_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
 
-        layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q,   "weight", i), {n_embd, n_embd_head_k * n_head});
-        layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K,   "weight", i), {n_embd, n_embd_k_gqa});
-        layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V,   "weight", i), {n_embd, n_embd_v_gqa});
+        use_mmap_buffer &= !merge_qkv(tn, i, 0, true);
+
         layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
         if (version > 1) {
             layer.attn_post_norm = create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd});
@@ -2524,7 +2523,7 @@ bool create_tensors_helper::create_smollm3_tensors(const LLM_TN & tn) {
     return use_mmap_buffer;
 }
 
-bool create_tensors_helper::merge_qkv(const LLM_TN & tn, int i, int bias) {
+bool create_tensors_helper::merge_qkv(const LLM_TN & tn, int i, int bias, bool ignore_attn_scale) {
     auto& hparams = model.hparams;
     const int64_t n_head        = hparams.n_head();
     const int64_t n_head_kv     = hparams.n_head_kv();
@@ -2547,7 +2546,7 @@ bool create_tensors_helper::merge_qkv(const LLM_TN & tn, int i, int bias) {
     GGML_ASSERT(wq && wk && wv);
 
     bool fused_qkv = false;
-    if (ml.merge_qkv && wq->type == wk->type && wq->type == wv->type && hparams.f_attention_scale == 0.0f) {
+    if (ml.merge_qkv && wq->type == wk->type && wq->type == wv->type && (ignore_attn_scale || hparams.f_attention_scale == 0.0f)) {
         GGML_ASSERT(wq->ne[0] == n_embd && wq->ne[1] == n_head * n_embd_head_k);
         GGML_ASSERT(wk->ne[0] == n_embd && wk->ne[1] == n_embd_gqa);
         GGML_ASSERT(wv->ne[0] == n_embd && wv->ne[1] == n_embd_gqa);
@@ -2585,7 +2584,7 @@ bool create_tensors_helper::merge_qkv(const LLM_TN & tn, int i, int bias) {
             }
         }
     }
-    if (!fused_qkv && ml.merge_qkv && wq->type == wk->type && hparams.f_attention_scale == 0.0f) {
+    if (!fused_qkv && ml.merge_qkv && wq->type == wk->type && (ignore_attn_scale || hparams.f_attention_scale == 0.0f)) {
         GGML_ASSERT(wq->ne[0] == n_embd && wq->ne[1] == n_head * n_embd_head_k);
         GGML_ASSERT(wk->ne[0] == n_embd && wk->ne[1] == n_embd_gqa);
         layer.wqk = ggml_new_tensor_2d(ctx_split, wq->type, n_embd, n_embd_head_k * (n_head + n_head_kv));
@@ -2623,7 +2622,7 @@ bool create_tensors_helper::merge_qkv(const LLM_TN & tn, int i, int bias) {
     if (!fused_qkv) {
         if (ml.merge_qkv) {
             printf("%s: did not merge Q, K, V in layer %d because %d, %d, %d\n", __func__, i,
-                    wq->type == wk->type, wq->type == wv->type, hparams.f_attention_scale == 0.0f);
+                    wq->type == wk->type, wq->type == wv->type, (ignore_attn_scale || hparams.f_attention_scale == 0.0f));
         }
         layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q,   "weight", i), {n_embd, n_embd_head_k * n_head});
         layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K,   "weight", i), {n_embd, n_embd_gqa});