From 7b5fb602d24259d77bbe3e5433aa47d9df4f2c74 Mon Sep 17 00:00:00 2001
From: Wei Fu <36355462+garrett4wade@users.noreply.github.com>
Date: Thu, 12 Sep 2024 09:34:09 +0800
Subject: [PATCH] Fix `tie_embedding` config in qwen2 (#69)

---
 realhf/api/from_hf/qwen2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/realhf/api/from_hf/qwen2.py b/realhf/api/from_hf/qwen2.py
index ad4f5d40..7e0243e0 100644
--- a/realhf/api/from_hf/qwen2.py
+++ b/realhf/api/from_hf/qwen2.py
@@ -52,6 +52,7 @@ def convert_config_qwen2(
         apply_rotary=True,
         rotary_base=hf_config.rope_theta,
         rotary_interleaved=False,
+        tied_embedding=hf_config.tie_word_embeddings,
     )
 
 
@@ -70,6 +71,7 @@ def convert_config_back_qwen2(
         hidden_act=config.activation_function,
         attention_dropout=config.attn_pdrop,
         rope_theta=config.rotary_base,
+        tie_word_embeddings=config.tied_embedding,
         architectures=["Qwen2ForCausalLM"],
     )