shujakuin/cl-lmm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

delete debug infomation

Browse Source
This commit is contained in:
YunyaoZhou 2025-01-02 21:52:32 +08:00
parent f230886c3f
commit ee8cc6f81d
Signed by: shujakuin
GPG Key ID: 418C3CA28E350CCF
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/peft_library/tuners/mmoelora/mmoelora.py
View File

@ -56,9 +56,7 @@ class MMOELoraModel(LoraModel):
self.peft_config = config self.peft_config = config
# self.add_adapter(adapter_name, self.peft_config[adapter_name]) # self.add_adapter(adapter_name, self.peft_config[adapter_name])
import sys; print(__file__, sys._getframe().f_lineno)
self.add_adapter(adapter_name, config=self.peft_config[adapter_name]) self.add_adapter(adapter_name, config=self.peft_config[adapter_name])
import sys; print(__file__, sys._getframe().f_lineno)
def add_adapter(self, adapter_name, config=None): def add_adapter(self, adapter_name, config=None):
if config is not None: # get the lora config if config is not None: # get the lora config
@ -71,13 +69,14 @@ class MMOELoraModel(LoraModel):
self.peft_config[adapter_name] = config # subsititue the original config self.peft_config[adapter_name] = config # subsititue the original config
self._find_and_replace(adapter_name) self._find_and_replace(adapter_name)
if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none": if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none":
raise ValueError( raise ValueError(
"MMOELoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters." "MMOELoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters."
) )
print(self.peft_config) print(self.peft_config)
self.mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) self.mark_only_lora_as_trainable(
self.model, self.peft_config[adapter_name].bias
)
if self.peft_config[adapter_name].inference_mode: if self.peft_config[adapter_name].inference_mode:
_freeze_adapter(self.model, adapter_name) _freeze_adapter(self.model, adapter_name)
@ -95,7 +94,11 @@ class MMOELoraModel(LoraModel):
p.requires_grad = True p.requires_grad = True
elif bias == "lora_only": elif bias == "lora_only":
for m in model.modules(): for m in model.modules():
if isinstance(m, LoraLayer) and hasattr(m, "bias") and m.bias is not None: if (
isinstance(m, LoraLayer)
and hasattr(m, "bias")
and m.bias is not None
):
m.bias.requires_grad = True m.bias.requires_grad = True
else: else:
raise NotImplementedError raise NotImplementedError
@ -405,7 +408,11 @@ class MMOELoraLinear(nn.Module, MMOELoraLayer):
) )
for i in range(self.expert_num): for i in range(self.expert_num):
result += ( result += (
self.lora_B[self._active_adapter].loraB[i](self.lora_A[self._active_adapter].loraA[i](self.lora_dropout[self._active_adapter](x))) self.lora_B[self._active_adapter].loraB[i](
self.lora_A[self._active_adapter].loraA[i](
self.lora_dropout[self._active_adapter](x)
)
)
* self.scaling[self._active_adapter] * self.scaling[self._active_adapter]
* expert_weight[..., i].view(-1, 1, 1) * expert_weight[..., i].view(-1, 1, 1)
) )