From ee8cc6f81d658efc6bb6054a3f90108d51e2b106 Mon Sep 17 00:00:00 2001 From: YunyaoZhou Date: Thu, 2 Jan 2025 21:52:32 +0800 Subject: [PATCH] delete debug infomation --- src/peft_library/tuners/mmoelora/mmoelora.py | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/peft_library/tuners/mmoelora/mmoelora.py b/src/peft_library/tuners/mmoelora/mmoelora.py index c7938fd..a9c3000 100644 --- a/src/peft_library/tuners/mmoelora/mmoelora.py +++ b/src/peft_library/tuners/mmoelora/mmoelora.py @@ -56,9 +56,7 @@ class MMOELoraModel(LoraModel): self.peft_config = config # self.add_adapter(adapter_name, self.peft_config[adapter_name]) - import sys; print(__file__, sys._getframe().f_lineno) self.add_adapter(adapter_name, config=self.peft_config[adapter_name]) - import sys; print(__file__, sys._getframe().f_lineno) def add_adapter(self, adapter_name, config=None): if config is not None: # get the lora config @@ -71,18 +69,19 @@ class MMOELoraModel(LoraModel): self.peft_config[adapter_name] = config # subsititue the original config self._find_and_replace(adapter_name) - if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none": raise ValueError( "MMOELoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters." ) print(self.peft_config) - self.mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) + self.mark_only_lora_as_trainable( + self.model, self.peft_config[adapter_name].bias + ) if self.peft_config[adapter_name].inference_mode: _freeze_adapter(self.model, adapter_name) - - def mark_only_lora_as_trainable(self,model: nn.Module, bias: str = "none") -> None: + + def mark_only_lora_as_trainable(self, model: nn.Module, bias: str = "none") -> None: """Only activate the LoRA layer as trainable""" for n, p in model.named_parameters(): if "lora_" not in n: @@ -95,7 +94,11 @@ class MMOELoraModel(LoraModel): p.requires_grad = True elif bias == "lora_only": for m in model.modules(): - if isinstance(m, LoraLayer) and hasattr(m, "bias") and m.bias is not None: + if ( + isinstance(m, LoraLayer) + and hasattr(m, "bias") + and m.bias is not None + ): m.bias.requires_grad = True else: raise NotImplementedError @@ -378,7 +381,7 @@ class MMOELoraLinear(nn.Module, MMOELoraLayer): def forward(self, x: torch.Tensor, **kwargs): # task_id = kwargs["task_id"] - for k,v in kwargs.items(): + for k, v in kwargs.items(): print(k, v.shape) task_id = torch.tensor([0] * len(x), dtype=torch.long).to(x.device) previous_dtype = x.dtype @@ -405,7 +408,11 @@ class MMOELoraLinear(nn.Module, MMOELoraLayer): ) for i in range(self.expert_num): result += ( - self.lora_B[self._active_adapter].loraB[i](self.lora_A[self._active_adapter].loraA[i](self.lora_dropout[self._active_adapter](x))) + self.lora_B[self._active_adapter].loraB[i]( + self.lora_A[self._active_adapter].loraA[i]( + self.lora_dropout[self._active_adapter](x) + ) + ) * self.scaling[self._active_adapter] * expert_weight[..., i].view(-1, 1, 1) )