From 68c3e053fb794c9eecdd3a36d4862be86ef37fb0 Mon Sep 17 00:00:00 2001 From: YunyaoZhou Date: Wed, 15 Jan 2025 21:56:20 +0800 Subject: [PATCH] =?UTF-8?q?feat=E2=9C=A8:=20=E6=B7=BB=E5=8A=A0=E5=AD=90?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E6=94=AF=E6=8C=81=EF=BC=8C=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E8=AE=AD=E7=BB=83=E5=92=8C=E8=AF=84?= =?UTF-8?q?=E4=BC=B0=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitmodules | 6 ++++++ src/.vscode/settings.json | 6 ++++++ src/dataset_library/CHEM.py | 6 +++--- src/evaluation.py | 4 ++++ src/evaluation.sh | 2 +- src/model_library/factory.py | 2 ++ src/peft_repo | 1 + src/train.py | 12 ++++++------ src/train.sh | 4 ++-- src/transformers_repo | 1 + 10 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 .gitmodules create mode 100644 src/.vscode/settings.json create mode 160000 src/peft_repo create mode 160000 src/transformers_repo diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e27a49e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "src/transformers_repo"] + path = src/transformers_repo + url = git@github.com:Shujakuinkuraudo/transformers.git +[submodule "src/peft_repo"] + path = src/peft_repo + url = git@github.com:Shujakuinkuraudo/peft.git diff --git a/src/.vscode/settings.json b/src/.vscode/settings.json new file mode 100644 index 0000000..1261ba3 --- /dev/null +++ b/src/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "python.analysis.extraPaths": [ + "transformers_repo/src/", + "peft_repo/src/" + ] +} \ No newline at end of file diff --git a/src/dataset_library/CHEM.py b/src/dataset_library/CHEM.py index 27a8849..357f584 100644 --- a/src/dataset_library/CHEM.py +++ b/src/dataset_library/CHEM.py @@ -25,9 +25,9 @@ class CHEMDataset(Dataset): def _vis_processor(self, image: Image.Image): width, height = image.size - if width > 800 or height > 800: + if width > 600 or height > 600: max_size = max(width, height) - ratio = 800 / max_size + ratio = 600 / max_size new_width = int(width * ratio) new_height = int(height * ratio) image = image.resize((new_width, new_height), Image.Resampling.BILINEAR) @@ -69,7 +69,7 @@ class CHEMDataset(Dataset): return processed_data def __len__(self): - return len(self.data) + return len(self.data) // 60 def __getitem__(self, index): sample = self.data[index] diff --git a/src/evaluation.py b/src/evaluation.py index 1a1e41f..1a0a16f 100644 --- a/src/evaluation.py +++ b/src/evaluation.py @@ -1,3 +1,7 @@ +import sys +sys.path.insert(0, "./transformers_repo/src/") +sys.path.insert(0, "./peft_repo/src/") + import torch from dataset_library.factory import get_dataset from transformers import ( diff --git a/src/evaluation.sh b/src/evaluation.sh index 0e0655f..598c97b 100755 --- a/src/evaluation.sh +++ b/src/evaluation.sh @@ -9,7 +9,7 @@ accelerate launch --config_file configs/accelerate_configs/deepspeed_zero2.yaml --per_device_train_batch_size 1 \ --per_device_eval_batch_size 2 \ --gradient_accumulation_steps 8 \ - --output_dir checkpoint/qwen2/ \ + --output_dir checkpoint/qwen2mmoe/ \ --bf16 \ --torch_dtype bfloat16 # --eval_strategy epoch \ diff --git a/src/model_library/factory.py b/src/model_library/factory.py index 74ed698..85f85a9 100644 --- a/src/model_library/factory.py +++ b/src/model_library/factory.py @@ -5,6 +5,8 @@ from trl import ( get_quantization_config, ) from utils.args import ContinualModelConfig +import transformers +print(transformers.__version__) def get_model(model_args: ContinualModelConfig): diff --git a/src/peft_repo b/src/peft_repo new file mode 160000 index 0000000..de88c70 --- /dev/null +++ b/src/peft_repo @@ -0,0 +1 @@ +Subproject commit de88c703065fdd3a05521da1054c0463d16ea33c diff --git a/src/train.py b/src/train.py index 7f68bfb..5808322 100644 --- a/src/train.py +++ b/src/train.py @@ -1,5 +1,8 @@ -from dataset_library.factory import get_dataset +import sys +sys.path.insert(0, "./transformers_repo/src/") +sys.path.insert(0, "./peft_repo/src/") +from dataset_library.factory import get_dataset from transformers import ( TrainingArguments, @@ -8,8 +11,6 @@ from transformers import ( from trl import ( TrlParser, ) -from peft_library import get_peft_model - from utils.trainer import ContinualTrainer from utils.args import ContinualScriptArguments, ContinualModelConfig import logging @@ -49,10 +50,9 @@ if __name__ == "__main__": peft_config = MMOELoraConfig(target_modules=model_args.lora_target_modules) - model = get_peft_model(model, peft_config) + # model = get_peft_model(model, peft_config) + model.add_adapter(peft_config) - if accelerator.is_local_main_process: - model.print_trainable_parameters() elif model_args.peft_type == "LORA": from peft.tuners.lora import LoraConfig diff --git a/src/train.sh b/src/train.sh index 50b26f7..3c56c2f 100755 --- a/src/train.sh +++ b/src/train.sh @@ -3,13 +3,13 @@ accelerate launch --config_file configs/accelerate_configs/deepspeed_zero2.yaml train.py \ --dataset_name CHEM \ --use_peft \ - --peft_type LORA \ + --peft_type MMOELORA \ --model_name_or_path Qwen/Qwen2-VL-7B-Instruct \ --lora_target_modules q_proj v_proj \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 2 \ --gradient_accumulation_steps 4 \ - --output_dir checkpoint/qwen2/ \ + --output_dir checkpoint/qwen2mmoe/ \ --bf16 \ --torch_dtype bfloat16 \ --logging_steps 30 diff --git a/src/transformers_repo b/src/transformers_repo new file mode 160000 index 0000000..6bc0fbc --- /dev/null +++ b/src/transformers_repo @@ -0,0 +1 @@ +Subproject commit 6bc0fbcfa7acb6ac4937e7456a76c2f7975fefec