From d686cbc254c654c4cefbda7f3316a1a0ae576e3d Mon Sep 17 00:00:00 2001 From: YunyaoZhou Date: Tue, 3 Jun 2025 20:25:20 +0800 Subject: [PATCH] =?UTF-8?q?feat=E2=9C=A8:=20=E6=B7=BB=E5=8A=A0MOELORA?= =?UTF-8?q?=E6=94=AF=E6=8C=81=EF=BC=8C=E4=BC=98=E5=8C=96=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E5=92=8C=E8=AF=84=E4=BC=B0=E8=84=9A=E6=9C=AC=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=8B=BC=E5=86=99=E9=94=99=E8=AF=AF=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E4=BB=A3=E7=A0=81=E5=8F=AF=E8=AF=BB=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/settings.json | 2 +- src/evaluation.py | 104 ++++++++--------------------------- src/model_library/factory.py | 78 ++++++++++++++++++-------- src/peft_repo | 2 +- src/scripts/eval_omni.sh | 15 +++++ src/scripts/train_omni.sh | 5 +- src/test_evalutae.py | 31 +++++++++++ src/train.py | 21 +++---- src/utils/args.py | 2 +- src/utils/evaluate_tool.py | 94 ++++++++++++++++++++++++++----- src/utils/trainer.py | 5 +- 11 files changed, 221 insertions(+), 138 deletions(-) create mode 100755 src/scripts/eval_omni.sh create mode 100644 src/test_evalutae.py diff --git a/.vscode/settings.json b/.vscode/settings.json index e957b23..cbd3cbc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,5 +15,5 @@ "python.analysis.typeCheckingMode": "basic", "python.analysis.userFileIndexingLimit": 10000, "python.analysis.usePullDiagnostics": false, - "python.analysis.importFormat": "relative" + "python.analysis.importFormat": "relative", } \ No newline at end of file diff --git a/src/evaluation.py b/src/evaluation.py index dc25bbd..41c04d3 100644 --- a/src/evaluation.py +++ b/src/evaluation.py @@ -19,99 +19,39 @@ from trl import ( get_quantization_config, ) -from utils.args import ContinualScriptArguments, ContinualModelConfig +from utils.args import ( + ContinualScriptArguments, + ContinualModelConfig, + ContinualRegularizationArguments, +) +from typing import TYPE_CHECKING if __name__ == "__main__": parser = TrlParser( - (ContinualScriptArguments, TrainingArguments, ContinualModelConfig) + ( + ContinualScriptArguments, + TrainingArguments, + ContinualModelConfig, + ContinualRegularizationArguments, + ) ) - script_args, training_args, model_args = parser.parse_args_and_config() + script_args, training_args, model_args, reg_args = parser.parse_args_and_config() # for type hint - if 0 == 1: + if TYPE_CHECKING: script_args = ContinualScriptArguments() training_args = TrainingArguments() - model_args = ModelConfig() + model_args = ContinualModelConfig() + reg_args = ContinualRegularizationArguments() + training_args.gradient_checkpointing_kwargs = dict(use_reentrant=False) training_args.remove_unused_columns = False - training_args.dataset_kwargs = {"skip_prepare_dataset": True} from model_library.factory import get_model - if model_args.model_name_or_path == "Qwen/Qwen2.5-VL-3B-Instruct": - torch_dtype = ( - model_args.torch_dtype - if model_args.torch_dtype in ["auto", None] - else getattr(torch, model_args.torch_dtype) - ) - quantization_config = get_quantization_config(model_args) - model_kwargs = dict( - attn_implementation=model_args.attn_implementation, - torch_dtype=torch_dtype, - quantization_config=quantization_config, - ) - from transformers import Qwen2_5_VLProcessor, Qwen2_5_VLForConditionalGeneration - - model = Qwen2_5_VLForConditionalGeneration.from_pretrained( - training_args.output_dir, - **model_kwargs, - ) - - processor = Qwen2_5_VLProcessor.from_pretrained( - model_args.model_name_or_path, - trust_remote_code=model_args.trust_remote_code, - padding_side="left", - ) - - from model_library.qwen2vl import collate_fn_for_train, collate_fn_for_evaluate - from functools import partial - - collate_fn_for_train = partial(collate_fn_for_train, processor=processor) - collate_fn_for_evaluate = partial(collate_fn_for_evaluate, processor=processor) - - elif model_args.model_name_or_path == "Qwen/Qwen2-VL-7B-Instruct": - torch_dtype = ( - model_args.torch_dtype - if model_args.torch_dtype in ["auto", None] - else getattr(torch, model_args.torch_dtype) - ) - - quantization_config = get_quantization_config(model_args) - model_kwargs = dict( - attn_implementation=model_args.attn_implementation, - torch_dtype=torch_dtype, - quantization_config=quantization_config, - ) - from transformers import ( - Qwen2VLProcessor, - Qwen2VLForConditionalGeneration, - AutoModelForVision2Seq, - AutoModel, - ) - from peft.peft_model import PeftModelForCausalLM - from model_library.qwen2vl import Qwen2VLForConditionalGeneration_modified - - model = Qwen2VLForConditionalGeneration.from_pretrained( - training_args.output_dir, - **model_kwargs, - ) - - # from peft_library import get_peft_model - - processor = Qwen2VLProcessor.from_pretrained( - model_args.model_name_or_path, - trust_remote_code=model_args.trust_remote_code, - padding_side="left", - ) - from model_library.qwen2vl import ( - collate_fn_for_train, - collate_fn_for_evaluate, - ) - from functools import partial - - collate_fn_for_train = partial(collate_fn_for_train, processor=processor) - collate_fn_for_evaluate = partial(collate_fn_for_evaluate, processor=processor) - + model, processor, collate_fn_for_train, collate_fn_for_evaluate = get_model( + model_args=model_args, training_args=training_args + ) ################ # Dataset ################ @@ -139,6 +79,6 @@ if __name__ == "__main__": collate_fn=collate_fn_for_evaluate, ) val_dataloader = accelerator.prepare_data_loader(val_dataloader) - from utils.evaluate_tool import evaluate_rouge, evalute_save + from utils.evaluate_tool import evaluate_rouge, evaluate_save - evalute_save(model, val_dataloader, processor, accelerator) + evaluate_save(model, val_dataloader, processor, accelerator) diff --git a/src/model_library/factory.py b/src/model_library/factory.py index c474f80..44bae4e 100644 --- a/src/model_library/factory.py +++ b/src/model_library/factory.py @@ -5,9 +5,12 @@ from trl import ( get_quantization_config, ) from utils.args import ContinualModelConfig +from transformers import TrainingArguments -def get_model(model_args: ContinualModelConfig): +def get_model( + model_args: ContinualModelConfig, training_args: TrainingArguments = None +): torch_dtype = ( model_args.torch_dtype if model_args.torch_dtype in ["auto", None] @@ -26,12 +29,20 @@ def get_model(model_args: ContinualModelConfig): from transformers import Qwen2VLProcessor, Qwen2VLForConditionalGeneration # from .qwen2vl import Qwen2VLForConditionalGeneration_modified + if training_args is not None: + model = Qwen2VLForConditionalGeneration.from_pretrained( + training_args.output_dir, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) + + else: + model = Qwen2VLForConditionalGeneration.from_pretrained( + model_args.model_name_or_path, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) - model = Qwen2VLForConditionalGeneration.from_pretrained( - model_args.model_name_or_path, - trust_remote_code=model_args.trust_remote_code, - **model_kwargs, - ) processor = Qwen2VLProcessor.from_pretrained( model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code, @@ -49,11 +60,18 @@ def get_model(model_args: ContinualModelConfig): if model_args.model_name_or_path == "Qwen/Qwen2-Audio-7B-Instruct": from transformers import Qwen2AudioProcessor, Qwen2AudioForConditionalGeneration - model = Qwen2AudioForConditionalGeneration.from_pretrained( - model_args.model_name_or_path, - trust_remote_code=model_args.trust_remote_code, - **model_kwargs, - ) + if training_args is not None: + model = Qwen2AudioForConditionalGeneration.from_pretrained( + training_args.output_dir, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) + else: + model = Qwen2AudioForConditionalGeneration.from_pretrained( + model_args.model_name_or_path, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) processor = Qwen2AudioProcessor.from_pretrained( model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code, @@ -71,11 +89,18 @@ def get_model(model_args: ContinualModelConfig): if model_args.model_name_or_path == "Qwen/Qwen2.5-VL-3B-Instruct": from transformers import Qwen2_5_VLProcessor, Qwen2_5_VLForConditionalGeneration - model = Qwen2_5_VLForConditionalGeneration.from_pretrained( - model_args.model_name_or_path, - trust_remote_code=model_args.trust_remote_code, - **model_kwargs, - ) + if training_args is not None: + model = Qwen2_5_VLForConditionalGeneration.from_pretrained( + training_args.output_dir, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) + else: + model = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_args.model_name_or_path, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) processor = Qwen2_5_VLProcessor.from_pretrained( model_args.model_name_or_path, @@ -88,18 +113,25 @@ def get_model(model_args: ContinualModelConfig): collate_fn_for_train = partial(collate_fn_for_train, processor=processor) collate_fn_for_evaluate = partial(collate_fn_for_evaluate, processor=processor) - + if model_args.model_name_or_path == "Qwen/Qwen2.5-Omni-3B": from transformers.models.qwen2_5_omni import ( Qwen2_5OmniThinkerForConditionalGeneration, - Qwen2_5OmniProcessor + Qwen2_5OmniProcessor, ) - model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( - model_args.model_name_or_path, - trust_remote_code=model_args.trust_remote_code, - **model_kwargs, - ) + if training_args is not None: + model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( + training_args.output_dir, + **model_kwargs, + ) + + else: + model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( + model_args.model_name_or_path, + trust_remote_code=model_args.trust_remote_code, + **model_kwargs, + ) processor = Qwen2_5OmniProcessor.from_pretrained( model_args.model_name_or_path, diff --git a/src/peft_repo b/src/peft_repo index 317d957..f58e3bd 160000 --- a/src/peft_repo +++ b/src/peft_repo @@ -1 +1 @@ -Subproject commit 317d957cc101c4cb064066a1b228526a55f6e927 +Subproject commit f58e3bd57f3f6cf2f713edaac4b8a54ecafe8e20 diff --git a/src/scripts/eval_omni.sh b/src/scripts/eval_omni.sh new file mode 100755 index 0000000..dc70208 --- /dev/null +++ b/src/scripts/eval_omni.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +accelerate launch --config_file configs/accelerate_configs/deepspeed_zero1.yaml evaluation.py \ + --dataset_name textvqa \ + --use_peft \ + --peft_type MOELORA \ + --model_name_or_path Qwen/Qwen2.5-Omni-3B \ + --lora_target_modules .*model\.layers.*proj\|.*merger.*0\|.*merger.*1 \ + --per_device_train_batch_size 3 \ + --per_device_eval_batch_size 2 \ + --gradient_accumulation_steps 2 \ + --output_dir ./checkpoint/qwen2_5omni_moelora/ \ + --bf16 \ + --torch_dtype bfloat16 +# --eval_strategy epoch \ diff --git a/src/scripts/train_omni.sh b/src/scripts/train_omni.sh index 1b8c7b1..147f6a5 100755 --- a/src/scripts/train_omni.sh +++ b/src/scripts/train_omni.sh @@ -18,7 +18,8 @@ accelerate launch --config_file configs/accelerate_configs/deepspeed_zero1.yaml --lr_scheduler_type cosine \ --bf16 \ --torch_dtype bfloat16 \ - --logging_steps 10 \ + --logging_steps 100 \ --gradient_checkpointing \ --weight_decay 0.1 \ - # --resume_from_checkpoint /root/autodl-tmp/zhouyunyao/projects/CL-LMM/src/checkpoint/qwen2_alllinear/checkpoint-1000 \ No newline at end of file + --eval_strategy steps \ + # --resume_from_checkpoint /root/autodl-tmp/zhouyunyao/projects/CL-LMM/src/checkpoint/qwen2_5omni_moelora/checkpoint-1500 \ No newline at end of file diff --git a/src/test_evalutae.py b/src/test_evalutae.py new file mode 100644 index 0000000..71e1e68 --- /dev/null +++ b/src/test_evalutae.py @@ -0,0 +1,31 @@ +import evaluate + +# Bleu_1, Bleu_2, Bleu_3, Bleu_4, METEOR, ROUGE_L, and CIDEr +example = { + "generated": "The cat sat on the mat.", + "target": "The cat is sitting on the mat.", + "original": "The cat is sitting on the mat.", +} +evaluate_bleu = evaluate.load("bleu") +evaluate_rouge = evaluate.load("rouge") +evaluate_meteor = evaluate.load("meteor") + +evaluate_bleu.add_batch( + predictions=[example["generated"]], + references=[[example["target"]]], +) +evaluate_rouge.add_batch( + predictions=[example["generated"]], + references=[[example["target"]]], +) +evaluate_meteor.add_batch( + predictions=[example["generated"]], + references=[[example["target"]]], +) + +bleu = evaluate_bleu.compute() +rouge = evaluate_rouge.compute() +meteor = evaluate_meteor.compute() + +comprehensive_results = sum(bleu['precisions']) + rouge['rougeL'] + meteor['meteor'] +print("Comprehensive Results:", comprehensive_results/6) \ No newline at end of file diff --git a/src/train.py b/src/train.py index 79e262c..23c0280 100644 --- a/src/train.py +++ b/src/train.py @@ -16,7 +16,7 @@ from utils.trainer import ContinualTrainer from utils.args import ( ContinualScriptArguments, ContinualModelConfig, - ContiunalRegularizationArguments, + ContinualRegularizationArguments, ) import logging from typing import TYPE_CHECKING @@ -31,8 +31,8 @@ if __name__ == "__main__": ContinualScriptArguments, TrainingArguments, ContinualModelConfig, - ContiunalRegularizationArguments, - ) # type: ignore + ContinualRegularizationArguments, + ) # type: ignore ) script_args, training_args, model_args, reg_args = parser.parse_args_and_config() # for type hint @@ -40,7 +40,7 @@ if __name__ == "__main__": script_args = ContinualScriptArguments() training_args = TrainingArguments() model_args = ContinualModelConfig() - reg_args = ContiunalRegularizationArguments() + reg_args = ContinualRegularizationArguments() training_args.gradient_checkpointing_kwargs = dict(use_reentrant=False) training_args.remove_unused_columns = False @@ -48,7 +48,7 @@ if __name__ == "__main__": from model_library.factory import get_model model, processor, collate_fn_for_train, collate_fn_for_evaluate = get_model( - model_args + model_args=model_args ) ################ # Dataset @@ -100,9 +100,9 @@ if __name__ == "__main__": model=model, args=training_args, data_collator=collate_fn_for_train, - train_dataset=dataset[script_args.dataset_train_split], # type: ignore + train_dataset=dataset[script_args.dataset_train_split], # type: ignore eval_dataset=( - dataset[script_args.dataset_test_split] # type: ignore + dataset[script_args.dataset_test_split] # type: ignore if training_args.eval_strategy != "no" else None ), @@ -113,7 +113,8 @@ if __name__ == "__main__": if accelerator.is_local_main_process: print("Saving model") - trainer.save_model(training_args.output_dir) + # trainer.save_model(training_args.output_dir) + model.save_pretrained(training_args.output_dir) if accelerator.is_local_main_process: print("Model saved") @@ -131,6 +132,6 @@ if __name__ == "__main__": # ) # val_dataloader = accelerator.prepare(val_dataloader) - # from utils.evaluate_tool import evaluate_rouge + # from utils.evaluate_tool import evaluate_save - # evaluate_rouge(model, val_dataloader, processor) + # evaluate_save(model, val_dataloader, processor, accelerator) diff --git a/src/utils/args.py b/src/utils/args.py index 0180178..360eea8 100644 --- a/src/utils/args.py +++ b/src/utils/args.py @@ -21,7 +21,7 @@ class ContinualModelConfig(ModelConfig): @dataclass -class ContiunalRegularizationArguments: +class ContinualRegularizationArguments: """Regularization arguments for continual learning.""" # EWC diff --git a/src/utils/evaluate_tool.py b/src/utils/evaluate_tool.py index 664064f..87ebe47 100644 --- a/src/utils/evaluate_tool.py +++ b/src/utils/evaluate_tool.py @@ -1,5 +1,6 @@ import evaluate from accelerate import Accelerator +from typing import TYPE_CHECKING def evaluate_rouge(model, val_dataloader, processor, accelerator: Accelerator = None): @@ -7,10 +8,7 @@ def evaluate_rouge(model, val_dataloader, processor, accelerator: Accelerator = for batch in val_dataloader: completion = model.generate( - input_ids=batch["input_ids"], - attention_mask=batch["attention_mask"], - pixel_values=batch["pixel_values"], - image_grid_thw=batch["image_grid_thw"], + **batch, max_length=1000, ) target = batch["answers_ids"] @@ -27,7 +25,7 @@ def evaluate_rouge(model, val_dataloader, processor, accelerator: Accelerator = print(glue.compute()) -def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = None): +def evaluate_save(model, val_dataloader, processor, accelerator: Accelerator = None): import os mtime = 0 @@ -53,6 +51,7 @@ def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = No answers = [] completion = model.generate( **batch, + # max_new_tokens=30, max_length=1000, ) generated_text = [ @@ -63,20 +62,17 @@ def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = No generated_text, skip_special_tokens=True ) target_text = processor.tokenizer.batch_decode(target, skip_special_tokens=True) - for i in range(len(generated_text)): - answers.append( - { - "generated": generated_text[i], - "target": target_text[i], - "original": str(origianl[i]), - } - ) import json world_size = accelerator.process_index - with open(f"results/{mtime}/answers_{world_size}.jsonl", "a") as f: - for answer in answers: + for i in range(len(generated_text)): + answer = { + "generated": generated_text[i], + "target": target_text[i], + "original": str(origianl[i]), + } + with open(f"results/{mtime}/answers_{world_size}.jsonl", "a") as f: f.write(json.dumps(answer) + "\n") if accelerator.is_local_main_process: @@ -97,3 +93,71 @@ def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = No # delete file for file in files: os.remove(f"results/{mtime}/{file}") + + +def evaluate_from_jsonl_directory(directory_path): + """ + 从指定目录读取所有jsonl文件并计算综合评估结果 + + Args: + directory_path: 包含jsonl文件的目录路径 + + Returns: + dict: 包含各项指标和综合结果的字典 + """ + import os + import json + + # 初始化评估器 + evaluate_bleu = evaluate.load("bleu") + evaluate_rouge = evaluate.load("rouge") + evaluate_meteor = evaluate.load("meteor") + + # 读取目录下所有jsonl文件 + all_data = [] + for file in os.listdir(directory_path): + if file.endswith(".jsonl"): + file_path = os.path.join(directory_path, file) + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + all_data.append(data) + + if not all_data: + print(f"未在目录 {directory_path} 中找到有效的jsonl数据") + return None + + # 准备数据 + predictions = [item["generated"] for item in all_data] + references = [[item["target"]] for item in all_data] + + # 批量添加数据 + evaluate_bleu.add_batch(predictions=predictions, references=references) + evaluate_rouge.add_batch(predictions=predictions, references=references) + evaluate_meteor.add_batch(predictions=predictions, references=references) + + # 计算结果 + bleu = evaluate_bleu.compute() + rouge = evaluate_rouge.compute() + meteor = evaluate_meteor.compute() + + # 计算综合结果 + comprehensive_score = (sum(bleu["precisions"]) + rouge["rougeL"] + meteor["meteor"]) / 6 + + results = { + "bleu": bleu, + "rouge": rouge, + "meteor": meteor, + "comprehensive_score": comprehensive_score, + "total_samples": len(all_data), + } + + print(f"评估完成，共处理 {len(all_data)} 条数据") + print(f"BLEU分数: {bleu}") + print(f"ROUGE分数: {rouge}") + print(f"METEOR分数: {meteor}") + print(f"综合分数: {comprehensive_score}") + + return results diff --git a/src/utils/trainer.py b/src/utils/trainer.py index cc7acc0..a55ee39 100644 --- a/src/utils/trainer.py +++ b/src/utils/trainer.py @@ -5,7 +5,7 @@ from transformers.trainer import * from transformers import ( TrainingArguments, ) -from .args import ContiunalRegularizationArguments +from .args import ContinualRegularizationArguments from peft_library.regularizations import EWC, LWF from torch.nn import CrossEntropyLoss @@ -41,7 +41,7 @@ class ContinualTrainer(Trainer): train_dataset, eval_dataset, accelerator, - reg_args: ContiunalRegularizationArguments = None, + reg_args: ContinualRegularizationArguments = None, ): self.accelerator = accelerator super().__init__( @@ -155,4 +155,3 @@ class ContinualTrainer(Trainer): self.optimizer = smp.DistributedOptimizer(self.optimizer) return self.optimizer -