feat✨: 添加MOELORA支持,优化训练和评估脚本,修复拼写错误,提升代码可读性
This commit is contained in:
parent
b84ebb03c7
commit
d686cbc254
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@ -15,5 +15,5 @@
|
||||
"python.analysis.typeCheckingMode": "basic",
|
||||
"python.analysis.userFileIndexingLimit": 10000,
|
||||
"python.analysis.usePullDiagnostics": false,
|
||||
"python.analysis.importFormat": "relative"
|
||||
"python.analysis.importFormat": "relative",
|
||||
}
|
@ -19,99 +19,39 @@ from trl import (
|
||||
get_quantization_config,
|
||||
)
|
||||
|
||||
from utils.args import ContinualScriptArguments, ContinualModelConfig
|
||||
from utils.args import (
|
||||
ContinualScriptArguments,
|
||||
ContinualModelConfig,
|
||||
ContinualRegularizationArguments,
|
||||
)
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = TrlParser(
|
||||
(ContinualScriptArguments, TrainingArguments, ContinualModelConfig)
|
||||
(
|
||||
ContinualScriptArguments,
|
||||
TrainingArguments,
|
||||
ContinualModelConfig,
|
||||
ContinualRegularizationArguments,
|
||||
)
|
||||
script_args, training_args, model_args = parser.parse_args_and_config()
|
||||
)
|
||||
script_args, training_args, model_args, reg_args = parser.parse_args_and_config()
|
||||
# for type hint
|
||||
if 0 == 1:
|
||||
if TYPE_CHECKING:
|
||||
script_args = ContinualScriptArguments()
|
||||
training_args = TrainingArguments()
|
||||
model_args = ModelConfig()
|
||||
model_args = ContinualModelConfig()
|
||||
reg_args = ContinualRegularizationArguments()
|
||||
|
||||
training_args.gradient_checkpointing_kwargs = dict(use_reentrant=False)
|
||||
training_args.remove_unused_columns = False
|
||||
training_args.dataset_kwargs = {"skip_prepare_dataset": True}
|
||||
|
||||
from model_library.factory import get_model
|
||||
|
||||
if model_args.model_name_or_path == "Qwen/Qwen2.5-VL-3B-Instruct":
|
||||
torch_dtype = (
|
||||
model_args.torch_dtype
|
||||
if model_args.torch_dtype in ["auto", None]
|
||||
else getattr(torch, model_args.torch_dtype)
|
||||
model, processor, collate_fn_for_train, collate_fn_for_evaluate = get_model(
|
||||
model_args=model_args, training_args=training_args
|
||||
)
|
||||
quantization_config = get_quantization_config(model_args)
|
||||
model_kwargs = dict(
|
||||
attn_implementation=model_args.attn_implementation,
|
||||
torch_dtype=torch_dtype,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
from transformers import Qwen2_5_VLProcessor, Qwen2_5_VLForConditionalGeneration
|
||||
|
||||
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||||
training_args.output_dir,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
processor = Qwen2_5_VLProcessor.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
padding_side="left",
|
||||
)
|
||||
|
||||
from model_library.qwen2vl import collate_fn_for_train, collate_fn_for_evaluate
|
||||
from functools import partial
|
||||
|
||||
collate_fn_for_train = partial(collate_fn_for_train, processor=processor)
|
||||
collate_fn_for_evaluate = partial(collate_fn_for_evaluate, processor=processor)
|
||||
|
||||
elif model_args.model_name_or_path == "Qwen/Qwen2-VL-7B-Instruct":
|
||||
torch_dtype = (
|
||||
model_args.torch_dtype
|
||||
if model_args.torch_dtype in ["auto", None]
|
||||
else getattr(torch, model_args.torch_dtype)
|
||||
)
|
||||
|
||||
quantization_config = get_quantization_config(model_args)
|
||||
model_kwargs = dict(
|
||||
attn_implementation=model_args.attn_implementation,
|
||||
torch_dtype=torch_dtype,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
from transformers import (
|
||||
Qwen2VLProcessor,
|
||||
Qwen2VLForConditionalGeneration,
|
||||
AutoModelForVision2Seq,
|
||||
AutoModel,
|
||||
)
|
||||
from peft.peft_model import PeftModelForCausalLM
|
||||
from model_library.qwen2vl import Qwen2VLForConditionalGeneration_modified
|
||||
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
training_args.output_dir,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
# from peft_library import get_peft_model
|
||||
|
||||
processor = Qwen2VLProcessor.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
padding_side="left",
|
||||
)
|
||||
from model_library.qwen2vl import (
|
||||
collate_fn_for_train,
|
||||
collate_fn_for_evaluate,
|
||||
)
|
||||
from functools import partial
|
||||
|
||||
collate_fn_for_train = partial(collate_fn_for_train, processor=processor)
|
||||
collate_fn_for_evaluate = partial(collate_fn_for_evaluate, processor=processor)
|
||||
|
||||
################
|
||||
# Dataset
|
||||
################
|
||||
@ -139,6 +79,6 @@ if __name__ == "__main__":
|
||||
collate_fn=collate_fn_for_evaluate,
|
||||
)
|
||||
val_dataloader = accelerator.prepare_data_loader(val_dataloader)
|
||||
from utils.evaluate_tool import evaluate_rouge, evalute_save
|
||||
from utils.evaluate_tool import evaluate_rouge, evaluate_save
|
||||
|
||||
evalute_save(model, val_dataloader, processor, accelerator)
|
||||
evaluate_save(model, val_dataloader, processor, accelerator)
|
||||
|
@ -5,9 +5,12 @@ from trl import (
|
||||
get_quantization_config,
|
||||
)
|
||||
from utils.args import ContinualModelConfig
|
||||
from transformers import TrainingArguments
|
||||
|
||||
|
||||
def get_model(model_args: ContinualModelConfig):
|
||||
def get_model(
|
||||
model_args: ContinualModelConfig, training_args: TrainingArguments = None
|
||||
):
|
||||
torch_dtype = (
|
||||
model_args.torch_dtype
|
||||
if model_args.torch_dtype in ["auto", None]
|
||||
@ -26,12 +29,20 @@ def get_model(model_args: ContinualModelConfig):
|
||||
from transformers import Qwen2VLProcessor, Qwen2VLForConditionalGeneration
|
||||
|
||||
# from .qwen2vl import Qwen2VLForConditionalGeneration_modified
|
||||
if training_args is not None:
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
training_args.output_dir,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
processor = Qwen2VLProcessor.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
@ -49,6 +60,13 @@ def get_model(model_args: ContinualModelConfig):
|
||||
if model_args.model_name_or_path == "Qwen/Qwen2-Audio-7B-Instruct":
|
||||
from transformers import Qwen2AudioProcessor, Qwen2AudioForConditionalGeneration
|
||||
|
||||
if training_args is not None:
|
||||
model = Qwen2AudioForConditionalGeneration.from_pretrained(
|
||||
training_args.output_dir,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
**model_kwargs,
|
||||
)
|
||||
else:
|
||||
model = Qwen2AudioForConditionalGeneration.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
@ -71,6 +89,13 @@ def get_model(model_args: ContinualModelConfig):
|
||||
if model_args.model_name_or_path == "Qwen/Qwen2.5-VL-3B-Instruct":
|
||||
from transformers import Qwen2_5_VLProcessor, Qwen2_5_VLForConditionalGeneration
|
||||
|
||||
if training_args is not None:
|
||||
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||||
training_args.output_dir,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
**model_kwargs,
|
||||
)
|
||||
else:
|
||||
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
@ -92,9 +117,16 @@ def get_model(model_args: ContinualModelConfig):
|
||||
if model_args.model_name_or_path == "Qwen/Qwen2.5-Omni-3B":
|
||||
from transformers.models.qwen2_5_omni import (
|
||||
Qwen2_5OmniThinkerForConditionalGeneration,
|
||||
Qwen2_5OmniProcessor
|
||||
Qwen2_5OmniProcessor,
|
||||
)
|
||||
|
||||
if training_args is not None:
|
||||
model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(
|
||||
training_args.output_dir,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
|
@ -1 +1 @@
|
||||
Subproject commit 317d957cc101c4cb064066a1b228526a55f6e927
|
||||
Subproject commit f58e3bd57f3f6cf2f713edaac4b8a54ecafe8e20
|
15
src/scripts/eval_omni.sh
Executable file
15
src/scripts/eval_omni.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
accelerate launch --config_file configs/accelerate_configs/deepspeed_zero1.yaml evaluation.py \
|
||||
--dataset_name textvqa \
|
||||
--use_peft \
|
||||
--peft_type MOELORA \
|
||||
--model_name_or_path Qwen/Qwen2.5-Omni-3B \
|
||||
--lora_target_modules .*model\.layers.*proj\|.*merger.*0\|.*merger.*1 \
|
||||
--per_device_train_batch_size 3 \
|
||||
--per_device_eval_batch_size 2 \
|
||||
--gradient_accumulation_steps 2 \
|
||||
--output_dir ./checkpoint/qwen2_5omni_moelora/ \
|
||||
--bf16 \
|
||||
--torch_dtype bfloat16
|
||||
# --eval_strategy epoch \
|
@ -18,7 +18,8 @@ accelerate launch --config_file configs/accelerate_configs/deepspeed_zero1.yaml
|
||||
--lr_scheduler_type cosine \
|
||||
--bf16 \
|
||||
--torch_dtype bfloat16 \
|
||||
--logging_steps 10 \
|
||||
--logging_steps 100 \
|
||||
--gradient_checkpointing \
|
||||
--weight_decay 0.1 \
|
||||
# --resume_from_checkpoint /root/autodl-tmp/zhouyunyao/projects/CL-LMM/src/checkpoint/qwen2_alllinear/checkpoint-1000
|
||||
--eval_strategy steps \
|
||||
# --resume_from_checkpoint /root/autodl-tmp/zhouyunyao/projects/CL-LMM/src/checkpoint/qwen2_5omni_moelora/checkpoint-1500
|
31
src/test_evalutae.py
Normal file
31
src/test_evalutae.py
Normal file
@ -0,0 +1,31 @@
|
||||
import evaluate
|
||||
|
||||
# Bleu_1, Bleu_2, Bleu_3, Bleu_4, METEOR, ROUGE_L, and CIDEr
|
||||
example = {
|
||||
"generated": "The cat sat on the mat.",
|
||||
"target": "The cat is sitting on the mat.",
|
||||
"original": "The cat is sitting on the mat.",
|
||||
}
|
||||
evaluate_bleu = evaluate.load("bleu")
|
||||
evaluate_rouge = evaluate.load("rouge")
|
||||
evaluate_meteor = evaluate.load("meteor")
|
||||
|
||||
evaluate_bleu.add_batch(
|
||||
predictions=[example["generated"]],
|
||||
references=[[example["target"]]],
|
||||
)
|
||||
evaluate_rouge.add_batch(
|
||||
predictions=[example["generated"]],
|
||||
references=[[example["target"]]],
|
||||
)
|
||||
evaluate_meteor.add_batch(
|
||||
predictions=[example["generated"]],
|
||||
references=[[example["target"]]],
|
||||
)
|
||||
|
||||
bleu = evaluate_bleu.compute()
|
||||
rouge = evaluate_rouge.compute()
|
||||
meteor = evaluate_meteor.compute()
|
||||
|
||||
comprehensive_results = sum(bleu['precisions']) + rouge['rougeL'] + meteor['meteor']
|
||||
print("Comprehensive Results:", comprehensive_results/6)
|
15
src/train.py
15
src/train.py
@ -16,7 +16,7 @@ from utils.trainer import ContinualTrainer
|
||||
from utils.args import (
|
||||
ContinualScriptArguments,
|
||||
ContinualModelConfig,
|
||||
ContiunalRegularizationArguments,
|
||||
ContinualRegularizationArguments,
|
||||
)
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
@ -31,7 +31,7 @@ if __name__ == "__main__":
|
||||
ContinualScriptArguments,
|
||||
TrainingArguments,
|
||||
ContinualModelConfig,
|
||||
ContiunalRegularizationArguments,
|
||||
ContinualRegularizationArguments,
|
||||
) # type: ignore
|
||||
)
|
||||
script_args, training_args, model_args, reg_args = parser.parse_args_and_config()
|
||||
@ -40,7 +40,7 @@ if __name__ == "__main__":
|
||||
script_args = ContinualScriptArguments()
|
||||
training_args = TrainingArguments()
|
||||
model_args = ContinualModelConfig()
|
||||
reg_args = ContiunalRegularizationArguments()
|
||||
reg_args = ContinualRegularizationArguments()
|
||||
|
||||
training_args.gradient_checkpointing_kwargs = dict(use_reentrant=False)
|
||||
training_args.remove_unused_columns = False
|
||||
@ -48,7 +48,7 @@ if __name__ == "__main__":
|
||||
from model_library.factory import get_model
|
||||
|
||||
model, processor, collate_fn_for_train, collate_fn_for_evaluate = get_model(
|
||||
model_args
|
||||
model_args=model_args
|
||||
)
|
||||
################
|
||||
# Dataset
|
||||
@ -113,7 +113,8 @@ if __name__ == "__main__":
|
||||
|
||||
if accelerator.is_local_main_process:
|
||||
print("Saving model")
|
||||
trainer.save_model(training_args.output_dir)
|
||||
# trainer.save_model(training_args.output_dir)
|
||||
model.save_pretrained(training_args.output_dir)
|
||||
|
||||
if accelerator.is_local_main_process:
|
||||
print("Model saved")
|
||||
@ -131,6 +132,6 @@ if __name__ == "__main__":
|
||||
# )
|
||||
# val_dataloader = accelerator.prepare(val_dataloader)
|
||||
|
||||
# from utils.evaluate_tool import evaluate_rouge
|
||||
# from utils.evaluate_tool import evaluate_save
|
||||
|
||||
# evaluate_rouge(model, val_dataloader, processor)
|
||||
# evaluate_save(model, val_dataloader, processor, accelerator)
|
||||
|
@ -21,7 +21,7 @@ class ContinualModelConfig(ModelConfig):
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContiunalRegularizationArguments:
|
||||
class ContinualRegularizationArguments:
|
||||
"""Regularization arguments for continual learning."""
|
||||
|
||||
# EWC
|
||||
|
@ -1,5 +1,6 @@
|
||||
import evaluate
|
||||
from accelerate import Accelerator
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
||||
def evaluate_rouge(model, val_dataloader, processor, accelerator: Accelerator = None):
|
||||
@ -7,10 +8,7 @@ def evaluate_rouge(model, val_dataloader, processor, accelerator: Accelerator =
|
||||
|
||||
for batch in val_dataloader:
|
||||
completion = model.generate(
|
||||
input_ids=batch["input_ids"],
|
||||
attention_mask=batch["attention_mask"],
|
||||
pixel_values=batch["pixel_values"],
|
||||
image_grid_thw=batch["image_grid_thw"],
|
||||
**batch,
|
||||
max_length=1000,
|
||||
)
|
||||
target = batch["answers_ids"]
|
||||
@ -27,7 +25,7 @@ def evaluate_rouge(model, val_dataloader, processor, accelerator: Accelerator =
|
||||
print(glue.compute())
|
||||
|
||||
|
||||
def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = None):
|
||||
def evaluate_save(model, val_dataloader, processor, accelerator: Accelerator = None):
|
||||
import os
|
||||
|
||||
mtime = 0
|
||||
@ -53,6 +51,7 @@ def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = No
|
||||
answers = []
|
||||
completion = model.generate(
|
||||
**batch,
|
||||
# max_new_tokens=30,
|
||||
max_length=1000,
|
||||
)
|
||||
generated_text = [
|
||||
@ -63,20 +62,17 @@ def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = No
|
||||
generated_text, skip_special_tokens=True
|
||||
)
|
||||
target_text = processor.tokenizer.batch_decode(target, skip_special_tokens=True)
|
||||
for i in range(len(generated_text)):
|
||||
answers.append(
|
||||
{
|
||||
"generated": generated_text[i],
|
||||
"target": target_text[i],
|
||||
"original": str(origianl[i]),
|
||||
}
|
||||
)
|
||||
import json
|
||||
|
||||
world_size = accelerator.process_index
|
||||
|
||||
for i in range(len(generated_text)):
|
||||
answer = {
|
||||
"generated": generated_text[i],
|
||||
"target": target_text[i],
|
||||
"original": str(origianl[i]),
|
||||
}
|
||||
with open(f"results/{mtime}/answers_{world_size}.jsonl", "a") as f:
|
||||
for answer in answers:
|
||||
f.write(json.dumps(answer) + "\n")
|
||||
|
||||
if accelerator.is_local_main_process:
|
||||
@ -97,3 +93,71 @@ def evalute_save(model, val_dataloader, processor, accelerator: Accelerator = No
|
||||
# delete file
|
||||
for file in files:
|
||||
os.remove(f"results/{mtime}/{file}")
|
||||
|
||||
|
||||
def evaluate_from_jsonl_directory(directory_path):
|
||||
"""
|
||||
从指定目录读取所有jsonl文件并计算综合评估结果
|
||||
|
||||
Args:
|
||||
directory_path: 包含jsonl文件的目录路径
|
||||
|
||||
Returns:
|
||||
dict: 包含各项指标和综合结果的字典
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
|
||||
# 初始化评估器
|
||||
evaluate_bleu = evaluate.load("bleu")
|
||||
evaluate_rouge = evaluate.load("rouge")
|
||||
evaluate_meteor = evaluate.load("meteor")
|
||||
|
||||
# 读取目录下所有jsonl文件
|
||||
all_data = []
|
||||
for file in os.listdir(directory_path):
|
||||
if file.endswith(".jsonl"):
|
||||
file_path = os.path.join(directory_path, file)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
data = json.loads(line)
|
||||
all_data.append(data)
|
||||
|
||||
if not all_data:
|
||||
print(f"未在目录 {directory_path} 中找到有效的jsonl数据")
|
||||
return None
|
||||
|
||||
# 准备数据
|
||||
predictions = [item["generated"] for item in all_data]
|
||||
references = [[item["target"]] for item in all_data]
|
||||
|
||||
# 批量添加数据
|
||||
evaluate_bleu.add_batch(predictions=predictions, references=references)
|
||||
evaluate_rouge.add_batch(predictions=predictions, references=references)
|
||||
evaluate_meteor.add_batch(predictions=predictions, references=references)
|
||||
|
||||
# 计算结果
|
||||
bleu = evaluate_bleu.compute()
|
||||
rouge = evaluate_rouge.compute()
|
||||
meteor = evaluate_meteor.compute()
|
||||
|
||||
# 计算综合结果
|
||||
comprehensive_score = (sum(bleu["precisions"]) + rouge["rougeL"] + meteor["meteor"]) / 6
|
||||
|
||||
results = {
|
||||
"bleu": bleu,
|
||||
"rouge": rouge,
|
||||
"meteor": meteor,
|
||||
"comprehensive_score": comprehensive_score,
|
||||
"total_samples": len(all_data),
|
||||
}
|
||||
|
||||
print(f"评估完成,共处理 {len(all_data)} 条数据")
|
||||
print(f"BLEU分数: {bleu}")
|
||||
print(f"ROUGE分数: {rouge}")
|
||||
print(f"METEOR分数: {meteor}")
|
||||
print(f"综合分数: {comprehensive_score}")
|
||||
|
||||
return results
|
||||
|
@ -5,7 +5,7 @@ from transformers.trainer import *
|
||||
from transformers import (
|
||||
TrainingArguments,
|
||||
)
|
||||
from .args import ContiunalRegularizationArguments
|
||||
from .args import ContinualRegularizationArguments
|
||||
from peft_library.regularizations import EWC, LWF
|
||||
from torch.nn import CrossEntropyLoss
|
||||
|
||||
@ -41,7 +41,7 @@ class ContinualTrainer(Trainer):
|
||||
train_dataset,
|
||||
eval_dataset,
|
||||
accelerator,
|
||||
reg_args: ContiunalRegularizationArguments = None,
|
||||
reg_args: ContinualRegularizationArguments = None,
|
||||
):
|
||||
self.accelerator = accelerator
|
||||
super().__init__(
|
||||
@ -155,4 +155,3 @@ class ContinualTrainer(Trainer):
|
||||
self.optimizer = smp.DistributedOptimizer(self.optimizer)
|
||||
|
||||
return self.optimizer
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user