The usage code sample doesn't work
#1
by
chenbowen184
- opened
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
from peft import PeftModel, PeftConfig
prompt = "Write a story about an lamb named Dolly that went to the zoo."
peft_model_id = 'coniferlabs/flan-ul2-dolly-lora'
config = PeftConfig.from_pretrained(peft_model_id)
model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path, device_map="auto", load_in_8bit=True)
model = PeftModel.from_pretrained(model, peft_model_id, device_map={'': 0})
tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
model.eval()
tokenized_text = tokenizer.encode(prompt, return_tensors="pt").to("cuda")
outputs = model.generate(input_ids=tokenized_text, parameters={"min_length": 10, "max_length": 250})
tokenizer.batch_decode(outputs, skip_special_tokens=True)
###
gets the following error
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ in <cell line: 14>:14 │
│ │
│ /usr/local/lib/python3.10/dist-packages/peft/peft_model.py:952 in generate │
│ │
│ 949 │ │ ) │
│ 950 │ │ try: │
│ 951 │ │ │ if not isinstance(peft_config, PromptLearningConfig): │
│ ❱ 952 │ │ │ │ outputs = self.base_model.generate(**kwargs) │
│ 953 │ │ │ else: │
│ 954 │ │ │ │ if "input_ids" not in kwargs: │
│ 955 │ │ │ │ │ raise ValueError("input_ids must be provided for Peft model generati │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py:115 in decorate_context │
│ │
│ 112 │ @functools.wraps(func) │
│ 113 │ def decorate_context(*args, **kwargs): │
│ 114 │ │ with ctx_factory(): │
│ ❱ 115 │ │ │ return func(*args, **kwargs) │
│ 116 │ │
│ 117 │ return decorate_context │
│ 118 │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1271 in generate │
│ │
│ 1268 │ │ generation_config = copy.deepcopy(generation_config) │
│ 1269 │ │ model_kwargs = generation_config.update(**kwargs) # All unused kwargs must be m │
│ 1270 │ │ generation_config.validate() │
│ ❱ 1271 │ │ self._validate_model_kwargs(model_kwargs.copy()) │
│ 1272 │ │ │
│ 1273 │ │ # 2. Set generation parameters if not already defined │
│ 1274 │ │ logits_processor = logits_processor if logits_processor is not None else LogitsP │
│ │
│ /usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1144 in │
│ _validate_model_kwargs │
│ │
│ 1141 │ │ │ │ unused_model_args.append(key) │
│ 1142 │ │ │
│ 1143 │ │ if unused_model_args: │
│ ❱ 1144 │ │ │ raise ValueError( │
│ 1145 │ │ │ │ f"The following `model_kwargs` are not used by the model: {unused_model_ │
│ 1146 │ │ │ │ " generate arguments will also show up in this list)" │
│ 1147 │ │ │ ) │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
ValueError: The following `model_kwargs` are not used by the model: ['parameters'] (note: typos in the generate
arguments will also show up in this list)
Any ideas?