xinsu commited on
Commit
a54ab16
1 Parent(s): fea3f11

Add the parameters for the tokenizer

Browse files
Files changed (1) hide show
  1. processing_phi3_v.py +1 -1
processing_phi3_v.py CHANGED
@@ -150,7 +150,7 @@ class Phi3VProcessor(ProcessorMixin):
150
  return BatchFeature(data={**model_inputs})
151
 
152
  pattern = r"<\|image_\d+\|>"
153
- prompt_chunks = [self.tokenizer(chunk).input_ids for chunk in re.split(pattern, texts)]
154
 
155
  if 'num_img_tokens' in images:
156
  num_img_tokens = images['num_img_tokens']
 
150
  return BatchFeature(data={**model_inputs})
151
 
152
  pattern = r"<\|image_\d+\|>"
153
+ prompt_chunks = [self.tokenizer(chunk, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length).input_ids for chunk in re.split(pattern, texts)]
154
 
155
  if 'num_img_tokens' in images:
156
  num_img_tokens = images['num_img_tokens']