rooa commited on
Commit
d2cc612
1 Parent(s): 8f0af50
Files changed (1) hide show
  1. tokenization_xgen.py +6 -2
tokenization_xgen.py CHANGED
@@ -6,11 +6,15 @@
6
 
7
  from typing import List, Optional
8
 
9
- import tiktoken
10
-
11
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
12
  from transformers.utils import logging
13
 
 
 
 
 
 
 
14
  logger = logging.get_logger(__name__)
15
 
16
  MAX_MODEL_INPUT_SIZES = {
 
6
 
7
  from typing import List, Optional
8
 
 
 
9
  from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
10
  from transformers.utils import logging
11
 
12
+ try:
13
+ import tiktoken
14
+ except ModuleNotFoundError as e:
15
+ raise ModuleNotFoundError("XGen requires the installation of tiktoken. Please install it via `pip install tiktoken`.") from e
16
+
17
+
18
  logger = logging.get_logger(__name__)
19
 
20
  MAX_MODEL_INPUT_SIZES = {