simonJJJ commited on
Commit
33eda72
1 Parent(s): ba9726f

Upload folder using huggingface_hub

Browse files
config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "./",
3
+ "architectures": [
4
+ "QWenLMHeadModel"
5
+ ],
6
+ "attn_dropout_prob": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_qwen.QWenConfig",
9
+ "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"
10
+ },
11
+ "bf16": true,
12
+ "emb_dropout_prob": 0.0,
13
+ "fp16": false,
14
+ "fp32": false,
15
+ "hidden_size": 4096,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 22016,
18
+ "kv_channels": 128,
19
+ "layer_norm_epsilon": 1e-06,
20
+ "max_position_embeddings": 8192,
21
+ "model_type": "qwen",
22
+ "no_bias": true,
23
+ "num_attention_heads": 32,
24
+ "num_hidden_layers": 32,
25
+ "onnx_safe": null,
26
+ "rotary_emb_base": 10000,
27
+ "rotary_pct": 1.0,
28
+ "scale_attn_weights": true,
29
+ "seq_length": 2048,
30
+ "tie_word_embeddings": false,
31
+ "tokenizer_type": "QWenTokenizer",
32
+ "torch_dtype": "bfloat16",
33
+ "transformers_version": "4.31.0",
34
+ "use_cache": true,
35
+ "use_dynamic_ntk": true,
36
+ "use_flash_attn": false,
37
+ "use_logn_attn": true,
38
+ "visual": {
39
+ "heads": 16,
40
+ "image_size": 448,
41
+ "image_start_id": 151857,
42
+ "layers": 48,
43
+ "mlp_ratio": 4.9231,
44
+ "output_dim": 4096,
45
+ "patch_size": 14,
46
+ "width": 1664
47
+ },
48
+ "vocab_size": 151936
49
+ }
configuration_qwen.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from transformers import PretrainedConfig
7
+
8
+
9
+ class QWenConfig(PretrainedConfig):
10
+ model_type = "qwen"
11
+ keys_to_ignore_at_inference = ["past_key_values"]
12
+
13
+ def __init__(
14
+ self,
15
+ vocab_size=151936,
16
+ hidden_size=4096,
17
+ num_hidden_layers=32,
18
+ num_attention_heads=32,
19
+ emb_dropout_prob=0.0,
20
+ attn_dropout_prob=0.0,
21
+ layer_norm_epsilon=1e-6,
22
+ initializer_range=0.02,
23
+ max_position_embeddings=8192,
24
+ scale_attn_weights=True,
25
+ use_cache=True,
26
+ bf16=False,
27
+ fp16=False,
28
+ fp32=False,
29
+ kv_channels=128,
30
+ rotary_pct=1.0,
31
+ rotary_emb_base=10000,
32
+ use_dynamic_ntk=True,
33
+ use_logn_attn=True,
34
+ use_flash_attn="auto",
35
+ intermediate_size=22016,
36
+ no_bias=True,
37
+ tie_word_embeddings=False,
38
+ **kwargs,
39
+ ):
40
+ self.vocab_size = vocab_size
41
+ self.hidden_size = hidden_size
42
+ self.intermediate_size = intermediate_size
43
+ self.num_hidden_layers = num_hidden_layers
44
+ self.num_attention_heads = num_attention_heads
45
+ self.emb_dropout_prob = emb_dropout_prob
46
+ self.attn_dropout_prob = attn_dropout_prob
47
+ self.layer_norm_epsilon = layer_norm_epsilon
48
+ self.initializer_range = initializer_range
49
+ self.scale_attn_weights = scale_attn_weights
50
+ self.use_cache = use_cache
51
+ self.max_position_embeddings = max_position_embeddings
52
+ self.bf16 = bf16
53
+ self.fp16 = fp16
54
+ self.fp32 = fp32
55
+ self.kv_channels = kv_channels
56
+ self.rotary_pct = rotary_pct
57
+ self.rotary_emb_base = rotary_emb_base
58
+ self.use_dynamic_ntk = use_dynamic_ntk
59
+ self.use_logn_attn = use_logn_attn
60
+ self.use_flash_attn = use_flash_attn
61
+ self.no_bias = no_bias
62
+ super().__init__(
63
+ tie_word_embeddings=tie_word_embeddings,
64
+ **kwargs
65
+ )
generation_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "chat_format": "chatml",
3
+ "do_sample": true,
4
+ "eos_token_id": 151643,
5
+ "max_new_tokens": 512,
6
+ "max_window_size": 6144,
7
+ "pad_token_id": 151643,
8
+ "top_k": 0,
9
+ "top_p": 0.5,
10
+ "transformers_version": "4.31.0"
11
+ }
modeling_qwen.py ADDED
@@ -0,0 +1,1264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import importlib
7
+ import math
8
+ from typing import TYPE_CHECKING, Optional, Tuple, Union, Callable, List, Any, Generator
9
+
10
+ import torch
11
+ import torch.nn.functional as F
12
+ import torch.utils.checkpoint
13
+ from torch.cuda.amp import autocast
14
+
15
+ from torch.nn import CrossEntropyLoss
16
+ from transformers import PreTrainedTokenizer, GenerationConfig, StoppingCriteriaList
17
+ from transformers.generation.logits_process import LogitsProcessorList
18
+
19
+ if TYPE_CHECKING:
20
+ from transformers.generation.streamers import BaseStreamer
21
+ from transformers.generation.utils import GenerateOutput
22
+ from transformers.modeling_outputs import (
23
+ BaseModelOutputWithPast,
24
+ CausalLMOutputWithPast,
25
+ )
26
+ from transformers.modeling_utils import PreTrainedModel
27
+ from transformers.utils import logging
28
+
29
+ try:
30
+ from einops import rearrange
31
+ except ImportError:
32
+ rearrange = None
33
+ from torch import nn
34
+
35
+ SUPPORT_CUDA = torch.cuda.is_available()
36
+ SUPPORT_BF16 = SUPPORT_CUDA and torch.cuda.is_bf16_supported()
37
+ SUPPORT_FP16 = SUPPORT_CUDA and torch.cuda.get_device_capability(0)[0] >= 7
38
+
39
+ from .configuration_qwen import QWenConfig
40
+ from .qwen_generation_utils import (
41
+ HistoryType,
42
+ make_context,
43
+ decode_tokens,
44
+ get_stop_words_ids,
45
+ StopWordsLogitsProcessor,
46
+ )
47
+ from .visual import VisionTransformer
48
+
49
+
50
+ logger = logging.get_logger(__name__)
51
+
52
+ _CHECKPOINT_FOR_DOC = "qwen"
53
+ _CONFIG_FOR_DOC = "QWenConfig"
54
+
55
+ QWen_PRETRAINED_MODEL_ARCHIVE_LIST = ["qwen-7b"]
56
+
57
+ _ERROR_BAD_CHAT_FORMAT = """\
58
+ We detect you are probably using the pretrained model (rather than chat model) for chatting, since the chat_format in generation_config is not "chatml".
59
+ If you are directly using the model downloaded from Huggingface, please make sure you are using our "Qwen/Qwen-7B-Chat" Huggingface model (rather than "Qwen/Qwen-7B") when you call model.chat().
60
+ 我们检测到您可能在使用预训练模型(而非chat模型)进行多轮chat,因为您当前在generation_config指定的chat_format,并未设置为我们在对话中所支持的"chatml"格式。
61
+ 如果您在直接使用我们从Huggingface提供的模型,请确保您在调用model.chat()时,使用的是"Qwen/Qwen-7B-Chat"模型(而非"Qwen/Qwen-7B"预训练模型)。
62
+ """
63
+
64
+ _SENTINEL = object()
65
+ _ERROR_STREAM_IN_CHAT = """\
66
+ Pass argument `stream` to model.chat() is buggy, deprecated, and marked for removal. Please use model.chat_stream(...) instead of model.chat(..., stream=True).
67
+ 向model.chat()传入参数stream的用法可能存在Bug,该用法已被废弃,将在未来被移除。请使用model.chat_stream(...)代替model.chat(..., stream=True)。
68
+ """
69
+
70
+ apply_rotary_emb_func = None
71
+ rms_norm = None
72
+ flash_attn_unpadded_func = None
73
+
74
+
75
+ def _import_flash_attn():
76
+ global apply_rotary_emb_func, rms_norm, flash_attn_unpadded_func
77
+ try:
78
+ from flash_attn.layers.rotary import apply_rotary_emb_func as __apply_rotary_emb_func
79
+ apply_rotary_emb_func = __apply_rotary_emb_func
80
+ except ImportError:
81
+ logger.warn(
82
+ "Warning: import flash_attn rotary fail, please install FlashAttention rotary to get higher efficiency "
83
+ "https://github.com/Dao-AILab/flash-attention/tree/main/csrc/rotary"
84
+ )
85
+
86
+ try:
87
+ from flash_attn.ops.rms_norm import rms_norm as __rms_norm
88
+ rms_norm = __rms_norm
89
+ except ImportError:
90
+ logger.warn(
91
+ "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency "
92
+ "https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm"
93
+ )
94
+
95
+ try:
96
+ import flash_attn
97
+ if not hasattr(flash_attn, '__version__'):
98
+ from flash_attn.flash_attn_interface import flash_attn_unpadded_func as __flash_attn_unpadded_func
99
+ else:
100
+ if int(flash_attn.__version__.split(".")[0]) >= 2:
101
+ from flash_attn.flash_attn_interface import flash_attn_varlen_func as __flash_attn_unpadded_func
102
+ else:
103
+ from flash_attn.flash_attn_interface import flash_attn_unpadded_func as __flash_attn_unpadded_func
104
+ flash_attn_unpadded_func = __flash_attn_unpadded_func
105
+ except ImportError:
106
+ logger.warn(
107
+ "Warning: import flash_attn fail, please install FlashAttention to get higher efficiency "
108
+ "https://github.com/Dao-AILab/flash-attention"
109
+ )
110
+
111
+ # Copied from transformers.models.bart.modeling_bart._make_causal_mask
112
+ def _make_causal_mask(
113
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
114
+ ):
115
+ """
116
+ Make causal mask used for bi-directional self-attention.
117
+ """
118
+ bsz, tgt_len = input_ids_shape
119
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
120
+ mask_cond = torch.arange(mask.size(-1), device=device)
121
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
122
+ mask = mask.to(dtype)
123
+
124
+ if past_key_values_length > 0:
125
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
126
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
127
+
128
+
129
+ # Copied from transformers.models.bart.modeling_bart._expand_mask
130
+ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
131
+ """
132
+ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
133
+ """
134
+ bsz, src_len = mask.size()
135
+ tgt_len = tgt_len if tgt_len is not None else src_len
136
+
137
+ expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
138
+
139
+ inverted_mask = 1.0 - expanded_mask
140
+
141
+ return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
142
+
143
+
144
+ class FlashSelfAttention(torch.nn.Module):
145
+ def __init__(
146
+ self,
147
+ causal=False,
148
+ softmax_scale=None,
149
+ attention_dropout=0.0,
150
+ ):
151
+ super().__init__()
152
+ assert flash_attn_unpadded_func is not None, (
153
+ "Please install FlashAttention first, " "e.g., with pip install flash-attn"
154
+ )
155
+ assert (
156
+ rearrange is not None
157
+ ), "Please install einops first, e.g., with pip install einops"
158
+ self.causal = causal
159
+ self.softmax_scale = softmax_scale
160
+ self.dropout_p = attention_dropout
161
+
162
+ def forward(self, q, k, v):
163
+ assert all((i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v)))
164
+ assert all((i.is_cuda for i in (q, k, v)))
165
+ batch_size, seqlen_q = q.shape[0], q.shape[1]
166
+ seqlen_k = k.shape[1]
167
+ q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
168
+ cu_seqlens_q = torch.arange(
169
+ 0,
170
+ (batch_size + 1) * seqlen_q,
171
+ step=seqlen_q,
172
+ dtype=torch.int32,
173
+ device=q.device,
174
+ )
175
+
176
+ if self.training:
177
+ assert seqlen_k == seqlen_q
178
+
179
+ is_causal = self.causal
180
+ cu_seqlens_k = cu_seqlens_q
181
+ else:
182
+ is_causal = seqlen_q == seqlen_k
183
+ cu_seqlens_k = torch.arange(
184
+ 0,
185
+ (batch_size + 1) * seqlen_k,
186
+ step=seqlen_k,
187
+ dtype=torch.int32,
188
+ device=q.device,
189
+ )
190
+ self.dropout_p = 0
191
+ output = flash_attn_unpadded_func(
192
+ q,
193
+ k,
194
+ v,
195
+ cu_seqlens_q,
196
+ cu_seqlens_k,
197
+ seqlen_q,
198
+ seqlen_k,
199
+ self.dropout_p,
200
+ softmax_scale=self.softmax_scale,
201
+ causal=is_causal,
202
+ )
203
+
204
+ output = rearrange(output, "(b s) ... -> b s ...", b=batch_size)
205
+ return output
206
+
207
+
208
+ class QWenAttention(nn.Module):
209
+ def __init__(self, config):
210
+ super().__init__()
211
+
212
+ max_positions = config.max_position_embeddings
213
+ self.register_buffer(
214
+ "bias",
215
+ torch.tril(
216
+ torch.ones((max_positions, max_positions), dtype=torch.bool)
217
+ ).view(1, 1, max_positions, max_positions),
218
+ persistent=False,
219
+ )
220
+ self.register_buffer("masked_bias", torch.tensor(-1e4), persistent=False)
221
+ self.seq_length = config.seq_length
222
+
223
+ self.hidden_size = config.hidden_size
224
+ self.split_size = config.hidden_size
225
+ self.num_heads = config.num_attention_heads
226
+ self.head_dim = self.hidden_size // self.num_heads
227
+
228
+ self.use_flash_attn = config.use_flash_attn
229
+ self.scale_attn_weights = True
230
+
231
+ self.projection_size = config.kv_channels * config.num_attention_heads
232
+
233
+ assert self.projection_size % config.num_attention_heads == 0
234
+ self.hidden_size_per_attention_head = (
235
+ self.projection_size // config.num_attention_heads
236
+ )
237
+
238
+ self.c_attn = nn.Linear(config.hidden_size, 3 * self.projection_size)
239
+
240
+ self.c_proj = nn.Linear(
241
+ config.hidden_size, self.projection_size, bias=not config.no_bias
242
+ )
243
+
244
+ self.is_fp32 = not (config.bf16 or config.fp16)
245
+ if (
246
+ self.use_flash_attn
247
+ and flash_attn_unpadded_func is not None
248
+ and not self.is_fp32
249
+ ):
250
+ self.core_attention_flash = FlashSelfAttention(
251
+ causal=True, attention_dropout=config.attn_dropout_prob
252
+ )
253
+
254
+ self.bf16 = config.bf16
255
+
256
+ if config.rotary_pct == 1.0:
257
+ self.rotary_ndims = None
258
+ else:
259
+ assert config.rotary_pct < 1
260
+ self.rotary_ndims = int(
261
+ self.hidden_size_per_attention_head * config.rotary_pct
262
+ )
263
+ dim = (
264
+ self.rotary_ndims
265
+ if self.rotary_ndims is not None
266
+ else self.hidden_size_per_attention_head
267
+ )
268
+ self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base)
269
+
270
+ self.use_dynamic_ntk = config.use_dynamic_ntk
271
+ self.use_logn_attn = config.use_logn_attn
272
+
273
+ logn_list = [
274
+ math.log(i, self.seq_length) if i > self.seq_length else 1
275
+ for i in range(1, 32768)
276
+ ]
277
+ self.logn_tensor = torch.tensor(logn_list)[None, :, None, None]
278
+ self._ntk_cached = 1.0
279
+
280
+ self.attn_dropout = nn.Dropout(config.attn_dropout_prob)
281
+
282
+ def _attn(self, query, key, value, attention_mask=None, head_mask=None):
283
+ attn_weights = torch.matmul(query, key.transpose(-1, -2))
284
+
285
+ if self.scale_attn_weights:
286
+ attn_weights = attn_weights / torch.full(
287
+ [],
288
+ value.size(-1) ** 0.5,
289
+ dtype=attn_weights.dtype,
290
+ device=attn_weights.device,
291
+ )
292
+
293
+ query_length, key_length = query.size(-2), key.size(-2)
294
+ # causal_mask = self.bias[
295
+ # :, :, key_length - query_length : key_length, :key_length
296
+ # ]
297
+ # mask_value = torch.finfo(attn_weights.dtype).min
298
+ # mask_value = torch.full([], mask_value, dtype=attn_weights.dtype).to(
299
+ # attn_weights.device
300
+ # )
301
+ # attn_weights = torch.where(
302
+ # causal_mask, attn_weights.to(attn_weights.dtype), mask_value
303
+ # )
304
+ attn_weights = attn_weights + attention_mask
305
+
306
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
307
+
308
+ attn_weights = attn_weights.type(value.dtype)
309
+ attn_weights = self.attn_dropout(attn_weights)
310
+
311
+ if head_mask is not None:
312
+ attn_weights = attn_weights * head_mask
313
+
314
+ attn_output = torch.matmul(attn_weights, value)
315
+ attn_output = attn_output.transpose(1, 2)
316
+
317
+ return attn_output, attn_weights
318
+
319
+ def _upcast_and_reordered_attn(
320
+ self, query, key, value, attention_mask=None, head_mask=None
321
+ ):
322
+ bsz, num_heads, q_seq_len, dk = query.size()
323
+ _, _, k_seq_len, _ = key.size()
324
+
325
+ attn_weights = torch.empty(
326
+ bsz * num_heads,
327
+ q_seq_len,
328
+ k_seq_len,
329
+ dtype=torch.float32,
330
+ device=query.device,
331
+ )
332
+
333
+ scale_factor = 1.0
334
+ if self.scale_attn_weights:
335
+ scale_factor /= float(value.size(-1)) ** 0.5
336
+
337
+ with autocast(enabled=False):
338
+ q, k = query.reshape(-1, q_seq_len, dk), key.transpose(-1, -2).reshape(
339
+ -1, dk, k_seq_len
340
+ )
341
+ attn_weights = torch.baddbmm(
342
+ attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor
343
+ )
344
+ attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, k_seq_len)
345
+
346
+ query_length, key_length = query.size(-2), key.size(-2)
347
+ causal_mask = self.bias[
348
+ :, :, key_length - query_length : key_length, :key_length
349
+ ]
350
+ mask_value = torch.finfo(attn_weights.dtype).min
351
+ mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(
352
+ attn_weights.device
353
+ )
354
+ attn_weights = torch.where(causal_mask, attn_weights, mask_value)
355
+
356
+ if attention_mask is not None:
357
+ attn_weights = attn_weights + attention_mask
358
+
359
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
360
+
361
+ if attn_weights.dtype != torch.float32:
362
+ raise RuntimeError(
363
+ "Error with upcasting, attn_weights does not have dtype torch.float32"
364
+ )
365
+ attn_weights = attn_weights.type(value.dtype)
366
+ attn_weights = self.attn_dropout(attn_weights)
367
+
368
+ if head_mask is not None:
369
+ attn_weights = attn_weights * head_mask
370
+
371
+ attn_output = torch.matmul(attn_weights, value)
372
+
373
+ return attn_output, attn_weights
374
+
375
+ def _split_heads(self, tensor, num_heads, attn_head_size):
376
+ new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
377
+ tensor = tensor.view(new_shape)
378
+ return tensor
379
+
380
+ def _merge_heads(self, tensor, num_heads, attn_head_size):
381
+ tensor = tensor.contiguous()
382
+ new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
383
+ return tensor.view(new_shape)
384
+
385
+ def forward(
386
+ self,
387
+ hidden_states: Optional[Tuple[torch.FloatTensor]],
388
+ layer_past: Optional[Tuple[torch.Tensor]] = None,
389
+ attention_mask: Optional[torch.FloatTensor] = None,
390
+ head_mask: Optional[torch.FloatTensor] = None,
391
+ encoder_hidden_states: Optional[torch.Tensor] = None,
392
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
393
+ output_attentions: Optional[bool] = False,
394
+ use_cache: Optional[bool] = False,
395
+ ):
396
+
397
+ mixed_x_layer = self.c_attn(hidden_states)
398
+ query, key, value = mixed_x_layer.split(self.split_size, dim=2)
399
+
400
+ query = self._split_heads(query, self.num_heads, self.head_dim)
401
+ key = self._split_heads(key, self.num_heads, self.head_dim)
402
+ value = self._split_heads(value, self.num_heads, self.head_dim)
403
+
404
+ kv_seq_len = hidden_states.size()[1]
405
+ if layer_past:
406
+ # layer past[0] shape: bs * seq_len * head_num * dim
407
+ kv_seq_len += layer_past[0].shape[1]
408
+ if (
409
+ self.use_dynamic_ntk
410
+ and kv_seq_len == hidden_states.size()[1]
411
+ and not self.training
412
+ ):
413
+ context_value = math.log(kv_seq_len / self.seq_length, 2) + 1
414
+ ntk_alpha = 2 ** math.ceil(context_value) - 1
415
+ ntk_alpha = max(ntk_alpha, 1)
416
+ self._ntk_cached = ntk_alpha
417
+ else:
418
+ ntk_alpha = self._ntk_cached
419
+ rotary_pos_emb = self.rotary_emb(kv_seq_len, ntk_alpha=ntk_alpha).to(
420
+ hidden_states.device
421
+ )
422
+
423
+ if rotary_pos_emb is not None:
424
+ if isinstance(rotary_pos_emb, tuple):
425
+ rotary_pos_emb = rotary_pos_emb
426
+ else:
427
+ rotary_pos_emb = (rotary_pos_emb,) * 2
428
+
429
+ if rotary_pos_emb is not None:
430
+ q_pos_emb, k_pos_emb = rotary_pos_emb
431
+ # Slice the pos emb for current inference
432
+ cur_len = query.shape[1]
433
+ q_pos_emb = q_pos_emb[:, -cur_len:, :, :]
434
+ k_pos_emb = k_pos_emb[:, -cur_len:, :, :]
435
+ query = apply_rotary_pos_emb(query, q_pos_emb)
436
+ key = apply_rotary_pos_emb(key, k_pos_emb)
437
+
438
+ if layer_past is not None:
439
+ past_key, past_value = layer_past[0], layer_past[1]
440
+ key = torch.cat((past_key, key), dim=1)
441
+ value = torch.cat((past_value, value), dim=1)
442
+
443
+ if use_cache:
444
+ present = (key, value)
445
+ else:
446
+ present = None
447
+
448
+ if self.use_logn_attn and not self.training:
449
+ if self.logn_tensor.device != query.device or self.logn_tensor.dtype != query.dtype:
450
+ self.logn_tensor = self.logn_tensor.to(query.device).type_as(query)
451
+ seq_start = key.size(1) - query.size(1)
452
+ seq_end = key.size(1)
453
+ logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :]
454
+ query = query * logn_tensor.expand_as(query)
455
+
456
+ if (
457
+ self.use_flash_attn
458
+ and flash_attn_unpadded_func is not None
459
+ and not self.is_fp32
460
+ and query.is_cuda
461
+ ):
462
+ q, k, v = query, key, value
463
+ context_layer = self.core_attention_flash(q, k, v)
464
+
465
+ context_layer = rearrange(
466
+ context_layer, "b s h d -> b s (h d)"
467
+ ).contiguous()
468
+ else:
469
+ query = query.permute(0, 2, 1, 3)
470
+ key = key.permute(0, 2, 1, 3)
471
+ value = value.permute(0, 2, 1, 3)
472
+ attn_output, attn_weight = self._attn(
473
+ query, key, value, attention_mask, head_mask
474
+ )
475
+ context_layer = self._merge_heads(
476
+ attn_output, self.num_heads, self.head_dim
477
+ )
478
+
479
+ attn_output = self.c_proj(context_layer)
480
+ outputs = (attn_output, present)
481
+ if output_attentions:
482
+ if (
483
+ self.use_flash_attn
484
+ and flash_attn_unpadded_func is not None
485
+ and not self.is_fp32
486
+ ):
487
+ raise ValueError("Cannot output attentions while using flash-attn")
488
+ else:
489
+ outputs += (attn_weight,)
490
+
491
+ return outputs
492
+
493
+
494
+ class QWenMLP(nn.Module):
495
+ def __init__(self, config):
496
+ super().__init__()
497
+ self.w1 = nn.Linear(
498
+ config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
499
+ )
500
+ self.w2 = nn.Linear(
501
+ config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
502
+ )
503
+ ff_dim_in = config.intermediate_size // 2
504
+ self.c_proj = nn.Linear(ff_dim_in, config.hidden_size, bias=not config.no_bias)
505
+
506
+ def forward(self, hidden_states):
507
+ a1 = self.w1(hidden_states)
508
+ a2 = self.w2(hidden_states)
509
+ intermediate_parallel = a1 * F.silu(a2)
510
+ output = self.c_proj(intermediate_parallel)
511
+ return output
512
+
513
+
514
+ class QWenBlock(nn.Module):
515
+ def __init__(self, config):
516
+ super().__init__()
517
+ hidden_size = config.hidden_size
518
+ self.bf16 = config.bf16
519
+
520
+ self.ln_1 = RMSNorm(
521
+ hidden_size,
522
+ eps=config.layer_norm_epsilon,
523
+ )
524
+ self.attn = QWenAttention(config)
525
+ self.ln_2 = RMSNorm(
526
+ hidden_size,
527
+ eps=config.layer_norm_epsilon,
528
+ )
529
+
530
+ self.mlp = QWenMLP(config)
531
+
532
+ def forward(
533
+ self,
534
+ hidden_states: Optional[Tuple[torch.FloatTensor]],
535
+ layer_past: Optional[Tuple[torch.Tensor]] = None,
536
+ attention_mask: Optional[torch.FloatTensor] = None,
537
+ head_mask: Optional[torch.FloatTensor] = None,
538
+ encoder_hidden_states: Optional[torch.Tensor] = None,
539
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
540
+ use_cache: Optional[bool] = False,
541
+ output_attentions: Optional[bool] = False,
542
+ ):
543
+ layernorm_output = self.ln_1(hidden_states)
544
+
545
+ attn_outputs = self.attn(
546
+ layernorm_output,
547
+ layer_past=layer_past,
548
+ attention_mask=attention_mask,
549
+ head_mask=head_mask,
550
+ use_cache=use_cache,
551
+ output_attentions=output_attentions,
552
+ )
553
+ attn_output = attn_outputs[0]
554
+
555
+ outputs = attn_outputs[1:]
556
+
557
+ residual = hidden_states
558
+ layernorm_input = attn_output + residual
559
+
560
+ layernorm_output = self.ln_2(layernorm_input)
561
+
562
+ residual = layernorm_input
563
+ mlp_output = self.mlp(layernorm_output)
564
+ hidden_states = residual + mlp_output
565
+
566
+ if use_cache:
567
+ outputs = (hidden_states,) + outputs
568
+ else:
569
+ outputs = (hidden_states,) + outputs[1:]
570
+
571
+ return outputs
572
+
573
+
574
+ class QWenPreTrainedModel(PreTrainedModel):
575
+ config_class = QWenConfig
576
+ base_model_prefix = "transformer"
577
+ is_parallelizable = False
578
+ supports_gradient_checkpointing = True
579
+ _no_split_modules = ["QWenBlock"]
580
+
581
+ def __init__(self, *inputs, **kwargs):
582
+ super().__init__(*inputs, **kwargs)
583
+
584
+ def _init_weights(self, module):
585
+ """Initialize the weights."""
586
+ if isinstance(module, nn.Linear):
587
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
588
+ if module.bias is not None:
589
+ module.bias.data.zero_()
590
+ elif isinstance(module, nn.Embedding):
591
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
592
+ if module.padding_idx is not None:
593
+ module.weight.data[module.padding_idx].zero_()
594
+ elif isinstance(module, RMSNorm):
595
+ module.weight.data.fill_(1.0)
596
+
597
+ for name, p in module.named_parameters():
598
+ if name == "c_proj.weight":
599
+ p.data.normal_(
600
+ mean=0.0,
601
+ std=(
602
+ self.config.initializer_range
603
+ / math.sqrt(2 * self.config.num_hidden_layers)
604
+ ),
605
+ )
606
+
607
+ def _set_gradient_checkpointing(self, module, value=False):
608
+ if isinstance(module, QWenModel):
609
+ module.gradient_checkpointing = value
610
+
611
+
612
+ class QWenModel(QWenPreTrainedModel):
613
+ _keys_to_ignore_on_load_missing = ["attn.masked_bias"]
614
+
615
+ def __init__(self, config):
616
+ super().__init__(config)
617
+ self.vocab_size = config.vocab_size
618
+ self.num_hidden_layers = config.num_hidden_layers
619
+ self.embed_dim = config.hidden_size
620
+
621
+ self.gradient_checkpointing = False
622
+
623
+ self.wte = nn.Embedding(self.vocab_size, self.embed_dim)
624
+
625
+ self.drop = nn.Dropout(config.emb_dropout_prob)
626
+ self.h = nn.ModuleList(
627
+ [
628
+ QWenBlock(
629
+ config,
630
+ )
631
+ for i in range(config.num_hidden_layers)
632
+ ]
633
+ )
634
+ self.ln_f = RMSNorm(
635
+ self.embed_dim,
636
+ eps=config.layer_norm_epsilon,
637
+ )
638
+
639
+ self.visual = VisionTransformer(**config.visual)
640
+
641
+ self.post_init()
642
+
643
+ def get_input_embeddings(self):
644
+ return self.wte
645
+
646
+ def set_input_embeddings(self, new_embeddings):
647
+ self.wte = new_embeddings
648
+
649
+ # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
650
+ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
651
+ # create causal mask
652
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
653
+ combined_attention_mask = None
654
+ if input_shape[-1] > 1:
655
+ combined_attention_mask = _make_causal_mask(
656
+ input_shape,
657
+ inputs_embeds.dtype,
658
+ device=inputs_embeds.device,
659
+ past_key_values_length=past_key_values_length,
660
+ )
661
+
662
+ if attention_mask is not None:
663
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
664
+ expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(
665
+ inputs_embeds.device
666
+ )
667
+ combined_attention_mask = (
668
+ expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
669
+ )
670
+
671
+ return combined_attention_mask
672
+
673
+
674
+ def forward(
675
+ self,
676
+ input_ids: Optional[torch.LongTensor] = None,
677
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
678
+ attention_mask: Optional[torch.FloatTensor] = None,
679
+ token_type_ids: Optional[torch.LongTensor] = None,
680
+ position_ids: Optional[torch.LongTensor] = None,
681
+ head_mask: Optional[torch.FloatTensor] = None,
682
+ inputs_embeds: Optional[torch.FloatTensor] = None,
683
+ encoder_hidden_states: Optional[torch.Tensor] = None,
684
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
685
+ use_cache: Optional[bool] = None,
686
+ output_attentions: Optional[bool] = None,
687
+ output_hidden_states: Optional[bool] = None,
688
+ return_dict: Optional[bool] = None,
689
+ ):
690
+ if past_key_values is None and torch.any(input_ids == self.config.visual['image_start_id']):
691
+ bos_pos = torch.where(input_ids == self.config.visual['image_start_id'])
692
+ eos_pos = torch.where(input_ids == self.config.visual['image_start_id'] + 1)
693
+ assert (bos_pos[0] == eos_pos[0]).all()
694
+ img_pos = torch.stack((bos_pos[0], bos_pos[1], eos_pos[1]), dim=1)
695
+ images = []
696
+ for i, a, b in img_pos:
697
+ image = input_ids[i][a + 1 : b - 1].tolist()
698
+ image = image[ : image.index(self.config.visual['image_start_id'] + 2)]
699
+ images.append(bytes(image).decode('utf-8'))
700
+
701
+ images = self.visual.encode(images)
702
+ assert images.shape[0] == len(images)
703
+ else:
704
+ images = None
705
+
706
+ output_attentions = (
707
+ output_attentions
708
+ if output_attentions is not None
709
+ else self.config.output_attentions
710
+ )
711
+ output_hidden_states = (
712
+ output_hidden_states
713
+ if output_hidden_states is not None
714
+ else self.config.output_hidden_states
715
+ )
716
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
717
+ return_dict = (
718
+ return_dict if return_dict is not None else self.config.use_return_dict
719
+ )
720
+
721
+ if input_ids is not None and inputs_embeds is not None:
722
+ raise ValueError(
723
+ "You cannot specify both input_ids and inputs_embeds at the same time"
724
+ )
725
+ elif input_ids is not None:
726
+ input_shape = input_ids.size()
727
+ input_ids = input_ids.view(-1, input_shape[-1])
728
+ batch_size = input_ids.shape[0]
729
+ elif inputs_embeds is not None:
730
+ input_shape = inputs_embeds.size()[:-1]
731
+ batch_size = inputs_embeds.shape[0]
732
+ else:
733
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
734
+
735
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
736
+
737
+ if token_type_ids is not None:
738
+ token_type_ids = token_type_ids.view(-1, input_shape[-1])
739
+ if position_ids is not None:
740
+ position_ids = position_ids.view(-1, input_shape[-1])
741
+
742
+ if past_key_values is None:
743
+ past_length = 0
744
+ past_key_values = tuple([None] * len(self.h))
745
+ else:
746
+ past_length = past_key_values[0][0].size(-2)
747
+
748
+ if position_ids is None:
749
+ position_ids = torch.arange(
750
+ past_length,
751
+ input_shape[-1] + past_length,
752
+ dtype=torch.long,
753
+ device=device,
754
+ )
755
+ position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
756
+
757
+ encoder_attention_mask = None
758
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
759
+
760
+ if inputs_embeds is None:
761
+ inputs_embeds = self.wte(input_ids)
762
+
763
+ if batch_size <= 0:
764
+ raise ValueError("batch_size has to be defined and > 0")
765
+ attention_mask = self._prepare_decoder_attention_mask(
766
+ attention_mask, input_shape, inputs_embeds, past_length
767
+ )
768
+
769
+ hidden_states = inputs_embeds
770
+
771
+ hidden_states = self.drop(hidden_states)
772
+ if images is not None:
773
+ for idx, (i, a, b) in enumerate(img_pos):
774
+ hidden_states[i][a + 1 : b] = images[idx]
775
+ output_shape = input_shape + (hidden_states.size(-1),)
776
+
777
+ if self.gradient_checkpointing and self.training:
778
+ if use_cache:
779
+ logger.warning_once(
780
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
781
+ )
782
+ use_cache = False
783
+
784
+ presents = () if use_cache else None
785
+ all_self_attentions = () if output_attentions else None
786
+ all_hidden_states = () if output_hidden_states else None
787
+ for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
788
+
789
+ if output_hidden_states:
790
+ all_hidden_states = all_hidden_states + (hidden_states,)
791
+
792
+ if self.gradient_checkpointing and self.training:
793
+
794
+ def create_custom_forward(module):
795
+ def custom_forward(*inputs):
796
+ # None for past_key_value
797
+ return module(*inputs, use_cache, output_attentions)
798
+
799
+ return custom_forward
800
+
801
+ outputs = torch.utils.checkpoint.checkpoint(
802
+ create_custom_forward(block),
803
+ hidden_states,
804
+ None,
805
+ attention_mask,
806
+ head_mask[i],
807
+ encoder_hidden_states,
808
+ encoder_attention_mask,
809
+ )
810
+ else:
811
+ outputs = block(
812
+ hidden_states,
813
+ layer_past=layer_past,
814
+ attention_mask=attention_mask,
815
+ head_mask=head_mask[i],
816
+ encoder_hidden_states=encoder_hidden_states,
817
+ encoder_attention_mask=encoder_attention_mask,
818
+ use_cache=use_cache,
819
+ output_attentions=output_attentions,
820
+ )
821
+
822
+ hidden_states = outputs[0]
823
+ if use_cache is True:
824
+ presents = presents + (outputs[2 if output_attentions else 1],)
825
+
826
+ if output_attentions:
827
+ all_self_attentions = all_self_attentions + (outputs[1],)
828
+
829
+ hidden_states = self.ln_f(hidden_states)
830
+ hidden_states = hidden_states.view(output_shape)
831
+ # Add last hidden state
832
+ if output_hidden_states:
833
+ all_hidden_states = all_hidden_states + (hidden_states,)
834
+
835
+ if not return_dict:
836
+ return tuple(
837
+ v for v in [hidden_states, presents, all_hidden_states] if v is not None
838
+ )
839
+
840
+ return BaseModelOutputWithPast(
841
+ last_hidden_state=hidden_states,
842
+ past_key_values=presents,
843
+ hidden_states=all_hidden_states,
844
+ attentions=all_self_attentions,
845
+ )
846
+
847
+
848
+ class QWenLMHeadModel(QWenPreTrainedModel):
849
+ _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.rotary_emb\.inv_freq"]
850
+ _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.attn\.masked_bias"]
851
+
852
+ def __init__(self, config):
853
+ super().__init__(config)
854
+ assert (
855
+ config.bf16 + config.fp16 + config.fp32 <= 1
856
+ ), "Only one of \"bf16\", \"fp16\", \"fp32\" can be true"
857
+
858
+ autoset_precision = config.bf16 + config.fp16 + config.fp32 == 0
859
+
860
+ if autoset_precision:
861
+ if SUPPORT_BF16:
862
+ logger.warn(
863
+ "The model is automatically converting to bf16 for faster inference. "
864
+ "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
865
+ )
866
+ config.bf16 = True
867
+ elif SUPPORT_FP16:
868
+ logger.warn(
869
+ "The model is automatically converting to fp16 for faster inference. "
870
+ "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
871
+ )
872
+ config.fp16 = True
873
+ else:
874
+ config.fp32 = True
875
+
876
+ if config.bf16 and SUPPORT_CUDA and not SUPPORT_BF16:
877
+ logger.warn("Your device does NOT seem to support bf16, you can switch to fp16 or fp32 by by passing fp16/fp32=True in \"AutoModelForCausalLM.from_pretrained\".")
878
+ if config.fp16 and SUPPORT_CUDA and not SUPPORT_FP16:
879
+ logger.warn("Your device does NOT support faster inference with fp16, please switch to fp32 which is likely to be faster")
880
+ if config.fp32:
881
+ if SUPPORT_BF16:
882
+ logger.warn("Your device support faster inference by passing bf16=True in \"AutoModelForCausalLM.from_pretrained\".")
883
+ elif SUPPORT_FP16:
884
+ logger.warn("Your device support faster inference by passing fp16=True in \"AutoModelForCausalLM.from_pretrained\".")
885
+
886
+ if config.use_flash_attn == "auto":
887
+ if config.bf16 or config.fp16:
888
+ logger.warn("Try importing flash-attention for faster inference...")
889
+ config.use_flash_attn = True
890
+ else:
891
+ config.use_flash_attn = False
892
+ if config.use_flash_attn and config.fp32:
893
+ logger.warn("Flash attention will be disabled because it does NOT support fp32.")
894
+
895
+ if config.use_flash_attn:
896
+ _import_flash_attn()
897
+
898
+ self.transformer = QWenModel(config)
899
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
900
+
901
+ if config.bf16:
902
+ self.transformer.bfloat16()
903
+ self.lm_head.bfloat16()
904
+ if config.fp16:
905
+ self.transformer.half()
906
+ self.lm_head.half()
907
+ self.post_init()
908
+
909
+ def get_output_embeddings(self):
910
+ return self.lm_head
911
+
912
+ def set_output_embeddings(self, new_embeddings):
913
+ self.lm_head = new_embeddings
914
+
915
+ def prepare_inputs_for_generation(
916
+ self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs
917
+ ):
918
+ token_type_ids = kwargs.get("token_type_ids", None)
919
+ if past_key_values:
920
+ input_ids = input_ids[:, -1].unsqueeze(-1)
921
+ if token_type_ids is not None:
922
+ token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
923
+
924
+ attention_mask = kwargs.get("attention_mask", None)
925
+ position_ids = kwargs.get("position_ids", None)
926
+
927
+ if attention_mask is not None and position_ids is None:
928
+ position_ids = attention_mask.long().cumsum(-1) - 1
929
+ position_ids.masked_fill_(attention_mask == 0, 1)
930
+ if past_key_values:
931
+ position_ids = position_ids[:, -1].unsqueeze(-1)
932
+ else:
933
+ position_ids = None
934
+
935
+ if inputs_embeds is not None and past_key_values is None:
936
+ model_inputs = {"inputs_embeds": inputs_embeds}
937
+ else:
938
+ model_inputs = {"input_ids": input_ids}
939
+
940
+ model_inputs.update(
941
+ {
942
+ "past_key_values": past_key_values,
943
+ "use_cache": kwargs.get("use_cache"),
944
+ "position_ids": position_ids,
945
+ "attention_mask": attention_mask,
946
+ "token_type_ids": token_type_ids,
947
+ }
948
+ )
949
+ return model_inputs
950
+
951
+ def forward(
952
+ self,
953
+ input_ids: Optional[torch.LongTensor] = None,
954
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
955
+ attention_mask: Optional[torch.FloatTensor] = None,
956
+ token_type_ids: Optional[torch.LongTensor] = None,
957
+ position_ids: Optional[torch.LongTensor] = None,
958
+ head_mask: Optional[torch.FloatTensor] = None,
959
+ inputs_embeds: Optional[torch.FloatTensor] = None,
960
+ encoder_hidden_states: Optional[torch.Tensor] = None,
961
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
962
+ labels: Optional[torch.LongTensor] = None,
963
+ use_cache: Optional[bool] = None,
964
+ output_attentions: Optional[bool] = None,
965
+ output_hidden_states: Optional[bool] = None,
966
+ return_dict: Optional[bool] = None,
967
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
968
+
969
+ return_dict = (
970
+ return_dict if return_dict is not None else self.config.use_return_dict
971
+ )
972
+
973
+ transformer_outputs = self.transformer(
974
+ input_ids,
975
+ past_key_values=past_key_values,
976
+ attention_mask=attention_mask,
977
+ token_type_ids=token_type_ids,
978
+ position_ids=position_ids,
979
+ head_mask=head_mask,
980
+ inputs_embeds=inputs_embeds,
981
+ encoder_hidden_states=encoder_hidden_states,
982
+ encoder_attention_mask=encoder_attention_mask,
983
+ use_cache=use_cache,
984
+ output_attentions=output_attentions,
985
+ output_hidden_states=output_hidden_states,
986
+ return_dict=return_dict,
987
+ )
988
+ hidden_states = transformer_outputs[0]
989
+
990
+ lm_logits = self.lm_head(hidden_states)
991
+
992
+ loss = None
993
+ if labels is not None:
994
+ labels = labels.to(lm_logits.device)
995
+ shift_logits = lm_logits[..., :-1, :].contiguous()
996
+ shift_labels = labels[..., 1:].contiguous()
997
+ loss_fct = CrossEntropyLoss()
998
+ loss = loss_fct(
999
+ shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
1000
+ )
1001
+
1002
+ if not return_dict:
1003
+ output = (lm_logits,) + transformer_outputs[1:]
1004
+ return ((loss,) + output) if loss is not None else output
1005
+
1006
+ return CausalLMOutputWithPast(
1007
+ loss=loss,
1008
+ logits=lm_logits,
1009
+ past_key_values=transformer_outputs.past_key_values,
1010
+ hidden_states=transformer_outputs.hidden_states,
1011
+ attentions=transformer_outputs.attentions,
1012
+ )
1013
+
1014
+ @staticmethod
1015
+ def _reorder_cache(
1016
+ past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
1017
+ ) -> Tuple[Tuple[torch.Tensor]]:
1018
+
1019
+ return tuple(
1020
+ tuple(
1021
+ past_state.index_select(0, beam_idx.to(past_state.device))
1022
+ for past_state in layer_past
1023
+ )
1024
+ for layer_past in past_key_values
1025
+ )
1026
+
1027
+ def chat(
1028
+ self,
1029
+ tokenizer: PreTrainedTokenizer,
1030
+ query: str,
1031
+ history: Optional[HistoryType],
1032
+ system: str = "You are a helpful assistant.",
1033
+ append_history: bool = True,
1034
+ stream: Optional[bool] = _SENTINEL,
1035
+ stop_words_ids: Optional[List[List[int]]] = None,
1036
+ **kwargs,
1037
+ ) -> Tuple[str, HistoryType]:
1038
+ assert stream is _SENTINEL, _ERROR_STREAM_IN_CHAT
1039
+ assert self.generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
1040
+ if history is None:
1041
+ history = []
1042
+ if stop_words_ids is None:
1043
+ stop_words_ids = []
1044
+
1045
+ max_window_size = kwargs.get('max_window_size', None)
1046
+ if max_window_size is None:
1047
+ max_window_size = self.generation_config.max_window_size
1048
+ raw_text, context_tokens = make_context(
1049
+ tokenizer,
1050
+ query,
1051
+ history=history,
1052
+ system=system,
1053
+ max_window_size=max_window_size,
1054
+ chat_format=self.generation_config.chat_format,
1055
+ )
1056
+
1057
+ stop_words_ids.extend(get_stop_words_ids(
1058
+ self.generation_config.chat_format, tokenizer
1059
+ ))
1060
+ input_ids = torch.tensor([context_tokens]).to(self.device)
1061
+ outputs = self.generate(
1062
+ input_ids,
1063
+ stop_words_ids = stop_words_ids,
1064
+ return_dict_in_generate = False,
1065
+ **kwargs,
1066
+ )
1067
+
1068
+ response = decode_tokens(
1069
+ outputs[0],
1070
+ tokenizer,
1071
+ raw_text_len=len(raw_text),
1072
+ context_length=len(context_tokens),
1073
+ chat_format=self.generation_config.chat_format,
1074
+ verbose=False,
1075
+ errors='replace'
1076
+ )
1077
+
1078
+ if append_history:
1079
+ history.append((query, response))
1080
+
1081
+ return response, history
1082
+
1083
+ def chat_stream(
1084
+ self,
1085
+ tokenizer: PreTrainedTokenizer,
1086
+ query: str,
1087
+ history: Optional[HistoryType],
1088
+ system: str = "You are a helpful assistant.",
1089
+ stop_words_ids: Optional[List[List[int]]] = None,
1090
+ logits_processor: Optional[LogitsProcessorList] = None,
1091
+ **kwargs,
1092
+ ) -> Generator[str, Any, None]:
1093
+ assert self.generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
1094
+ if history is None:
1095
+ history = []
1096
+ if stop_words_ids is None:
1097
+ stop_words_ids = []
1098
+
1099
+ max_window_size = kwargs.get('max_window_size', None)
1100
+ if max_window_size is None:
1101
+ max_window_size = self.generation_config.max_window_size
1102
+ raw_text, context_tokens = make_context(
1103
+ tokenizer,
1104
+ query,
1105
+ history=history,
1106
+ system=system,
1107
+ max_window_size=max_window_size,
1108
+ chat_format=self.generation_config.chat_format,
1109
+ )
1110
+
1111
+ stop_words_ids.extend(get_stop_words_ids(
1112
+ self.generation_config.chat_format, tokenizer
1113
+ ))
1114
+ if stop_words_ids is not None:
1115
+ stop_words_logits_processor = StopWordsLogitsProcessor(
1116
+ stop_words_ids=stop_words_ids,
1117
+ eos_token_id=self.generation_config.eos_token_id,
1118
+ )
1119
+ if logits_processor is None:
1120
+ logits_processor = LogitsProcessorList([stop_words_logits_processor])
1121
+ else:
1122
+ logits_processor.append(stop_words_logits_processor)
1123
+ input_ids = torch.tensor([context_tokens]).to(self.device)
1124
+
1125
+ from transformers_stream_generator.main import NewGenerationMixin, StreamGenerationConfig
1126
+ self.__class__.generate_stream = NewGenerationMixin.generate
1127
+ self.__class__.sample_stream = NewGenerationMixin.sample_stream
1128
+ stream_config = StreamGenerationConfig(**self.generation_config.to_dict(), do_stream=True)
1129
+ def stream_generator():
1130
+ outputs = []
1131
+ for token in self.generate_stream(
1132
+ input_ids,
1133
+ return_dict_in_generate=False,
1134
+ generation_config=stream_config,
1135
+ logits_processor=logits_processor,
1136
+ seed=-1,
1137
+ **kwargs):
1138
+ outputs.append(token.item())
1139
+ yield tokenizer.decode(outputs, skip_special_tokens=True, errors='ignore')
1140
+
1141
+ return stream_generator()
1142
+
1143
+ def generate(
1144
+ self,
1145
+ inputs: Optional[torch.Tensor] = None,
1146
+ generation_config: Optional[GenerationConfig] = None,
1147
+ logits_processor: Optional[LogitsProcessorList] = None,
1148
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
1149
+ prefix_allowed_tokens_fn: Optional[
1150
+ Callable[[int, torch.Tensor], List[int]]
1151
+ ] = None,
1152
+ synced_gpus: Optional[bool] = None,
1153
+ assistant_model: Optional["PreTrainedModel"] = None,
1154
+ streamer: Optional["BaseStreamer"] = None,
1155
+ **kwargs,
1156
+ ) -> Union[GenerateOutput, torch.LongTensor]:
1157
+ # Process stop_words_ids.
1158
+ stop_words_ids = kwargs.pop("stop_words_ids", None)
1159
+ if stop_words_ids is None and generation_config is not None:
1160
+ stop_words_ids = getattr(generation_config, "stop_words_ids", None)
1161
+ if stop_words_ids is None:
1162
+ stop_words_ids = getattr(self.generation_config, "stop_words_ids", None)
1163
+
1164
+ if stop_words_ids is not None:
1165
+ stop_words_logits_processor = StopWordsLogitsProcessor(
1166
+ stop_words_ids=stop_words_ids,
1167
+ eos_token_id=self.generation_config.eos_token_id,
1168
+ )
1169
+ if logits_processor is None:
1170
+ logits_processor = LogitsProcessorList([stop_words_logits_processor])
1171
+ else:
1172
+ logits_processor.append(stop_words_logits_processor)
1173
+
1174
+ return super().generate(
1175
+ inputs,
1176
+ generation_config=generation_config,
1177
+ logits_processor=logits_processor,
1178
+ stopping_criteria=stopping_criteria,
1179
+ prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
1180
+ synced_gpus=synced_gpus,
1181
+ assistant_model=assistant_model,
1182
+ streamer=streamer,
1183
+ **kwargs,
1184
+ )
1185
+
1186
+
1187
+ class RotaryEmbedding(torch.nn.Module):
1188
+ def __init__(self, dim, base=10000):
1189
+ super().__init__()
1190
+ self.dim = dim
1191
+ self.base = base
1192
+ self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
1193
+ if importlib.util.find_spec("einops") is None:
1194
+ raise RuntimeError("einops is required for Rotary Embedding")
1195
+
1196
+ self._rotary_pos_emb_cache = None
1197
+ self._seq_len_cached = 0
1198
+ self._ntk_alpha_cached = 1.0
1199
+
1200
+ def update_rotary_pos_emb_cache(self, max_seq_len, offset=0, ntk_alpha=1.0):
1201
+ seqlen = max_seq_len + offset
1202
+ if seqlen > self._seq_len_cached or ntk_alpha != self._ntk_alpha_cached:
1203
+ base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
1204
+ self.inv_freq = 1.0 / (
1205
+ base
1206
+ ** (
1207
+ torch.arange(0, self.dim, 2, device=self.inv_freq.device).float()
1208
+ / self.dim
1209
+ )
1210
+ )
1211
+ self._seq_len_cached = max(2 * seqlen, 16)
1212
+ self._ntk_alpha_cached = ntk_alpha
1213
+ seq = torch.arange(self._seq_len_cached, device=self.inv_freq.device)
1214
+ freqs = torch.outer(seq.type_as(self.inv_freq), self.inv_freq)
1215
+ emb = torch.cat((freqs, freqs), dim=-1)
1216
+ from einops import rearrange
1217
+
1218
+ self._rotary_pos_emb_cache = rearrange(emb, "n d -> 1 n 1 d")
1219
+
1220
+ def forward(self, max_seq_len, offset=0, ntk_alpha=1.0):
1221
+ self.update_rotary_pos_emb_cache(max_seq_len, offset, ntk_alpha)
1222
+ return self._rotary_pos_emb_cache[:, offset : offset + max_seq_len]
1223
+
1224
+
1225
+ def _rotate_half(x):
1226
+ from einops import rearrange
1227
+
1228
+ x = rearrange(x, "... (j d) -> ... j d", j=2)
1229
+ x1, x2 = x.unbind(dim=-2)
1230
+ return torch.cat((-x2, x1), dim=-1)
1231
+
1232
+
1233
+ def apply_rotary_pos_emb(t, freqs):
1234
+ if apply_rotary_emb_func is not None and t.is_cuda:
1235
+ t_ = t.float()
1236
+ freqs = freqs.squeeze(0).squeeze(1)
1237
+ cos = freqs[:, : freqs.shape[-1] // 2].cos()
1238
+ sin = freqs[:, : freqs.shape[-1] // 2].sin()
1239
+ output = apply_rotary_emb_func(t_, cos, sin).type_as(t)
1240
+ return output
1241
+ else:
1242
+ rot_dim = freqs.shape[-1]
1243
+ t_, t_pass_ = t[..., :rot_dim], t[..., rot_dim:]
1244
+ t_ = t_.float()
1245
+ t_pass_ = t_pass_.float()
1246
+ t_ = (t_ * freqs.cos()) + (_rotate_half(t_) * freqs.sin())
1247
+ return torch.cat((t_, t_pass_), dim=-1).type_as(t)
1248
+
1249
+
1250
+ class RMSNorm(torch.nn.Module):
1251
+ def __init__(self, dim: int, eps: float = 1e-6):
1252
+ super().__init__()
1253
+ self.eps = eps
1254
+ self.weight = nn.Parameter(torch.ones(dim))
1255
+
1256
+ def _norm(self, x):
1257
+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
1258
+
1259
+ def forward(self, x):
1260
+ if rms_norm is not None and x.is_cuda:
1261
+ return rms_norm(x, self.weight, self.eps)
1262
+ else:
1263
+ output = self._norm(x.float()).type_as(x)
1264
+ return output * self.weight
pytorch_model-00001-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f85037d904705766b58b76c5265a568591f288925a081488a0ec3f3f752c7fce
3
+ size 1964070447
pytorch_model-00002-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe54b166888161c8598615eec862e6ad2c86b3d3eaac501daa6f86d3b0174850
3
+ size 1933792141
pytorch_model-00003-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ce49783fa46b607f112df9d27e7be2f313041cd3f8414226f6d8bdbeae86f4c
3
+ size 1933792141
pytorch_model-00004-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e64e1b5d87d41e2da7dd6b42ad5dd92feac963d01ab5481e86b4da198a670264
3
+ size 1990406779
pytorch_model-00005-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d1b139edfde201c5ff4b592b968bc2baf00cba4b42ecc5e858a1fc898d3c434
3
+ size 1923281531
pytorch_model-00006-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:206f37b9ee4747f53a5c66c1a388c7bcb8236315bec83e9ca105b350acae0781
3
+ size 1933783675
pytorch_model-00007-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8cf880e42cb11a45985f8b1e2caa0ba9afa98ab13ce38be5150641bb4cbb85fd
3
+ size 1933792205
pytorch_model-00008-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f8c5659e3d3934ccc92519039168ca84c945dec227f8477fbcafd2426264156
3
+ size 1975364669
pytorch_model-00009-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02b652b95c6dffc2754d2cc3d8166b35c542726a571b979fc2b59417c218f196
3
+ size 1994925011
pytorch_model-00010-of-00010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:508bc4b5caab78e08d9961000cd00411f0e56f0f016828bcf9e78c132d960c24
3
+ size 1730968463
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,860 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 19313870336
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00010-of-00010.bin",
7
+ "transformer.h.0.attn.c_attn.bias": "pytorch_model-00001-of-00010.bin",
8
+ "transformer.h.0.attn.c_attn.weight": "pytorch_model-00001-of-00010.bin",
9
+ "transformer.h.0.attn.c_proj.weight": "pytorch_model-00001-of-00010.bin",
10
+ "transformer.h.0.ln_1.weight": "pytorch_model-00001-of-00010.bin",
11
+ "transformer.h.0.ln_2.weight": "pytorch_model-00001-of-00010.bin",
12
+ "transformer.h.0.mlp.c_proj.weight": "pytorch_model-00001-of-00010.bin",
13
+ "transformer.h.0.mlp.w1.weight": "pytorch_model-00001-of-00010.bin",
14
+ "transformer.h.0.mlp.w2.weight": "pytorch_model-00001-of-00010.bin",
15
+ "transformer.h.1.attn.c_attn.bias": "pytorch_model-00001-of-00010.bin",
16
+ "transformer.h.1.attn.c_attn.weight": "pytorch_model-00001-of-00010.bin",
17
+ "transformer.h.1.attn.c_proj.weight": "pytorch_model-00001-of-00010.bin",
18
+ "transformer.h.1.ln_1.weight": "pytorch_model-00001-of-00010.bin",
19
+ "transformer.h.1.ln_2.weight": "pytorch_model-00001-of-00010.bin",
20
+ "transformer.h.1.mlp.c_proj.weight": "pytorch_model-00002-of-00010.bin",
21
+ "transformer.h.1.mlp.w1.weight": "pytorch_model-00001-of-00010.bin",
22
+ "transformer.h.1.mlp.w2.weight": "pytorch_model-00001-of-00010.bin",
23
+ "transformer.h.10.attn.c_attn.bias": "pytorch_model-00003-of-00010.bin",
24
+ "transformer.h.10.attn.c_attn.weight": "pytorch_model-00003-of-00010.bin",
25
+ "transformer.h.10.attn.c_proj.weight": "pytorch_model-00003-of-00010.bin",
26
+ "transformer.h.10.ln_1.weight": "pytorch_model-00003-of-00010.bin",
27
+ "transformer.h.10.ln_2.weight": "pytorch_model-00003-of-00010.bin",
28
+ "transformer.h.10.mlp.c_proj.weight": "pytorch_model-00003-of-00010.bin",
29
+ "transformer.h.10.mlp.w1.weight": "pytorch_model-00003-of-00010.bin",
30
+ "transformer.h.10.mlp.w2.weight": "pytorch_model-00003-of-00010.bin",
31
+ "transformer.h.11.attn.c_attn.bias": "pytorch_model-00003-of-00010.bin",
32
+ "transformer.h.11.attn.c_attn.weight": "pytorch_model-00003-of-00010.bin",
33
+ "transformer.h.11.attn.c_proj.weight": "pytorch_model-00003-of-00010.bin",
34
+ "transformer.h.11.ln_1.weight": "pytorch_model-00003-of-00010.bin",
35
+ "transformer.h.11.ln_2.weight": "pytorch_model-00003-of-00010.bin",
36
+ "transformer.h.11.mlp.c_proj.weight": "pytorch_model-00004-of-00010.bin",
37
+ "transformer.h.11.mlp.w1.weight": "pytorch_model-00004-of-00010.bin",
38
+ "transformer.h.11.mlp.w2.weight": "pytorch_model-00004-of-00010.bin",
39
+ "transformer.h.12.attn.c_attn.bias": "pytorch_model-00004-of-00010.bin",
40
+ "transformer.h.12.attn.c_attn.weight": "pytorch_model-00004-of-00010.bin",
41
+ "transformer.h.12.attn.c_proj.weight": "pytorch_model-00004-of-00010.bin",
42
+ "transformer.h.12.ln_1.weight": "pytorch_model-00004-of-00010.bin",
43
+ "transformer.h.12.ln_2.weight": "pytorch_model-00004-of-00010.bin",
44
+ "transformer.h.12.mlp.c_proj.weight": "pytorch_model-00004-of-00010.bin",
45
+ "transformer.h.12.mlp.w1.weight": "pytorch_model-00004-of-00010.bin",
46
+ "transformer.h.12.mlp.w2.weight": "pytorch_model-00004-of-00010.bin",
47
+ "transformer.h.13.attn.c_attn.bias": "pytorch_model-00004-of-00010.bin",
48
+ "transformer.h.13.attn.c_attn.weight": "pytorch_model-00004-of-00010.bin",
49
+ "transformer.h.13.attn.c_proj.weight": "pytorch_model-00004-of-00010.bin",
50
+ "transformer.h.13.ln_1.weight": "pytorch_model-00004-of-00010.bin",
51
+ "transformer.h.13.ln_2.weight": "pytorch_model-00004-of-00010.bin",
52
+ "transformer.h.13.mlp.c_proj.weight": "pytorch_model-00004-of-00010.bin",
53
+ "transformer.h.13.mlp.w1.weight": "pytorch_model-00004-of-00010.bin",
54
+ "transformer.h.13.mlp.w2.weight": "pytorch_model-00004-of-00010.bin",
55
+ "transformer.h.14.attn.c_attn.bias": "pytorch_model-00004-of-00010.bin",
56
+ "transformer.h.14.attn.c_attn.weight": "pytorch_model-00004-of-00010.bin",
57
+ "transformer.h.14.attn.c_proj.weight": "pytorch_model-00004-of-00010.bin",
58
+ "transformer.h.14.ln_1.weight": "pytorch_model-00004-of-00010.bin",
59
+ "transformer.h.14.ln_2.weight": "pytorch_model-00004-of-00010.bin",
60
+ "transformer.h.14.mlp.c_proj.weight": "pytorch_model-00004-of-00010.bin",
61
+ "transformer.h.14.mlp.w1.weight": "pytorch_model-00004-of-00010.bin",
62
+ "transformer.h.14.mlp.w2.weight": "pytorch_model-00004-of-00010.bin",
63
+ "transformer.h.15.attn.c_attn.bias": "pytorch_model-00004-of-00010.bin",
64
+ "transformer.h.15.attn.c_attn.weight": "pytorch_model-00004-of-00010.bin",
65
+ "transformer.h.15.attn.c_proj.weight": "pytorch_model-00004-of-00010.bin",
66
+ "transformer.h.15.ln_1.weight": "pytorch_model-00004-of-00010.bin",
67
+ "transformer.h.15.ln_2.weight": "pytorch_model-00004-of-00010.bin",
68
+ "transformer.h.15.mlp.c_proj.weight": "pytorch_model-00004-of-00010.bin",
69
+ "transformer.h.15.mlp.w1.weight": "pytorch_model-00004-of-00010.bin",
70
+ "transformer.h.15.mlp.w2.weight": "pytorch_model-00004-of-00010.bin",
71
+ "transformer.h.16.attn.c_attn.bias": "pytorch_model-00004-of-00010.bin",
72
+ "transformer.h.16.attn.c_attn.weight": "pytorch_model-00004-of-00010.bin",
73
+ "transformer.h.16.attn.c_proj.weight": "pytorch_model-00005-of-00010.bin",
74
+ "transformer.h.16.ln_1.weight": "pytorch_model-00004-of-00010.bin",
75
+ "transformer.h.16.ln_2.weight": "pytorch_model-00005-of-00010.bin",
76
+ "transformer.h.16.mlp.c_proj.weight": "pytorch_model-00005-of-00010.bin",
77
+ "transformer.h.16.mlp.w1.weight": "pytorch_model-00005-of-00010.bin",
78
+ "transformer.h.16.mlp.w2.weight": "pytorch_model-00005-of-00010.bin",
79
+ "transformer.h.17.attn.c_attn.bias": "pytorch_model-00005-of-00010.bin",
80
+ "transformer.h.17.attn.c_attn.weight": "pytorch_model-00005-of-00010.bin",
81
+ "transformer.h.17.attn.c_proj.weight": "pytorch_model-00005-of-00010.bin",
82
+ "transformer.h.17.ln_1.weight": "pytorch_model-00005-of-00010.bin",
83
+ "transformer.h.17.ln_2.weight": "pytorch_model-00005-of-00010.bin",
84
+ "transformer.h.17.mlp.c_proj.weight": "pytorch_model-00005-of-00010.bin",
85
+ "transformer.h.17.mlp.w1.weight": "pytorch_model-00005-of-00010.bin",
86
+ "transformer.h.17.mlp.w2.weight": "pytorch_model-00005-of-00010.bin",
87
+ "transformer.h.18.attn.c_attn.bias": "pytorch_model-00005-of-00010.bin",
88
+ "transformer.h.18.attn.c_attn.weight": "pytorch_model-00005-of-00010.bin",
89
+ "transformer.h.18.attn.c_proj.weight": "pytorch_model-00005-of-00010.bin",
90
+ "transformer.h.18.ln_1.weight": "pytorch_model-00005-of-00010.bin",
91
+ "transformer.h.18.ln_2.weight": "pytorch_model-00005-of-00010.bin",
92
+ "transformer.h.18.mlp.c_proj.weight": "pytorch_model-00005-of-00010.bin",
93
+ "transformer.h.18.mlp.w1.weight": "pytorch_model-00005-of-00010.bin",
94
+ "transformer.h.18.mlp.w2.weight": "pytorch_model-00005-of-00010.bin",
95
+ "transformer.h.19.attn.c_attn.bias": "pytorch_model-00005-of-00010.bin",
96
+ "transformer.h.19.attn.c_attn.weight": "pytorch_model-00005-of-00010.bin",
97
+ "transformer.h.19.attn.c_proj.weight": "pytorch_model-00005-of-00010.bin",
98
+ "transformer.h.19.ln_1.weight": "pytorch_model-00005-of-00010.bin",
99
+ "transformer.h.19.ln_2.weight": "pytorch_model-00005-of-00010.bin",
100
+ "transformer.h.19.mlp.c_proj.weight": "pytorch_model-00005-of-00010.bin",
101
+ "transformer.h.19.mlp.w1.weight": "pytorch_model-00005-of-00010.bin",
102
+ "transformer.h.19.mlp.w2.weight": "pytorch_model-00005-of-00010.bin",
103
+ "transformer.h.2.attn.c_attn.bias": "pytorch_model-00002-of-00010.bin",
104
+ "transformer.h.2.attn.c_attn.weight": "pytorch_model-00002-of-00010.bin",
105
+ "transformer.h.2.attn.c_proj.weight": "pytorch_model-00002-of-00010.bin",
106
+ "transformer.h.2.ln_1.weight": "pytorch_model-00002-of-00010.bin",
107
+ "transformer.h.2.ln_2.weight": "pytorch_model-00002-of-00010.bin",
108
+ "transformer.h.2.mlp.c_proj.weight": "pytorch_model-00002-of-00010.bin",
109
+ "transformer.h.2.mlp.w1.weight": "pytorch_model-00002-of-00010.bin",
110
+ "transformer.h.2.mlp.w2.weight": "pytorch_model-00002-of-00010.bin",
111
+ "transformer.h.20.attn.c_attn.bias": "pytorch_model-00005-of-00010.bin",
112
+ "transformer.h.20.attn.c_attn.weight": "pytorch_model-00005-of-00010.bin",
113
+ "transformer.h.20.attn.c_proj.weight": "pytorch_model-00005-of-00010.bin",
114
+ "transformer.h.20.ln_1.weight": "pytorch_model-00005-of-00010.bin",
115
+ "transformer.h.20.ln_2.weight": "pytorch_model-00005-of-00010.bin",
116
+ "transformer.h.20.mlp.c_proj.weight": "pytorch_model-00005-of-00010.bin",
117
+ "transformer.h.20.mlp.w1.weight": "pytorch_model-00005-of-00010.bin",
118
+ "transformer.h.20.mlp.w2.weight": "pytorch_model-00005-of-00010.bin",
119
+ "transformer.h.21.attn.c_attn.bias": "pytorch_model-00006-of-00010.bin",
120
+ "transformer.h.21.attn.c_attn.weight": "pytorch_model-00006-of-00010.bin",
121
+ "transformer.h.21.attn.c_proj.weight": "pytorch_model-00006-of-00010.bin",
122
+ "transformer.h.21.ln_1.weight": "pytorch_model-00005-of-00010.bin",
123
+ "transformer.h.21.ln_2.weight": "pytorch_model-00006-of-00010.bin",
124
+ "transformer.h.21.mlp.c_proj.weight": "pytorch_model-00006-of-00010.bin",
125
+ "transformer.h.21.mlp.w1.weight": "pytorch_model-00006-of-00010.bin",
126
+ "transformer.h.21.mlp.w2.weight": "pytorch_model-00006-of-00010.bin",
127
+ "transformer.h.22.attn.c_attn.bias": "pytorch_model-00006-of-00010.bin",
128
+ "transformer.h.22.attn.c_attn.weight": "pytorch_model-00006-of-00010.bin",
129
+ "transformer.h.22.attn.c_proj.weight": "pytorch_model-00006-of-00010.bin",
130
+ "transformer.h.22.ln_1.weight": "pytorch_model-00006-of-00010.bin",
131
+ "transformer.h.22.ln_2.weight": "pytorch_model-00006-of-00010.bin",
132
+ "transformer.h.22.mlp.c_proj.weight": "pytorch_model-00006-of-00010.bin",
133
+ "transformer.h.22.mlp.w1.weight": "pytorch_model-00006-of-00010.bin",
134
+ "transformer.h.22.mlp.w2.weight": "pytorch_model-00006-of-00010.bin",
135
+ "transformer.h.23.attn.c_attn.bias": "pytorch_model-00006-of-00010.bin",
136
+ "transformer.h.23.attn.c_attn.weight": "pytorch_model-00006-of-00010.bin",
137
+ "transformer.h.23.attn.c_proj.weight": "pytorch_model-00006-of-00010.bin",
138
+ "transformer.h.23.ln_1.weight": "pytorch_model-00006-of-00010.bin",
139
+ "transformer.h.23.ln_2.weight": "pytorch_model-00006-of-00010.bin",
140
+ "transformer.h.23.mlp.c_proj.weight": "pytorch_model-00006-of-00010.bin",
141
+ "transformer.h.23.mlp.w1.weight": "pytorch_model-00006-of-00010.bin",
142
+ "transformer.h.23.mlp.w2.weight": "pytorch_model-00006-of-00010.bin",
143
+ "transformer.h.24.attn.c_attn.bias": "pytorch_model-00006-of-00010.bin",
144
+ "transformer.h.24.attn.c_attn.weight": "pytorch_model-00006-of-00010.bin",
145
+ "transformer.h.24.attn.c_proj.weight": "pytorch_model-00006-of-00010.bin",
146
+ "transformer.h.24.ln_1.weight": "pytorch_model-00006-of-00010.bin",
147
+ "transformer.h.24.ln_2.weight": "pytorch_model-00006-of-00010.bin",
148
+ "transformer.h.24.mlp.c_proj.weight": "pytorch_model-00006-of-00010.bin",
149
+ "transformer.h.24.mlp.w1.weight": "pytorch_model-00006-of-00010.bin",
150
+ "transformer.h.24.mlp.w2.weight": "pytorch_model-00006-of-00010.bin",
151
+ "transformer.h.25.attn.c_attn.bias": "pytorch_model-00006-of-00010.bin",
152
+ "transformer.h.25.attn.c_attn.weight": "pytorch_model-00006-of-00010.bin",
153
+ "transformer.h.25.attn.c_proj.weight": "pytorch_model-00006-of-00010.bin",
154
+ "transformer.h.25.ln_1.weight": "pytorch_model-00006-of-00010.bin",
155
+ "transformer.h.25.ln_2.weight": "pytorch_model-00006-of-00010.bin",
156
+ "transformer.h.25.mlp.c_proj.weight": "pytorch_model-00007-of-00010.bin",
157
+ "transformer.h.25.mlp.w1.weight": "pytorch_model-00006-of-00010.bin",
158
+ "transformer.h.25.mlp.w2.weight": "pytorch_model-00006-of-00010.bin",
159
+ "transformer.h.26.attn.c_attn.bias": "pytorch_model-00007-of-00010.bin",
160
+ "transformer.h.26.attn.c_attn.weight": "pytorch_model-00007-of-00010.bin",
161
+ "transformer.h.26.attn.c_proj.weight": "pytorch_model-00007-of-00010.bin",
162
+ "transformer.h.26.ln_1.weight": "pytorch_model-00007-of-00010.bin",
163
+ "transformer.h.26.ln_2.weight": "pytorch_model-00007-of-00010.bin",
164
+ "transformer.h.26.mlp.c_proj.weight": "pytorch_model-00007-of-00010.bin",
165
+ "transformer.h.26.mlp.w1.weight": "pytorch_model-00007-of-00010.bin",
166
+ "transformer.h.26.mlp.w2.weight": "pytorch_model-00007-of-00010.bin",
167
+ "transformer.h.27.attn.c_attn.bias": "pytorch_model-00007-of-00010.bin",
168
+ "transformer.h.27.attn.c_attn.weight": "pytorch_model-00007-of-00010.bin",
169
+ "transformer.h.27.attn.c_proj.weight": "pytorch_model-00007-of-00010.bin",
170
+ "transformer.h.27.ln_1.weight": "pytorch_model-00007-of-00010.bin",
171
+ "transformer.h.27.ln_2.weight": "pytorch_model-00007-of-00010.bin",
172
+ "transformer.h.27.mlp.c_proj.weight": "pytorch_model-00007-of-00010.bin",
173
+ "transformer.h.27.mlp.w1.weight": "pytorch_model-00007-of-00010.bin",
174
+ "transformer.h.27.mlp.w2.weight": "pytorch_model-00007-of-00010.bin",
175
+ "transformer.h.28.attn.c_attn.bias": "pytorch_model-00007-of-00010.bin",
176
+ "transformer.h.28.attn.c_attn.weight": "pytorch_model-00007-of-00010.bin",
177
+ "transformer.h.28.attn.c_proj.weight": "pytorch_model-00007-of-00010.bin",
178
+ "transformer.h.28.ln_1.weight": "pytorch_model-00007-of-00010.bin",
179
+ "transformer.h.28.ln_2.weight": "pytorch_model-00007-of-00010.bin",
180
+ "transformer.h.28.mlp.c_proj.weight": "pytorch_model-00007-of-00010.bin",
181
+ "transformer.h.28.mlp.w1.weight": "pytorch_model-00007-of-00010.bin",
182
+ "transformer.h.28.mlp.w2.weight": "pytorch_model-00007-of-00010.bin",
183
+ "transformer.h.29.attn.c_attn.bias": "pytorch_model-00007-of-00010.bin",
184
+ "transformer.h.29.attn.c_attn.weight": "pytorch_model-00007-of-00010.bin",
185
+ "transformer.h.29.attn.c_proj.weight": "pytorch_model-00007-of-00010.bin",
186
+ "transformer.h.29.ln_1.weight": "pytorch_model-00007-of-00010.bin",
187
+ "transformer.h.29.ln_2.weight": "pytorch_model-00007-of-00010.bin",
188
+ "transformer.h.29.mlp.c_proj.weight": "pytorch_model-00007-of-00010.bin",
189
+ "transformer.h.29.mlp.w1.weight": "pytorch_model-00007-of-00010.bin",
190
+ "transformer.h.29.mlp.w2.weight": "pytorch_model-00007-of-00010.bin",
191
+ "transformer.h.3.attn.c_attn.bias": "pytorch_model-00002-of-00010.bin",
192
+ "transformer.h.3.attn.c_attn.weight": "pytorch_model-00002-of-00010.bin",
193
+ "transformer.h.3.attn.c_proj.weight": "pytorch_model-00002-of-00010.bin",
194
+ "transformer.h.3.ln_1.weight": "pytorch_model-00002-of-00010.bin",
195
+ "transformer.h.3.ln_2.weight": "pytorch_model-00002-of-00010.bin",
196
+ "transformer.h.3.mlp.c_proj.weight": "pytorch_model-00002-of-00010.bin",
197
+ "transformer.h.3.mlp.w1.weight": "pytorch_model-00002-of-00010.bin",
198
+ "transformer.h.3.mlp.w2.weight": "pytorch_model-00002-of-00010.bin",
199
+ "transformer.h.30.attn.c_attn.bias": "pytorch_model-00007-of-00010.bin",
200
+ "transformer.h.30.attn.c_attn.weight": "pytorch_model-00007-of-00010.bin",
201
+ "transformer.h.30.attn.c_proj.weight": "pytorch_model-00007-of-00010.bin",
202
+ "transformer.h.30.ln_1.weight": "pytorch_model-00007-of-00010.bin",
203
+ "transformer.h.30.ln_2.weight": "pytorch_model-00007-of-00010.bin",
204
+ "transformer.h.30.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
205
+ "transformer.h.30.mlp.w1.weight": "pytorch_model-00007-of-00010.bin",
206
+ "transformer.h.30.mlp.w2.weight": "pytorch_model-00008-of-00010.bin",
207
+ "transformer.h.31.attn.c_attn.bias": "pytorch_model-00008-of-00010.bin",
208
+ "transformer.h.31.attn.c_attn.weight": "pytorch_model-00008-of-00010.bin",
209
+ "transformer.h.31.attn.c_proj.weight": "pytorch_model-00008-of-00010.bin",
210
+ "transformer.h.31.ln_1.weight": "pytorch_model-00008-of-00010.bin",
211
+ "transformer.h.31.ln_2.weight": "pytorch_model-00008-of-00010.bin",
212
+ "transformer.h.31.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
213
+ "transformer.h.31.mlp.w1.weight": "pytorch_model-00008-of-00010.bin",
214
+ "transformer.h.31.mlp.w2.weight": "pytorch_model-00008-of-00010.bin",
215
+ "transformer.h.4.attn.c_attn.bias": "pytorch_model-00002-of-00010.bin",
216
+ "transformer.h.4.attn.c_attn.weight": "pytorch_model-00002-of-00010.bin",
217
+ "transformer.h.4.attn.c_proj.weight": "pytorch_model-00002-of-00010.bin",
218
+ "transformer.h.4.ln_1.weight": "pytorch_model-00002-of-00010.bin",
219
+ "transformer.h.4.ln_2.weight": "pytorch_model-00002-of-00010.bin",
220
+ "transformer.h.4.mlp.c_proj.weight": "pytorch_model-00002-of-00010.bin",
221
+ "transformer.h.4.mlp.w1.weight": "pytorch_model-00002-of-00010.bin",
222
+ "transformer.h.4.mlp.w2.weight": "pytorch_model-00002-of-00010.bin",
223
+ "transformer.h.5.attn.c_attn.bias": "pytorch_model-00002-of-00010.bin",
224
+ "transformer.h.5.attn.c_attn.weight": "pytorch_model-00002-of-00010.bin",
225
+ "transformer.h.5.attn.c_proj.weight": "pytorch_model-00002-of-00010.bin",
226
+ "transformer.h.5.ln_1.weight": "pytorch_model-00002-of-00010.bin",
227
+ "transformer.h.5.ln_2.weight": "pytorch_model-00002-of-00010.bin",
228
+ "transformer.h.5.mlp.c_proj.weight": "pytorch_model-00002-of-00010.bin",
229
+ "transformer.h.5.mlp.w1.weight": "pytorch_model-00002-of-00010.bin",
230
+ "transformer.h.5.mlp.w2.weight": "pytorch_model-00002-of-00010.bin",
231
+ "transformer.h.6.attn.c_attn.bias": "pytorch_model-00002-of-00010.bin",
232
+ "transformer.h.6.attn.c_attn.weight": "pytorch_model-00002-of-00010.bin",
233
+ "transformer.h.6.attn.c_proj.weight": "pytorch_model-00002-of-00010.bin",
234
+ "transformer.h.6.ln_1.weight": "pytorch_model-00002-of-00010.bin",
235
+ "transformer.h.6.ln_2.weight": "pytorch_model-00002-of-00010.bin",
236
+ "transformer.h.6.mlp.c_proj.weight": "pytorch_model-00003-of-00010.bin",
237
+ "transformer.h.6.mlp.w1.weight": "pytorch_model-00002-of-00010.bin",
238
+ "transformer.h.6.mlp.w2.weight": "pytorch_model-00003-of-00010.bin",
239
+ "transformer.h.7.attn.c_attn.bias": "pytorch_model-00003-of-00010.bin",
240
+ "transformer.h.7.attn.c_attn.weight": "pytorch_model-00003-of-00010.bin",
241
+ "transformer.h.7.attn.c_proj.weight": "pytorch_model-00003-of-00010.bin",
242
+ "transformer.h.7.ln_1.weight": "pytorch_model-00003-of-00010.bin",
243
+ "transformer.h.7.ln_2.weight": "pytorch_model-00003-of-00010.bin",
244
+ "transformer.h.7.mlp.c_proj.weight": "pytorch_model-00003-of-00010.bin",
245
+ "transformer.h.7.mlp.w1.weight": "pytorch_model-00003-of-00010.bin",
246
+ "transformer.h.7.mlp.w2.weight": "pytorch_model-00003-of-00010.bin",
247
+ "transformer.h.8.attn.c_attn.bias": "pytorch_model-00003-of-00010.bin",
248
+ "transformer.h.8.attn.c_attn.weight": "pytorch_model-00003-of-00010.bin",
249
+ "transformer.h.8.attn.c_proj.weight": "pytorch_model-00003-of-00010.bin",
250
+ "transformer.h.8.ln_1.weight": "pytorch_model-00003-of-00010.bin",
251
+ "transformer.h.8.ln_2.weight": "pytorch_model-00003-of-00010.bin",
252
+ "transformer.h.8.mlp.c_proj.weight": "pytorch_model-00003-of-00010.bin",
253
+ "transformer.h.8.mlp.w1.weight": "pytorch_model-00003-of-00010.bin",
254
+ "transformer.h.8.mlp.w2.weight": "pytorch_model-00003-of-00010.bin",
255
+ "transformer.h.9.attn.c_attn.bias": "pytorch_model-00003-of-00010.bin",
256
+ "transformer.h.9.attn.c_attn.weight": "pytorch_model-00003-of-00010.bin",
257
+ "transformer.h.9.attn.c_proj.weight": "pytorch_model-00003-of-00010.bin",
258
+ "transformer.h.9.ln_1.weight": "pytorch_model-00003-of-00010.bin",
259
+ "transformer.h.9.ln_2.weight": "pytorch_model-00003-of-00010.bin",
260
+ "transformer.h.9.mlp.c_proj.weight": "pytorch_model-00003-of-00010.bin",
261
+ "transformer.h.9.mlp.w1.weight": "pytorch_model-00003-of-00010.bin",
262
+ "transformer.h.9.mlp.w2.weight": "pytorch_model-00003-of-00010.bin",
263
+ "transformer.ln_f.weight": "pytorch_model-00008-of-00010.bin",
264
+ "transformer.visual.attn_pool.attn.in_proj_bias": "pytorch_model-00010-of-00010.bin",
265
+ "transformer.visual.attn_pool.attn.in_proj_weight": "pytorch_model-00010-of-00010.bin",
266
+ "transformer.visual.attn_pool.attn.out_proj.bias": "pytorch_model-00010-of-00010.bin",
267
+ "transformer.visual.attn_pool.attn.out_proj.weight": "pytorch_model-00010-of-00010.bin",
268
+ "transformer.visual.attn_pool.kv_proj.weight": "pytorch_model-00010-of-00010.bin",
269
+ "transformer.visual.attn_pool.ln_kv.bias": "pytorch_model-00010-of-00010.bin",
270
+ "transformer.visual.attn_pool.ln_kv.weight": "pytorch_model-00010-of-00010.bin",
271
+ "transformer.visual.attn_pool.ln_q.bias": "pytorch_model-00010-of-00010.bin",
272
+ "transformer.visual.attn_pool.ln_q.weight": "pytorch_model-00010-of-00010.bin",
273
+ "transformer.visual.attn_pool.pos_embed": "pytorch_model-00010-of-00010.bin",
274
+ "transformer.visual.attn_pool.query": "pytorch_model-00010-of-00010.bin",
275
+ "transformer.visual.conv1.weight": "pytorch_model-00008-of-00010.bin",
276
+ "transformer.visual.ln_post.bias": "pytorch_model-00010-of-00010.bin",
277
+ "transformer.visual.ln_post.weight": "pytorch_model-00010-of-00010.bin",
278
+ "transformer.visual.ln_pre.bias": "pytorch_model-00008-of-00010.bin",
279
+ "transformer.visual.ln_pre.weight": "pytorch_model-00008-of-00010.bin",
280
+ "transformer.visual.positional_embedding": "pytorch_model-00008-of-00010.bin",
281
+ "transformer.visual.proj": "pytorch_model-00008-of-00010.bin",
282
+ "transformer.visual.transformer.resblocks.0.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
283
+ "transformer.visual.transformer.resblocks.0.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
284
+ "transformer.visual.transformer.resblocks.0.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
285
+ "transformer.visual.transformer.resblocks.0.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
286
+ "transformer.visual.transformer.resblocks.0.ln_1.bias": "pytorch_model-00008-of-00010.bin",
287
+ "transformer.visual.transformer.resblocks.0.ln_1.weight": "pytorch_model-00008-of-00010.bin",
288
+ "transformer.visual.transformer.resblocks.0.ln_2.bias": "pytorch_model-00008-of-00010.bin",
289
+ "transformer.visual.transformer.resblocks.0.ln_2.weight": "pytorch_model-00008-of-00010.bin",
290
+ "transformer.visual.transformer.resblocks.0.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
291
+ "transformer.visual.transformer.resblocks.0.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
292
+ "transformer.visual.transformer.resblocks.0.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
293
+ "transformer.visual.transformer.resblocks.0.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
294
+ "transformer.visual.transformer.resblocks.1.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
295
+ "transformer.visual.transformer.resblocks.1.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
296
+ "transformer.visual.transformer.resblocks.1.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
297
+ "transformer.visual.transformer.resblocks.1.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
298
+ "transformer.visual.transformer.resblocks.1.ln_1.bias": "pytorch_model-00008-of-00010.bin",
299
+ "transformer.visual.transformer.resblocks.1.ln_1.weight": "pytorch_model-00008-of-00010.bin",
300
+ "transformer.visual.transformer.resblocks.1.ln_2.bias": "pytorch_model-00008-of-00010.bin",
301
+ "transformer.visual.transformer.resblocks.1.ln_2.weight": "pytorch_model-00008-of-00010.bin",
302
+ "transformer.visual.transformer.resblocks.1.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
303
+ "transformer.visual.transformer.resblocks.1.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
304
+ "transformer.visual.transformer.resblocks.1.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
305
+ "transformer.visual.transformer.resblocks.1.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
306
+ "transformer.visual.transformer.resblocks.10.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
307
+ "transformer.visual.transformer.resblocks.10.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
308
+ "transformer.visual.transformer.resblocks.10.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
309
+ "transformer.visual.transformer.resblocks.10.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
310
+ "transformer.visual.transformer.resblocks.10.ln_1.bias": "pytorch_model-00008-of-00010.bin",
311
+ "transformer.visual.transformer.resblocks.10.ln_1.weight": "pytorch_model-00008-of-00010.bin",
312
+ "transformer.visual.transformer.resblocks.10.ln_2.bias": "pytorch_model-00008-of-00010.bin",
313
+ "transformer.visual.transformer.resblocks.10.ln_2.weight": "pytorch_model-00008-of-00010.bin",
314
+ "transformer.visual.transformer.resblocks.10.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
315
+ "transformer.visual.transformer.resblocks.10.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
316
+ "transformer.visual.transformer.resblocks.10.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
317
+ "transformer.visual.transformer.resblocks.10.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
318
+ "transformer.visual.transformer.resblocks.11.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
319
+ "transformer.visual.transformer.resblocks.11.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
320
+ "transformer.visual.transformer.resblocks.11.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
321
+ "transformer.visual.transformer.resblocks.11.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
322
+ "transformer.visual.transformer.resblocks.11.ln_1.bias": "pytorch_model-00008-of-00010.bin",
323
+ "transformer.visual.transformer.resblocks.11.ln_1.weight": "pytorch_model-00008-of-00010.bin",
324
+ "transformer.visual.transformer.resblocks.11.ln_2.bias": "pytorch_model-00008-of-00010.bin",
325
+ "transformer.visual.transformer.resblocks.11.ln_2.weight": "pytorch_model-00008-of-00010.bin",
326
+ "transformer.visual.transformer.resblocks.11.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
327
+ "transformer.visual.transformer.resblocks.11.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
328
+ "transformer.visual.transformer.resblocks.11.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
329
+ "transformer.visual.transformer.resblocks.11.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
330
+ "transformer.visual.transformer.resblocks.12.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
331
+ "transformer.visual.transformer.resblocks.12.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
332
+ "transformer.visual.transformer.resblocks.12.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
333
+ "transformer.visual.transformer.resblocks.12.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
334
+ "transformer.visual.transformer.resblocks.12.ln_1.bias": "pytorch_model-00008-of-00010.bin",
335
+ "transformer.visual.transformer.resblocks.12.ln_1.weight": "pytorch_model-00008-of-00010.bin",
336
+ "transformer.visual.transformer.resblocks.12.ln_2.bias": "pytorch_model-00008-of-00010.bin",
337
+ "transformer.visual.transformer.resblocks.12.ln_2.weight": "pytorch_model-00008-of-00010.bin",
338
+ "transformer.visual.transformer.resblocks.12.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
339
+ "transformer.visual.transformer.resblocks.12.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
340
+ "transformer.visual.transformer.resblocks.12.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
341
+ "transformer.visual.transformer.resblocks.12.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
342
+ "transformer.visual.transformer.resblocks.13.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
343
+ "transformer.visual.transformer.resblocks.13.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
344
+ "transformer.visual.transformer.resblocks.13.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
345
+ "transformer.visual.transformer.resblocks.13.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
346
+ "transformer.visual.transformer.resblocks.13.ln_1.bias": "pytorch_model-00008-of-00010.bin",
347
+ "transformer.visual.transformer.resblocks.13.ln_1.weight": "pytorch_model-00008-of-00010.bin",
348
+ "transformer.visual.transformer.resblocks.13.ln_2.bias": "pytorch_model-00008-of-00010.bin",
349
+ "transformer.visual.transformer.resblocks.13.ln_2.weight": "pytorch_model-00008-of-00010.bin",
350
+ "transformer.visual.transformer.resblocks.13.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
351
+ "transformer.visual.transformer.resblocks.13.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
352
+ "transformer.visual.transformer.resblocks.13.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
353
+ "transformer.visual.transformer.resblocks.13.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
354
+ "transformer.visual.transformer.resblocks.14.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
355
+ "transformer.visual.transformer.resblocks.14.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
356
+ "transformer.visual.transformer.resblocks.14.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
357
+ "transformer.visual.transformer.resblocks.14.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
358
+ "transformer.visual.transformer.resblocks.14.ln_1.bias": "pytorch_model-00008-of-00010.bin",
359
+ "transformer.visual.transformer.resblocks.14.ln_1.weight": "pytorch_model-00008-of-00010.bin",
360
+ "transformer.visual.transformer.resblocks.14.ln_2.bias": "pytorch_model-00008-of-00010.bin",
361
+ "transformer.visual.transformer.resblocks.14.ln_2.weight": "pytorch_model-00008-of-00010.bin",
362
+ "transformer.visual.transformer.resblocks.14.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
363
+ "transformer.visual.transformer.resblocks.14.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
364
+ "transformer.visual.transformer.resblocks.14.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
365
+ "transformer.visual.transformer.resblocks.14.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
366
+ "transformer.visual.transformer.resblocks.15.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
367
+ "transformer.visual.transformer.resblocks.15.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
368
+ "transformer.visual.transformer.resblocks.15.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
369
+ "transformer.visual.transformer.resblocks.15.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
370
+ "transformer.visual.transformer.resblocks.15.ln_1.bias": "pytorch_model-00008-of-00010.bin",
371
+ "transformer.visual.transformer.resblocks.15.ln_1.weight": "pytorch_model-00008-of-00010.bin",
372
+ "transformer.visual.transformer.resblocks.15.ln_2.bias": "pytorch_model-00008-of-00010.bin",
373
+ "transformer.visual.transformer.resblocks.15.ln_2.weight": "pytorch_model-00008-of-00010.bin",
374
+ "transformer.visual.transformer.resblocks.15.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
375
+ "transformer.visual.transformer.resblocks.15.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
376
+ "transformer.visual.transformer.resblocks.15.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
377
+ "transformer.visual.transformer.resblocks.15.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
378
+ "transformer.visual.transformer.resblocks.16.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
379
+ "transformer.visual.transformer.resblocks.16.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
380
+ "transformer.visual.transformer.resblocks.16.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
381
+ "transformer.visual.transformer.resblocks.16.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
382
+ "transformer.visual.transformer.resblocks.16.ln_1.bias": "pytorch_model-00008-of-00010.bin",
383
+ "transformer.visual.transformer.resblocks.16.ln_1.weight": "pytorch_model-00008-of-00010.bin",
384
+ "transformer.visual.transformer.resblocks.16.ln_2.bias": "pytorch_model-00008-of-00010.bin",
385
+ "transformer.visual.transformer.resblocks.16.ln_2.weight": "pytorch_model-00008-of-00010.bin",
386
+ "transformer.visual.transformer.resblocks.16.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
387
+ "transformer.visual.transformer.resblocks.16.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
388
+ "transformer.visual.transformer.resblocks.16.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
389
+ "transformer.visual.transformer.resblocks.16.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
390
+ "transformer.visual.transformer.resblocks.17.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
391
+ "transformer.visual.transformer.resblocks.17.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
392
+ "transformer.visual.transformer.resblocks.17.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
393
+ "transformer.visual.transformer.resblocks.17.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
394
+ "transformer.visual.transformer.resblocks.17.ln_1.bias": "pytorch_model-00008-of-00010.bin",
395
+ "transformer.visual.transformer.resblocks.17.ln_1.weight": "pytorch_model-00008-of-00010.bin",
396
+ "transformer.visual.transformer.resblocks.17.ln_2.bias": "pytorch_model-00008-of-00010.bin",
397
+ "transformer.visual.transformer.resblocks.17.ln_2.weight": "pytorch_model-00008-of-00010.bin",
398
+ "transformer.visual.transformer.resblocks.17.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
399
+ "transformer.visual.transformer.resblocks.17.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
400
+ "transformer.visual.transformer.resblocks.17.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
401
+ "transformer.visual.transformer.resblocks.17.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
402
+ "transformer.visual.transformer.resblocks.18.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
403
+ "transformer.visual.transformer.resblocks.18.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
404
+ "transformer.visual.transformer.resblocks.18.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
405
+ "transformer.visual.transformer.resblocks.18.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
406
+ "transformer.visual.transformer.resblocks.18.ln_1.bias": "pytorch_model-00009-of-00010.bin",
407
+ "transformer.visual.transformer.resblocks.18.ln_1.weight": "pytorch_model-00009-of-00010.bin",
408
+ "transformer.visual.transformer.resblocks.18.ln_2.bias": "pytorch_model-00009-of-00010.bin",
409
+ "transformer.visual.transformer.resblocks.18.ln_2.weight": "pytorch_model-00009-of-00010.bin",
410
+ "transformer.visual.transformer.resblocks.18.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
411
+ "transformer.visual.transformer.resblocks.18.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
412
+ "transformer.visual.transformer.resblocks.18.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
413
+ "transformer.visual.transformer.resblocks.18.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
414
+ "transformer.visual.transformer.resblocks.19.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
415
+ "transformer.visual.transformer.resblocks.19.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
416
+ "transformer.visual.transformer.resblocks.19.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
417
+ "transformer.visual.transformer.resblocks.19.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
418
+ "transformer.visual.transformer.resblocks.19.ln_1.bias": "pytorch_model-00009-of-00010.bin",
419
+ "transformer.visual.transformer.resblocks.19.ln_1.weight": "pytorch_model-00009-of-00010.bin",
420
+ "transformer.visual.transformer.resblocks.19.ln_2.bias": "pytorch_model-00009-of-00010.bin",
421
+ "transformer.visual.transformer.resblocks.19.ln_2.weight": "pytorch_model-00009-of-00010.bin",
422
+ "transformer.visual.transformer.resblocks.19.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
423
+ "transformer.visual.transformer.resblocks.19.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
424
+ "transformer.visual.transformer.resblocks.19.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
425
+ "transformer.visual.transformer.resblocks.19.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
426
+ "transformer.visual.transformer.resblocks.2.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
427
+ "transformer.visual.transformer.resblocks.2.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
428
+ "transformer.visual.transformer.resblocks.2.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
429
+ "transformer.visual.transformer.resblocks.2.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
430
+ "transformer.visual.transformer.resblocks.2.ln_1.bias": "pytorch_model-00008-of-00010.bin",
431
+ "transformer.visual.transformer.resblocks.2.ln_1.weight": "pytorch_model-00008-of-00010.bin",
432
+ "transformer.visual.transformer.resblocks.2.ln_2.bias": "pytorch_model-00008-of-00010.bin",
433
+ "transformer.visual.transformer.resblocks.2.ln_2.weight": "pytorch_model-00008-of-00010.bin",
434
+ "transformer.visual.transformer.resblocks.2.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
435
+ "transformer.visual.transformer.resblocks.2.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
436
+ "transformer.visual.transformer.resblocks.2.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
437
+ "transformer.visual.transformer.resblocks.2.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
438
+ "transformer.visual.transformer.resblocks.20.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
439
+ "transformer.visual.transformer.resblocks.20.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
440
+ "transformer.visual.transformer.resblocks.20.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
441
+ "transformer.visual.transformer.resblocks.20.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
442
+ "transformer.visual.transformer.resblocks.20.ln_1.bias": "pytorch_model-00009-of-00010.bin",
443
+ "transformer.visual.transformer.resblocks.20.ln_1.weight": "pytorch_model-00009-of-00010.bin",
444
+ "transformer.visual.transformer.resblocks.20.ln_2.bias": "pytorch_model-00009-of-00010.bin",
445
+ "transformer.visual.transformer.resblocks.20.ln_2.weight": "pytorch_model-00009-of-00010.bin",
446
+ "transformer.visual.transformer.resblocks.20.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
447
+ "transformer.visual.transformer.resblocks.20.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
448
+ "transformer.visual.transformer.resblocks.20.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
449
+ "transformer.visual.transformer.resblocks.20.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
450
+ "transformer.visual.transformer.resblocks.21.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
451
+ "transformer.visual.transformer.resblocks.21.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
452
+ "transformer.visual.transformer.resblocks.21.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
453
+ "transformer.visual.transformer.resblocks.21.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
454
+ "transformer.visual.transformer.resblocks.21.ln_1.bias": "pytorch_model-00009-of-00010.bin",
455
+ "transformer.visual.transformer.resblocks.21.ln_1.weight": "pytorch_model-00009-of-00010.bin",
456
+ "transformer.visual.transformer.resblocks.21.ln_2.bias": "pytorch_model-00009-of-00010.bin",
457
+ "transformer.visual.transformer.resblocks.21.ln_2.weight": "pytorch_model-00009-of-00010.bin",
458
+ "transformer.visual.transformer.resblocks.21.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
459
+ "transformer.visual.transformer.resblocks.21.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
460
+ "transformer.visual.transformer.resblocks.21.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
461
+ "transformer.visual.transformer.resblocks.21.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
462
+ "transformer.visual.transformer.resblocks.22.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
463
+ "transformer.visual.transformer.resblocks.22.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
464
+ "transformer.visual.transformer.resblocks.22.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
465
+ "transformer.visual.transformer.resblocks.22.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
466
+ "transformer.visual.transformer.resblocks.22.ln_1.bias": "pytorch_model-00009-of-00010.bin",
467
+ "transformer.visual.transformer.resblocks.22.ln_1.weight": "pytorch_model-00009-of-00010.bin",
468
+ "transformer.visual.transformer.resblocks.22.ln_2.bias": "pytorch_model-00009-of-00010.bin",
469
+ "transformer.visual.transformer.resblocks.22.ln_2.weight": "pytorch_model-00009-of-00010.bin",
470
+ "transformer.visual.transformer.resblocks.22.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
471
+ "transformer.visual.transformer.resblocks.22.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
472
+ "transformer.visual.transformer.resblocks.22.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
473
+ "transformer.visual.transformer.resblocks.22.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
474
+ "transformer.visual.transformer.resblocks.23.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
475
+ "transformer.visual.transformer.resblocks.23.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
476
+ "transformer.visual.transformer.resblocks.23.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
477
+ "transformer.visual.transformer.resblocks.23.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
478
+ "transformer.visual.transformer.resblocks.23.ln_1.bias": "pytorch_model-00009-of-00010.bin",
479
+ "transformer.visual.transformer.resblocks.23.ln_1.weight": "pytorch_model-00009-of-00010.bin",
480
+ "transformer.visual.transformer.resblocks.23.ln_2.bias": "pytorch_model-00009-of-00010.bin",
481
+ "transformer.visual.transformer.resblocks.23.ln_2.weight": "pytorch_model-00009-of-00010.bin",
482
+ "transformer.visual.transformer.resblocks.23.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
483
+ "transformer.visual.transformer.resblocks.23.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
484
+ "transformer.visual.transformer.resblocks.23.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
485
+ "transformer.visual.transformer.resblocks.23.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
486
+ "transformer.visual.transformer.resblocks.24.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
487
+ "transformer.visual.transformer.resblocks.24.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
488
+ "transformer.visual.transformer.resblocks.24.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
489
+ "transformer.visual.transformer.resblocks.24.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
490
+ "transformer.visual.transformer.resblocks.24.ln_1.bias": "pytorch_model-00009-of-00010.bin",
491
+ "transformer.visual.transformer.resblocks.24.ln_1.weight": "pytorch_model-00009-of-00010.bin",
492
+ "transformer.visual.transformer.resblocks.24.ln_2.bias": "pytorch_model-00009-of-00010.bin",
493
+ "transformer.visual.transformer.resblocks.24.ln_2.weight": "pytorch_model-00009-of-00010.bin",
494
+ "transformer.visual.transformer.resblocks.24.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
495
+ "transformer.visual.transformer.resblocks.24.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
496
+ "transformer.visual.transformer.resblocks.24.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
497
+ "transformer.visual.transformer.resblocks.24.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
498
+ "transformer.visual.transformer.resblocks.25.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
499
+ "transformer.visual.transformer.resblocks.25.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
500
+ "transformer.visual.transformer.resblocks.25.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
501
+ "transformer.visual.transformer.resblocks.25.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
502
+ "transformer.visual.transformer.resblocks.25.ln_1.bias": "pytorch_model-00009-of-00010.bin",
503
+ "transformer.visual.transformer.resblocks.25.ln_1.weight": "pytorch_model-00009-of-00010.bin",
504
+ "transformer.visual.transformer.resblocks.25.ln_2.bias": "pytorch_model-00009-of-00010.bin",
505
+ "transformer.visual.transformer.resblocks.25.ln_2.weight": "pytorch_model-00009-of-00010.bin",
506
+ "transformer.visual.transformer.resblocks.25.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
507
+ "transformer.visual.transformer.resblocks.25.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
508
+ "transformer.visual.transformer.resblocks.25.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
509
+ "transformer.visual.transformer.resblocks.25.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
510
+ "transformer.visual.transformer.resblocks.26.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
511
+ "transformer.visual.transformer.resblocks.26.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
512
+ "transformer.visual.transformer.resblocks.26.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
513
+ "transformer.visual.transformer.resblocks.26.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
514
+ "transformer.visual.transformer.resblocks.26.ln_1.bias": "pytorch_model-00009-of-00010.bin",
515
+ "transformer.visual.transformer.resblocks.26.ln_1.weight": "pytorch_model-00009-of-00010.bin",
516
+ "transformer.visual.transformer.resblocks.26.ln_2.bias": "pytorch_model-00009-of-00010.bin",
517
+ "transformer.visual.transformer.resblocks.26.ln_2.weight": "pytorch_model-00009-of-00010.bin",
518
+ "transformer.visual.transformer.resblocks.26.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
519
+ "transformer.visual.transformer.resblocks.26.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
520
+ "transformer.visual.transformer.resblocks.26.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
521
+ "transformer.visual.transformer.resblocks.26.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
522
+ "transformer.visual.transformer.resblocks.27.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
523
+ "transformer.visual.transformer.resblocks.27.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
524
+ "transformer.visual.transformer.resblocks.27.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
525
+ "transformer.visual.transformer.resblocks.27.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
526
+ "transformer.visual.transformer.resblocks.27.ln_1.bias": "pytorch_model-00009-of-00010.bin",
527
+ "transformer.visual.transformer.resblocks.27.ln_1.weight": "pytorch_model-00009-of-00010.bin",
528
+ "transformer.visual.transformer.resblocks.27.ln_2.bias": "pytorch_model-00009-of-00010.bin",
529
+ "transformer.visual.transformer.resblocks.27.ln_2.weight": "pytorch_model-00009-of-00010.bin",
530
+ "transformer.visual.transformer.resblocks.27.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
531
+ "transformer.visual.transformer.resblocks.27.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
532
+ "transformer.visual.transformer.resblocks.27.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
533
+ "transformer.visual.transformer.resblocks.27.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
534
+ "transformer.visual.transformer.resblocks.28.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
535
+ "transformer.visual.transformer.resblocks.28.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
536
+ "transformer.visual.transformer.resblocks.28.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
537
+ "transformer.visual.transformer.resblocks.28.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
538
+ "transformer.visual.transformer.resblocks.28.ln_1.bias": "pytorch_model-00009-of-00010.bin",
539
+ "transformer.visual.transformer.resblocks.28.ln_1.weight": "pytorch_model-00009-of-00010.bin",
540
+ "transformer.visual.transformer.resblocks.28.ln_2.bias": "pytorch_model-00009-of-00010.bin",
541
+ "transformer.visual.transformer.resblocks.28.ln_2.weight": "pytorch_model-00009-of-00010.bin",
542
+ "transformer.visual.transformer.resblocks.28.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
543
+ "transformer.visual.transformer.resblocks.28.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
544
+ "transformer.visual.transformer.resblocks.28.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
545
+ "transformer.visual.transformer.resblocks.28.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
546
+ "transformer.visual.transformer.resblocks.29.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
547
+ "transformer.visual.transformer.resblocks.29.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
548
+ "transformer.visual.transformer.resblocks.29.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
549
+ "transformer.visual.transformer.resblocks.29.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
550
+ "transformer.visual.transformer.resblocks.29.ln_1.bias": "pytorch_model-00009-of-00010.bin",
551
+ "transformer.visual.transformer.resblocks.29.ln_1.weight": "pytorch_model-00009-of-00010.bin",
552
+ "transformer.visual.transformer.resblocks.29.ln_2.bias": "pytorch_model-00009-of-00010.bin",
553
+ "transformer.visual.transformer.resblocks.29.ln_2.weight": "pytorch_model-00009-of-00010.bin",
554
+ "transformer.visual.transformer.resblocks.29.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
555
+ "transformer.visual.transformer.resblocks.29.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
556
+ "transformer.visual.transformer.resblocks.29.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
557
+ "transformer.visual.transformer.resblocks.29.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
558
+ "transformer.visual.transformer.resblocks.3.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
559
+ "transformer.visual.transformer.resblocks.3.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
560
+ "transformer.visual.transformer.resblocks.3.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
561
+ "transformer.visual.transformer.resblocks.3.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
562
+ "transformer.visual.transformer.resblocks.3.ln_1.bias": "pytorch_model-00008-of-00010.bin",
563
+ "transformer.visual.transformer.resblocks.3.ln_1.weight": "pytorch_model-00008-of-00010.bin",
564
+ "transformer.visual.transformer.resblocks.3.ln_2.bias": "pytorch_model-00008-of-00010.bin",
565
+ "transformer.visual.transformer.resblocks.3.ln_2.weight": "pytorch_model-00008-of-00010.bin",
566
+ "transformer.visual.transformer.resblocks.3.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
567
+ "transformer.visual.transformer.resblocks.3.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
568
+ "transformer.visual.transformer.resblocks.3.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
569
+ "transformer.visual.transformer.resblocks.3.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
570
+ "transformer.visual.transformer.resblocks.30.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
571
+ "transformer.visual.transformer.resblocks.30.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
572
+ "transformer.visual.transformer.resblocks.30.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
573
+ "transformer.visual.transformer.resblocks.30.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
574
+ "transformer.visual.transformer.resblocks.30.ln_1.bias": "pytorch_model-00009-of-00010.bin",
575
+ "transformer.visual.transformer.resblocks.30.ln_1.weight": "pytorch_model-00009-of-00010.bin",
576
+ "transformer.visual.transformer.resblocks.30.ln_2.bias": "pytorch_model-00009-of-00010.bin",
577
+ "transformer.visual.transformer.resblocks.30.ln_2.weight": "pytorch_model-00009-of-00010.bin",
578
+ "transformer.visual.transformer.resblocks.30.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
579
+ "transformer.visual.transformer.resblocks.30.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
580
+ "transformer.visual.transformer.resblocks.30.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
581
+ "transformer.visual.transformer.resblocks.30.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
582
+ "transformer.visual.transformer.resblocks.31.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
583
+ "transformer.visual.transformer.resblocks.31.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
584
+ "transformer.visual.transformer.resblocks.31.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
585
+ "transformer.visual.transformer.resblocks.31.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
586
+ "transformer.visual.transformer.resblocks.31.ln_1.bias": "pytorch_model-00009-of-00010.bin",
587
+ "transformer.visual.transformer.resblocks.31.ln_1.weight": "pytorch_model-00009-of-00010.bin",
588
+ "transformer.visual.transformer.resblocks.31.ln_2.bias": "pytorch_model-00009-of-00010.bin",
589
+ "transformer.visual.transformer.resblocks.31.ln_2.weight": "pytorch_model-00009-of-00010.bin",
590
+ "transformer.visual.transformer.resblocks.31.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
591
+ "transformer.visual.transformer.resblocks.31.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
592
+ "transformer.visual.transformer.resblocks.31.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
593
+ "transformer.visual.transformer.resblocks.31.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
594
+ "transformer.visual.transformer.resblocks.32.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
595
+ "transformer.visual.transformer.resblocks.32.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
596
+ "transformer.visual.transformer.resblocks.32.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
597
+ "transformer.visual.transformer.resblocks.32.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
598
+ "transformer.visual.transformer.resblocks.32.ln_1.bias": "pytorch_model-00009-of-00010.bin",
599
+ "transformer.visual.transformer.resblocks.32.ln_1.weight": "pytorch_model-00009-of-00010.bin",
600
+ "transformer.visual.transformer.resblocks.32.ln_2.bias": "pytorch_model-00009-of-00010.bin",
601
+ "transformer.visual.transformer.resblocks.32.ln_2.weight": "pytorch_model-00009-of-00010.bin",
602
+ "transformer.visual.transformer.resblocks.32.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
603
+ "transformer.visual.transformer.resblocks.32.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
604
+ "transformer.visual.transformer.resblocks.32.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
605
+ "transformer.visual.transformer.resblocks.32.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
606
+ "transformer.visual.transformer.resblocks.33.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
607
+ "transformer.visual.transformer.resblocks.33.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
608
+ "transformer.visual.transformer.resblocks.33.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
609
+ "transformer.visual.transformer.resblocks.33.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
610
+ "transformer.visual.transformer.resblocks.33.ln_1.bias": "pytorch_model-00009-of-00010.bin",
611
+ "transformer.visual.transformer.resblocks.33.ln_1.weight": "pytorch_model-00009-of-00010.bin",
612
+ "transformer.visual.transformer.resblocks.33.ln_2.bias": "pytorch_model-00009-of-00010.bin",
613
+ "transformer.visual.transformer.resblocks.33.ln_2.weight": "pytorch_model-00009-of-00010.bin",
614
+ "transformer.visual.transformer.resblocks.33.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
615
+ "transformer.visual.transformer.resblocks.33.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
616
+ "transformer.visual.transformer.resblocks.33.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
617
+ "transformer.visual.transformer.resblocks.33.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
618
+ "transformer.visual.transformer.resblocks.34.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
619
+ "transformer.visual.transformer.resblocks.34.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
620
+ "transformer.visual.transformer.resblocks.34.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
621
+ "transformer.visual.transformer.resblocks.34.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
622
+ "transformer.visual.transformer.resblocks.34.ln_1.bias": "pytorch_model-00009-of-00010.bin",
623
+ "transformer.visual.transformer.resblocks.34.ln_1.weight": "pytorch_model-00009-of-00010.bin",
624
+ "transformer.visual.transformer.resblocks.34.ln_2.bias": "pytorch_model-00009-of-00010.bin",
625
+ "transformer.visual.transformer.resblocks.34.ln_2.weight": "pytorch_model-00009-of-00010.bin",
626
+ "transformer.visual.transformer.resblocks.34.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
627
+ "transformer.visual.transformer.resblocks.34.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
628
+ "transformer.visual.transformer.resblocks.34.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
629
+ "transformer.visual.transformer.resblocks.34.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
630
+ "transformer.visual.transformer.resblocks.35.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
631
+ "transformer.visual.transformer.resblocks.35.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
632
+ "transformer.visual.transformer.resblocks.35.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
633
+ "transformer.visual.transformer.resblocks.35.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
634
+ "transformer.visual.transformer.resblocks.35.ln_1.bias": "pytorch_model-00009-of-00010.bin",
635
+ "transformer.visual.transformer.resblocks.35.ln_1.weight": "pytorch_model-00009-of-00010.bin",
636
+ "transformer.visual.transformer.resblocks.35.ln_2.bias": "pytorch_model-00009-of-00010.bin",
637
+ "transformer.visual.transformer.resblocks.35.ln_2.weight": "pytorch_model-00009-of-00010.bin",
638
+ "transformer.visual.transformer.resblocks.35.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
639
+ "transformer.visual.transformer.resblocks.35.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
640
+ "transformer.visual.transformer.resblocks.35.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
641
+ "transformer.visual.transformer.resblocks.35.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
642
+ "transformer.visual.transformer.resblocks.36.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
643
+ "transformer.visual.transformer.resblocks.36.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
644
+ "transformer.visual.transformer.resblocks.36.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
645
+ "transformer.visual.transformer.resblocks.36.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
646
+ "transformer.visual.transformer.resblocks.36.ln_1.bias": "pytorch_model-00009-of-00010.bin",
647
+ "transformer.visual.transformer.resblocks.36.ln_1.weight": "pytorch_model-00009-of-00010.bin",
648
+ "transformer.visual.transformer.resblocks.36.ln_2.bias": "pytorch_model-00009-of-00010.bin",
649
+ "transformer.visual.transformer.resblocks.36.ln_2.weight": "pytorch_model-00009-of-00010.bin",
650
+ "transformer.visual.transformer.resblocks.36.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
651
+ "transformer.visual.transformer.resblocks.36.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
652
+ "transformer.visual.transformer.resblocks.36.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
653
+ "transformer.visual.transformer.resblocks.36.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
654
+ "transformer.visual.transformer.resblocks.37.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
655
+ "transformer.visual.transformer.resblocks.37.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
656
+ "transformer.visual.transformer.resblocks.37.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
657
+ "transformer.visual.transformer.resblocks.37.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
658
+ "transformer.visual.transformer.resblocks.37.ln_1.bias": "pytorch_model-00009-of-00010.bin",
659
+ "transformer.visual.transformer.resblocks.37.ln_1.weight": "pytorch_model-00009-of-00010.bin",
660
+ "transformer.visual.transformer.resblocks.37.ln_2.bias": "pytorch_model-00009-of-00010.bin",
661
+ "transformer.visual.transformer.resblocks.37.ln_2.weight": "pytorch_model-00009-of-00010.bin",
662
+ "transformer.visual.transformer.resblocks.37.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
663
+ "transformer.visual.transformer.resblocks.37.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
664
+ "transformer.visual.transformer.resblocks.37.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
665
+ "transformer.visual.transformer.resblocks.37.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
666
+ "transformer.visual.transformer.resblocks.38.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
667
+ "transformer.visual.transformer.resblocks.38.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
668
+ "transformer.visual.transformer.resblocks.38.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
669
+ "transformer.visual.transformer.resblocks.38.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
670
+ "transformer.visual.transformer.resblocks.38.ln_1.bias": "pytorch_model-00009-of-00010.bin",
671
+ "transformer.visual.transformer.resblocks.38.ln_1.weight": "pytorch_model-00009-of-00010.bin",
672
+ "transformer.visual.transformer.resblocks.38.ln_2.bias": "pytorch_model-00009-of-00010.bin",
673
+ "transformer.visual.transformer.resblocks.38.ln_2.weight": "pytorch_model-00009-of-00010.bin",
674
+ "transformer.visual.transformer.resblocks.38.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
675
+ "transformer.visual.transformer.resblocks.38.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
676
+ "transformer.visual.transformer.resblocks.38.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
677
+ "transformer.visual.transformer.resblocks.38.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
678
+ "transformer.visual.transformer.resblocks.39.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
679
+ "transformer.visual.transformer.resblocks.39.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
680
+ "transformer.visual.transformer.resblocks.39.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
681
+ "transformer.visual.transformer.resblocks.39.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
682
+ "transformer.visual.transformer.resblocks.39.ln_1.bias": "pytorch_model-00009-of-00010.bin",
683
+ "transformer.visual.transformer.resblocks.39.ln_1.weight": "pytorch_model-00009-of-00010.bin",
684
+ "transformer.visual.transformer.resblocks.39.ln_2.bias": "pytorch_model-00009-of-00010.bin",
685
+ "transformer.visual.transformer.resblocks.39.ln_2.weight": "pytorch_model-00009-of-00010.bin",
686
+ "transformer.visual.transformer.resblocks.39.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
687
+ "transformer.visual.transformer.resblocks.39.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
688
+ "transformer.visual.transformer.resblocks.39.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
689
+ "transformer.visual.transformer.resblocks.39.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
690
+ "transformer.visual.transformer.resblocks.4.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
691
+ "transformer.visual.transformer.resblocks.4.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
692
+ "transformer.visual.transformer.resblocks.4.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
693
+ "transformer.visual.transformer.resblocks.4.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
694
+ "transformer.visual.transformer.resblocks.4.ln_1.bias": "pytorch_model-00008-of-00010.bin",
695
+ "transformer.visual.transformer.resblocks.4.ln_1.weight": "pytorch_model-00008-of-00010.bin",
696
+ "transformer.visual.transformer.resblocks.4.ln_2.bias": "pytorch_model-00008-of-00010.bin",
697
+ "transformer.visual.transformer.resblocks.4.ln_2.weight": "pytorch_model-00008-of-00010.bin",
698
+ "transformer.visual.transformer.resblocks.4.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
699
+ "transformer.visual.transformer.resblocks.4.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
700
+ "transformer.visual.transformer.resblocks.4.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
701
+ "transformer.visual.transformer.resblocks.4.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
702
+ "transformer.visual.transformer.resblocks.40.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
703
+ "transformer.visual.transformer.resblocks.40.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
704
+ "transformer.visual.transformer.resblocks.40.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
705
+ "transformer.visual.transformer.resblocks.40.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
706
+ "transformer.visual.transformer.resblocks.40.ln_1.bias": "pytorch_model-00009-of-00010.bin",
707
+ "transformer.visual.transformer.resblocks.40.ln_1.weight": "pytorch_model-00009-of-00010.bin",
708
+ "transformer.visual.transformer.resblocks.40.ln_2.bias": "pytorch_model-00009-of-00010.bin",
709
+ "transformer.visual.transformer.resblocks.40.ln_2.weight": "pytorch_model-00009-of-00010.bin",
710
+ "transformer.visual.transformer.resblocks.40.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
711
+ "transformer.visual.transformer.resblocks.40.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
712
+ "transformer.visual.transformer.resblocks.40.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
713
+ "transformer.visual.transformer.resblocks.40.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
714
+ "transformer.visual.transformer.resblocks.41.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
715
+ "transformer.visual.transformer.resblocks.41.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
716
+ "transformer.visual.transformer.resblocks.41.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
717
+ "transformer.visual.transformer.resblocks.41.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
718
+ "transformer.visual.transformer.resblocks.41.ln_1.bias": "pytorch_model-00009-of-00010.bin",
719
+ "transformer.visual.transformer.resblocks.41.ln_1.weight": "pytorch_model-00009-of-00010.bin",
720
+ "transformer.visual.transformer.resblocks.41.ln_2.bias": "pytorch_model-00009-of-00010.bin",
721
+ "transformer.visual.transformer.resblocks.41.ln_2.weight": "pytorch_model-00009-of-00010.bin",
722
+ "transformer.visual.transformer.resblocks.41.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
723
+ "transformer.visual.transformer.resblocks.41.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
724
+ "transformer.visual.transformer.resblocks.41.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
725
+ "transformer.visual.transformer.resblocks.41.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
726
+ "transformer.visual.transformer.resblocks.42.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
727
+ "transformer.visual.transformer.resblocks.42.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
728
+ "transformer.visual.transformer.resblocks.42.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
729
+ "transformer.visual.transformer.resblocks.42.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
730
+ "transformer.visual.transformer.resblocks.42.ln_1.bias": "pytorch_model-00009-of-00010.bin",
731
+ "transformer.visual.transformer.resblocks.42.ln_1.weight": "pytorch_model-00009-of-00010.bin",
732
+ "transformer.visual.transformer.resblocks.42.ln_2.bias": "pytorch_model-00009-of-00010.bin",
733
+ "transformer.visual.transformer.resblocks.42.ln_2.weight": "pytorch_model-00009-of-00010.bin",
734
+ "transformer.visual.transformer.resblocks.42.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
735
+ "transformer.visual.transformer.resblocks.42.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
736
+ "transformer.visual.transformer.resblocks.42.mlp.c_proj.bias": "pytorch_model-00009-of-00010.bin",
737
+ "transformer.visual.transformer.resblocks.42.mlp.c_proj.weight": "pytorch_model-00009-of-00010.bin",
738
+ "transformer.visual.transformer.resblocks.43.attn.in_proj.bias": "pytorch_model-00009-of-00010.bin",
739
+ "transformer.visual.transformer.resblocks.43.attn.in_proj.weight": "pytorch_model-00009-of-00010.bin",
740
+ "transformer.visual.transformer.resblocks.43.attn.out_proj.bias": "pytorch_model-00009-of-00010.bin",
741
+ "transformer.visual.transformer.resblocks.43.attn.out_proj.weight": "pytorch_model-00009-of-00010.bin",
742
+ "transformer.visual.transformer.resblocks.43.ln_1.bias": "pytorch_model-00009-of-00010.bin",
743
+ "transformer.visual.transformer.resblocks.43.ln_1.weight": "pytorch_model-00009-of-00010.bin",
744
+ "transformer.visual.transformer.resblocks.43.ln_2.bias": "pytorch_model-00009-of-00010.bin",
745
+ "transformer.visual.transformer.resblocks.43.ln_2.weight": "pytorch_model-00009-of-00010.bin",
746
+ "transformer.visual.transformer.resblocks.43.mlp.c_fc.bias": "pytorch_model-00009-of-00010.bin",
747
+ "transformer.visual.transformer.resblocks.43.mlp.c_fc.weight": "pytorch_model-00009-of-00010.bin",
748
+ "transformer.visual.transformer.resblocks.43.mlp.c_proj.bias": "pytorch_model-00010-of-00010.bin",
749
+ "transformer.visual.transformer.resblocks.43.mlp.c_proj.weight": "pytorch_model-00010-of-00010.bin",
750
+ "transformer.visual.transformer.resblocks.44.attn.in_proj.bias": "pytorch_model-00010-of-00010.bin",
751
+ "transformer.visual.transformer.resblocks.44.attn.in_proj.weight": "pytorch_model-00010-of-00010.bin",
752
+ "transformer.visual.transformer.resblocks.44.attn.out_proj.bias": "pytorch_model-00010-of-00010.bin",
753
+ "transformer.visual.transformer.resblocks.44.attn.out_proj.weight": "pytorch_model-00010-of-00010.bin",
754
+ "transformer.visual.transformer.resblocks.44.ln_1.bias": "pytorch_model-00010-of-00010.bin",
755
+ "transformer.visual.transformer.resblocks.44.ln_1.weight": "pytorch_model-00010-of-00010.bin",
756
+ "transformer.visual.transformer.resblocks.44.ln_2.bias": "pytorch_model-00010-of-00010.bin",
757
+ "transformer.visual.transformer.resblocks.44.ln_2.weight": "pytorch_model-00010-of-00010.bin",
758
+ "transformer.visual.transformer.resblocks.44.mlp.c_fc.bias": "pytorch_model-00010-of-00010.bin",
759
+ "transformer.visual.transformer.resblocks.44.mlp.c_fc.weight": "pytorch_model-00010-of-00010.bin",
760
+ "transformer.visual.transformer.resblocks.44.mlp.c_proj.bias": "pytorch_model-00010-of-00010.bin",
761
+ "transformer.visual.transformer.resblocks.44.mlp.c_proj.weight": "pytorch_model-00010-of-00010.bin",
762
+ "transformer.visual.transformer.resblocks.45.attn.in_proj.bias": "pytorch_model-00010-of-00010.bin",
763
+ "transformer.visual.transformer.resblocks.45.attn.in_proj.weight": "pytorch_model-00010-of-00010.bin",
764
+ "transformer.visual.transformer.resblocks.45.attn.out_proj.bias": "pytorch_model-00010-of-00010.bin",
765
+ "transformer.visual.transformer.resblocks.45.attn.out_proj.weight": "pytorch_model-00010-of-00010.bin",
766
+ "transformer.visual.transformer.resblocks.45.ln_1.bias": "pytorch_model-00010-of-00010.bin",
767
+ "transformer.visual.transformer.resblocks.45.ln_1.weight": "pytorch_model-00010-of-00010.bin",
768
+ "transformer.visual.transformer.resblocks.45.ln_2.bias": "pytorch_model-00010-of-00010.bin",
769
+ "transformer.visual.transformer.resblocks.45.ln_2.weight": "pytorch_model-00010-of-00010.bin",
770
+ "transformer.visual.transformer.resblocks.45.mlp.c_fc.bias": "pytorch_model-00010-of-00010.bin",
771
+ "transformer.visual.transformer.resblocks.45.mlp.c_fc.weight": "pytorch_model-00010-of-00010.bin",
772
+ "transformer.visual.transformer.resblocks.45.mlp.c_proj.bias": "pytorch_model-00010-of-00010.bin",
773
+ "transformer.visual.transformer.resblocks.45.mlp.c_proj.weight": "pytorch_model-00010-of-00010.bin",
774
+ "transformer.visual.transformer.resblocks.46.attn.in_proj.bias": "pytorch_model-00010-of-00010.bin",
775
+ "transformer.visual.transformer.resblocks.46.attn.in_proj.weight": "pytorch_model-00010-of-00010.bin",
776
+ "transformer.visual.transformer.resblocks.46.attn.out_proj.bias": "pytorch_model-00010-of-00010.bin",
777
+ "transformer.visual.transformer.resblocks.46.attn.out_proj.weight": "pytorch_model-00010-of-00010.bin",
778
+ "transformer.visual.transformer.resblocks.46.ln_1.bias": "pytorch_model-00010-of-00010.bin",
779
+ "transformer.visual.transformer.resblocks.46.ln_1.weight": "pytorch_model-00010-of-00010.bin",
780
+ "transformer.visual.transformer.resblocks.46.ln_2.bias": "pytorch_model-00010-of-00010.bin",
781
+ "transformer.visual.transformer.resblocks.46.ln_2.weight": "pytorch_model-00010-of-00010.bin",
782
+ "transformer.visual.transformer.resblocks.46.mlp.c_fc.bias": "pytorch_model-00010-of-00010.bin",
783
+ "transformer.visual.transformer.resblocks.46.mlp.c_fc.weight": "pytorch_model-00010-of-00010.bin",
784
+ "transformer.visual.transformer.resblocks.46.mlp.c_proj.bias": "pytorch_model-00010-of-00010.bin",
785
+ "transformer.visual.transformer.resblocks.46.mlp.c_proj.weight": "pytorch_model-00010-of-00010.bin",
786
+ "transformer.visual.transformer.resblocks.47.attn.in_proj.bias": "pytorch_model-00010-of-00010.bin",
787
+ "transformer.visual.transformer.resblocks.47.attn.in_proj.weight": "pytorch_model-00010-of-00010.bin",
788
+ "transformer.visual.transformer.resblocks.47.attn.out_proj.bias": "pytorch_model-00010-of-00010.bin",
789
+ "transformer.visual.transformer.resblocks.47.attn.out_proj.weight": "pytorch_model-00010-of-00010.bin",
790
+ "transformer.visual.transformer.resblocks.47.ln_1.bias": "pytorch_model-00010-of-00010.bin",
791
+ "transformer.visual.transformer.resblocks.47.ln_1.weight": "pytorch_model-00010-of-00010.bin",
792
+ "transformer.visual.transformer.resblocks.47.ln_2.bias": "pytorch_model-00010-of-00010.bin",
793
+ "transformer.visual.transformer.resblocks.47.ln_2.weight": "pytorch_model-00010-of-00010.bin",
794
+ "transformer.visual.transformer.resblocks.47.mlp.c_fc.bias": "pytorch_model-00010-of-00010.bin",
795
+ "transformer.visual.transformer.resblocks.47.mlp.c_fc.weight": "pytorch_model-00010-of-00010.bin",
796
+ "transformer.visual.transformer.resblocks.47.mlp.c_proj.bias": "pytorch_model-00010-of-00010.bin",
797
+ "transformer.visual.transformer.resblocks.47.mlp.c_proj.weight": "pytorch_model-00010-of-00010.bin",
798
+ "transformer.visual.transformer.resblocks.5.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
799
+ "transformer.visual.transformer.resblocks.5.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
800
+ "transformer.visual.transformer.resblocks.5.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
801
+ "transformer.visual.transformer.resblocks.5.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
802
+ "transformer.visual.transformer.resblocks.5.ln_1.bias": "pytorch_model-00008-of-00010.bin",
803
+ "transformer.visual.transformer.resblocks.5.ln_1.weight": "pytorch_model-00008-of-00010.bin",
804
+ "transformer.visual.transformer.resblocks.5.ln_2.bias": "pytorch_model-00008-of-00010.bin",
805
+ "transformer.visual.transformer.resblocks.5.ln_2.weight": "pytorch_model-00008-of-00010.bin",
806
+ "transformer.visual.transformer.resblocks.5.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
807
+ "transformer.visual.transformer.resblocks.5.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
808
+ "transformer.visual.transformer.resblocks.5.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
809
+ "transformer.visual.transformer.resblocks.5.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
810
+ "transformer.visual.transformer.resblocks.6.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
811
+ "transformer.visual.transformer.resblocks.6.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
812
+ "transformer.visual.transformer.resblocks.6.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
813
+ "transformer.visual.transformer.resblocks.6.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
814
+ "transformer.visual.transformer.resblocks.6.ln_1.bias": "pytorch_model-00008-of-00010.bin",
815
+ "transformer.visual.transformer.resblocks.6.ln_1.weight": "pytorch_model-00008-of-00010.bin",
816
+ "transformer.visual.transformer.resblocks.6.ln_2.bias": "pytorch_model-00008-of-00010.bin",
817
+ "transformer.visual.transformer.resblocks.6.ln_2.weight": "pytorch_model-00008-of-00010.bin",
818
+ "transformer.visual.transformer.resblocks.6.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
819
+ "transformer.visual.transformer.resblocks.6.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
820
+ "transformer.visual.transformer.resblocks.6.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
821
+ "transformer.visual.transformer.resblocks.6.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
822
+ "transformer.visual.transformer.resblocks.7.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
823
+ "transformer.visual.transformer.resblocks.7.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
824
+ "transformer.visual.transformer.resblocks.7.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
825
+ "transformer.visual.transformer.resblocks.7.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
826
+ "transformer.visual.transformer.resblocks.7.ln_1.bias": "pytorch_model-00008-of-00010.bin",
827
+ "transformer.visual.transformer.resblocks.7.ln_1.weight": "pytorch_model-00008-of-00010.bin",
828
+ "transformer.visual.transformer.resblocks.7.ln_2.bias": "pytorch_model-00008-of-00010.bin",
829
+ "transformer.visual.transformer.resblocks.7.ln_2.weight": "pytorch_model-00008-of-00010.bin",
830
+ "transformer.visual.transformer.resblocks.7.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
831
+ "transformer.visual.transformer.resblocks.7.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
832
+ "transformer.visual.transformer.resblocks.7.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
833
+ "transformer.visual.transformer.resblocks.7.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
834
+ "transformer.visual.transformer.resblocks.8.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
835
+ "transformer.visual.transformer.resblocks.8.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
836
+ "transformer.visual.transformer.resblocks.8.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
837
+ "transformer.visual.transformer.resblocks.8.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
838
+ "transformer.visual.transformer.resblocks.8.ln_1.bias": "pytorch_model-00008-of-00010.bin",
839
+ "transformer.visual.transformer.resblocks.8.ln_1.weight": "pytorch_model-00008-of-00010.bin",
840
+ "transformer.visual.transformer.resblocks.8.ln_2.bias": "pytorch_model-00008-of-00010.bin",
841
+ "transformer.visual.transformer.resblocks.8.ln_2.weight": "pytorch_model-00008-of-00010.bin",
842
+ "transformer.visual.transformer.resblocks.8.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
843
+ "transformer.visual.transformer.resblocks.8.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
844
+ "transformer.visual.transformer.resblocks.8.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
845
+ "transformer.visual.transformer.resblocks.8.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
846
+ "transformer.visual.transformer.resblocks.9.attn.in_proj.bias": "pytorch_model-00008-of-00010.bin",
847
+ "transformer.visual.transformer.resblocks.9.attn.in_proj.weight": "pytorch_model-00008-of-00010.bin",
848
+ "transformer.visual.transformer.resblocks.9.attn.out_proj.bias": "pytorch_model-00008-of-00010.bin",
849
+ "transformer.visual.transformer.resblocks.9.attn.out_proj.weight": "pytorch_model-00008-of-00010.bin",
850
+ "transformer.visual.transformer.resblocks.9.ln_1.bias": "pytorch_model-00008-of-00010.bin",
851
+ "transformer.visual.transformer.resblocks.9.ln_1.weight": "pytorch_model-00008-of-00010.bin",
852
+ "transformer.visual.transformer.resblocks.9.ln_2.bias": "pytorch_model-00008-of-00010.bin",
853
+ "transformer.visual.transformer.resblocks.9.ln_2.weight": "pytorch_model-00008-of-00010.bin",
854
+ "transformer.visual.transformer.resblocks.9.mlp.c_fc.bias": "pytorch_model-00008-of-00010.bin",
855
+ "transformer.visual.transformer.resblocks.9.mlp.c_fc.weight": "pytorch_model-00008-of-00010.bin",
856
+ "transformer.visual.transformer.resblocks.9.mlp.c_proj.bias": "pytorch_model-00008-of-00010.bin",
857
+ "transformer.visual.transformer.resblocks.9.mlp.c_proj.weight": "pytorch_model-00008-of-00010.bin",
858
+ "transformer.wte.weight": "pytorch_model-00001-of-00010.bin"
859
+ }
860
+ }
qwen_generation_utils.py ADDED
@@ -0,0 +1,420 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ """Generation support."""
7
+
8
+ from typing import Tuple, List, Union, Iterable
9
+
10
+ import numpy as np
11
+ import torch
12
+ import torch.nn.functional as F
13
+ from transformers import PreTrainedTokenizer
14
+ from transformers import logging
15
+ from transformers.generation import LogitsProcessor
16
+
17
+ logger = logging.get_logger(__name__)
18
+
19
+ # Types.
20
+ HistoryType = List[Tuple[str, str]]
21
+ TokensType = List[int]
22
+ BatchTokensType = List[List[int]]
23
+
24
+
25
+ def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType:
26
+ for tokens in batch:
27
+ context_length = len(tokens)
28
+ if context_length < seq_length:
29
+ tokens.extend([pad_id] * (seq_length - context_length))
30
+ return batch
31
+
32
+
33
+ def get_ltor_masks_and_position_ids(
34
+ data,
35
+ eod_token,
36
+ reset_position_ids,
37
+ reset_attention_mask,
38
+ eod_mask_loss,
39
+ ):
40
+ """Build masks and position id for left to right model."""
41
+
42
+ # Extract batch size and sequence length.
43
+ micro_batch_size, seq_length = data.size()
44
+
45
+ # Attention mask (lower triangular).
46
+ if reset_attention_mask:
47
+ att_mask_batch = micro_batch_size
48
+ else:
49
+ att_mask_batch = 1
50
+ attention_mask = torch.tril(
51
+ torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)
52
+ ).view(att_mask_batch, 1, seq_length, seq_length)
53
+
54
+ # Loss mask.
55
+ loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device)
56
+ if eod_mask_loss:
57
+ loss_mask[data == eod_token] = 0.0
58
+
59
+ # Position ids.
60
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
61
+ position_ids = position_ids.unsqueeze(0).expand_as(data)
62
+ # We need to clone as the ids will be modifed based on batch index.
63
+ if reset_position_ids:
64
+ position_ids = position_ids.clone()
65
+
66
+ if reset_position_ids or reset_attention_mask:
67
+ # Loop through the batches:
68
+ for b in range(micro_batch_size):
69
+
70
+ # Find indecies where EOD token is.
71
+ eod_index = position_ids[b, data[b] == eod_token]
72
+ # Detach indecies from positions if going to modify positions.
73
+ if reset_position_ids:
74
+ eod_index = eod_index.clone()
75
+
76
+ # Loop through EOD indecies:
77
+ prev_index = 0
78
+ for j in range(eod_index.size()[0]):
79
+ i = eod_index[j]
80
+ # Mask attention loss.
81
+ if reset_attention_mask:
82
+ attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0
83
+ # Reset positions.
84
+ if reset_position_ids:
85
+ position_ids[b, (i + 1) :] -= i + 1 - prev_index
86
+ prev_index = i + 1
87
+
88
+ # Convert attention mask to binary:
89
+ attention_mask = attention_mask < 0.5
90
+
91
+ return attention_mask, loss_mask, position_ids
92
+
93
+
94
+ def get_batch(context_tokens: torch.LongTensor, eod_id: int):
95
+ """Generate batch from context tokens."""
96
+ # Move to GPU.
97
+ tokens = context_tokens.contiguous().to(context_tokens.device)
98
+ # Get the attention mask and postition ids.
99
+ attention_mask, _, position_ids = get_ltor_masks_and_position_ids(
100
+ tokens,
101
+ eod_id,
102
+ reset_position_ids=False,
103
+ reset_attention_mask=False,
104
+ eod_mask_loss=False,
105
+ )
106
+ return tokens, attention_mask, position_ids
107
+
108
+
109
+ def get_stop_words_ids(chat_format, tokenizer):
110
+ if chat_format == "raw":
111
+ stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]]
112
+ elif chat_format == "chatml":
113
+ stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]]
114
+ else:
115
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
116
+ return stop_words_ids
117
+
118
+
119
+ def make_context(
120
+ tokenizer: PreTrainedTokenizer,
121
+ query: str,
122
+ history: List[Tuple[str, str]] = None,
123
+ system: str = "",
124
+ max_window_size: int = 6144,
125
+ chat_format: str = "chatml",
126
+ ):
127
+ if history is None:
128
+ history = []
129
+
130
+ if chat_format == "chatml":
131
+ im_start, im_end = "<|im_start|>", "<|im_end|>"
132
+ im_start_tokens = [tokenizer.im_start_id]
133
+ im_end_tokens = [tokenizer.im_end_id]
134
+ nl_tokens = tokenizer.encode("\n")
135
+
136
+ def _tokenize_str(role, content):
137
+ return f"{role}\n{content}", tokenizer.encode(
138
+ role, allowed_special=set(tokenizer.IMAGE_ST)
139
+ ) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST))
140
+
141
+ system_text, system_tokens_part = _tokenize_str("system", system)
142
+ system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
143
+
144
+ raw_text = ""
145
+ context_tokens = []
146
+
147
+ for turn_query, turn_response in reversed(history):
148
+ query_text, query_tokens_part = _tokenize_str("user", turn_query)
149
+ query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
150
+ if turn_response is not None:
151
+ response_text, response_tokens_part = _tokenize_str(
152
+ "assistant", turn_response
153
+ )
154
+ response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
155
+
156
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
157
+ prev_chat = (
158
+ f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
159
+ )
160
+ else:
161
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens
162
+ prev_chat = f"\n{im_start}{query_text}{im_end}\n"
163
+
164
+ current_context_size = (
165
+ len(system_tokens) + len(next_context_tokens) + len(context_tokens)
166
+ )
167
+ if current_context_size < max_window_size:
168
+ context_tokens = next_context_tokens + context_tokens
169
+ raw_text = prev_chat + raw_text
170
+ else:
171
+ break
172
+
173
+ context_tokens = system_tokens + context_tokens
174
+ raw_text = f"{im_start}{system_text}{im_end}" + raw_text
175
+ context_tokens += (
176
+ nl_tokens
177
+ + im_start_tokens
178
+ + _tokenize_str("user", query)[1]
179
+ + im_end_tokens
180
+ + nl_tokens
181
+ + im_start_tokens
182
+ + tokenizer.encode("assistant")
183
+ + nl_tokens
184
+ )
185
+ raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
186
+
187
+ elif chat_format == "raw":
188
+ raw_text = query
189
+ context_tokens = tokenizer.encode(raw_text)
190
+ else:
191
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
192
+
193
+ return raw_text, context_tokens
194
+
195
+
196
+ def _decode_default(
197
+ tokens: List[int],
198
+ *,
199
+ stop_words: List[str],
200
+ eod_words: List[str],
201
+ tokenizer: PreTrainedTokenizer,
202
+ raw_text_len: int,
203
+ verbose: bool = False,
204
+ return_end_reason: bool = False,
205
+ errors: str='replace',
206
+ ):
207
+ trim_decode_tokens = tokenizer.decode(tokens, errors=errors)[raw_text_len:]
208
+ if verbose:
209
+ print("\nRaw Generate: ", trim_decode_tokens)
210
+
211
+ end_reason = f"Gen length {len(tokens)}"
212
+ for stop_word in stop_words:
213
+ trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
214
+ for eod_word in eod_words:
215
+ if eod_word in trim_decode_tokens:
216
+ end_reason = f"Gen {eod_word!r}"
217
+ trim_decode_tokens = trim_decode_tokens.split(eod_word)[0]
218
+ trim_decode_tokens = trim_decode_tokens.strip()
219
+ if verbose:
220
+ print("\nEnd Reason:", end_reason)
221
+ print("\nGenerate: ", trim_decode_tokens)
222
+
223
+ if return_end_reason:
224
+ return trim_decode_tokens, end_reason
225
+ else:
226
+ return trim_decode_tokens
227
+
228
+
229
+ def _decode_chatml(
230
+ tokens: List[int],
231
+ *,
232
+ stop_words: List[str],
233
+ eod_token_ids: List[int],
234
+ tokenizer: PreTrainedTokenizer,
235
+ raw_text_len: int,
236
+ context_length: int,
237
+ verbose: bool = False,
238
+ return_end_reason: bool = False,
239
+ errors: str='replace'
240
+ ):
241
+ end_reason = f"Gen length {len(tokens)}"
242
+ eod_token_idx = context_length
243
+ for eod_token_idx in range(context_length, len(tokens)):
244
+ if tokens[eod_token_idx] in eod_token_ids:
245
+ end_reason = f"Gen {tokenizer.decode([tokens[eod_token_idx]])!r}"
246
+ break
247
+
248
+ trim_decode_tokens = tokenizer.decode(tokens[:eod_token_idx], errors=errors)[raw_text_len:]
249
+ if verbose:
250
+ print("\nRaw Generate w/o EOD:", tokenizer.decode(tokens, errors=errors)[raw_text_len:])
251
+ print("\nRaw Generate:", trim_decode_tokens)
252
+ print("\nEnd Reason:", end_reason)
253
+ for stop_word in stop_words:
254
+ trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
255
+ trim_decode_tokens = trim_decode_tokens.strip()
256
+ if verbose:
257
+ print("\nGenerate:", trim_decode_tokens)
258
+
259
+ if return_end_reason:
260
+ return trim_decode_tokens, end_reason
261
+ else:
262
+ return trim_decode_tokens
263
+
264
+
265
+ def decode_tokens(
266
+ tokens: Union[torch.LongTensor, TokensType],
267
+ tokenizer: PreTrainedTokenizer,
268
+ raw_text_len: int,
269
+ context_length: int,
270
+ chat_format: str,
271
+ verbose: bool = False,
272
+ return_end_reason: bool = False,
273
+ errors: str="replace",
274
+ ) -> str:
275
+ if torch.is_tensor(tokens):
276
+ tokens = tokens.cpu().numpy().tolist()
277
+
278
+ if chat_format == "chatml":
279
+ return _decode_chatml(
280
+ tokens,
281
+ stop_words=[],
282
+ eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id],
283
+ tokenizer=tokenizer,
284
+ raw_text_len=raw_text_len,
285
+ context_length=context_length,
286
+ verbose=verbose,
287
+ return_end_reason=return_end_reason,
288
+ errors=errors,
289
+ )
290
+ elif chat_format == "raw":
291
+ return _decode_default(
292
+ tokens,
293
+ stop_words=["<|endoftext|>"],
294
+ eod_words=["<|endoftext|>"],
295
+ tokenizer=tokenizer,
296
+ raw_text_len=raw_text_len,
297
+ verbose=verbose,
298
+ return_end_reason=return_end_reason,
299
+ errors=errors,
300
+ )
301
+ else:
302
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
303
+
304
+
305
+ class StopWordsLogitsProcessor(LogitsProcessor):
306
+ """
307
+ :class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration.
308
+
309
+ Args:
310
+ stop_words_ids (:obj:`List[List[int]]`):
311
+ List of list of token ids of stop ids. In order to get the tokens of the words
312
+ that should not appear in the generated text, use :obj:`tokenizer(bad_word,
313
+ add_prefix_space=True).input_ids`.
314
+ eos_token_id (:obj:`int`):
315
+ The id of the `end-of-sequence` token.
316
+ """
317
+
318
+ def __init__(self, stop_words_ids: Iterable[Iterable[int]], eos_token_id: int):
319
+
320
+ if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0:
321
+ raise ValueError(
322
+ f"`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}."
323
+ )
324
+ if any(not isinstance(bad_word_ids, list) for bad_word_ids in stop_words_ids):
325
+ raise ValueError(
326
+ f"`stop_words_ids` has to be a list of lists, but is {stop_words_ids}."
327
+ )
328
+ if any(
329
+ any(
330
+ (not isinstance(token_id, (int, np.integer)) or token_id < 0)
331
+ for token_id in stop_word_ids
332
+ )
333
+ for stop_word_ids in stop_words_ids
334
+ ):
335
+ raise ValueError(
336
+ f"Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}."
337
+ )
338
+
339
+ self.stop_words_ids = list(
340
+ filter(
341
+ lambda bad_token_seq: bad_token_seq != [eos_token_id], stop_words_ids
342
+ )
343
+ )
344
+ self.eos_token_id = eos_token_id
345
+ for stop_token_seq in self.stop_words_ids:
346
+ assert (
347
+ len(stop_token_seq) > 0
348
+ ), "Stop words token sequences {} cannot have an empty list".format(
349
+ stop_words_ids
350
+ )
351
+
352
+ def __call__(
353
+ self, input_ids: torch.LongTensor, scores: torch.FloatTensor
354
+ ) -> torch.FloatTensor:
355
+ stopped_samples = self._calc_stopped_samples(input_ids)
356
+ for i, should_stop in enumerate(stopped_samples):
357
+ if should_stop:
358
+ scores[i, self.eos_token_id] = float(2**15)
359
+ return scores
360
+
361
+ def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool:
362
+ if len(tokens) == 0:
363
+ # if bad word tokens is just one token always ban it
364
+ return True
365
+ elif len(tokens) > len(prev_tokens):
366
+ # if bad word tokens are longer then prev input_ids they can't be equal
367
+ return False
368
+ elif prev_tokens[-len(tokens) :].tolist() == tokens:
369
+ # if tokens match
370
+ return True
371
+ else:
372
+ return False
373
+
374
+ def _calc_stopped_samples(self, prev_input_ids: Iterable[int]) -> Iterable[int]:
375
+ stopped_samples = []
376
+ for prev_input_ids_slice in prev_input_ids:
377
+ match = False
378
+ for stop_token_seq in self.stop_words_ids:
379
+ if self._tokens_match(prev_input_ids_slice, stop_token_seq):
380
+ # if tokens do not match continue
381
+ match = True
382
+ break
383
+ stopped_samples.append(match)
384
+
385
+ return stopped_samples
386
+
387
+
388
+ def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float("Inf")):
389
+ """This function has been mostly taken from huggingface conversational
390
+ ai code at
391
+ https://medium.com/huggingface/how-to-build-a-state-of-the-art-
392
+ conversational-ai-with-transfer-learning-2d818ac26313"""
393
+
394
+ if top_k > 0:
395
+ # Remove all tokens with a probability less than the
396
+ # last token of the top-k
397
+ indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
398
+ logits[indices_to_remove] = filter_value
399
+
400
+ if top_p > 0.0:
401
+ # Cconvert to 1D
402
+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
403
+ cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
404
+
405
+ # Remove tokens with cumulative probability above the threshold
406
+ sorted_indices_to_remove = cumulative_probs > top_p
407
+ # Shift the indices to the right to keep also the first token
408
+ # above the threshold
409
+ sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
410
+ sorted_indices_to_remove[..., 0] = 0
411
+ for i in range(sorted_indices.size(0)):
412
+ indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]]
413
+ logits[i][indices_to_remove] = filter_value
414
+
415
+ return logits
416
+
417
+
418
+ def switch(val1, val2, boolean):
419
+ boolean = boolean.type_as(val1)
420
+ return (1 - boolean) * val1 + boolean * val2
visual.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import OrderedDict
2
+ import math
3
+ import requests
4
+ from io import BytesIO
5
+ from functools import partial
6
+ from PIL import Image
7
+ from typing import Callable, Optional, Sequence, Tuple, List
8
+
9
+ import torch
10
+ from torch import nn
11
+ from torch.nn import functional as F
12
+ from torch.utils.checkpoint import checkpoint
13
+ from torch.nn.init import trunc_normal_
14
+ from torchvision import transforms
15
+ from torchvision.transforms import InterpolationMode
16
+
17
+
18
+ def get_abs_pos(abs_pos, tgt_size):
19
+ # abs_pos: L, C
20
+ # tgt_size: M
21
+ # return: M, C
22
+ src_size = int(math.sqrt(abs_pos.size(0)))
23
+ tgt_size = int(math.sqrt(tgt_size))
24
+ dtype = abs_pos.dtype
25
+
26
+ if src_size != tgt_size:
27
+ return F.interpolate(
28
+ abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
29
+ size=(tgt_size, tgt_size),
30
+ mode="bicubic",
31
+ align_corners=False,
32
+ ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
33
+ else:
34
+ return abs_pos
35
+
36
+
37
+ class Resampler(nn.Module):
38
+ def __init__(
39
+ self,
40
+ grid_size,
41
+ embed_dim,
42
+ num_heads,
43
+ kv_dim=None,
44
+ norm_layer=nn.LayerNorm
45
+ ):
46
+ super().__init__()
47
+ self.num_queries = grid_size ** 2
48
+ self.embed_dim = embed_dim
49
+ self.num_heads = num_heads
50
+
51
+ self.pos_embed = nn.Parameter(torch.randn(embed_dim, grid_size)).requires_grad_(False)
52
+
53
+ self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
54
+ trunc_normal_(self.query, std=.02)
55
+
56
+ if kv_dim is not None and kv_dim != embed_dim:
57
+ self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
58
+ else:
59
+ self.kv_proj = nn.Identity()
60
+
61
+ self.attn = nn.MultiheadAttention(embed_dim, num_heads)
62
+ self.ln_q = norm_layer(embed_dim)
63
+ self.ln_kv = norm_layer(embed_dim)
64
+
65
+ self.apply(self._init_weights)
66
+
67
+ def _init_weights(self, m):
68
+ if isinstance(m, nn.Linear):
69
+ trunc_normal_(m.weight, std=.02)
70
+ if isinstance(m, nn.Linear) and m.bias is not None:
71
+ nn.init.constant_(m.bias, 0)
72
+ elif isinstance(m, nn.LayerNorm):
73
+ nn.init.constant_(m.bias, 0)
74
+ nn.init.constant_(m.weight, 1.0)
75
+
76
+ def forward(self, x, attn_mask=None):
77
+
78
+ pos_embed = get_abs_pos(self.pos_embed, x.size(1))
79
+
80
+ x = self.kv_proj(x)
81
+ x = self.ln_kv(x).permute(1, 0, 2)
82
+
83
+ N = x.shape[1]
84
+ q = self.ln_q(self.query)
85
+ out = self.attn(
86
+ self._repeat(q, N) + self.pos_embed.unsqueeze(1),
87
+ x + pos_embed.unsqueeze(1),
88
+ x,
89
+ attn_mask=attn_mask)[0]
90
+ return out.permute(1, 0, 2)
91
+
92
+ def _repeat(self, query, N: int):
93
+ return query.unsqueeze(1).repeat(1, N, 1)
94
+
95
+
96
+ class VisualAttention(nn.Module):
97
+ """self-attention layer class.
98
+
99
+ Self-attention layer takes input with size [s, b, h]
100
+ and returns output of the same size.
101
+ """
102
+
103
+ def __init__(self, embed_dim, num_heads,
104
+ bias=True, kdim=None, vdim=None):
105
+ super(VisualAttention, self).__init__()
106
+ self.embed_dim = embed_dim
107
+ self.kdim = kdim if kdim is not None else embed_dim
108
+ self.vdim = vdim if vdim is not None else embed_dim
109
+ self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
110
+
111
+ self.num_heads = num_heads
112
+
113
+ # Per attention head and per partition values.
114
+ assert embed_dim % num_heads == 0
115
+ self.hidden_size_per_attention_head = embed_dim // num_heads
116
+ self.num_attention_heads_per_partition = num_heads
117
+ self.hidden_size_per_partition = embed_dim
118
+
119
+ # Strided linear layer.
120
+ assert self._qkv_same_embed_dim, 'Only Support SelfAttention Currently'
121
+ self.in_proj = nn.Linear(embed_dim, 3 * embed_dim)
122
+ self.out_proj = nn.Linear(embed_dim, embed_dim)
123
+ self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
124
+
125
+ def forward(self, query, key, value, attn_mask = None):
126
+ # query/key/value: [sq, b, h]
127
+ sq, b, _ = query.size()
128
+
129
+ assert query is key, 'Only Support Self-Attention Currently'
130
+ sk = sq
131
+ mixed_x_layer = self.in_proj(query)
132
+
133
+ # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
134
+ new_tensor_shape = mixed_x_layer.size()[:-1] + \
135
+ (self.num_attention_heads_per_partition,
136
+ 3 * self.hidden_size_per_attention_head)
137
+ mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
138
+
139
+ # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
140
+ query_layer, key_layer, value_layer = mixed_x_layer.split(
141
+ self.hidden_size_per_attention_head, dim=-1)
142
+
143
+ # [sq, b, np, hn] -> [sq, b * np, hn]
144
+ query_layer = query_layer.view(sq,
145
+ b * self.num_attention_heads_per_partition,
146
+ self.hidden_size_per_attention_head).transpose(0, 1)
147
+ # [sk, b, np, hn] -> [sk, b * np, hn]
148
+ key_layer = key_layer.view(sk,
149
+ b * self.num_attention_heads_per_partition,
150
+ self.hidden_size_per_attention_head).transpose(0, 1)
151
+
152
+ q_scaled = query_layer / self.norm_factor
153
+ if attn_mask is not None:
154
+ attention_probs = torch.baddbmm(attn_mask, q_scaled, key_layer.transpose(-2, -1))
155
+ else:
156
+ attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
157
+ attention_probs = attention_probs.softmax(dim=-1)
158
+
159
+ value_layer = value_layer.view(sk,
160
+ b * self.num_attention_heads_per_partition,
161
+ self.hidden_size_per_attention_head).transpose(0, 1)
162
+
163
+ # matmul: [b * np, sq, hn]
164
+ context_layer = torch.bmm(attention_probs, value_layer)
165
+
166
+ # change view [b, np, sq, hn]
167
+ context_layer = context_layer.view(b,
168
+ self.num_attention_heads_per_partition,
169
+ sq, self.hidden_size_per_attention_head)
170
+
171
+ # [b, np, sq, hn] --> [sq, b, np, hn]
172
+ context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
173
+
174
+ # [sq, b, np, hn] --> [sq, b, hp]
175
+ new_context_layer_shape = context_layer.size()[:-2] + \
176
+ (self.hidden_size_per_partition,)
177
+ context_layer = context_layer.view(*new_context_layer_shape)
178
+
179
+ output = self.out_proj(context_layer)
180
+
181
+ return output
182
+
183
+
184
+ class VisualAttentionBlock(nn.Module):
185
+ def __init__(
186
+ self,
187
+ d_model: int,
188
+ n_head: int,
189
+ mlp_ratio: float = 4.0,
190
+ act_layer: Callable = nn.GELU,
191
+ norm_layer: Callable = nn.LayerNorm,
192
+ is_cross_attention: bool = False,
193
+ ):
194
+ super().__init__()
195
+
196
+ self.ln_1 = norm_layer(d_model)
197
+ if is_cross_attention:
198
+ self.ln_1_kv = norm_layer(d_model)
199
+
200
+ self.ln_2 = norm_layer(d_model)
201
+ mlp_width = int(d_model * mlp_ratio)
202
+ self.attn = VisualAttention(d_model, n_head)
203
+ self.mlp = nn.Sequential(OrderedDict([
204
+ ("c_fc", nn.Linear(d_model, mlp_width)),
205
+ ("gelu", act_layer()),
206
+ ("c_proj", nn.Linear(mlp_width, d_model))
207
+ ]))
208
+
209
+ def attention(
210
+ self,
211
+ q_x: torch.Tensor,
212
+ k_x: Optional[torch.Tensor] = None,
213
+ v_x: Optional[torch.Tensor] = None,
214
+ attn_mask: Optional[torch.Tensor] = None,
215
+ ):
216
+ k_x = k_x if k_x is not None else q_x
217
+ v_x = v_x if v_x is not None else q_x
218
+
219
+ attn_mask = attn_mask.to(q_x.dtype) if attn_mask is not None else None
220
+ return self.attn(q_x, k_x, v_x, attn_mask=attn_mask)
221
+
222
+ def forward(
223
+ self,
224
+ q_x: torch.Tensor,
225
+ k_x: Optional[torch.Tensor] = None,
226
+ v_x: Optional[torch.Tensor] = None,
227
+ attn_mask: Optional[torch.Tensor] = None,
228
+ ):
229
+ k_x = self.ln_1_kv(k_x) if hasattr(self, "ln_1_kv") and k_x is not None else None
230
+ v_x = self.ln_1_kv(v_x) if hasattr(self, "ln_1_kv") and v_x is not None else None
231
+
232
+ x = q_x + self.attention(q_x=self.ln_1(q_x), k_x=k_x, v_x=v_x, attn_mask=attn_mask)
233
+ x = x + self.mlp(self.ln_2(x))
234
+ return x
235
+
236
+
237
+ class Transformer(nn.Module):
238
+ def __init__(
239
+ self,
240
+ width: int,
241
+ layers: int,
242
+ heads: int,
243
+ mlp_ratio: float = 4.0,
244
+ act_layer: Callable = nn.GELU,
245
+ norm_layer: Callable = nn.LayerNorm,
246
+ ):
247
+ super().__init__()
248
+ self.width = width
249
+ self.layers = layers
250
+ self.grad_checkpointing = False
251
+
252
+ self.resblocks = nn.ModuleList([
253
+ VisualAttentionBlock(
254
+ width, heads, mlp_ratio, act_layer=act_layer, norm_layer=norm_layer)
255
+ for _ in range(layers)
256
+ ])
257
+
258
+ def get_cast_dtype(self) -> torch.dtype:
259
+ return self.resblocks[0].mlp.c_fc.weight.dtype
260
+
261
+ def get_cast_device(self) -> torch.device:
262
+ return self.resblocks[0].mlp.c_fc.weight.device
263
+
264
+ def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None):
265
+ for r in self.resblocks:
266
+ if self.grad_checkpointing and not torch.jit.is_scripting():
267
+ # TODO: handle kwargs https://github.com/pytorch/pytorch/issues/79887#issuecomment-1161758372
268
+ x = checkpoint(r, x, None, None, attn_mask)
269
+ else:
270
+ x = r(x, attn_mask=attn_mask)
271
+ return x
272
+
273
+
274
+ class VisionTransformer(nn.Module):
275
+
276
+ def __init__(
277
+ self,
278
+ image_size: int,
279
+ patch_size: int,
280
+ width: int,
281
+ layers: int,
282
+ heads: int,
283
+ mlp_ratio: float,
284
+ n_queries: int = 256,
285
+ output_dim: int = 512,
286
+ **kwargs
287
+ ):
288
+ super().__init__()
289
+ image_height, image_width = self.image_size = (image_size, image_size)
290
+ patch_height, patch_width = self.patch_size = (patch_size, patch_size)
291
+ self.grid_size = (image_height // patch_height, image_width // patch_width)
292
+ self.output_dim = output_dim
293
+
294
+ mean = (0.48145466, 0.4578275, 0.40821073)
295
+ std = (0.26862954, 0.26130258, 0.27577711)
296
+ self.image_transform = transforms.Compose([
297
+ transforms.Resize(
298
+ (image_size, image_size),
299
+ interpolation=InterpolationMode.BICUBIC
300
+ ),
301
+ transforms.ToTensor(),
302
+ transforms.Normalize(mean=mean, std=std),
303
+ ])
304
+
305
+ self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
306
+
307
+ # class embeddings and positional embeddings
308
+ scale = width ** -0.5
309
+ self.positional_embedding = nn.Parameter(scale * torch.randn(self.grid_size[0] * self.grid_size[1], width))
310
+
311
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
312
+ act_layer = nn.GELU
313
+
314
+ self.ln_pre = norm_layer(width)
315
+ self.transformer = Transformer(
316
+ width,
317
+ layers,
318
+ heads,
319
+ mlp_ratio,
320
+ act_layer=act_layer,
321
+ norm_layer=norm_layer,
322
+ )
323
+
324
+ self.attn_pool = Resampler(
325
+ grid_size=int(math.sqrt(n_queries)),
326
+ embed_dim=output_dim,
327
+ num_heads=output_dim // 128,
328
+ kv_dim=width,
329
+ norm_layer=norm_layer,
330
+ )
331
+ self.ln_post = norm_layer(output_dim)
332
+ self.proj = nn.Parameter((output_dim** -0.5) * torch.randn(output_dim, output_dim))
333
+
334
+ @torch.jit.ignore
335
+ def set_grad_checkpointing(self, enable=True):
336
+ self.transformer.grad_checkpointing = enable
337
+
338
+ def forward(self, x: torch.Tensor):
339
+ x = x.to(
340
+ dtype=self.transformer.get_cast_dtype(),
341
+ device=self.transformer.get_cast_device(),
342
+ )
343
+ # to patches
344
+ x = self.conv1(x) # shape = [*, width, grid, grid]
345
+ x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]
346
+ x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
347
+
348
+ x = x + get_abs_pos(self.positional_embedding, x.size(1))
349
+
350
+ x = self.ln_pre(x)
351
+
352
+ x = x.permute(1, 0, 2) # NLD -> LND
353
+ x = self.transformer(x)
354
+ x = x.permute(1, 0, 2) # LND -> NLD
355
+
356
+ if self.attn_pool:
357
+ x = self.attn_pool(x)
358
+ x = self.ln_post(x)
359
+ x = x @ self.proj
360
+
361
+ return x
362
+
363
+ def encode(self, image_paths: List[str]):
364
+ images = []
365
+ for image_path in image_paths:
366
+ if image_path.startswith("http://") or image_path.startswith("https://"):
367
+ image = Image.open(requests.get(image_path, stream=True).raw)
368
+ elif image_path.startswith("oss://"):
369
+ raise NotImplementedError
370
+ else:
371
+ image = Image.open(image_path)
372
+ image = image.convert("RGB")
373
+ images.append(self.image_transform(image))
374
+ images = torch.stack(images, dim=0)
375
+ return self(images)