Spaces:
Running
Running
# coding=utf-8 | |
# Copyright 2022 rinna Co., Ltd. | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
""" CLOOB model configuration""" | |
import logging | |
import copy | |
import os | |
from typing import Union | |
from transformers import AutoConfig, PretrainedConfig | |
logger = logging.getLogger(__name__) | |
class CLOOBTextConfig(PretrainedConfig): | |
model_type = "cloob_text_model" | |
def __init__( | |
self, | |
vocab_size=49408, | |
hidden_size=512, | |
intermediate_size=2048, | |
num_hidden_layers=12, | |
num_attention_heads=8, | |
max_position_embeddings=77, | |
hidden_act="quick_gelu", | |
layer_norm_eps=0.00001, | |
dropout=0.0, | |
attention_dropout=0.0, | |
initializer_range=0.02, | |
initializer_factor=1.0, | |
pad_token_id=1, | |
bos_token_id=0, | |
eos_token_id=2, | |
**kwargs | |
): | |
super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) | |
self.vocab_size = vocab_size | |
self.hidden_size = hidden_size | |
self.intermediate_size = intermediate_size | |
self.dropout = dropout | |
self.num_hidden_layers = num_hidden_layers | |
self.num_attention_heads = num_attention_heads | |
self.max_position_embeddings = max_position_embeddings | |
self.layer_norm_eps = layer_norm_eps | |
self.hidden_act = hidden_act | |
self.initializer_range = initializer_range | |
self.initializer_factor = initializer_factor | |
self.attention_dropout = attention_dropout | |
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": | |
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) | |
# get the text config dict if we are loading from CLIPConfig | |
if config_dict.get("model_type") == "clip": | |
config_dict = config_dict["text_config"] | |
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: | |
logger.warning( | |
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " | |
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." | |
) | |
return cls.from_dict(config_dict, **kwargs) | |
class CLOOBVisionConfig(PretrainedConfig): | |
model_type = "cloob_vision_model" | |
def __init__( | |
self, | |
hidden_size=768, | |
intermediate_size=3072, | |
num_hidden_layers=12, | |
num_attention_heads=12, | |
image_size=224, | |
patch_size=32, | |
hidden_act="quick_gelu", | |
layer_norm_eps=0.00001, | |
dropout=0.0, | |
attention_dropout=0.0, | |
initializer_range=0.02, | |
initializer_factor=1.0, | |
**kwargs | |
): | |
super().__init__(**kwargs) | |
self.hidden_size = hidden_size | |
self.intermediate_size = intermediate_size | |
self.dropout = dropout | |
self.num_hidden_layers = num_hidden_layers | |
self.num_attention_heads = num_attention_heads | |
self.patch_size = patch_size | |
self.image_size = image_size | |
self.initializer_range = initializer_range | |
self.initializer_factor = initializer_factor | |
self.attention_dropout = attention_dropout | |
self.layer_norm_eps = layer_norm_eps | |
self.hidden_act = hidden_act | |
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": | |
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) | |
# get the vision config dict if we are loading from CLIPConfig | |
if config_dict.get("model_type") == "clip": | |
config_dict = config_dict["vision_config"] | |
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: | |
logger.warning( | |
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " | |
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." | |
) | |
return cls.from_dict(config_dict, **kwargs) | |
class CLOOBConfig(PretrainedConfig): | |
model_type = "cloob" | |
is_composition = True | |
def __init__( | |
self, | |
text_config=None, | |
vision_config=None, | |
projection_dim=512, | |
init_inv_tau=30.0, | |
scale_hopfield=15.0, | |
**kwargs | |
): | |
super().__init__(text_config=text_config, vision_config=vision_config, **kwargs) | |
if vision_config is None: | |
raise ValueError("`vision_config` can not be `None`.") | |
if text_config is None: | |
raise ValueError("`text_config` can not be `None`.") | |
vision_model_type = vision_config.pop("model_type") | |
text_model_type = text_config.pop("model_type") | |
if vision_model_type == "cloob_vision_model": | |
self.vision_config = CLOOBVisionConfig(**vision_config) | |
else: | |
self.vision_config = AutoConfig.for_model( | |
vision_model_type, **vision_config | |
) | |
if text_model_type == "cloob_text_model": | |
self.text_config = CLOOBTextConfig(**text_config) | |
else: | |
self.text_config = AutoConfig.for_model( | |
text_model_type, **text_config | |
) | |
self.projection_dim = projection_dim | |
self.initializer_factor = 1.0 | |
self.init_inv_tau = init_inv_tau | |
self.scale_hopfield = scale_hopfield | |
def from_text_vision_configs(cls, text_config: CLOOBTextConfig, vision_config: CLOOBVisionConfig, **kwargs): | |
r""" | |
Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model | |
configuration. | |
Returns: | |
[`CLIPConfig`]: An instance of a configuration object | |
""" | |
return cls(text_config_dict=text_config.to_dict(), vision_config_dict=vision_config.to_dict(), **kwargs) | |
def to_dict(self): | |
""" | |
Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. | |
Returns: | |
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, | |
""" | |
output = copy.deepcopy(self.__dict__) | |
output["text_config"] = self.text_config.to_dict() | |
output["vision_config"] = self.vision_config.to_dict() | |
output["model_type"] = self.__class__.model_type | |
return output | |