Init

.gitignore

@@ -0,0 +1,2 @@
+env
+__pycache__

TUTORIAL.md

@@ -0,0 +1,5 @@
+# Write a custom models
+
+Ref: https://huggingface.co/docs/transformers/custom_models
+
+## Writing a custom configuration

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+timm

resnet_config/configuration_resnet.py

@@ -0,0 +1,79 @@
+from typing import List
+
+from transformers import PretrainedConfig
+
+"""
+The configuration of a model is an object that 
+will contain all the necessary information to build the model.
+
+The three important things to remember when writing you own configuration are the following:
+
+- you have to inherit from PretrainedConfig,
+- the __init__ of your PretrainedConfig must accept any kwargs,
+- those kwargs need to be passed to the superclass __init__.
+"""
+
+
+class ResnetConfig(PretrainedConfig):
+
+    """
+    Defining a model_type for your configuration (here model_type="resnet") is not mandatory,
+    unless you want to register your model with the auto classes (see last section)."""
+
+    model_type = "resnet"
+
+    def __init__(
+        self,
+        block_type="bottleneck",
+        layers: List[int] = [3, 4, 6, 3],
+        num_classes: int = 1000,
+        input_channels: int = 3,
+        cardinality: int = 1,
+        base_width: int = 64,
+        stem_width: int = 64,
+        stem_type: str = "",
+        avg_down: bool = False,
+        **kwargs,
+    ):
+        if block_type not in ["basic", "bottleneck"]:
+            raise ValueError(
+                f"`block_type` must be 'basic' or bottleneck', got {block_type}."
+            )
+        if stem_type not in ["", "deep", "deep-tiered"]:
+            raise ValueError(
+                f"`stem_type` must be '', 'deep' or 'deep-tiered', got {stem_type}."
+            )
+
+        self.block_type = block_type
+        self.layers = layers
+        self.num_classes = num_classes
+        self.input_channels = input_channels
+        self.cardinality = cardinality
+        self.base_width = base_width
+        self.stem_width = stem_width
+        self.stem_type = stem_type
+        self.avg_down = avg_down
+        super().__init__(**kwargs)
+
+
+if __name__ == "__main__":
+    """
+    With this done, you can easily create and save your configuration like
+    you would do with any other model config of the library.
+    Here is how we can create a resnet50d config and save it:
+    """
+    resnet50d_config = ResnetConfig(
+        block_type="bottleneck", stem_width=32, stem_type="deep", avg_down=True
+    )
+    resnet50d_config.save_pretrained("custom-resnet")
+
+    """
+    This will save a file named config.json inside the folder custom-resnet. 
+    You can then reload your config with the from_pretrained method:
+    """
+    resnet50d_config = ResnetConfig.from_pretrained("custom-resnet")
+
+    """
+    You can also use any other method of the PretrainedConfig class, 
+    like push_to_hub() to directly upload your config to the Hub.
+    """

resnet_config/custom-resnet/config.json

@@ -0,0 +1,18 @@
+{
+  "avg_down": true,
+  "base_width": 64,
+  "block_type": "bottleneck",
+  "cardinality": 1,
+  "input_channels": 3,
+  "layers": [
+    3,
+    4,
+    6,
+    3
+  ],
+  "model_type": "resnet",
+  "num_classes": 1000,
+  "stem_type": "deep",
+  "stem_width": 32,
+  "transformers_version": "4.26.1"
+}

resnet_config/modeling_resnet.py

@@ -0,0 +1,58 @@
+from typing import Dict
+
+import torch
+from timm.models.resnet import BasicBlock, Bottleneck, ResNet
+from torch import Tensor, nn
+from transformers import PreTrainedModel
+
+from .configuration_resnet import ResnetConfig
+
+BLOCK_MAPPING = {"basic": BasicBlock, "bottleneck": Bottleneck}
+
+
+class ResnetModel(PreTrainedModel):
+    config_class = ResnetConfig
+
+    def __init__(self, config: ResnetConfig):
+        super().__init__(config)
+        block_layer = BLOCK_MAPPING[config.block_type]
+        self.model = ResNet(
+            block_layer,
+            config.layers,
+            num_classes=config.num_classes,
+            in_chans=config.input_channels,
+            cardinality=config.cardinality,
+            base_width=config.base_width,
+            stem_width=config.stem_width,
+            stem_type=config.stem_type,
+            avg_down=config.avg_down,
+        )
+
+    def forward(self, tensor):
+        return self.model.forward_features(tensor)
+
+
+class ResnetModelForImageClassification(PreTrainedModel):
+    config_class = ResnetConfig
+
+    def __init__(self, config: ResnetConfig):
+        super().__init__(config)
+        block_layer = BLOCK_MAPPING[config.block_type]
+        self.model = ResNet(
+            block_layer,
+            config.layers,
+            num_classes=config.num_classes,
+            in_chans=config.input_channels,
+            cardinality=config.cardinality,
+            base_width=config.base_width,
+            stem_width=config.stem_width,
+            stem_type=config.stem_type,
+            avg_down=config.avg_down,
+        )
+
+    def forward(self, tensor: Tensor, labels=None) -> Dict[str, Tensor]:
+        logits = self.model(tensor)
+        if labels is not None:
+            loss = nn.cross_entropy(logits, labels)
+            return {"loss": loss, "logits": logits}
+        return {"logits": logits}