init repo

README.md

@@ -1,8 +1,8 @@
 ---
 title: Stamp Detection
-emoji: 🏢
-colorFrom: pink
-colorTo: yellow
+emoji: 📜
+colorFrom: yellow
+colorTo: gray
 sdk: gradio
 sdk_version: 3.35.2
 app_file: app.py

app.py

@@ -0,0 +1,58 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../app.ipynb.
+
+# %% auto 0
+__all__ = ['device', 'model', 'transforms', 'image', 'result', 'examples', 'intf', 'detect_stamps']
+
+# %% ../app.ipynb 1
+from model import YOLOStamp
+from utils import *
+import torch
+import gradio as gr
+import albumentations as A 
+from albumentations.pytorch.transforms import ToTensorV2
+from PIL import Image
+
+# %% ../app.ipynb 2
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+model = YOLOStamp()
+model.load_state_dict(torch.load('model.pth', map_location=torch.device('cpu')))
+model = model.to(device)
+model.eval()
+
+# %% ../app.ipynb 3
+transforms = A.Compose([
+        A.Resize(height=448, width=448),
+        A.Normalize(),
+        ToTensorV2(p=1.0),
+    ])
+
+# %% ../app.ipynb 7
+def detect_stamps(image):
+    shape = image.size[:2]
+    image = image.convert('RGB')
+    image = np.array(image)
+    image = transforms(image=image)['image']
+
+    output = model(image.unsqueeze(0).to(device))[0]
+    boxes = output_tensor_to_boxes(output.detach().cpu())
+    boxes = nonmax_suppression(boxes)
+    img = image.permute(1, 2, 0).cpu().numpy()
+    img = visualize_bbox(img.copy(), boxes=boxes, draw_center=False)
+
+    
+    img = cv2.resize(img, dsize=shape)
+
+    return Image.fromarray((255. * (img * np.array(STD) + np.array(MEAN))).astype(np.uint8))
+
+# %% ../app.ipynb 9
+image = gr.inputs.Image(type="pil")
+result = gr.outputs.Image(type="pil")
+examples = ['./examples/1.jpg', './examples/2.jpg', './examples/3.jpg']
+
+intf = gr.Interface(fn=detect_stamps, 
+                    inputs=image, 
+                    outputs=result, 
+                    title='Stamp detection',
+                    examples=examples)
+intf.launch(inline=False)

constants.py

@@ -0,0 +1,25 @@
+# shape of input image to YOLO
+W, H =  448, 448
+# grid size after last convolutional layer of YOLO
+S = 7 
+# anchors of YOLO model
+ANCHORS = [[1.08,1.19],
+            [3.42,4.41],
+            [16.62,10.52]]
+# number of anchors boxes
+BOX = len(ANCHORS)
+# maximum number of stamps on image
+STAMP_NB_MAX = 10
+# minimal confidence of presence a stamp in the grid cell
+OUTPUT_THRESH = 0.76
+# maximal iou score to consider boxes different
+IOU_THRESH = 0.3
+# path to folder containing images
+IMAGE_FOLDER = './data/images'
+# path to .cvs file containing annotations
+ANNOTATIONS_PATH = './data/all_annotations.csv'
+# standard deviation and mean of pixel values for normalization
+STD = (0.229, 0.224, 0.225)
+MEAN = (0.485, 0.456, 0.406)
+# box color to show the bounding box on image
+BOX_COLOR = (0, 0, 255)

model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3e633824bea418bf44bb82f8d5a667b9d7d29b913079c4d8b6ca217f6f383e7
+size 502816

model.py

@@ -0,0 +1,80 @@
+import torch
+import torch.nn as nn
+
+from constants import *
+
+"""
+    Class for custom activation.
+"""
+class SymReLU(nn.Module):
+    def __init__(self, inplace: bool = False):
+        super().__init__()
+        self.inplace = inplace
+
+    def forward(self, input):
+        return torch.min(torch.max(input, -torch.ones_like(input)), torch.ones_like(input))
+
+    def extra_repr(self) -> str:
+        inplace_str = 'inplace=True' if self.inplace else ''
+        return inplace_str
+
+
+"""
+    Class implementing YOLO-Stamp architecture described in https://link.springer.com/article/10.1134/S1054661822040046.
+"""
+class YOLOStamp(nn.Module):
+    def __init__(
+            self,
+            anchors=ANCHORS,
+            in_channels=3,
+    ):
+        super().__init__()
+        
+        self.register_buffer('anchors', torch.tensor(anchors))
+
+        self.act = SymReLU()
+        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
+        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm1 = nn.BatchNorm2d(num_features=8)
+        self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm2 = nn.BatchNorm2d(num_features=16)
+        self.conv3 = nn.Conv2d(in_channels=16, out_channels=16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm3 = nn.BatchNorm2d(num_features=16)
+        self.conv4 = nn.Conv2d(in_channels=16, out_channels=16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm4 = nn.BatchNorm2d(num_features=16)
+        self.conv5 = nn.Conv2d(in_channels=16, out_channels=16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm5 = nn.BatchNorm2d(num_features=16)
+        self.conv6 = nn.Conv2d(in_channels=16, out_channels=24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm6 = nn.BatchNorm2d(num_features=24)
+        self.conv7 = nn.Conv2d(in_channels=24, out_channels=24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm7 = nn.BatchNorm2d(num_features=24)
+        self.conv8 = nn.Conv2d(in_channels=24, out_channels=48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm8 = nn.BatchNorm2d(num_features=48)
+        self.conv9 = nn.Conv2d(in_channels=48, out_channels=48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm9 = nn.BatchNorm2d(num_features=48)
+        self.conv10 = nn.Conv2d(in_channels=48, out_channels=48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm10 = nn.BatchNorm2d(num_features=48)
+        self.conv11 = nn.Conv2d(in_channels=48, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        self.norm11 = nn.BatchNorm2d(num_features=64)
+        self.conv12 = nn.Conv2d(in_channels=64, out_channels=256, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0))
+        self.norm12 = nn.BatchNorm2d(num_features=256)
+        self.conv13 = nn.Conv2d(in_channels=256, out_channels=len(anchors) * 5, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0))
+    
+    def forward(self, x, head=True):
+        x = x.type(self.conv1.weight.dtype)
+        x = self.act(self.pool(self.norm1(self.conv1(x))))
+        x = self.act(self.pool(self.norm2(self.conv2(x))))
+        x = self.act(self.pool(self.norm3(self.conv3(x))))
+        x = self.act(self.pool(self.norm4(self.conv4(x))))
+        x = self.act(self.pool(self.norm5(self.conv5(x))))
+        x = self.act(self.norm6(self.conv6(x)))
+        x = self.act(self.norm7(self.conv7(x)))
+        x = self.act(self.pool(self.norm8(self.conv8(x))))
+        x = self.act(self.norm9(self.conv9(x)))
+        x = self.act(self.norm10(self.conv10(x)))
+        x = self.act(self.norm11(self.conv11(x)))
+        x = self.act(self.norm12(self.conv12(x)))
+        x = self.conv13(x)
+        nb, _, nh, nw= x.shape
+        x = x.permute(0, 2, 3, 1).view(nb, nh, nw, self.anchors.shape[0], 5)
+        return x

requirements.txt

@@ -0,0 +1,4 @@
+torch
+opencv-python
+pillow
+albumentations

utils.py

@@ -0,0 +1,259 @@
+import torch
+import cv2
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import matplotlib.pyplot as plt
+from constants import *
+
+
+def output_tensor_to_boxes(boxes_tensor):
+    """
+        Converts the YOLO output tensor to list of boxes with probabilites.
+
+        Arguments:
+        boxes_tensor -- tensor of shape (S, S, BOX, 5)
+
+        Returns:
+        boxes -- list of shape (None, 5)
+
+        Note: "None" is here because you don't know the exact number of selected boxes, as it depends on the threshold. 
+        For example, the actual output size of scores would be (10, 5) if there are 10 boxes
+    """
+    cell_w, cell_h = W/S, H/S
+    boxes = []
+    
+    for i in range(S):
+        for j in range(S):
+            for b in range(BOX):
+                anchor_wh = torch.tensor(ANCHORS[b])
+                data = boxes_tensor[i,j,b]
+                xy = torch.sigmoid(data[:2])
+                wh = torch.exp(data[2:4])*anchor_wh
+                obj_prob = torch.sigmoid(data[4])
+                
+                if obj_prob > OUTPUT_THRESH:
+                    x_center, y_center, w, h = xy[0], xy[1], wh[0], wh[1]
+                    x, y = x_center+j-w/2, y_center+i-h/2
+                    x,y,w,h = x*cell_w, y*cell_h, w*cell_w, h*cell_h
+                    box = [x,y,w,h, obj_prob]
+                    boxes.append(box)
+    return boxes
+
+
+def plot_img(img, size=(7,7)):
+    plt.figure(figsize=size)
+    plt.imshow(img)
+    plt.show()
+
+
+def plot_normalized_img(img, std=STD, mean=MEAN, size=(7,7)):
+    mean = mean if isinstance(mean, np.ndarray) else np.array(mean)
+    std = std if isinstance(std, np.ndarray) else np.array(std)
+    plt.figure(figsize=size)
+    plt.imshow((255. * (img * std + mean)).astype(np.uint))
+    plt.show()
+    
+
+def visualize_bbox(img, boxes, thickness=2, color=BOX_COLOR, draw_center=True):
+    """
+        Draws boxes on the given image.
+
+        Arguments:
+        img -- torch.Tensor of shape (3, W, H) or numpy.ndarray of shape (W, H, 3)
+        boxes -- list of shape (None, 5)
+        thickness -- number specifying the thickness of box border
+        color -- RGB tuple of shape (3,) specifying the color of boxes
+        draw_center -- boolean specifying whether to draw center or not
+
+        Returns:
+        img_copy -- numpy.ndarray of shape(W, H, 3) containing image with bouning boxes
+    """
+    img_copy = img.cpu().permute(1,2,0).numpy() if isinstance(img, torch.Tensor) else img.copy()
+    for box in boxes:
+        x,y,w,h = int(box[0]), int(box[1]), int(box[2]), int(box[3])
+        img_copy = cv2.rectangle(
+            img_copy,
+            (x,y),(x+w, y+h),
+            color, thickness)
+        if draw_center:
+            center = (x+w//2, y+h//2)
+            img_copy = cv2.circle(img_copy, center=center, radius=3, color=(0,255,0), thickness=2)
+    return img_copy
+
+
+def read_data(annotations=Path(ANNOTATIONS_PATH)):
+    """
+        Reads annotations data from .csv file. Must contain columns: image_name, bbox_x, bbox_y, bbox_width, bbox_height.
+
+        Arguments:
+        annotations_path -- string or Path specifying path of annotations file
+
+        Returns:
+        data -- list of dictionaries containing path, number of boxes and boxes itself
+    """
+    data = []
+
+    boxes = pd.read_csv(annotations)
+    image_names = boxes['image_name'].unique()
+
+    for image_name in image_names:
+        cur_boxes = boxes[boxes['image_name'] == image_name]
+        img_data = {
+            'file_path': image_name,
+            'box_nb': len(cur_boxes),
+            'boxes': []}
+        stamp_nb = img_data['box_nb']
+        if stamp_nb <= STAMP_NB_MAX:
+            img_data['boxes'] = cur_boxes[['bbox_x', 'bbox_y','bbox_width','bbox_height']].values
+        data.append(img_data)
+    return data
+
+
+def boxes_to_tensor(boxes):
+    """
+        Convert list of boxes (and labels) to tensor format
+        
+        Arguments:
+        boxes -- list of boxes
+
+        Returns:
+        boxes_tensor -- tensor of shape (S, S, BOX, 5)
+    """
+    boxes_tensor = torch.zeros((S, S, BOX, 5))
+    cell_w, cell_h = W/S, H/S
+    for i, box in enumerate(boxes):
+        x, y, w, h = box
+        # normalize xywh with cell_size
+        x, y, w, h = x / cell_w, y / cell_h, w / cell_w, h / cell_h
+        center_x, center_y = x + w / 2, y + h / 2
+        grid_x = int(np.floor(center_x))
+        grid_y = int(np.floor(center_y))
+        
+        if grid_x < S and grid_y < S:
+            boxes_tensor[grid_y, grid_x, :, 0:4] = torch.tensor(BOX * [[center_x - grid_x, center_y - grid_y, w, h]])
+            boxes_tensor[grid_y, grid_x, :, 4]  = torch.tensor(BOX * [1.])
+    return boxes_tensor
+
+
+def target_tensor_to_boxes(boxes_tensor, output_threshold=OUTPUT_THRESH):
+    """
+        Recover target tensor (tensor output of dataset) to bboxes.
+        Arguments:
+            boxes_tensor -- tensor of shape (S, S, BOX, 5)
+        Returns:
+            boxes -- list of boxes, each box is [x, y, w, h]
+    """
+    cell_w, cell_h = W/S, H/S
+    boxes = []
+    for i in range(S):
+        for j in range(S):
+            for b in range(BOX):
+                data = boxes_tensor[i,j,b]
+                x_center,y_center, w, h, obj_prob = data[0], data[1], data[2], data[3], data[4]
+                if obj_prob > output_threshold:
+                    x, y = x_center+j-w/2, y_center+i-h/2
+                    x,y,w,h = x*cell_w, y*cell_h, w*cell_w, h*cell_h
+                    box = [x,y,w,h]
+                    boxes.append(box)
+    return boxes    
+
+
+def overlap(interval_1, interval_2):
+    """
+        Calculates length of overlap between two intervals.
+
+        Arguments:
+        interval_1 -- list or tuple of shape (2,) containing endpoints of the first interval
+        interval_2 -- list or tuple of shape (2, 2) containing endpoints of the second interval
+
+        Returns:
+        overlap -- length of overlap
+    """
+    x1, x2 = interval_1
+    x3, x4 = interval_2
+    if x3 < x1:
+        if x4 < x1:
+            return 0
+        else:
+            return min(x2,x4) - x1
+    else:
+        if x2 < x3:
+            return 0
+        else:
+            return min(x2,x4) - x3
+
+
+def compute_iou(box1, box2):
+    """
+        Compute IOU between box1 and box2.
+
+        Argmunets:
+        box1 -- list of shape (5, ). Represents the first box
+        box2 -- list of shape (5, ). Represents the second box
+        Each box is [x, y, w, h, prob]
+
+        Returns:
+        iou -- intersection over union score between two boxes
+    """
+    x1,y1,w1,h1 = box1[0], box1[1], box1[2], box1[3]
+    x2,y2,w2,h2 = box2[0], box2[1], box2[2], box2[3]
+
+    area1, area2 = w1*h1, w2*h2
+    intersect_w = overlap((x1,x1+w1), (x2,x2+w2))
+    intersect_h = overlap((y1,y1+h1), (y2,y2+w2))
+    if intersect_w == w1 and intersect_h == h1 or intersect_w == w2 and intersect_h == h2:
+        return 1.
+    intersect_area = intersect_w*intersect_h
+    iou = intersect_area/(area1 + area2 - intersect_area)
+    return iou
+
+
+def nonmax_suppression(boxes, iou_thresh = IOU_THRESH):
+    """
+        Removes ovelap bboxes
+
+        Arguments:
+        boxes -- list of shape (None, 5)
+        iou_thresh -- maximal value of iou when boxes are considered different
+        Each box is [x, y, w, h, prob]
+
+        Returns:
+        boxes -- list of shape (None, 5) with removed overlapping boxes
+    """
+    boxes = sorted(boxes, key=lambda x: x[4], reverse=True)
+    for i, current_box in enumerate(boxes):
+        if current_box[4] <= 0:
+            continue
+        for j in range(i+1, len(boxes)):
+            iou = compute_iou(current_box, boxes[j])
+            if iou > iou_thresh:
+                boxes[j][4] = 0
+    boxes = [box for box in boxes if box[4] > 0]
+    return boxes
+
+
+    
+def yolo_head(yolo_output):
+    """
+        Converts a yolo output tensor to separate tensors of coordinates, shapes and probabilities.
+
+        Arguments:
+        yolo_output -- tensor of shape (batch_size, S, S, BOX, 5)
+
+        Returns:
+        xy -- tensor of shape (batch_size, S, S, BOX, 2) containing coordinates of centers of found boxes for each anchor in each grid cell
+        wh -- tensor of shape (batch_size, S, S, BOX, 2) containing width and height of found boxes for each anchor in each grid cell
+        prob -- tensor of shape (batch_size, S, S, BOX, 1) containing the probability of presence of boxes for each anchor in each grid cell
+    """
+    xy = torch.sigmoid(yolo_output[..., 0:2])
+    anchors_wh = torch.tensor(ANCHORS, device=yolo_output.device).view(1, 1, 1, len(ANCHORS), 2)
+    wh = torch.exp(yolo_output[..., 2:4]) * anchors_wh
+    prob = torch.sigmoid(yolo_output[..., 4:5])
+    return xy, wh, prob
+
+def process_target(target):
+    xy = target[..., 0:2]
+    wh = target[..., 2:4]
+    prob = target[..., 4:5]
+    return xy, wh, prob