Implement gaussian blurring.

main.py

@@ -182,14 +182,18 @@ if __name__ == "__main__":
 
     ctr = -1
     pred = -1
-    move = True
-    rotate = True
+    move = False
+    rotate = False
+    resize = False
+    squeeze = False
+    gauss = True
     transform_interval = 1
     angle_step = 5
+    tv_factor = 1
     frame_read = False
     fix_frame = False
-    patch_transformer.maxangle = math.pi
-    patch_transformer.minangle = - math.pi
+    patch_transformer.maxangle = 5/180 * math.pi
+    patch_transformer.minangle = - 5/180 * math.pi
     loss = None
     while True:
         if not (fix_frame and frame_read):
@@ -212,8 +216,8 @@ if __name__ == "__main__":
                 # transform patch (every transform_interval of frames)
                 if ctr % 1 == 0:
                     trans_patch = patch_transformer(patch.cuda(), torch.ones([1, 14, 5]).cuda(), img_size_x, img_size_y,
-                                                    do_rotate=rotate, rand_loc=move)
-                    trans_patch = torch.transpose(trans_patch[0][0].T, 0, 1)
+                                                    do_rotate=rotate, rand_loc=move, rand_size=resize,
+                                                    rand_squeeze=squeeze, gauss=gauss)
 
                     # extract bounding box (x1, y1, x2, y2)
                     bounding_box = extract_bounding_box(trans_patch)
@@ -229,7 +233,9 @@ if __name__ == "__main__":
                     # debug_preds()
                     pass
 
-                iou, pred = get_best_prediction(bounding_box, raw_results, 15)  # get cats
+                # iou, pred = get_best_prediction(bounding_box, raw_results, 15)  # get cats
+                iou, pred = get_best_prediction(bounding_box, raw_results, 12)  # get parking meters
+                iou, pred = get_best_prediction(bounding_box, raw_results, 11)  # get stop signs
                 # pred = get_best_prediction(bounding_box, raw_results, 42)  # get forked
 
                 # pred = get_avg_prediction(raw_results, 15)  # make everything cats
@@ -244,7 +250,7 @@ if __name__ == "__main__":
 
                     # total variation loss component
                     tv_loss = total_variation(patch)
-                    loss += tv_loss
+                    loss += tv_factor * tv_loss
 
                     # IoU loss component (low iou = high loss)
                     loss += 0.1 * (1 - iou)
@@ -284,6 +290,15 @@ if __name__ == "__main__":
             if key == ord("o"):
                 rotate = not rotate
                 print("Rotate: {}".format(rotate))
+            if key == ord("t"):
+                resize = not resize
+                print("Resize: {}".format(resize))
+            if key == ord("z"):
+                squeeze = not squeeze
+                print("Squeeze: {}".format(squeeze))
+            if key == ord("g"):
+                gauss = not gauss
+                print("Gauss: {}".format(gauss))
             if key == ord("+"):
                 transform_interval += 1
                 print("Transform Interval: {}".format(transform_interval))
@@ -304,6 +319,12 @@ if __name__ == "__main__":
             if key == ord("f"):
                 fix_frame = not fix_frame
                 print("Fix Frame: {}".format(fix_frame))
+            if key == ord("a"):
+                tv_factor += 1
+                print("Total Variation Loss Factor: {}".format(tv_factor))
+            if key == ord("y"):
+                tv_factor -= 1
+                print("Total Variation Loss Factor: {}".format(tv_factor))
 
         # calculate FPS
         fps += 1

patch_transformer.py

 import math
+import random
+
+import torchvision
 from torch.nn.modules.utils import _pair, _quadruple
 import torch.nn.functional as F
 import numpy as np
 import torch
-from torch import nn
+from torch import nn, Tensor
+
 
 class PatchApplier(nn.Module):
     """PatchApplier: applies adversarial patches to images.
@@ -60,7 +64,8 @@ class MedianPool2d(nn.Module):
     def forward(self, x):
         # using existing pytorch functions and tensor ops so that we get autograd,
         # would likely be more efficient to implement from scratch at C/Cuda level
-        x = F.pad(x, self._padding(x), mode='reflect')
+        # x = F.pad(x, self._padding(x), mode='reflect')
+        x = F.pad(x, self._padding(x), mode='replicate')
         x = x.unfold(2, self.k[0], self.stride[0]).unfold(3, self.k[1], self.stride[1])
         x = x.contiguous().view(x.size()[:4] + (-1,)).median(dim=-1)[0]
         return x
@@ -83,6 +88,8 @@ class PatchTransformer(nn.Module):
         self.noise_factor = 0.10
         self.minangle = -20 / 180 * math.pi
         self.maxangle = 20 / 180 * math.pi
+        self.minsize = 0.5
+        self.maxsize = 1.5
         self.medianpooler = MedianPool2d(7, same=True)
         '''
         kernel = torch.cuda.FloatTensor([[0.003765, 0.015019, 0.023792, 0.015019, 0.003765],                                                                                    
@@ -93,7 +100,8 @@ class PatchTransformer(nn.Module):
         self.kernel = kernel.unsqueeze(0).unsqueeze(0).expand(3,3,-1,-1)
         '''
 
-    def forward(self, adv_patch, lab_batch, img_size_x, img_size_y, do_rotate=True, rand_loc=True):
+    def forward(self, adv_patch, lab_batch, img_size_x, img_size_y,
+                do_rotate=True, rand_loc=True, rand_size=True, gauss=True, max_sigma=6, rand_squeeze=True):
         # adv_patch = F.conv2d(adv_patch.unsqueeze(0),self.kernel,padding=(2,2))
         adv_patch = self.medianpooler(adv_patch.unsqueeze(0))
         # Determine size of padding
@@ -168,8 +176,14 @@ class PatchTransformer(nn.Module):
             off_y = targetoff_y * (torch.cuda.FloatTensor(targetoff_y.size()).uniform_(-0.4, 0.4))
             target_y = target_y + off_y
         target_y = target_y - 0.05
-        scale = target_size / current_patch_size
-        scale = scale.view(anglesize)
+
+        resize_factor_x = random.random() * (self.maxsize - self.minsize) + self.minsize if rand_size else 1
+        resize_factor_y = random.random() * (self.maxsize - self.minsize) + self.minsize if rand_squeeze \
+            else resize_factor_x
+        scale_x = resize_factor_x * target_size / current_patch_size
+        scale_y = resize_factor_y * target_size / current_patch_size
+        scale_x = scale_x.view(anglesize)
+        scale_y = scale_y.view(anglesize)
 
         s = adv_batch.size()
         adv_batch = adv_batch.view(s[0] * s[1], s[2], s[3], s[4])
@@ -180,14 +194,16 @@ class PatchTransformer(nn.Module):
         sin = torch.sin(angle)
         cos = torch.cos(angle)
 
+
+
         # Theta = rotation,rescale matrix
         theta = torch.cuda.FloatTensor(anglesize, 2, 3).fill_(0)
-        theta[:, 0, 0] = cos / scale
-        theta[:, 0, 1] = sin / scale
-        theta[:, 0, 2] = tx * cos / scale + ty * sin / scale
-        theta[:, 1, 0] = -sin / scale
-        theta[:, 1, 1] = cos / scale
-        theta[:, 1, 2] = -tx * sin / scale + ty * cos / scale
+        theta[:, 0, 0] = cos / scale_x
+        theta[:, 0, 1] = sin / scale_x
+        theta[:, 0, 2] = tx * cos / scale_x + ty * sin / scale_y
+        theta[:, 1, 0] = -sin / scale_y
+        theta[:, 1, 1] = cos / scale_y
+        theta[:, 1, 2] = -tx * sin / scale_y + ty * cos / scale_x
 
         b_sh = adv_batch.shape
         grid = F.affine_grid(theta, adv_batch.shape, align_corners=False)
@@ -219,4 +235,55 @@ class PatchTransformer(nn.Module):
         # img.show()
         # exit()
 
+        # Fix dimensions
+        adv_batch_t = torch.transpose(adv_batch_t[0][0].T, 0, 1)
+
+        # Gaussian Filter
+        if gauss:
+            adv_batch_t = self.gauss_filter(adv_batch_t, random.random() * max_sigma)
+
         return adv_batch_t #* msk_batch_t
+
+    def gauss_filter(self, img, sigma):
+
+        channels = 3
+
+        # Set these to whatever you want for your gaussian filter
+        kernel_size = 15
+        padding = 7
+
+        # Create a x, y coordinate grid of shape (kernel_size, kernel_size, 2)
+        x_cord = torch.arange(kernel_size)
+        x_grid = x_cord.repeat(kernel_size).view(kernel_size, kernel_size)
+        y_grid = x_grid.t()
+        xy_grid = torch.stack([x_grid, y_grid], dim=-1)
+
+        mean = (kernel_size - 1) / 2.
+        variance = sigma ** 2.
+
+        # Calculate the 2-dimensional gaussian kernel which is
+        # the product of two gaussian distributions for two different
+        # variables (in this case called x and y)
+        gaussian_kernel = (1. / (2. * math.pi * variance)) * \
+                          torch.exp(
+                              -torch.sum((xy_grid - mean) ** 2., dim=-1) / \
+                              (2 * variance)
+                          )
+        # Make sure sum of values in gaussian kernel equals 1.
+        gaussian_kernel = gaussian_kernel / torch.sum(gaussian_kernel)
+
+        # Reshape to 2d depthwise convolutional weight
+        # gaussian_kernel = gaussian_kernel.view(1, 1, kernel_size, kernel_size)
+        gaussian_kernel = gaussian_kernel.view(kernel_size, kernel_size)
+        gaussian_kernel = gaussian_kernel.repeat(channels, 1, 1, 1)
+
+        gaussian_filter = nn.Conv2d(in_channels=channels, out_channels=channels,
+                                    kernel_size=kernel_size, groups=channels, bias=False, padding=padding,
+                                    padding_mode='replicate')
+
+        gaussian_filter.weight.data = gaussian_kernel
+        gaussian_filter.weight.requires_grad = False
+
+        gaussian_filter.cuda()
+
+        return gaussian_filter.forward(img.T.unsqueeze(0)).squeeze(0).T.flip(-1)