From 4de808fb493b0b3d7796e70d3429525dbc38e32f Mon Sep 17 00:00:00 2001 From: ax06udyx <raunak.samanta@fau.de> Date: Sat, 30 Nov 2024 13:42:49 +0100 Subject: [PATCH] updated RNN --- .DS_Store | Bin 10244 -> 10244 bytes exercise3_material/.DS_Store | Bin 6148 -> 6148 bytes exercise3_material/log.txt | 24 ++ exercise3_material/src_to_implement/.DS_Store | Bin 6148 -> 6148 bytes .../Layers/BatchNormalization.py | 316 ++++++++------- .../src_to_implement/Layers/Conv.py | 376 +++++++++--------- .../src_to_implement/Layers/Conv_o.py | 182 +++++++++ .../src_to_implement/Layers/RNN.py | 3 - .../BatchNormalization.cpython-310.pyc | Bin 4240 -> 4459 bytes .../Layers/__pycache__/Conv.cpython-310.pyc | Bin 4783 -> 5085 bytes .../__pycache__/Dropout.cpython-310.pyc | Bin 1065 -> 1065 bytes .../Layers/__pycache__/RNN.cpython-310.pyc | Bin 3352 -> 3352 bytes .../src_to_implement/NeuralNetwork.py | 154 ++++--- .../src_to_implement/NeuralNetwork_o.py | 65 +++ .../__pycache__/NeuralNetwork.cpython-310.pyc | Bin 2357 -> 3078 bytes exercise3_material/src_to_implement/log.txt | 121 ++++++ 16 files changed, 842 insertions(+), 399 deletions(-) create mode 100644 exercise3_material/log.txt create mode 100644 exercise3_material/src_to_implement/Layers/Conv_o.py create mode 100644 exercise3_material/src_to_implement/NeuralNetwork_o.py diff --git a/.DS_Store b/.DS_Store index fa8005c2b30ab41ea97aadce479a53c5ccd08a7d..eb933bd1cee7ec0edb0283b1ebf9beb2da7c01a4 100644 GIT binary patch delta 163 zcmZn(XbIS0BsRHTLTK_{fdp=e>S|*XLmdS}1Cz-c#H1(JiI`2kASygLL6Uc~qIdx# zOp%F^sg8o7xj9VHW?#uVMr2da6-?%n0hz+Hd8f1=%Vt&uT{dLZxa^#)DE<Th5+*UK delta 172 zcmZn(XbIS0B*wUN@^j%r9?9xzGh=fd1w$hf%gO7-q#3&>*NezA_DudSsxdiUl6SL$ zcmd<&AHpf@Mg}?xMn;B{*NX*CPLL4Z950#8NXU?UVbRGKq=hHPOY&^qF73y%nO#Af OjgTg&&65?x9|HhgCN=c{ diff --git a/exercise3_material/.DS_Store b/exercise3_material/.DS_Store index d3df6722b35ef6e33cce6e83464c26a32b62a855..fecaaf70e24e232810e41e67d501e67ce1c9db53 100644 GIT binary patch delta 87 zcmZoMXfc@J&&aVcU^g=($7UWDRz`Vth8%``hI9r!h7yJfh7!-5{N$vZ{3Hej1_1^J rru4~7ti6ovlP9o7^K)><3rJK~n;4nuC>WZXZ~n!4pK&ug$6tN`iGdZY delta 58 zcmZoMXfc@J&&a+pU^g=(`(_>%R>sNptaXeWllQSk^K){>3rJK~8yXqtC>R+TZq{JC O&p5F`Z!<f`Uw#0_1rN^v diff --git a/exercise3_material/log.txt b/exercise3_material/log.txt new file mode 100644 index 0000000..776ace8 --- /dev/null +++ b/exercise3_material/log.txt @@ -0,0 +1,24 @@ +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 64.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 62.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 60.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 66.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 62.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 64.0% +On the Iris dataset, we achieve an accuracy of: 96.0% diff --git a/exercise3_material/src_to_implement/.DS_Store b/exercise3_material/src_to_implement/.DS_Store index b768b11c944364b0e7124b9168311c0faddd8700..8df3b1a748f61f823bd55186f9eaf2a352391ce3 100644 GIT binary patch delta 32 ocmZoMXfc@J&&atkU^g=(=Vl(3m5iIevI;RxY;f4j&heKY0I91AXaE2J delta 79 zcmZoMXfc@J&&ahgU^g=(*Jd7;m5hc041Nr$45bW342cXmV0H;ZIYT}}5koeE9zy{` grDslla#Buy5(5K+00RT#&&|tN6`3}(bNuB801CkqvH$=8 diff --git a/exercise3_material/src_to_implement/Layers/BatchNormalization.py b/exercise3_material/src_to_implement/Layers/BatchNormalization.py index 430f413..de0823b 100644 --- a/exercise3_material/src_to_implement/Layers/BatchNormalization.py +++ b/exercise3_material/src_to_implement/Layers/BatchNormalization.py @@ -1,155 +1,163 @@ -from . import Base -import numpy as np -from . import Helpers - -class BatchNormalization(Base.BaseLayer): - def __init__(self, channels, alpha=0.8, epsilon=5e-11) -> None: - super().__init__() - self.trainable = True - self.input_tensor = None - self.output_tensor = None - self.channel_size = channels - self.check_conv = 0 - self.alpha = alpha - self.epsilon = epsilon - self.weights = None - self._optimizer = None - self._gradient_weights = None - self._bias_optimizer = None - self._gradient_bias = None - self.weights = None - self.bias = None - self.mean = None - self.variance = None - self.batch_size = None - self.channels = None - self.input_tensor = None - - self.initialize() - pass - - def initialize(self): - self.weights = np.ones(self.channel_size) - self.bias = np.zeros(self.channel_size) - - def forward(self, input_tensor): - self.input_tensor = input_tensor - self.check_conv = len(input_tensor.shape) == 4 - - - # Make sure to use the test mean. The test mean is computed during training time as a moving average. It is then kept fixed during test time. - - if self.check_conv: - self.mean = np.mean(self.input_tensor, axis=(0,2,3)) - self.variance = np.var(self.input_tensor, axis=(0,2,3)) - - self.batch_size = input_tensor.shape[0] - self.channels = input_tensor.shape[1] - if self.testing_phase: - x_tilda = (self.input_tensor - self.test_mean.reshape((1, self.channels, 1, 1)))/np.sqrt(self.test_variance.reshape((1, self.channels, 1, 1)) + self.epsilon) - - - else: - mean_k = np.mean(self.input_tensor, axis=(0,2,3)) - variance_k = np.var(self.input_tensor, axis=(0,2,3)) - - self.test_mean = self.alpha*self.mean.reshape((1, self.channels, 1, 1)) + (1-self.alpha)*mean_k.reshape(1, self.channels, 1, 1) - self.test_variance = self.alpha*self.variance.reshape(1, self.channels, 1, 1) + (1-self.alpha)*variance_k.reshape(1, self.channels, 1, 1) - - # store mean and variance for the next iteration - - self.mean = mean_k - self.variance = variance_k - x_tilda = (self.input_tensor - self.mean.reshape(1, self.channels, 1, 1)/np.sqrt(self.variance.reshape(1, self.channels, 1, 1)) + self.epsilon) - self.output_tensor = self.weights.reshape(1, self.channels, 1, 1) * x_tilda + self.bias.reshape(1, self.channels, 1, 1) - - else: - self.mean = np.mean(self.input_tensor, axis=0) - self.variance = np.var(self.input_tensor, axis=0) - - if self.testing_phase: - x_tilda = (self.input_tensor - self.test_mean)/np.sqrt(self.test_variance + self.epsilon) - - else: - mean_k = np.mean(input_tensor, axis=0) - variance_k = np.var(input_tensor, axis=0) - - self.test_mean = self.alpha*self.mean + (1-self.alpha)*mean_k - self.test_variance = self.alpha*self.variance + (1-self.alpha)*variance_k - - # store mean and variance for the next iteration - - self.mean = mean_k - self.variance = variance_k - x_tilda = (self.input_tensor - self.mean) / np.sqrt(self.variance + self.epsilon) - self.output_tensor = self.weights * x_tilda + self.bias - - - return self.output_tensor - - def backward(self, error_tensor): - - if self.check_conv: - self.error_tensor = Helpers.compute_bn_gradients(self.reformat(error_tensor), self.reformat(self.input_tensor), self.weights, self.mean, self.variance, self.epsilon) - self.error_tensor = self.reformat(self.error_tensor) - else: - self.error_tensor = Helpers.compute_bn_gradients(error_tensor, self.input_tensor, self.weights, self.mean, self.variance, self.epsilon) - self._gradient_weights = np.sum(self.input_tensor.T@error_tensor, axis=0) #verify this - self.gradient_bias = np.sum(error_tensor, axis=0) - - - if self._optimizer is not None: - self.weights = self.optimizer.calculate_update(self.weights, self._gradient_weights) - if self.bias_optimizer: - self.bias = self.bias_optimizer.calculate_update(self.bais, self._gradient_bias) - - return error_tensor - - def reformat(self, tensor): - is4D = len(tensor.shape) == 4 - - if is4D: - b, h, m, n = tensor.shape - output_tensor = tensor.reshape((b, h, m*n)) - output_tensor = np.transpose(output_tensor, (0, 2, 1)) - b, mn, h = output_tensor.shape - output_tensor = output_tensor.reshape((b*mn, h)) - - else: - b, h, m, n = self.input_tensor.shape - output_tensor = tensor.reshape((b, m*n, h)) - output_tensor = np.transpose(output_tensor, (0, 2, 1)) - output_tensor = output_tensor.reshape((b, h, m, n)) - return output_tensor - - @property - def bias_optimizer(self): - return self._bias_optimizer - - @bias_optimizer.setter - def bias_optimizer(self, value): - self._bias_optimizer = value - - @property - def optimizer(self): - return self._optimizer - - @optimizer.setter - def optimizer(self, value): - self._optimizer = value - - @property - def gradient_weights(self): - return self._gradient_weights - - @gradient_weights.setter - def gradient_weights(self, value): - self._gradient_weights = value - - @property - def gradient_bias(self): - return self._gradient_bias - - @gradient_bias.setter - def gradient_bias(self, value): +from . import Base +import numpy as np +from . import Helpers + +class BatchNormalization(Base.BaseLayer): + def __init__(self, channels, alpha=0.8, epsilon=5e-11) -> None: + super().__init__() + self.trainable = True + self.input_tensor = None + self.output_tensor = None + self.channel_size = channels + self.check_conv = 0 + self.alpha = alpha + self.epsilon = epsilon + self.weights = None + self._optimizer = None + self._gradient_weights = None + self._bias_optimizer = None + self._gradient_bias = None + self.weights = None + self.bias = None + self.mean = 0 + self.variance = 0 + self.batch_size = None + self.test_mean = 0 + self.test_variance = 1 + self.x_tilda = 0 + + self.input_tensor = None + + self.initialize(None, None) + pass + + def initialize(self, weights_initializer, bias_initializer): + self.weights = np.ones(self.channel_size) + self.bias = np.zeros(self.channel_size) + + + def forward(self, input_tensor): + self.input_tensor = input_tensor + + if len (input_tensor.shape) == 4: + self.check_conv = True + else: + self.check_conv = False + + + if self.check_conv: + self.mean = np.mean(input_tensor, axis=(0, 2, 3)) + self.variance = np.var(input_tensor, axis=(0, 2, 3)) + self.channel_size = input_tensor.shape[1] + if self.testing_phase: + self.x_tilda = (self.input_tensor-self.test_mean.reshape((1, self.channel_size, 1, 1)))/(self.test_variance.reshape((1, self.channel_size, 1, 1))+self.epsilon)**0.5 + return self.weights.reshape((1, self.channel_size, 1, 1)) * self.x_tilda + self.bias.reshape((1, self.channel_size, 1, 1)) + + new_mean = np.mean(self.input_tensor, axis=(0, 2, 3)) + new_var = np.var(self.input_tensor, axis=(0, 2, 3)) + + # Make sure to use the test mean. The test mean is computed during training time as a moving average. It is then kept fixed during test time. + + self.test_mean = self.alpha*self.mean.reshape((1, self.channel_size, 1, 1)) + (1 - self.alpha) * new_mean.reshape( + (1, self.channel_size, 1, 1)) + self.test_variance = self.alpha* self.variance.reshape((1, self.channel_size, 1, 1)) + ( + 1 - self.alpha) * new_var.reshape((1, self.channel_size, 1, 1)) + + self.mean = new_mean + self.variance = new_var + + self.x_tilda= (self.input_tensor - self.mean.reshape((1, self.channel_size, 1, 1))) / np.sqrt( + self.variance.reshape((1, self.channel_size, 1, 1)) + self.epsilon) + + return self.weights.reshape((1, self.channel_size, 1, 1)) * self.x_tilda + self.bias.reshape((1, self.channel_size, 1, 1)) + + else: #not convolutional + self.mean = np.mean(input_tensor, axis=0) + self.variance = np.var(input_tensor, axis=0) + if self.testing_phase: + self.x_tilda = (input_tensor - self.test_mean) / np.sqrt(self.test_variance + self.epsilon) + else: + self.test_mean = self.alpha * self.mean + (1-self.alpha)*self.mean + self.test_variance = self.alpha * self.variance + (1-self.alpha)*self.variance + + self.x_tilda = (self.input_tensor-self.mean)/np.sqrt(self.variance+self.epsilon) + return self.weights*self.x_tilda + self.bias + + + + def backward(self, error_tensor): + + if self.check_conv: + self.error_tensor = Helpers.compute_bn_gradients(self.reformat(error_tensor), self.reformat(self.input_tensor), self.weights, self.mean, self.variance, self.epsilon) + self.error_tensor = self.reformat(self.error_tensor) + self._gradient_weights = np.sum(error_tensor*self.x_tilda, axis=(0, 2, 3)) #verify this + self.gradient_bias = np.sum(error_tensor, axis=(0, 2 , 3)) + else: + self.error_tensor = Helpers.compute_bn_gradients(error_tensor, self.input_tensor, self.weights, self.mean, self.variance, self.epsilon) + self._gradient_weights = np.sum(error_tensor*self.x_tilda, axis=0) #verify this + self.gradient_bias = np.sum(error_tensor, axis=0) + + + if self._optimizer is not None: + self.weights = self.optimizer.calculate_update(self.weights, self._gradient_weights) + if self.bias_optimizer: + self.bias = self.bias_optimizer.calculate_update(self.bais, self._gradient_bias) + + return self.error_tensor + + def reformat(self, tensor): + + if len(tensor.shape) == 4: + b = tensor.shape[0] + h = tensor.shape[1] + m = tensor.shape[2] + n = tensor.shape[3] + output_tensor = tensor.reshape((b, h, m*n)) + output_tensor = np.transpose(output_tensor, (0, 2, 1)) + b, mn, h = output_tensor.shape + output_tensor = output_tensor.reshape((b*mn, h)) + + else: + b = self.input_tensor.shape[0] + h = self.input_tensor.shape[1] + m = self.input_tensor.shape[2] + n = self.input_tensor.shape[3] + output_tensor = tensor.reshape((b, m*n, h)) + output_tensor = np.transpose(output_tensor, (0, 2, 1)) + output_tensor = output_tensor.reshape((b, h, m, n)) + return output_tensor + + + #all the required setters and getters + + @property + def bias_optimizer(self): + return self._bias_optimizer + + @bias_optimizer.setter + def bias_optimizer(self, value): + self._bias_optimizer = value + + @property + def optimizer(self): + return self._optimizer + + @optimizer.setter + def optimizer(self, value): + self._optimizer = value + + @property + def gradient_weights(self): + return self._gradient_weights + + @gradient_weights.setter + def gradient_weights(self, value): + self._gradient_weights = value + + @property + def gradient_bias(self): + return self._gradient_bias + + @gradient_bias.setter + def gradient_bias(self, value): self._gradient_bias = value \ No newline at end of file diff --git a/exercise3_material/src_to_implement/Layers/Conv.py b/exercise3_material/src_to_implement/Layers/Conv.py index f91d7d4..54df6de 100644 --- a/exercise3_material/src_to_implement/Layers/Conv.py +++ b/exercise3_material/src_to_implement/Layers/Conv.py @@ -1,182 +1,194 @@ -from . import Base -from scipy import ndimage -from scipy import signal -import numpy as np - -#stride_shape - single value or tuple -#convolution_shape - 1D or 2D conv layer [c, m, n] -#num_kernels - integer value -class Conv(Base.BaseLayer): - - def __init__(self, stride_shape, convolution_shape, num_kernels) -> None: - super().__init__() - self.trainable = True - self._optimizer = None - self.weights = None - self.bias = None - self.gradient_weights = None - self.gradient_bias = None - self.stride_shape = stride_shape #single value or tuple - self.convolution_shape = convolution_shape #filter shape (c,m,n) - if len(self.convolution_shape) == 3: - self.c = self.convolution_shape[0] - self.m = self.convolution_shape[1] - self.n = self.convolution_shape[2] - else: - self.c = self.convolution_shape[0] - self.m = self.convolution_shape[1] - self.num_kernels = num_kernels - self.weights = np.random.uniform(0,1, (self.num_kernels, *convolution_shape)) - self.bias = np.random.uniform(0,1, (self.num_kernels,)) - pass - -#input shape - [batch, channels, y, x] -#output shape - [batch, num_kernels, y_o, x_o] -#y_o = (y + 2p - f)/s + 1 - def forward(self, input_tensor): - self.input_tensor = input_tensor - if len(self.stride_shape) == 2: - sy = self.stride_shape[0] - sx = self.stride_shape[1] - else: - sy = self.stride_shape[0] - sx = self.stride_shape[0] - - batch = input_tensor.shape[0] - - if len(self.convolution_shape) == 3: - y = input_tensor.shape[2] - x = input_tensor.shape[3] - padding_y = (self.m-1)/2 - padding_x = (self.n-1)/2 - self.padding = [padding_y, padding_x] - y_o = int((y + 2*padding_y - self.m)//sy + 1) - x_o = int((x + 2*padding_x - self.n)//sx + 1) - output_shape = (batch, self.num_kernels, y_o, x_o) - else: - y = input_tensor.shape[2] - padding_y = (self.m-1)/2 - self.padding = [padding_y] - y_o = int((y + 2*padding_y - self.m)//sy + 1) - output_shape = (batch, self.num_kernels, y_o) - - output_tensor = np.zeros(output_shape) - - - for ib in range(batch): - for ik in range(self.num_kernels): - if len(self.convolution_shape) == 3: - output_per_filter = np.zeros((y,x)) - else: - output_per_filter = np.zeros((y)) - for ic in range(self.c): - - output_per_filter += ndimage.convolve(self.input_tensor[ib, ic], self.weights[ik, ic], mode='constant', cval=0) - # output_per_filter += signal.correlate(input_tensor[ib, ic], self.weights[ik, ic], mode='same', method='direct') - - output_per_filter = output_per_filter[::sy,::sx] if len(self.convolution_shape) == 3 else output_per_filter[::sy] #striding - output_tensor[ib, ik] = output_per_filter + self.bias[ik] - - return output_tensor - - @property - def optimizer(self): - return self._optimizer - - @optimizer.setter - def optimizer(self, value): - self._optimizer = value - - @property - def gradient_weights(self): - return self._gradient_weights - - @gradient_weights.setter - def gradient_weights(self, value): - self._gradient_weights = value - - @property - def gradient_bias(self): - return self._gradient_bias - - @gradient_bias.setter - def gradient_bias(self, value): - self._gradient_bias = value - - def backward(self, error_tensor): - error_output = np.zeros_like(self.input_tensor) - if len(self.stride_shape) == 2: - sy = self.stride_shape[0] - sx = self.stride_shape[1] - else: - sy = self.stride_shape[0] - sx = self.stride_shape[0] - - T_weights = self.weights.copy() - T_weights = np.transpose(T_weights, axes=(1,0,2,3)) if len(self.convolution_shape) == 3 else np.transpose(T_weights, axes=(1,0,2)) - batch = self.input_tensor.shape[0] - nk, nc = T_weights.shape[:2] - - if len(self.convolution_shape) == 3: - y = self.input_tensor.shape[2] - x = self.input_tensor.shape[3] - else: - y = self.input_tensor.shape[2] - - for ib in range(batch): - for ik in range(nk): - error_per_channel = 0 - for ic in range(nc): - if len(self.convolution_shape) == 3: - err = np.zeros((y,x)) - err[::sy, ::sx] = error_tensor[ib, ic] - else: - err = np.zeros(y) - err[::sy] = error_tensor[ib, ic] - - error_per_channel += ndimage.correlate(err, T_weights[ik, ic], mode='constant', cval=0) - - error_output[ib, ik] = error_per_channel - - berror = error_tensor.sum(axis=0) - yerror = berror.sum(axis=1) - self.gradient_bias = yerror.sum(axis=1) if len(self.convolution_shape)==3 else yerror - - self.gradient_weights=np.zeros_like(self.weights) - for ib in range(batch): - for ic in range(self.input_tensor.shape[1]): - for ik in range(self.num_kernels): - if len(self.convolution_shape)==3: - error = np.zeros((y, x)) - error[::sy, ::sx] = error_tensor[ib, ik] - input = np.pad(self.input_tensor[ib, ic], - [(int(np.ceil(self.padding[0])), int(np.floor(self.padding[0]))), - (int(np.ceil(self.padding[1])), int(np.floor(self.padding[1])))]) - else: - error = np.zeros(y) - error[::sy] = error_tensor[ib, ik] - input = np.pad(self.input_tensor[ib, ic], [(int(np.ceil(self.padding[0])), int(np.floor(self.padding[0])))]) - buffer = ndimage.correlate(input, error, mode='constant') - expected_output_size = np.array(input.shape) - np.array(error.shape) + 1 - buffer = buffer[:expected_output_size[0], :expected_output_size[1]] if len(expected_output_size)==2 else buffer[:expected_output_size[0]] - - self.gradient_weights[ik, ic] += buffer - - - if self._optimizer is not None: - self.weights = self.optimizer.calculate_update(self.weights,self.gradient_weights) - self.bias = self.optimizer.calculate_update(self.bias,self.gradient_bias) - return error_output - - def initialize(self, weights_initializer, bias_initializer): - if len(self.convolution_shape) == 3: - fan_in = self.c * self.m * self.n - fan_out = self.num_kernels * self.m * self.n - self.weights = weights_initializer.initialize((self.num_kernels, self.c, self.m, self.n),fan_in, fan_out) - self.bias = bias_initializer.initialize((self.num_kernels,), 1, self.num_kernels) - else: - fan_in = self.c * self.m - fan_out = self.num_kernels * self.m - self.weights = weights_initializer.initialize((self.num_kernels, self.c, self.m),fan_in, fan_out) - self.bias = bias_initializer.initialize((self.num_kernels,), 1, self.num_kernels) - pass \ No newline at end of file +from Layers.Base import BaseLayer + +import numpy as np +from scipy import signal +from functools import reduce +import operator +from copy import deepcopy as copy + + +class Conv(BaseLayer): + def __init__(self, stride_shape, convolution_shape, num_kernels): + super().__init__() + self.trainable = True + self.stride_shape = (stride_shape[0], stride_shape[0]) if len(stride_shape) == 1 else stride_shape + # 1d as [channel,m], 2d as [channel,m,n] + self.convolution_shape = convolution_shape + self.num_kernels = num_kernels + # init weights as uniform random (will be initialized again with initialize method) + # shape for 2d conv: (num_kernels, channel, m, n) + self.weights = np.random.uniform(0, 1, (num_kernels, *convolution_shape)) + # bias shape: number of kernels + self.bias = np.random.rand(num_kernels) + + # grad parameters + self._gradient_weights = None + self._gradient_bias = None + + self._optimizer = None + self._bias_optimizer = None + + # conv_dim if it is 2d or 1d + self.conv_dim = 2 if len(convolution_shape) == 3 else 1 + + + def initialize(self, weights_initializer, bias_initializer): + self.weights = weights_initializer.initialize(self.weights.shape, + reduce(operator.mul, self.convolution_shape), + reduce(operator.mul, [self.num_kernels, *self.convolution_shape[1:]])) + + self.bias = bias_initializer.initialize(self.bias.shape, 1,self.num_kernels) + + self._optimizer=copy(self.optimizer) + self._bias_optimizer=copy(self.optimizer) + + def forward(self, input_tensor): + # if correlation is used in forward, we can use convole in backward + # or vice versa + # input_tensor shape (b,c,x,y) or (b,c,x) + self.input_tensor = input_tensor + ishape = input_tensor.shape + self.ishape = ishape + bsize, c, y, x = ishape if self.conv_dim==2 else (*ishape, None) + cx,cy = self.convolution_shape[-2:] + + sh, sw = self.stride_shape + + # new shape of y = (y-ky + 2*p)/sh + 1; y input size, ky kernel size, p padding size, sh stride size + # but we need o/p size same as i/p so p=(ky-1)/2 if sh==1 + # else we need to derive + pad=[(cx-1)/2] + out_shape = [int((y-cx+2*pad[0])/sh)+1] + if self.conv_dim==2: + pad.append((cy-1)/2) + out_shape.append(int((x-cy+2*pad[1])/sw)+1) + self.pad=pad + result = np.zeros((bsize, self.num_kernels, *out_shape)) + + # if used correlation in forward, should use convolve in backward + for cb in range(bsize): + for ck in range(self.num_kernels): + # sum outputs of correlation of this kernel with individual input channel of input + kout = np.zeros((y,x)) if x else np.zeros((y)) + for ch in range(c): + # correlate with this batch's this channel and this kernel's this channel + kout += signal.correlate(input_tensor[cb, ch], self.weights[ck, ch], mode='same', method='direct') + + kout = kout[::sh, ::sw] if self.conv_dim==2 else kout[::sh] + result[cb, ck] = kout + self.bias[ck] + + return result + + + def update_parameters(self, error_tensor): + # what is the grad of bias in this layer for this batch? + # we sum error tensor along axis of B,W,H (if 2d) + # B + berror = error_tensor.sum(axis=0) + # W + yerror = berror.sum(axis=1) + # H? + self._gradient_bias = yerror.sum(axis=1) if self.conv_dim==2 else yerror + + # what is the grad of weights in this layer for this batch? + batch_size, channels, y, x = self.ishape if self.conv_dim==2 else (*self.ishape, None) + sh, sw = self.stride_shape + cx, cy = self.convolution_shape[-2:] + + self.gradient_weights=np.zeros_like(self.weights) + for cb in range(batch_size): + for ch in range(channels): + for ck in range(self.num_kernels): + if self.conv_dim==2: + error = np.zeros((y, x)) + error[::sh, ::sw] = error_tensor[cb, ck] + inp = np.pad(self.input_tensor[cb, ch], + [(int(np.ceil(self.pad[0])), int(np.floor(self.pad[0]))), + (int(np.ceil(self.pad[1])), int(np.floor(self.pad[1])))] + # [int(np.ceil(self.pad[0])), int(np.floor(self.pad[1]))] + ) + else: + error = np.zeros(y) + error[::sh] = error_tensor[cb, ck] + inp = np.pad(self.input_tensor[cb, ch], [(int(np.ceil(self.pad[0])), int(np.floor(self.pad[0])))]) + + self.gradient_weights[ck, ch] += signal.correlate( + inp, error, mode='valid') + + if self.optimizer: + self.weights = self._optimizer.calculate_update(self.weights, self._gradient_weights) + self.bias = self._bias_optimizer.calculate_update(self.bias, self._gradient_bias) + + def error_this_layer(self, error_tensor): + # compute error in this layer + gradient=np.zeros_like(self.input_tensor) + sh,sw = self.stride_shape + + # input Conv2d weight shape: (num_kernel, channel, w, h), channel is channel of input data + # inner Conv2d weight shape: (num_kernel, input_channel, w, h) + # input channel is channel from previous layer + # while passing error backward, we calculate error cased by this layer's weights + # so transpose weight as : (input_channel, num_kernel, w, h) + nweight = self.weights.copy() + nweight = np.transpose(nweight, axes=(1,0,2,3)) if self.conv_dim==2 else np.transpose(nweight, axes=(1,0,2)) + ishape = self.input_tensor.shape + y,x = ishape[-2:] if self.conv_dim==2 else (ishape[-1],None) + + bsize = self.input_tensor.shape[0] + wk, wc = nweight.shape[:2] + + for cb in range(bsize): + for ck in range(wk): + grad = 0 + for c in range(wc): + if self.conv_dim==2: + err = np.zeros((y,x)) + err[::sh, ::sw] = error_tensor[cb, c] + else: + err = np.zeros(y) + err[::sh] = error_tensor[cb, ck] + # we used correlate on forward, use convolve now + grad += signal.convolve(err, nweight[ck, c], mode='same', method='direct') + + gradient[cb, ck] = grad + return gradient + + def backward(self, error_tensor): + self.update_parameters(error_tensor) + gradient = self.error_this_layer(error_tensor) + + + + return gradient + + @property + def gradient_weights(self): + return self._gradient_weights + + @gradient_weights.setter + def gradient_weights(self, value): + self._gradient_weights = value + + @property + def gradient_bias(self): + return self._gradient_bias + + @gradient_bias.setter + def gradient_bias(self, value): + self._gradient_bias = value + + @property + def optimizer(self): + return self._optimizer + + @optimizer.setter + def optimizer(self, value): + self._optimizer = value + + @property + def bias_optimizer(self): + return self._bias_optimizer + + @bias_optimizer.setter + def bias_optimizer(self, value): + self._bias_optimizer = value diff --git a/exercise3_material/src_to_implement/Layers/Conv_o.py b/exercise3_material/src_to_implement/Layers/Conv_o.py new file mode 100644 index 0000000..f91d7d4 --- /dev/null +++ b/exercise3_material/src_to_implement/Layers/Conv_o.py @@ -0,0 +1,182 @@ +from . import Base +from scipy import ndimage +from scipy import signal +import numpy as np + +#stride_shape - single value or tuple +#convolution_shape - 1D or 2D conv layer [c, m, n] +#num_kernels - integer value +class Conv(Base.BaseLayer): + + def __init__(self, stride_shape, convolution_shape, num_kernels) -> None: + super().__init__() + self.trainable = True + self._optimizer = None + self.weights = None + self.bias = None + self.gradient_weights = None + self.gradient_bias = None + self.stride_shape = stride_shape #single value or tuple + self.convolution_shape = convolution_shape #filter shape (c,m,n) + if len(self.convolution_shape) == 3: + self.c = self.convolution_shape[0] + self.m = self.convolution_shape[1] + self.n = self.convolution_shape[2] + else: + self.c = self.convolution_shape[0] + self.m = self.convolution_shape[1] + self.num_kernels = num_kernels + self.weights = np.random.uniform(0,1, (self.num_kernels, *convolution_shape)) + self.bias = np.random.uniform(0,1, (self.num_kernels,)) + pass + +#input shape - [batch, channels, y, x] +#output shape - [batch, num_kernels, y_o, x_o] +#y_o = (y + 2p - f)/s + 1 + def forward(self, input_tensor): + self.input_tensor = input_tensor + if len(self.stride_shape) == 2: + sy = self.stride_shape[0] + sx = self.stride_shape[1] + else: + sy = self.stride_shape[0] + sx = self.stride_shape[0] + + batch = input_tensor.shape[0] + + if len(self.convolution_shape) == 3: + y = input_tensor.shape[2] + x = input_tensor.shape[3] + padding_y = (self.m-1)/2 + padding_x = (self.n-1)/2 + self.padding = [padding_y, padding_x] + y_o = int((y + 2*padding_y - self.m)//sy + 1) + x_o = int((x + 2*padding_x - self.n)//sx + 1) + output_shape = (batch, self.num_kernels, y_o, x_o) + else: + y = input_tensor.shape[2] + padding_y = (self.m-1)/2 + self.padding = [padding_y] + y_o = int((y + 2*padding_y - self.m)//sy + 1) + output_shape = (batch, self.num_kernels, y_o) + + output_tensor = np.zeros(output_shape) + + + for ib in range(batch): + for ik in range(self.num_kernels): + if len(self.convolution_shape) == 3: + output_per_filter = np.zeros((y,x)) + else: + output_per_filter = np.zeros((y)) + for ic in range(self.c): + + output_per_filter += ndimage.convolve(self.input_tensor[ib, ic], self.weights[ik, ic], mode='constant', cval=0) + # output_per_filter += signal.correlate(input_tensor[ib, ic], self.weights[ik, ic], mode='same', method='direct') + + output_per_filter = output_per_filter[::sy,::sx] if len(self.convolution_shape) == 3 else output_per_filter[::sy] #striding + output_tensor[ib, ik] = output_per_filter + self.bias[ik] + + return output_tensor + + @property + def optimizer(self): + return self._optimizer + + @optimizer.setter + def optimizer(self, value): + self._optimizer = value + + @property + def gradient_weights(self): + return self._gradient_weights + + @gradient_weights.setter + def gradient_weights(self, value): + self._gradient_weights = value + + @property + def gradient_bias(self): + return self._gradient_bias + + @gradient_bias.setter + def gradient_bias(self, value): + self._gradient_bias = value + + def backward(self, error_tensor): + error_output = np.zeros_like(self.input_tensor) + if len(self.stride_shape) == 2: + sy = self.stride_shape[0] + sx = self.stride_shape[1] + else: + sy = self.stride_shape[0] + sx = self.stride_shape[0] + + T_weights = self.weights.copy() + T_weights = np.transpose(T_weights, axes=(1,0,2,3)) if len(self.convolution_shape) == 3 else np.transpose(T_weights, axes=(1,0,2)) + batch = self.input_tensor.shape[0] + nk, nc = T_weights.shape[:2] + + if len(self.convolution_shape) == 3: + y = self.input_tensor.shape[2] + x = self.input_tensor.shape[3] + else: + y = self.input_tensor.shape[2] + + for ib in range(batch): + for ik in range(nk): + error_per_channel = 0 + for ic in range(nc): + if len(self.convolution_shape) == 3: + err = np.zeros((y,x)) + err[::sy, ::sx] = error_tensor[ib, ic] + else: + err = np.zeros(y) + err[::sy] = error_tensor[ib, ic] + + error_per_channel += ndimage.correlate(err, T_weights[ik, ic], mode='constant', cval=0) + + error_output[ib, ik] = error_per_channel + + berror = error_tensor.sum(axis=0) + yerror = berror.sum(axis=1) + self.gradient_bias = yerror.sum(axis=1) if len(self.convolution_shape)==3 else yerror + + self.gradient_weights=np.zeros_like(self.weights) + for ib in range(batch): + for ic in range(self.input_tensor.shape[1]): + for ik in range(self.num_kernels): + if len(self.convolution_shape)==3: + error = np.zeros((y, x)) + error[::sy, ::sx] = error_tensor[ib, ik] + input = np.pad(self.input_tensor[ib, ic], + [(int(np.ceil(self.padding[0])), int(np.floor(self.padding[0]))), + (int(np.ceil(self.padding[1])), int(np.floor(self.padding[1])))]) + else: + error = np.zeros(y) + error[::sy] = error_tensor[ib, ik] + input = np.pad(self.input_tensor[ib, ic], [(int(np.ceil(self.padding[0])), int(np.floor(self.padding[0])))]) + buffer = ndimage.correlate(input, error, mode='constant') + expected_output_size = np.array(input.shape) - np.array(error.shape) + 1 + buffer = buffer[:expected_output_size[0], :expected_output_size[1]] if len(expected_output_size)==2 else buffer[:expected_output_size[0]] + + self.gradient_weights[ik, ic] += buffer + + + if self._optimizer is not None: + self.weights = self.optimizer.calculate_update(self.weights,self.gradient_weights) + self.bias = self.optimizer.calculate_update(self.bias,self.gradient_bias) + return error_output + + def initialize(self, weights_initializer, bias_initializer): + if len(self.convolution_shape) == 3: + fan_in = self.c * self.m * self.n + fan_out = self.num_kernels * self.m * self.n + self.weights = weights_initializer.initialize((self.num_kernels, self.c, self.m, self.n),fan_in, fan_out) + self.bias = bias_initializer.initialize((self.num_kernels,), 1, self.num_kernels) + else: + fan_in = self.c * self.m + fan_out = self.num_kernels * self.m + self.weights = weights_initializer.initialize((self.num_kernels, self.c, self.m),fan_in, fan_out) + self.bias = bias_initializer.initialize((self.num_kernels,), 1, self.num_kernels) + pass \ No newline at end of file diff --git a/exercise3_material/src_to_implement/Layers/RNN.py b/exercise3_material/src_to_implement/Layers/RNN.py index bea9121..e658f66 100644 --- a/exercise3_material/src_to_implement/Layers/RNN.py +++ b/exercise3_material/src_to_implement/Layers/RNN.py @@ -115,9 +115,6 @@ class RNN(Base.BaseLayer): self.output_FCLayer.weights = self.optimizer.calculate_update(self.output_FCLayer.weights, self.output_FCLayer_gradient_weights) self.weights = self.optimizer.calculate_update(self.weights, self.gradient_weights) - - - return gradient_inputs diff --git a/exercise3_material/src_to_implement/Layers/__pycache__/BatchNormalization.cpython-310.pyc b/exercise3_material/src_to_implement/Layers/__pycache__/BatchNormalization.cpython-310.pyc index df046cec3a85fe0fa9ac9477cea153c3145e6d13..2d221ce5bd067573fc841d5538c7e8d1204c36a0 100644 GIT binary patch literal 4459 zcmd1j<>g{vU|?9^(w3$t$-wX!#6iX^3=9ko3=9m#W(*7rDGVu$ISjdsQH+cXDNHHM zIZRQ^DGVtrISjchQ7nuM?hGlcDQqnaDQu~XS**>>QEVy9!3>)0FBusa7&IAgu{b3b zr@jOk@2AOli`^qNry#YcI2j~>j9H<a;uQ=G45<uJj42FJOexYSGVKg$j9?phqnK0J zQ#e`}qF7QmQ@C0fqF7V7Q+Qe!qS#V+Q}|jKqS#Y8QutE@S{Rxcqc~GJv$#?O;XIBM zp%h`T40kGb7FUW0oX3$Onj!|4;YsDm;z|*R^Egr@QY67Lys5ldTq#m8UONK|Llj>y zgQn~)A*aNW<P5+3qTIxs%&Nqa%>2CPvu1+8OZ(@$_g=qwzvi^<Ew-Z6lG36)zhqF* zLVd~rVuLU<D5z&JFff!bG&3w<T*$!4kiuBQ5YLptP{R<<3?^A>Kq9OuOf?MgY&9T~ zy@t7lA)W&)%LyjAz$7=A<N=esU=kwB2iD6ECI!Iaf?&2#3dkH`Fk1vH&RoN=Ks1FR zg<&BRBSQ*9FoPzGUq}(il||q<6}!b+Tw0J?bc-WCJ~J<~BtHHYXGu|FW?o`aPU<b5 z%)ElqlK7I;yyE<#TfF(DB}iPJ<c!3;ywsfd;>@blTU^N*sma;#$@zI@w^$Q%3NjLJ zv8NUkXXfPR-C{3K%}mcIDZa%OpI=atnVVUaT69Y=KD{V0B{MazBpxQsAD@(&Sd65G z51|GmbBhH+=B6g*-Qp-qEXqvGOHRGTl>~}5h%-1#Qj1IC!3uc6Ot>ociujVuoRq{{ zTp)jDf?_yTljRmmacWN5Ee@z(ii-po7#NBK85kH=GT!2hk5A4?EG~|ZU&-)mf_`Xm zYEiL%QDSLcVs>$2Zem_ZqP|Oha%paAUP-Zja7k%OrGBuhZ@i(tOKNICyiaOkQC?<V zy1r{gYEg1#ajJewYHERQ4oo-|Dr_8|n^=-sl$n^LUtE+NUy>i6nOl&P3NlsSC$SP_ zCuX|TE2u02WqdYJdNW{PVBlh8BNj68FtIRJNn<e-mIjhR2?Un3L2M9q23e&5PTC6? zYZ$Va7J`y7a|y!&7C4U;l*Cpt`f0M=V#+JH#gd<wT3o~j3Q{2k28LU#RjEb!#YN(v z(uBE4o`HcON*EgX;M50+m7*vCP&^}x6e)wu023-8mK*~EgA&L+pghOND8yJLi^V&L zh)!mL`G$c3oL4y+7#P4Fn#RPy0M3sLC5$zUSxn80DU8`nMUWiEoW~5#QOq?AS<FQr zN?5X3YZy~lQdoOIxt6trC5sJ-&koX`!X^pI=^Ql-S)4_wH4Iq{H4Ir?3%FC*Y8bLO zQrJ@1(pVNUN`ORp&_!z)vUtTAQrLwVq(Nrz)i7l7V^PB=0@5vjE+ftm%m8*FLly(X zjkQc*7b5r|e+bqvWU->_6=49oPf&uPh6z(n9ON3V8ip*8UUp1*xL#&VIj~+Ha4E+M zF6G#vVS?cX35F7u1wu6pSr|4z!et>NMWMo&!hsfVFh0mdoD&#}L_p~viwo`!9#nht z^q?WYiWFK5S*#)qU_Zjc2St`0Dho0T6l<(-(?PcKiZdYls)iv88n>Xh)a3Mg2}<4} zZY!C=CH6~DB7O<ViYpm!u_RVx7K6+1bV$;EU=LQLDO|(@%4*Cxsd=|pi!%}nQj7RO zSw;;+NPwlv5{rr?L0nMIyag^NGxO5p3o;UmQ;VcQO4y51AzEcX%0R_okt~P>%EjR9 z3oa*bu@n~;m1wenb28Z4D2}|;a&Yk*#SUVDY*Gek1QVdFdy73Szo<O1C`BEVeVrH> z7}yy3m^c{OnAn&F7zG%47}*%@7>yVOm;@NrSa=w@{!1}QF>?H8V`gKN1G9{n1(*dG zIT#ff<(T9c6_|t=tE8}GeyBAlIUf{&9I%}KhJgVTBMe0*C5#K0Y8VzU*D!*Lv<0AC z!UAGtvDPqTf$}XVt~eI5)iA_!g4Ho(af0OIIZL>*xWTE0M}#4TF@*_SF7QG`;e5V4 zWv~sPLK9R}Vlfq>qlO_1WDYZ$IWRuBJV;?EWh_doVaO6#0LqOZvv?PROQRZwETN)e z5DAtMu3^X$0n3Pj$|7*hU!(<!3tbQaN=BMukfZ@hWl<u@`MCw9C8_aAdGK1d_!dV| zY8t3LR059PB5?eJ5)mZ*fD*<np46hE{32*0p$L@BZZQ{^=0XyT2q-o2!fSd^xn3j; z65~X)1O$>3bCOGQ5=&C!OAAsGOHyy~fz%<|0Z}YTiJ8SkVjy>e>kcMJiZcfZfKwKj z04Fbwq{QTGQ2L4hr7tN^`eM>y<YN?L6k=3h5@Qr#<YE?KWMdRzWMjfw2g3BDBpgtp z0adl&D&row4y<9!VuV%}HXwEilO#he6PU#;$xzD-X0b>z)UtqCtdb11tTmv7##F;x z!&1X4!jQ(ikg=Angn0o=4O<G^LZ(`_8n!IP80K20TJ~Dz8ny*YHB2?^A`CUmX-puM zHEhA41k0Mm2(^nL3&e)l24O?&gRqfp1lw1`jLk-{EnpXcDkhK{KrMd1B27@>f>H^% zkSWpzadjXSH&_%b0B%FF73qT{L9LJ?V-O3R20+coyyAlV;#6?b(c}Qf*)29`W0EoH z7GuUO#@t(sc}2XSh5%D;9(pYg>ayg3q7f9cd`w)75GcaP!^p!Z#K^+M^Ap74VFZZ? zG2&>{z*2BB$gz;Jl7Rul24PTyeg?%Xs8$MQC{hKt{E9>&lsw2c;IsuMK+Y(#VPIfr zWnf_V3@S|6n5q=9ls%})7bPWubb&C~Vo(bXRD8sPo91B4i(ElgLlXEc*0RK$(p2<d z0lA<P<^nDzoc=0u0XZM)BT%L*asvr~y#gk{{;^_UVCV&jyE8B_RLNlR4Lqmf^hl8h z$N-Q{MLr;wCx`%B0Vd$~OazI0GcYg^=Vnmhfz8dg1mFb<wDmX@WH8tR%2+&ru9_Hs z_<~Ht<_|vx28Nj+N#gwhF1WCTy(I$!!(5PAV5ckKb2=zV5#w%uka5`D4YF$?ND}03 zO{OAHqAy|xWoV`%Q1Rxc$q(uT<|XE)#>d~{ijU9DPbtj-v3cU-3riDoATr=yM-iy~ zU1SO}%@#y}GU_dkf};F_)S{9~a2C16R-9T=l3G**N?%2wKq~^d;1(~mPnDdSlM^40 zl(ZE=27yxwX!rnxi$TQ=2ZI13hYSZRharaulOz`-3o{oZ6Eikih_MKyOOyQ;LlG#4 z-D1rv%`K<|*GNU6D()61X!ru$r2?x&vK*dFZ*kZ_{A~v+9g0B#!^6PA$ipbc1OV*+ B<huX> delta 2331 zcmaE@G(nLspO=@5fq{X+|7Bg8lK4ixIz}!g1_lOZ1_p-Wj>*#)lS~+D7~=U;K%@Ye z6a<q(V3N6pVS(^M21bSyhF}IwX1|akkVZ|B$&O6tl{k_!67%v>bBb?qW#(m;WG3ch zR;6mP++rzC%}FbgW?*2L9KgJlRfmCrfork=iwY+r8zTo33uBe^<Ul^*$$l)Hyo?~H zK{4ax6qZn#m5fEQ3=9lK3UERZ#FAlPV342uhsC0v6=WO(1A{Zj2VG1I3^j~34Dk#l zj5Ul|OwEibjM+>z4Drl03|Y)YS4&v3SZf%v7*d#0n0rCutR*a2Y$zh^Ablw;lAr+K z0JB-aY|a{nEUqGdklGrCEbaw7DIk+L>r+@$SkqV*GD<L%uq@!MVaVcxNx@`l7_#`q z85S~1Gt@HW<)AvDmKoInwJbFZSpqc-S*+-$h%kUH6p&!3VMdaxXI;o5!XOTEEjLsv zJCZb57Os&6T?VX=uZFpXA)d8{rG_D%9qK<6>ld;>y@qBo$V>GS3=sd-GSx6-38Gq1 z!z2Qd6+#sSTPj?`kR=?<01pioj0k~=fSklt^b6`RZbUeP?X8CetUcH$#uTtNXt2U~ z@IXZlMD$QYl7odP){uca0OT<KdRWYYd=K+4vJW-c{9dv!FfeE`gY(KuP;Pq($~7w) zZ?Pm+WEO*ki$Ez~Q>2K8fq~%`b53gBE!N_U#Ddf!eo$&x1`!fqsj|ePB1sTe3PeaV z)H5*L;w?!nF3HSGk1xnbEKZH$1aaeYQxo%Uu@|L+jk(2ATv${R1y&eemROXTn3tSd zBnUD>07M9Z2yqa>3vwzeB+r2y6UAN;Uy_-Vk{HDXGBG|oiVLPcKD$VPAr6)|6+zBp zPs=YVPb^9SIk?!4fq{XIQGk(+RftiDk%x(gk&V%TQGi8&(Td4}k&T6o(TY)zkxfGA zwGbm4lL8|jGasW2lMGl!j!}q_jk!t+Trx;;DsmlTU;rg0h9bquuAEj(c`B2abE<MM zWU*xN)G!K6?&6eI0i}l&hEm3&5Ku-~APCAuH4It23&9z^h9OI+C~>kNmnbhIxa@+2 zyVhhYu5ccZdv7rpm*(DL44FKiE7BQU=9qv2L;+5KeaMlNn4AsrVic&n5d(RVNrzE@ zQGrQ<QGk()S%i^|QG}6&30kNHaf{Y-f)v3@s1*zh3?&RTj9HA3vZ<D-hN;L9#E)UA zWv*qZWvykaVO+pe!(78s!&<{8!jQ(ikg=A%gn0o=4SNdHLZ({w8ul#480K2$T8>(l z8g{T+4iSbLmNX`i${O}yP)M<6F+xnNM=}y@U=0g4Bf-Xitpt^3AWI;jqz+0DkhEK* z0SXInN@Xho72`#^AOQmq0S-aVlA^@C;)49*RB%Gp<OHRoScY3{C8>GE`9)DInZ+h9 zw-}RdF=pIi%)P~!R|HCKQB1jcMX14CWX!<8kOK-~P%`IZWMSfC6k+6H<Y5$IWMLQh z17Zs?!mAEtUR_ls1_p+ej76d#$Ebk_G!v~D7#M0Nm-B|{fK(QNs*9COAonrcVl7L| zDNRLp0!TyS<e$7M`c5F@K)O~k772sc*vz+JU|?tkNxDq-;j`9t1u;Q-iabFqHxPm5 zB9M+wkhsU>8h%+DWLrc)!dPs%C6Hc}n39>AR}x>Inwg$aQrrtt=`xv@-wDY|FOVQM zE4>*Q7$$-w!B(0eTM0_aMOdsXGG}05m<rP4!oa{#r2x*65BSxQtn&fMVzUmUbtXvC zW3q{WbEFK2X#^syK?Ep{Z*de9<rkzDl~fji!uu9macW6PYEcm=Y>Ggx1zUfMH$FZ& zC$YFVIW;FIJ{~Ei<Uj@~PQD-@#wp3c!eP!K#3VTRy?}`pC_NT|%H3O>PKm{-K8cm7 TMMa>t0+MEMd?-%#6_f)2-Ot{v diff --git a/exercise3_material/src_to_implement/Layers/__pycache__/Conv.cpython-310.pyc b/exercise3_material/src_to_implement/Layers/__pycache__/Conv.cpython-310.pyc index 0a697e15a8ca2081823f22581d6ef79816c4fd8e..a4bc5324642d817258d596a07bc35c3f93005e25 100644 GIT binary patch literal 5085 zcmd1j<>g{vU|{&&(w6pHj)CDZh=Yt-7#J8F7#J9eLl_tsQW#Pga~N_NqZk=MY^EHh zDCQJ~6y_Y3T-GR7uoz1YTP}MPJDAOs!x6=i!jQt6!<ow!#mUIv&XB^E!rsD=!k)_5 z%pAp?!W7J)$?+0opC;oiPN&4;RG-Ak)FMAk##?N~ndy0nIUrV1YD#HxswU$tj+E5Y zg5>;y%4CohWXufZ6t7`mU`S<%VoYI(VrpkdV@zR8VQS%sVoqUBVQFEAVo705VQXQC zVohOB;b>uqVoTvn;c8)sVo%{t;b~!r;z;F8;Z5OdVQ6NI;!5Sp;!fd*^EgukQUt*= zJgGcc+$lnE9%qVhiU?SSH<dSwJ4F=E<4h4t5eLigrSfHQr%1qf?F=joQT)LSnv%Cz zob&U_l0g9t^#lWm&BnmM0P<DwJO&1a5{71m1&j+B7#UI+YZ&60N|<XHn;BD>vYCp+ zY8X?PB^e-O8dDx)4M+t`4O0z6JZlYe4MRLz340dD0?riX6s8*HbjBK{MT|iV3qdkm zU>WWj=7k_VJShw{4Dq~Rk`GMsgGqrB<{GAE#uS!prXsHt)*6O*!4!sIh7^zuV3sDE zU&u>F1_p*A5b+Y^>X%Fm3=En=w^)ly3sQ@2am2@G=4F<|$KT>CDN4-DOH9g1y~Ui9 zns<w*xTGjEB{jY{Be5X$mSA#zURi!lX-Q^&9#n`suQWG4JGCe;HK+I%Q(nO>wxYzm zl>FRV?4@~`Y57IDx7f>5Gt)Clif^%iWNxt}WhNHi5{yqTN=(U2%`1tAN%J9uz)HB{ z^9xEcb2F<_i*E79gM^R-I6y9mPsz;HWVyvsoSKtX#0T;qKZp=uU|?9uc#AVWJ~=0` zxHvw3CBv^K{m|mnqGJ7`#L~RP?Bc}S#JrM3eV6>?(%jU%l4AYflG2n){a{z$ctd@c z)YO7_pVY*nyv)3Geb<WAqU6ltRDDq5*3E$lr$U8|<8u>BQj0PZbM%XglH*JA<1=#$ za#BI2>Vq>&u|6pM=oM5JaWgP5fN-${0|NsWBO9X<BO4PNqZBhA6B{!dn61EE#S2yr zi;!ecY{AkUhz%+ZoIx2ri-CcmhH(Ky4MP@V7E=jx2}>4h4MP@NGlI>Y&Il4oVN79Y zWs+oA#MsOPPFIXIOkjN+Sxlhxn#B$l<tX8-VaVcY2BkG_BsLEyHEA;W6|sZT4DT&2 zP*TZE%mGD=2m=GdEmm;iC<5h*TO9cXsYQt;`9-&wb4zoI;PFu;22#laDjaTc!sE3_ z5)@xjAaj|EWI!r}p{W6!9-)@p5&%a&lC?$hAbns$0mM>dU|>)N1qvt;a53^Q$}qD1 z6Jz0F5@4+22Ztv@b22FD!$MRD6cY>#4B!x*#Rv+~8isg=8pbTfS|)HFWvXGwVoqVq zW-1b>Vaj7mX8`5T6o!e6K@2f0wam3FwXC&lH4IrSDNK-1j$x{0ujQy=$YPCQs^zTZ zs$oxImS9L>lx9e0tmQ6Yt6{BS2k~pTQ&=P!L>R;wYB;4Cni*4=#X%}~;Pxfea4%r5 z;ehF0$jHc0!?S?Bgsq0Hh68E>Gt>kwm<eE2+%*jG93`9!xIpDg7I!*R4G*|XVXWmX z;i+M6W^`eQ4XfoV0olb3=K0m~*RU0}f%UM~u%$6AWXxlM@D?)G3Y75Fuz=Nx)e6=K zl<;Qp)iBoZ)d;3BNrKWJe+@r`4Jy4;*n3&b8EORz`{6#>UL%mgkit;I*$S$T7;3m$ znL#@9*bw5947CC^0yPX-0yX@SpfKUB;j7_KW3pi=tS&66;RRJhenp%N3=A*-{r~?T zoEE{w*)5jh#N5<dY$=&VsmUcPnQpP<=BK3IV#`e}$;eO96uiZgnO9I+5?_*<SDar| zqy$Q2Y?+W0D-B9J;3Nu4H@BEG^Ga^9B^DH<=B3<XE=Wu%5(KGctx7G*FD?R=Dz{jR z67$kii$EFi7H4vPQBi6RDEEW10XT(g3W1Yxkv_;|1F-I-;>@blTa3xK7%OivR@`Dr z2I0zEOvM?un2O7bj6sSx^Gi#h<tbZHYH?{!$t|Yjq+3kM*-<Rn`K2IU23nfG#h#X5 zRGwIrq6<p*It&a9Y%FYyY>X<5LX2EYe5^{0T#Q1DYK&}*VvIb@e2g+oJd9dQ222u+ zOg~t-s(8S;0%`<GmH_#g7nUWMm_S(~ivd(k_ky!T76YiF>Se5D1{FNuN=Lkg877*? z2ugDCpiBd)kd#1)AdfvAluVcxF~TzrINQ{+gECGGQ!Pg=sOVu$VTETQE>Kok0Fvhb zSALu&T+k#qua>8TtA-Vv9H-avmT=WDWI+;fe=Q%}O-oAH7Vy-t)v$w-axH%iV+~IY zA1Kk)@Pm^g2Q(>if|8;QLkZgg-WrB1z8anyUXZF1{t~tYpybG#!YIkGkg=H&EFxIL zkOdb>V*-moR54?yf{LUur86yLsujr7gScoRBX-x2>xOhjkm)rHS)4U|Ah)wJNHUZN zWeL{^)bN8^bG^*w49pBR422<uzJ=~J3|S&Y|7#esL>GwFFl2#>De;A%h60Es0Tz?2 zVaQ?vi%EhCeh}8=@B`;HO~zX+i4~c}klY8#j76HD<jY!?n3I{J$yfwRi<(lmn2Squ zi@=!+RMQoK@&Quy3`$_PxWJhzJ|{CfwFs1ZZV8~b2Z~HVc?gv2icCN(P)32|D?^Z- zEXk>vIk#BTa`N+w!1=Mr45Ug8M1X3QTLQ_6ImxA<>=|ELkOIo3;vhLtWdq5og5a!r zizl_HD8C3=K18u4fw@s^m0<c7S5jh0az;Ej4|61EB<6wIIYpKrldM35Er_rK5!N8W z21Gc52vChu<OFglE5sz`%)A2hJS+%tS$siaQDSatNorAXFeo>JOyObSVUl6eU<T!4 zJ|-SU9!5S!E=E2^1x5izHAXc?rvGgJx!A-QZ5V}^*cf^KGks$E&n6_oD8b0`uSyJ@ zyD<#M$mj6Jga;$25xW4?Twq!V%I6GO%rQ)$>J5}b7qBj5s0BAJz!jlP2}2fp4QmQh z3Ue=09vhU$0^)&HF=cUpt3)=iYNHx9Sia6<tzk=HlVnI?NMnLmlT2A0DeRIAwVXAq zS)kqsBdDno!&D0@0XS<o!TCI-mZyZXh6~L0t>sN&ujMP@tbycuyIOuoBPW}wsH%iv z0ap!s4F@>a3xIPyKPW##D;j2Ku4jSd`aA)MdQcSyRtwJi2(>m0HGC!9Sv)lYHLNwf zpz27HA%z3fB>`7)e1*|9d^JpvGQ)<UFu2gS2GnEXhE!CVERcKxYIK8Zt|AUlLkFCP zQj0a2(Pcqd>E(Y=<p$QF1xiKQAOcjlYx06CqaqKG0JwGmRaxMyQ=|$K17$aG>picy zAip>jT+u<Y5IFaMYb<c1IX|Z?RTGl-i+n-plLMCXir|Skiaig~4uj-fL~=K0U|@)1 zD$kB$Do-u~r*m);X8{>eWC1dbIkl(=J(&xDb3lAaMrLt*4yXfJ1xn>05ArZ^F><l1 zF>)}fG4e18G4e3+FtRZ6fK$2>BNvkxqXH8j6Vn$K-YQXWnnpJXB}s!k3BsVf2X187 zFf3rGVFb5{7#1*sSfEy2FvChlKTW2hAW*nPf#L+x_yh+>Q8-8x+_VG}p!Vf0j-<ro zY*1x82jp;2*N=;li;;x|+E|ClCW9gl*0cn%K^Wwi&mb)|3|S1p3`No4-boRtFj~n7 zE`rd4tH_gqfnf;)1H)%fOO=fY>T^&KBlnk3d=63x!eGllxf<NdWB?W3U<-@lK(;c0 zt-A#(u1iy~*}4K|D;E<^zZAuS0to60aQPk&5&#Dlm;i?uFWkAHwmYcTUzEVWz)&Rw zjzttnoE|Ak1Q`snsVEi1N&*pJE5HQEo}x4c28InF@e~FI2IAZdsvU~3xVZ>acWeQf z1$HSP*ro8U3Nfxs2WiIUN|4DrK$66}5?oJVapf&OP=F$O!h1l*f*mXjb}*_eF|N)4 znSjmJnG6gJ2SAb_S8ED@MhWr~b5rBvZ*j%P=jNxB=788d@$rSFi8&A%a38lw5v0il zM8LCHQ4FYX;3y~p^$$xb!D;*!TXAYhNoo-|62YNg1hVoLFSO&HoSKspACHtyWkE)Q zqaBo=K*_vVkAZ=KgMo*ULxO{gLybe7Ly3uli;;zyi;;;L8!g0G1k$U?6U7bbo$G-{ zwTeJ_=N4;TX>LI!xSYMkTAZ9&Pzi2W+~Q0t%}XxH&(A3a_i>9rJxxfV%L;N3$UG!R f!qet04jV`y*@23IVo-U_!@vQq$~hQ$7)6)?+;MY< literal 4783 zcmd1j<>g{vU|={>>yY+JmVx0hh=Yt-7#J8F7#J9eofsGxQW#Pga~N_NqZk<(QkYVh zbC`0OqnN>LmK>&BmM9i5n>B|uip`xNg)N1>g&~DKl`)IGnK_Chg*BK#lj9{L0|SF5 z<1H4a#NyPKAgeSPZ?Wg4WacKOr)o0ZVk^!}&r8hlO9n|HV~|Tg{Nfe{28L9ID8>|q zD5eyVc7`;@6y_9`7M>{P6xI~B7KSL6RMr&s6pj{#X2vMCRJJVk6izsgHH9mM8!W?- z%8|vM!UN~Artqflfn_*TIkVVP_+h*hffT_OhA6HSp%md3hA8fK1{Q`Wo?r$|(OWFe z`FUlx*osn1N{jORl3`8+ITOTYV_;xl1_j<B1_p)_hGvEZj0+hU8B!Q)7~+{y7-|^e znZYCrm}CW$Y+#bT2Be0ghN*@jp0k9jh9QfynK6Yao2keg#7<$B1nK4mvsu7w9x$5~ z%;wF5@UW?3u3?DhE8)))SRe=rwi<>kzI4VKrbUcF3=2VWEU0p6j9^jL6oz01O*X%f zmmvQXfeBDVzGMQ$rqC_c;?jcDqFWsC@tJv<CGqjMI7^BWGxHLYa#C+`#pf54Waeg8 zr54>{FHg-(&nPLr#gdenSbR$$y(lpyGc~Uy9wyEU7Xd5aDK06>Oi7I|&PXgsy(O5O zpI4TjQ(BUlp9d9U&PmO?#h84HG4~c@-YxFD(%ks$)S|r9oZ?$dc?GxFiW2iu@^f#o zm*!=r<rn2@vfN@RPR&Ux5@KLrC=vz{5)2FsD;aNb#>Xe;Bo-IP$FF4g)ubO<oLW?@ zUzAvymzZ6gn46ealBn;JpIn-onpaY+A6!zJQmG&8>KkvU?~<BY5bu+kSd^EUm#*(x zky@0TS)8h$lA2nen*$S0g$f(T=O&h<7G);p=oc3y$Cu>CXXX~<q=HP<_erb-*$GNg zdIgn5+zbp1AY81*z`(%8$c6^_7<m|Z82K1^n0Of37-bm67+Dyrc)^-sk&q0E98g*W zVL^~73=9m;p!E2Uk%0l6!Wc>zYZ$Vani*3VvzdxSLF^PJNrqY`FpF7|p_Vxhl=_&E zL_zY*HH=xzU@;c32`tTwDJ<DcMR{OxMoETRRxpc2lA)HZh9QeJg;|0jg;AQJmc51{ ziw(}>s9~?+NC%~I_7aX7))dAXb`b_~29OR3h8iX*h7@LThGxcE&JvCqwiL!14yY`f z1VaroT$Zbb1(f=0IBU4lSZcZRRKa#bT!Csgdpamz5wHtpS{idLcL~=5?i%iejI}%^ zJT)xMj4lkZ3AMZ>;QR&VMbz@4guoWC4%QmBG^T}&wfuRk5I%?v*2N3fC0;8~!(YOe z#b3ja#ZbdrBap@f%DGtrHGB|u3Tp~e3R^FWIYTXf;XV`_Ue@rZFr+ZlFtsv+Y35dD zknTJ-gt#O_Eq@KjY{42nNl<w5)bQ5ur7_ts6fQ2DQv)iz*!_w)LCGJUm5VqS7#MDG zB<JT9mn7zutYo^ylAE8BdW$8wEHOt@=oU|AUO{O|d`W6vaeh&e2uLp|6WwA3=S@gn z5(9~fgUn$sNK8q|%uBz;oS9cr1j=JYk|1%`s??(V;#;gmiFxU%MWP_}ppp`7Z&`j$ zS!xj<NQ@s;tOyp#fY_=aeN4rbx0s46Zm}jMmLzA~VywKySP{huwIse0!K{d4u8hyW z#at1ee~Tx-v;^d4unTYTLU|A$++xa1y2X^49VG}AE=VnkPs_|HNiDj?l$l&453(9e zD1hu|Ps=YVPb^AN0c9UhUC74B$H>LV!^p+N#mL3P2O=5yz+!TYa*P5XF-8tXIj{;I zW{?a-Ooz#YS%Q)2D+_NG4>-3%4Ne9Xr=W}q!XP#XgEH%9P%Q#VCcz9V8U0o=7V&}$ zEJ)aZ9SA1CuH=N5v_T9E44*;ypN*-C4{Q%y3Z+;8DFI=yO`xU#q*wqI_eGYVAg}`2 z#ss$Q7He5zPH8Gy&=lD)FffF{tm9(B>5(D}ka-|OS2ErbjE5KW(4xP{8YB(&3zz_T zs7Qc;fguVcZp*;HP$dfX2)ag`{wcBp=>%C=1d6dDdk`0FHkg205C;->WME(*%IQ&j z2&aSUD6H-Vm9|MBW5MngLUlJt7cq`@0U3bJ(IC^)K$2ibgQ5?VXh0a00YT{l6rIIu zm_e<M1>h2XA!99533ClY7O12KmGu%JHn;=_i-KAdwJhMWo&`x1RMvyaR@Mb<3mIxz z!6vXn%X-rihAdE-&6>j2%aq3k<*|c!wXC4}Ba0Jk5?c*x7AL6b*vcfy5W`f<Udw@E zPCQgCsJ!NcvO)C*S01S5V9Mf50n3zd)v$rf&M;7^T*D4#``7ZMFxB#waMf^t`3|*w zC}z(nVOYQot{1?iaSdY)H>ezxWPp~H%+NBF1yqLGFysk9)PqWEY-(*7YIsX{vUqFw zYglV|ASR`7q%fs$f=g=N!txs48m1cV8Xj<2n_rk&!<faF!j!_*%UCN=Bap?H!VG2$ zqPX&AjUY@WkFiFuh9RE6gaI6HtP4T)l>prDm%x5!$O455vm^u9&&O-|z%4{UFmHb? z53<=47>m3ieyZUD*GNKOfAN57j2a<W{Q;(-VNk-bKnN5Iu#hSdE@4<8Qp1oX3X08z zjLnQ-5iw+uG$ybZL>03n1DYzRNE%Z*(?X_N;XFNvixx80Vs(uT19@&pX9SrJjvwI~ zp%ji3?q23vkrIXl;x)oqoC_JjEFlm}0^F{s6)j<|5rwpUCNLH)sS!zGr~&)371TNx z1?7oWCQwM_u_Mb#GSq^6Dga3VtPGM2H6rE=%nUXRg+B^E7Cx_G$dX85C}o_$SahX^ zAxm<B6sSE@!;mGgkg0|t9+WpAV$wAXS^Quz=^CbB22CEUb+RT)5vVSJG`hicHmIfs zb)ga~Qj0a2(Pb4uC4&;E0%g3#l30;htjSoU4C2Z_s$ec~6&#<FnVnh$s#_6tIk@h< z#gd#~PzkQj!L9hb;)49*)FMzTv`7nNv?hoESK6G(`9(#kIf*5yx0s7dbBjRb61d0) zSNOM>3ldX`G(fc{OLA&vPLVo@$(ojvpI>x~HL<8Du@YRDfh&+(0?COv$)zB(;!6us zK#gQ@8$b+F^YEk=73CK}8y`_n9;D(f(gC?g7gSMjhQRs}MS38+nDVl3G36x}>4Q`l zfCv*1VG2?!2r&^<>nCR<=H;d46q$kaFsBw3MX@D;Wuw?C!SpRwC<Sf;-C|2BO-oBH ziV{h!C`e5%Nll4|xFo(9)W-qWEnosv!QA3VN=(iMwH&fQEr%Qi1_n^89no^oV&q{I zV&r22iSjUkTJtR6c0C`X5+fIr7^4Cc9~09T9zGsM1+Xd+Mm|OXMm|ObMgc}OMm0vJ z|7`!cSi~4@7zG$*7_As3n3z5?{pS%DVH8312w-kV2DSD;^#lmBGB7ZJ+7{rtVi&lF zTEmdV2+0pcX5cb_A&XfA)EZ|IVW<Vw6QHz<5MildTmUMgL871@14so+8cPjx4GXx} z$X>$)kxyex0m-w1ML9r?3xw&&=GJ1En8r+uX`0M_;N~khD`;{;QZ%@C2X0}4YQ-W@ zvlNmpxj=o?%)}f}Y#~~ktdNu@3~i3YBPqHi0IIT4MA*_2^Wrn}Zn1;t{L&KiMo}aK z1H)=ijDzgvV-#W(VpIV2(b#zY@W6U&{NQE*!X8bgB6bD_20u;STO9H6d5O8H@$t8~ z;^TAkQ%Z9{Y@Yb|!qUVXhzz)cSY!@Lvm6CQ`30#(C6$n@#8#YIQj%H($%)`R839W6 zASd18g?4k3Q*(0S<B?n~3sM3~u|=RX0V-ySL1hRB10NHIB$FN&BMUPZBNH<!&A}|- z2v)Djd5fV4)DXGFTAZ9&P+0^jABsRJ>=tWYX>LI!r0C%U4f=rlxnNyL)+1#Y8;F1H UK!sj0s7cDh0P5;;Fp9AP0Jp_D;Q#;t diff --git a/exercise3_material/src_to_implement/Layers/__pycache__/Dropout.cpython-310.pyc b/exercise3_material/src_to_implement/Layers/__pycache__/Dropout.cpython-310.pyc index 55e7d8444d71d682326b5c82307a0416ab676b94..610ffb0a518f1921d2a3fb8c5dcce245df293fef 100644 GIT binary patch delta 20 bcmZ3<v66#3pO=@5fq{WRs;qG%w;T%qEOrC% delta 20 bcmZ3<v66#3pO=@5fq{V`*(rA;w;T%qEZGC` diff --git a/exercise3_material/src_to_implement/Layers/__pycache__/RNN.cpython-310.pyc b/exercise3_material/src_to_implement/Layers/__pycache__/RNN.cpython-310.pyc index d53a7162a2e4421e4084c7102e4f7af747776eb7..72c1568e112f4d122c9fa09557b1cfef22ec0701 100644 GIT binary patch delta 46 zcmbOsHA9LwpO=@5fq{V`Gq)`*eIu_9HzU(zEABYP%E>dia~W$Vv-8YjRG++zrw9N9 C-3%K5 delta 46 zcmbOsHA9LwpO=@5fq{WR;cR_c)<#|(ZbsJ0R@`xnHIrv@=Q1`-X6Kp5s5N;RPZ0nI CnhZJs diff --git a/exercise3_material/src_to_implement/NeuralNetwork.py b/exercise3_material/src_to_implement/NeuralNetwork.py index 9f49bdf..2dcb23e 100644 --- a/exercise3_material/src_to_implement/NeuralNetwork.py +++ b/exercise3_material/src_to_implement/NeuralNetwork.py @@ -1,60 +1,94 @@ -import copy - -class NeuralNetwork: - def __init__(self, optimizer, weights_initializer, bias_initializer) -> None: - self.optimizer = optimizer - self.loss = [] - self.layers = [] - self.data_layer = None - self.loss_layer = None - self.weights_initializer = weights_initializer - self.bias_initializer = bias_initializer - pass - - def forward(self): - loss_regularizer = 0 - self.input_tensor, self.label_tensor = self.data_layer.next() - for layer in self.layers: - self.input_tensor = layer.forward(self.input_tensor) - if self.optimizer.regularizer is not None: - loss_regularizer += self.optimizer.regularizer.norm(layer.weights) - loss = self.loss_layer.forward(self.input_tensor+loss_regularizer, self.label_tensor) - return loss - - def backward(self): - error = self.loss_layer.backward(self.label_tensor) - for layer in reversed(self.layers): - error = layer.backward(error) - pass - - def append_layer(self, layer): - if layer.trainable == True: - opti = copy.deepcopy(self.optimizer) - layer.optimizer = opti - layer.initialize(self.weights_initializer, self.bias_initializer) - self.layers.append(layer) - - def train(self, iterations): - self.testing_phase = False - for _ in range(iterations): - loss = self.forward() - self.loss.append(loss) - self.backward() - - def test(self, input_tensor): - self.data_layer = input_tensor - for layer in self.layers: - self.data_layer = layer.forward(self.data_layer) - return self.data_layer - - @property - def phase(self): - return self.phase - - @phase.setter - def phase(self, value): - self.phase = value - pass - - def norm(self, weights): - return self.loss_layer.norm(weights) \ No newline at end of file +from copy import deepcopy + +def save(filename, net): + import pickle + nnet=net + dlayer = nnet.data_layer + nnet.__setstate__({'data_layer': None}) + + with open(filename, 'wb') as f: + pickle.dump(nnet, f) + nnet.__setstate__({'data_layer': dlayer}) + + +def load(filename, data_layer): + import pickle + with open(filename, 'rb') as f: + net = pickle.load(f) + net.__setstate__({'data_layer': data_layer}) + + return net + +class NeuralNetwork: + def __init__(self, optimizer, weights_initializer, bias_initializer) -> None: + self.optimizer = optimizer + self.loss = [] + self.layers=[] + self.data_layer = None + self.loss_layer = None + self.weights_initializer = weights_initializer + self.bias_initializer = bias_initializer + + self._phase = None + + def __getstate__(self): + return self.__dict__.copy() + + def __setstate__(self, state): + self.__dict__.update(state) + return self.__dict__.copy() + + @property + def phase(self): + return self._phase + + @phase.setter + def phase(self, value): + self._phase = value + + def forward(self): + inp,op = self.data_layer.next() + self.label = op + regularization_loss = 0 + # print(inp) + for layer in self.layers: + inp = layer.forward(inp) + try: + regularization_loss += self.optimizer.regularizer.norm(layer.weights) + except: + pass + layer.testing_phase = True + + # inp = self.loss_layer.forward(inp, self.label) + self.pred=self.loss_layer.forward(inp+regularization_loss, op) + return self.pred + + def backward(self): + # loss = self.loss_layer.forward(self.pred, self.label) + loss = self.loss_layer.backward(self.label) + for layer in self.layers[::-1]: + loss = layer.backward(loss) + + + def append_layer(self, layer): + if layer.trainable: + layer.optimizer = deepcopy(self.optimizer) + layer.initialize(self.weights_initializer, self.bias_initializer) + + self.layers.append(layer) + + def train(self, iterations): + for i in range(iterations): + loss = self.forward() + self.backward() + self.loss.append(loss) + + def test(self, input_tensor): + inp = input_tensor #self.data_layer.next() + # print(inp.shape) + for layer in self.layers: + inp = layer.forward(inp) + # print(layer) + return inp + + diff --git a/exercise3_material/src_to_implement/NeuralNetwork_o.py b/exercise3_material/src_to_implement/NeuralNetwork_o.py new file mode 100644 index 0000000..a38d72b --- /dev/null +++ b/exercise3_material/src_to_implement/NeuralNetwork_o.py @@ -0,0 +1,65 @@ +import copy + +class NeuralNetwork: + def __init__(self, optimizer, weights_initializer, bias_initializer) -> None: + self.optimizer = optimizer + self.loss = [] + self.layers = [] + self.data_layer = None + self.loss_layer = None + self.weights_initializer = weights_initializer + self.bias_initializer = bias_initializer + pass + + def forward(self): + loss_regularizer = 0 + self.input_tensor, self.label_tensor = self.data_layer.next() + for layer in self.layers: + self.input_tensor = layer.forward(self.input_tensor) + #if self.optimizer.regularizer is not None: + try: + if layer.trainable: + loss_regularizer += self.optimizer.regularizer.norm(layer.weights) + except: + pass + + loss = self.loss_layer.forward(self.input_tensor, self.label_tensor) + return loss + loss_regularizer + + def backward(self): + error = self.loss_layer.backward(self.label_tensor) + for layer in reversed(self.layers): + error = layer.backward(error) + pass + + def append_layer(self, layer): + if layer.trainable == True: + opti = copy.deepcopy(self.optimizer) + layer.optimizer = opti + layer.initialize(self.weights_initializer, self.bias_initializer) + self.layers.append(layer) + + def train(self, iterations): + self.testing_phase = False + for _ in range(iterations): + loss = self.forward() + self.loss.append(loss) + self.backward() + + def test(self, input_tensor): + self.data_layer = input_tensor + for layer in self.layers: + self.data_layer = layer.forward(self.data_layer) + return self.data_layer + + @property + def phase(self): + return self.phase + + @phase.setter + def phase(self, value): + self.phase = value + pass + + def norm(self, weights): + return self.loss_layer.norm(weights) \ No newline at end of file diff --git a/exercise3_material/src_to_implement/__pycache__/NeuralNetwork.cpython-310.pyc b/exercise3_material/src_to_implement/__pycache__/NeuralNetwork.cpython-310.pyc index 56626f6b19db93890ac32aee063ad04b3cbcbccc..8582be15085fbe6387611208a5ad7b01636b789c 100644 GIT binary patch literal 3078 zcmd1j<>g{vU|`T`YD=r(Vqka-;vi#Y1_lNP1_p*=Jq8Ab6owSW9EM!RC`Lwx6s8pB z7KSLM6qXd$7KSKhcZL+U6!sQ|6!vDOD3%nCU<OUjmmrfg8E<i<q^1@m=ND8aGl4{) zn2mvffrEj8!5L(H2FQ5E6owpzTBaJtTIL$&EXG=v8s-H|DNHF0nT!h=85v5LYZy|P zo0<CgYFTTT7O>PX*RU>RVq^$sNMT@MU}0!xW@O0YFk~oJD`#Y60KrHGMh1u(H7pP_ zQW%06G+F$L7#SECZgHg~mL$gKBvz&t-C`<F(qz5GR*;#Tos(Jw^7bvB`1s<~lH!uY zlGOP4TP*nnsd=|pQc80RG}&%(q-Ex$<|XE)-eS&6Eh%DUU|@)1$;(SEiDFBE7|58m zlA(x)fq~&yi+*TvYEiL%QDSLcVs>$2Zem_ZqP|Oha%paAUP-Zja7k%OrGBuhZ@i(t zOKNICyiaOkQC?<Vy1r{gYEg1#ajHHj0CjU<!l_VU<M`aflGLKi#2o$NqU89J{P@h= zf}B*4srr7YrA3K3eyJtp`9;}!1(mm0iWAFHLH;Th00lfF3nLpNA0rPFAEV5FA*Lce z1_lO{=mB{N9z6j_(Nn@$!;r$%%+xPh%Ur{>fT@OgAt<U?7BHtU*DykY897R7Sb`Z0 z8HyEAB83Gh8kmZbzyYqw0uIL_c2L}~<m4x&6oJx+CTkHV0|NtC2}A)mhzm-$;P8bI zydW`<&p;l5dXa~bi&21)>mLh?z#kTtB9Lq{H1WYHP>6sNI4A|CGDI<^fD*M>3Udle z3riGp3Tp~m3qur33VRAi3qurZDq9L?3Req5Gh-BcDti`33O9_G!jr<=!Vtxo!k5C| z!VtxkB9J23!VtxsB9tQB!VtxiB9bE7!Vtxq!W7J)DSnF=C4z3T6{VJx7UlURv%p=( zzyM-{att_!m@qIf)G*dC#51Hb)G)*|f=Q+n5StlHveYouFvPRgFxN1|vw=m}L8*Nu zqn{?nEzbOclFZ!9s??%eEIIkb#kbhNsk9gzs<*g6LePvLT%MYlo>5X9pP83gl9`wT z)+3OVnTRUF7GIE&Se&ZKa*L%nH7Bh|1Qh-fAVLyEfD%3;Vs3H7gN=)i7YAi_P(-jX zB10~wDlwD*(SvD72IW*(n1I;u00AX{8ip)}1&j+Bf*Fd085kHenLswBWG0t@GAyXX zS;<%=jm14Y@$u>KtgQrc6UbOD#wsavS0I$3<PMN>P|^WMjTk6uz%Ho)7gS(ZF(F*V zTqFxh(rl##DWLo+2X-mgUAI`l#S0{)z+nbX$Dm}d3UU=F1%Og9rn?X&6GjMuLJ90H zkTD>)KtpFGW04dnEaX81*fKByvhfxxBsM{Ig1pAYR3(fW5@30ZAONWaTLq2`aOwo5 zIdEhYDI@!{EHS4v72TgI3=9mqFgv-JP<#R^e?TPx*!Ti?2rp!aVX9@UWdbL5rW8<B z!H~u5!VoK6%TmL#fCUr-wTx9fplp!En#Hz&y@n-=V<DqCLoIVSLmmsLT3}>|WJqDE zVTtD~<ga1K;sWWaVHRhoVOj{vLfkbBS=_-4n#_=DB1Dr1l1o_fQY%VsvF0QurREfY z;`|nST7FS^Vo?e>_21$yN=+}#Nh|{A7?!;JqTE~T(9Cp;w<NW=Br`7^l52`U+29sS zK~ZXoCMzT%FlXi!++xZvxFrlXC9xzkKQA7XH9?kErWT<Gp9TX1gCQvRK>3c3k&Tgs zk&B6ok%>`)k&Tgwk%Li$nT@eZ6g48DPDTl9P)Y_RHE>uP!Na<SA&Y4tV=ZG1Ll$!i zxb|;lmSk{Yh-C+b{{luxSQqkxqFj^7@8$ph|Nm>UKs?8hl$e|i4jXk)asvgICNtQp zMWP@9kh{_SW6Z$7-~{pysNUgY5@F<FWciO?Fu=^kNQ<Z*V9a7D5(HIMSxn80HH`60 zHH-_GYZ$UvY8bM>t%e!~aN+=GSda(6X%<vcmJ}ss<|QWOq!xivHzaDgkP1UkU>AW4 zIgqDru_YE1q~@h)GC@3t#b390AW9$wtS8KaY>YgN0*nGo=y@AiF^VTa9t2@<EP>i% zB@8u;&5SM#u>!SBHQ;ivmbr#u0n<WIvxgyz8Pr4qHJl1r5FTW`#afh@m!4V#N<2lN z5QoHv87Q-|fc<cbE3+iE2%P$gZ!u;<<iSM+n1GZQ;7|fJ3yMKmpNElyk%N(sk>xLX zxdB#(+Z&*=um+T!8L}V=i@An*0b>m)*)bKe)-XYfE=?wIh66iV6Y5Hy%)ElqlK7I; zyyE<#B2ah~VGDc~P|gd31v)5)@G!D4RSBWy3XmdArXnE*1_nP(K2R+JYTU)g-{Ojo z&&^LM%>gw5dE(;>OA~V-GDR|=WUK%pK;`T$j)J26g4Cjt$|6vyc8jezwWK7q2<&1= zFq?yvT7w8%5CINL1OW=wB0&ZQ22i3Z29?ts3|x#H+#JkIEL@B%%v_8d%p8Io>>TVo zY+!YMnyf{j(1WB>P^$;rzyZr4Wh}4_Aj5BQ*g!mD2dW>6L0OE4frE*ITZjVy15=9u literal 2357 zcmd1j<>g{vU|@J}o13<Xoq^#oh=Yuo85kHG7#J9er5G3(QW#Pga~Pr++!<1sQkYv9 zQka{Wq8L*cgBdhgUV?P_C4)$04A-B)z`&5o5XG3n5XF=tlER$A(!vtOoWh#I*1{0Q zlER+C(ZUeLn!=gF)xr?PmcpIF)4~wNp2C~L*TN9Rk;<9EpCZu0(99UcmCBXHogxV1 zr3j@6w=hKUq%Z|DXo}wA^-C=+O3d*~Eh*10%D%-`lv+|+l;@Ysg5)R=n~8ye!5QQ) zJq8Ab8paxic!qR_8isgAFv*kxVl#tDmKvrShIrN*<{E~0wiJe7hLwzdn(VhY^9xEc zb2F<_i*B*x<QEs;V#`UaOf4$D#g&p+k{A!>+~NYsK-t3OshQ~+CB^ZXd6^}di8)}c z0!f*PC?c9Hw^)i(bJB`f85kIfKyg#V&A`C0lA(x~fq~&yi+*TvYEiL%QDSLcVs>$2 zZem_ZqP|Oha%paAUP-Zja7k%OrGBuhZ@i(tOKNICyiaOkQC?<Vy1r{gYEg1#ajJew zYHERQ4oo-|Dr_8|n^=-sl$n^LUtE+NUy>i6nOl&P3Nlq6B}nxODsOSbgFO)+4+_a* z0R{#JHb!K~!dNASstKkcnGr1>K;8t$Ll!7~G1fBHFk~?-U|h%$0}6EzX0Bn#VsT-J zm8@l|VOqdi!;r<a5ENfbH4Is7S?nncrHn=Dpcn-47I4%sWpOTKG-s$~Eaa<U$l`+N zs9_XmsA0%rUdU9-T*Dm9pvmM{1agWdcM%5z1H&zrywr-4TRfS01*IkNC8>GE`9-&Q zauSnLbD+E;c91gmwEUv-#G({%?A+omN=+}#Nh|`#Buid?QSL2vXe1YLGB7Y`vJ~+! zFfc?3fKvgoVpeeSC}IP}F(_Wakz2$E;tDe`Fn}1vLJSNHEQ~^oT#S5-Jd7fYObud; zER0p6r~wOg5K6d$@*F5;z~QO`O5flxg@jx!V+lA!n;BggV%foA%2>m=kg=ArkO!2y zi$FRxS-~E-#gUYloDKFc$Thb(ic-r`i;7cIz@eeZ4EEkF*3_b+{GuXpkn7PrBf-GH z0K&zfK;{E`1T$^G>_`UXNm%kg@q-P>4~$t1DU8`nMPen43qWztypR#(`WnV~<{HKY zEHw;SptQlZkckl#)9ed4K)zyR0QpIi$uC5c^A=}GQDSCZVp2}(EtcebP&x+X1rVzU zlnifiA>~YX&I5b(7F%LLL26zK)XQMMMzMfO07%S%y$y=BTRaejkUTF3Dquh+aWV2R zvM~xU3NThlp~fGwa+Cl7MFR+fBhdt$IY5O}31bZ-IO+synQ9moFfC-L1w|+G0+t%) zg^VDtFJN8Bz{pU@4)wU3CfhCElGNgo%)IpYf{euC)LX1YiFxU%MWB=hNrLjAr~}0l zG{xTH$}CAON-W9D&nv#g7!Oed_9K`8dz2L%n5rNjf)Y6!BM&17BM&1NBg@|^VN^eY zRibzfC3e-ou?zBC4MP?vRdRyj6&%M5;GE7-$ObAEf*BxblNp?4!SMq211wSmLB7C} z3U9H1ybsC;;AF_g$i*nY$i`SDgyzcB;u4f>1S%gu7#stjtcv7_m5jGo!GWpC2+jm( z$)HGsfq}sY6gHqV&BlZtB4Dj3Ap%kd!eGllNd{D+#e-8X*w;nspaO*nY}qZ=vc#Oy zRBU#df$Rh&8#X2uMhpu;!49?n6k{OYEC6SHSimxYGkuXL!loiIP?&-v9ZY~?9xQJS zvIpc|9*l4Vm4lj0MIgKUG<iXFMP6cVYJB`HuK4)e{FKrh5Su4HzOXbg2O?9%53&WE z96^b&NCCtJrK}=Q(zwM@P?TSgT2xXA4%J(1#i=DFsYPICAO!)!eIRFnY%c~G#=*eF z$ic<I#lgnG&cV*a#>L3O%*DvVQUp?^$y6i*vJ1q<X4EYX8;FbTK<S_u6s{Z$LQDXI CAp6Py diff --git a/exercise3_material/src_to_implement/log.txt b/exercise3_material/src_to_implement/log.txt index 178c80c..022ddba 100644 --- a/exercise3_material/src_to_implement/log.txt +++ b/exercise3_material/src_to_implement/log.txt @@ -10,3 +10,124 @@ On the Iris dataset, we achieve an accuracy of: 94.0% On the Iris dataset, we achieve an accuracy of: 98.0% On the Iris dataset using Dropout, we achieve an accuracy of: 54.0% On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 66.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 57.99999999999999% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 66.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 57.99999999999999% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 64.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 62.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the UCI ML hand-written digits dataset using Batch_norm and L2 we achieve an accuracy of: 97.32888146911519% +On the UCI ML hand-written digits dataset using Batch_norm we achieve an accuracy of: 97.66277128547578% +On the UCI ML hand-written digits dataset using ADAM we achieve an accuracy of: 97.32888146911519% +On the UCI ML hand-written digits dataset using L1_regularizer we achieve an accuracy of: 87.31218697829716% +On the UCI ML hand-written digits dataset using L2_regularizer we achieve an accuracy of: 65.4424040066778% +On the UCI ML hand-written digits dataset using Dropout we achieve an accuracy of: 90.15025041736226% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 84.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 62.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the UCI ML hand-written digits dataset using Batch_norm and L2 we achieve an accuracy of: 95.99332220367279% +On the UCI ML hand-written digits dataset using Batch_norm we achieve an accuracy of: 98.49749582637729% +On the UCI ML hand-written digits dataset using ADAM we achieve an accuracy of: 95.99332220367279% +On the UCI ML hand-written digits dataset using L1_regularizer we achieve an accuracy of: 84.97495826377296% +On the UCI ML hand-written digits dataset using L2_regularizer we achieve an accuracy of: 86.64440734557596% +On the UCI ML hand-written digits dataset using Dropout we achieve an accuracy of: 86.47746243739566% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 86.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 60.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the UCI ML hand-written digits dataset using Batch_norm and L2 we achieve an accuracy of: 97.16193656093489% +On the UCI ML hand-written digits dataset using Batch_norm we achieve an accuracy of: 97.8297161936561% +On the UCI ML hand-written digits dataset using ADAM we achieve an accuracy of: 96.16026711185309% +On the UCI ML hand-written digits dataset using L1_regularizer we achieve an accuracy of: 76.29382303839732% +On the UCI ML hand-written digits dataset using L2_regularizer we achieve an accuracy of: 85.30884808013356% +On the UCI ML hand-written digits dataset using Dropout we achieve an accuracy of: 87.31218697829716% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 60.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 88.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 54.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 68.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 76.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 57.99999999999999% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 60.0% +On the Iris dataset, we achieve an accuracy of: 94.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 60.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 98.0% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the UCI ML hand-written digits dataset using Batch_norm and L2 we achieve an accuracy of: 96.661101836394% +On the UCI ML hand-written digits dataset using Batch_norm we achieve an accuracy of: 97.8297161936561% +On the UCI ML hand-written digits dataset using ADAM we achieve an accuracy of: 96.4941569282137% +On the UCI ML hand-written digits dataset using L1_regularizer we achieve an accuracy of: 86.81135225375625% +On the UCI ML hand-written digits dataset using L2_regularizer we achieve an accuracy of: 78.79799666110183% +On the UCI ML hand-written digits dataset using Dropout we achieve an accuracy of: 94.49081803005008% +On the Iris dataset, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 98.0% +On the Iris dataset using Batchnorm, we achieve an accuracy of: 98.0% +On the Iris dataset using Dropout, we achieve an accuracy of: 96.0% +On the Iris dataset, we achieve an accuracy of: 96.0% -- GitLab