final version

model v2/convert_bytes.py

+def convert(imgf, labelf, outf, n):
+    f = open(imgf, "rb")
+    o = open(outf, "w")
+    l = open(labelf, "rb")
+
+    f.read(16)
+    l.read(8)
+    images = []
+
+    for i in range(n):
+        image = [ord(l.read(1))]
+        for j in range(28*28):
+            image.append(ord(f.read(1)))
+        images.append(image)
+
+    for image in images:
+        o.write(",".join(str(pix) for pix in image)+"\n")
+    f.close()
+    o.close()
+    l.close()
+
+convert("train-images.idx3-ubyte", "train-labels.idx1-ubyte",
+        "mnist_train.csv", 60000)

model v2/data_gen.py

-import numpy as np
-import random
-
-csv_path='./data.txt'
-
-class Rectangle:
-    
-    def __init__(self, x, y, width, height):
-        
-        self.x=x
-        self.y=y
-        self.width=width
-        self.height=height
-        self.matrix_size=10
-        self.matrix=self._init_zeros_matrix()
-        self.fill()
-        
-    def _init_zeros_matrix(self):
-       return np.zeros(shape=(self.matrix_size, self.matrix_size), dtype=np.float32)
-     
-    def add_noise(self):
-        
-        new_matrix=np.zeros(shape=(self.matrix_size, self.matrix_size), dtype=np.float32)
-        
-        for i in range(10):
-            for j in range(10):
-                new_matrix[i][j]+=self.matrix[i][j]+np.random.poisson(lam=2.0)
-        return new_matrix
-        
-    def fill(self):
-        
-        for i in range(self.width):
-            for j in range(self.height):
-                self.matrix[self.x+i][self.y+j]=10.0
-                           
-    def get_matrix(self):
-        return self.matrix
-     
-def gen_data(n_samples):
-    
-    rectangles={0:Rectangle(0,0,2,2),
-                1:Rectangle(3,3,2,2),
-                2:Rectangle(5,5,2,2),
-                3:Rectangle(7,7,2,2),}
-    data=[]
-    
-    for i in range(0,n_samples):
-        label=i%4
-        rect=np.array(rectangles[label].add_noise())
-        rect_reshaped=np.reshape(rect,[1,100])
-        
-        data_sample=[]
-        data_sample.append(rect_reshaped)
-        data_sample.append(label)
-        data.append(data_sample)
-        
-    return data
-    
-

model v2/get_mnist.py

+import numpy as np
+mnist_path='mnist_train.csv'
+
+def get_mnist_data(n_samples=5000):
+
+    mnist_data=[]
+    
+    with open(mnist_path, 'r') as reader:
+        
+        for i,line in enumerate(reader):  
+                       
+            string_list=line.split(',')
+            float_list=[float(i) for i in string_list]
+            features=np.array(float_list[1:])/255.0
+            label=float_list[0]
+            
+            #if label>=3.0:
+             #   continue
+            
+            mnist_data.append([features,label])
+            
+            if i>n_samples:
+                break
+            
+    return mnist_data
+
+#data=get_mnist_data()
\ No newline at end of file

model v2/model_mnist.py

+import numpy as np
+from data_gen import gen_data
+from get_mnist import get_mnist_data
+import random
+
+N_INPUT = 784 # number of input features
+N_OUTPUT= 10    # number of output classes 
+
+val_after_iter= 10 # evaluate the training process after number of iterations
+#n_samples= 500     # how many data samples should be used during training
+
+learning_rate= 0.1   # learning rate
+
+class Model:
+    
+    def __init__(self, nodes=[N_INPUT, 50, N_OUTPUT]):
+        
+        self.nodes=nodes
+        self.num_layer=len(self.nodes)
+        self.weights=[]
+        
+        #initialize the weights for the NNs
+        for i in range(0,self.num_layer-1):
+            temp_weights=np.random.normal(loc=0.0, scale=0.4, size=(self.nodes[i], self.nodes[i+1]))
+            self.weights.append(temp_weights)
+    
+    def tanh(self, x, derivative):
+        '''Tangens hyperbolicus'''
+    
+        if not derivative:
+            return np.tanh(x)
+        else:
+            return (1-(np.tanh(x)**2))
+        
+    def sigmoid(self,x , derivative):
+        '''Sigmoid function. '''
+ 
+        if not derivative:
+            return 1/(1+np.exp(-x))
+        else: 
+            return self.sigmoid(x, False)*(1-self.sigmoid(x, False))
+    
+    def relu(self, x, derivative):
+        '''Rectifier linear unit. '''
+        
+        if not derivative:
+            return x*(x>0)
+        else:
+            return 1*(x>0)
+    
+    def activation(self, x, derivative=False, f='sigmoid'):
+        '''Activation function (sigmoid by default)
+        @param x: input data
+        @param derivative: boolean if the derivative of the activation is needed.'''
+        
+        if f=='sigmoid':
+            a=self.sigmoid
+        elif f=='tanh':
+            a=self.tanh
+        elif f=='relu':
+            a=self.relu
+        
+        if not derivative:
+            return a(x, False)
+        else: 
+            return a(x,True)
+       
+    def mean_squared_error(self, output, target):
+        '''Computes the mean squared error between the output of nn and the target. '''
+        return np.sum(np.power(target-output,2))/len(output) 
+    
+    def accuracy(self, output, target):
+        if np.argmax(output)==np.argmax(target):
+            return 1
+        else:
+            return 0
+           
+    def train(self, data):
+        '''Training of the neural network.
+        
+        @param data: matrix that contains 100 features and 1 label
+        '''
+        
+        random.shuffle(data)
+        
+        for epoch in range(0,1):
+            
+            outputs=[]
+            error=0
+            acc=0
+        
+            #iterate over the dataset
+            for n in range(0,len(data)):
+                
+                x=data[n][0] #take the features
+                label=data[n][1] # take the labels
+                
+                x=np.reshape(x, [784,1]) #bring the features into right shape
+                zeros=np.zeros(shape=[N_OUTPUT,1])
+                zeros[label]=1
+                y=zeros
+            
+                z_=[] # storage of neurons values before activation
+                a_=[] # storate of neurons values after activation 
+                a_.append(x)
+                delta=[] 
+                dEdW=[] # storage of weight gradient matrices
+                
+                #Forward step
+                for i in range(0, self.num_layer-1):
+                    z=np.dot(self.weights[i].T,a_[i])
+                    z_.append(z)
+                    a=self.activation(z_[i], False)
+                    a_.append(a)
+                outputs.append(a_[-1])
+                
+                #Backpropagation
+                
+                #comute the gradient matrix for the last matrix
+                temp_delta=-(y-a_[-1])*self.activation(z_[-1],True)
+                delta.append(temp_delta)
+                temp_dEdW=np.outer(a_[-2],temp_delta)
+                dEdW.append(temp_dEdW)
+                
+                # compute the gradient matrices for the rest 
+                for i in range(0, self.num_layer-2):
+                    temp_delta=np.dot(self.weights[self.num_layer-(i+2)],delta[i])*self.activation(z_[self.num_layer-(i+3)],True)
+                    delta.append(temp_delta)
+                    temp_dEdW=np.outer(a_[len(a_)-(i+3)],temp_delta)
+                    dEdW.append(temp_dEdW)
+                
+                #Updating weights
+                for i in range(0, self.num_layer-1):
+                    self.weights[i]=self.weights[i]-learning_rate*dEdW[self.num_layer-(i+2)]  
+                
+                #compute the mean squarred error
+                e_=self.mean_squared_error(a_[-1],y)
+                error+=e_
+                
+                acc+=self.accuracy(a_[-1], y)
+                
+                #make a evaluation of the error progress
+                if n>0 and n%val_after_iter==0:
+                    print('epoch_nr.: %i, n_sample: %i, avg_mse: %.3f, acc.: %.3f' %(epoch, n, (error/val_after_iter),(acc/n)))
+                    error=0
+                    #print(acc/n)
+
+                
+if __name__ == "__main__":    
+    
+    data=get_mnist_data()   
+    model=Model()
+    model.train(data)
+
+
+

model v2/model_v2.py → model v2/model_rectangle.py

@@ -141,5 +141,3 @@ if __name__ == "__main__":
     model=Model()
     model.train(data)
 
-
-