Back to Topics

from IPython.display import Image
Image(filename = "nn3.png", width=200)

Gradients:

$ { zh }_{ 1 }={ w }_{ 11 }{ x }_{ 1 }+{ w }_{ 21 }{ x }_{ 2 }+{ b }_{ 1 }\\ { zh }_{ 2 }={ w }_{ 12 }{ x }_{ 1 }+{ w }_{ 22 }{ x }_{ 2 }+{ b }_{ 1 }\\ { zh }_{ 3 }={ w }_{ 13 }{ x }_{ 1 }+{ w }_{ 23 }{ x }_{ 2 }+{ b }_{ 1 }\\ { h }_{ 1 }=\sigma ({ zh }_{ 1 })\\ { h }_{ 2 }=\sigma ({ zh }_{ 2 })\\ { h }_{ 3 }=\sigma ({ zh }_{ 3 })\\ { z }_{ out }={ w }_{ 1 }{ h }_{ 1 }+{ w }_{ 2 }{ h }_{ 2 }+{ w }_{ 3 }{ h }_{ 3 }+{ b }_{ 2 }\\ out=\sigma ({ z }_{ out })\\ \\ E=-t\cdot ln(out)-(1-t)\cdot ln(1-out) $

$ \frac { dE }{ { dw }_{ 1 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ { dw }_{ 1 } } -(1-t)\frac { 1 }{ 1-out } (-1)\frac { { d }_{ out } }{ { dw }_{ 1 } } \\ =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ { dw }_{ 1 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ { dw }_{ 1 } } \\ \frac { { d }_{ out } }{ { dw }_{ 1 } } =\frac { { d }_{ out } }{ { dz }_{ out } } \frac { { dz }_{ out } }{ { dw }_{ 1 } } =out(1-out){ h }_{ 1 }\\ Similarly:\\ \frac { dE }{ { dw }_{ 2 } } =out(1-out){ h }_{ 2 }\\ \frac { dE }{ { dw }_{ 3 } } =out(1-out){ h }_{ 3 } $

$ \frac { dE }{ { dw }_{ 11 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ { dw }_{ 11 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ { dw }_{ 11 } } \\ \frac { { d }_{ out } }{ { dw }_{ 11 } } =\frac { { d }_{ out } }{ { dz }_{ out } } \frac { { dz }_{ out } }{ { dh }_{ 1 } } \frac { { dh }_{ 1 } }{ { dzh }_{ 1 } } \frac { { dzh }_{ 1 } }{ { dw }_{ 11 } } =\\ ={ w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })out(1-out)\\ \frac { dE }{ { dw }_{ 11 } } =-t\frac { 1 }{ out } out(1-out){ w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })+\\ +(1-t)\frac { 1 }{ 1-out } out(1-out){ w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })=\\ -t(1-out){ w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })+(1-t)out\cdot { w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })=\\ ={ w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })(-t+t\cdot out+out-t\cdot out)=\\ ={ w }_{ 1 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })(out-t)\\ $

Similarly:

$ \frac { dE }{ { dw }_{ 12 } } ={ w }_{ 2 }{ x }_{ 1 }{ h }_{ 2 }(1-{ h }_{ 2 })(out-t)\\ \frac { dE }{ { dw }_{ 13 } } ={ w }_{ 3 }{ x }_{ 1 }{ h }_{ 3 }(1-{ h }_{ 3 })(out-t)\\ \frac { dE }{ { dw }_{ 21 } } ={ w }_{ 1 }{ x }_{ 2 }{ h }_{ 1 }(1-{ h }_{ 1 })(out-t)\\ \frac { dE }{ { dw }_{ 22 } } ={ w }_{ 2 }{ x }_{ 2 }{ h }_{ 2 }(1-{ h }_{ 2 })(out-t)\\ \frac { dE }{ { dw }_{ 23 } } ={ w }_{ 3 }{ x }_{ 2 }{ h }_{ 3 }(1-{ h }_{ 3 })(out-t)\\ \\ $

$ \frac { dE }{ db_{ 2 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ db_{ 2 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ { db }_{ 2 } } \\ \frac { { d }_{ out } }{ db_{ 2 } } =\frac { { d }_{ out } }{ { dz }_{ out } } \frac { { dz }_{ out } }{ { db }_{ 2 } } =out(1-out)\\ \frac { dE }{ { db }_{ 2 } } =-t\frac { 1 }{ out } out(1-out)+(1-t)\frac { 1 }{ 1-out } out(1-out)=\\ =-t(1-out)+(1-t)out=out-t\\ $

$ \frac { dE }{ { db }_{ 1 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ db_{ 1 } } -(1-t)\frac { 1 }{ 1-out } (-1)\frac { { d }_{ out } }{ { db }_{ 1 } } =\\ =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ db_{ 1 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ db_{ 1 } } \\ \frac { { d }_{ out } }{ { db }_{ 1 } } =\frac { { d }_{ out } }{ { dz }_{ out } } \frac { { dz }_{ out } }{ { db }_{ 1 } } =\frac { { d }_{ out } }{ { dz }_{ out } } (\frac { d }{ { db }_{ 1 } } { w }_{ 1 }{ h }_{ 1 }+\frac { d }{ { db }_{ 1 } } { w }_{ 2 }{ h }_{ 2 }+\frac { d }{ { db }_{ 1 } } { w }_{ 3 }{ h }_{ 3 })=\\ =\frac { { d }_{ out } }{ { dz }_{ 0 } } ({ w }_{ 1 }\frac { d{ h }_{ 1 } }{ { db }_{ 1 } } +{ w }_{ 2 }\frac { d{ h }_{ 2 } }{ { db }_{ 1 } } +{ w }_{ 3 }\frac { { dh }_{ 3 } }{ { db }_{ 1 } } )=\\ =\frac { { d }_{ out } }{ { dz }_{ 0 } } ({ w }_{ 1 }\frac { { dh }_{ 1 } }{ { dzh }_{ 1 } } \frac { { dzh }_{ 1 } }{ { db }_{ 1 } } +{ w }_{ 2 }\frac { { dh }_{ 2 } }{ { dzh }_{ 2 } } \frac { { dzh }_{ 2 } }{ { db }_{ 1 } } +{ w }_{ 3 }\frac { { dh }_{ 3 } }{ { dzh }_{ 3 } } \frac { { dzh }_{ 3 } }{ { db }_{ 1 } } )=\\ =out(1-out)({ w }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 2 }{ h }_{ 2 }(1-{ h }_{ 2 })+{ w }_{ 3 }{ h }_{ 3 }(1-{ h }_{ 3 }))\\ \\ \frac { dE }{ { db }_{ 1 } } =-t\frac { 1 }{ out } out(1-out)(...)+(1-t)\frac { 1 }{ 1-out } (...)=\\ =(out-t)({ w }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 2 }{ h }_{ 2 }(1-{ h }_{ 2 })+{ w }_{ 3 }{ h }_{ 3 }(1-{ h }_{ 3 }))\\ (1-t)\frac { 1 }{ 1-out } out(1-out)({ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 6 }{ h }_{ 2 }(1-{ h }_{ 2 }))=\\ =(...)(-t+t\cdot out+out-t\cdot out)=\\ =(out-t)({ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 6 }{ h }_{ 2 }(1-{ h }_{ 2 })) $

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

def plot_points(X, y, m="_"):
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'blue', edgecolor = 'k',zorder=2, marker=m)
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'red', edgecolor = 'k',zorder=2, marker=m )

def plot_zone(X, y):
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'blue', edgecolor = 'k',zorder=2, marker="+")
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'red', edgecolor = 'k',zorder=2, marker="_" )        
    
def boundary(net,xmin,xmax,ymin,ymax,X,T,filename='boundary.png'):
    more=1
    #plt.xlim(xmin-more,xmax+more)
    #plt.ylim(ymin-more,ymax+more)
    
    #samples
    #x = np.linspace(xmin, xmax, 200)
    #stepsize 
    #x = np.arange(xmin, xmax, 0.1)

    border = []
    xList = np.linspace(xmin, xmax, 50)
    for x in xList:
        yList = np.linspace(ymin, ymax, 50)
        for y in yList:
            out = net.output([x,y])
            if out >= 0.5:
                border.append([x,y,1]) 
            else:
                border.append([x,y,0])

    border = np.array(border)
    solutionX = border[:, [0,1]]
    solutionT = border[:, [2]]
    #plot_points(solutionX, solutionT)
    plot_zone(solutionX, solutionT)
    plot_points(X, T,m="o")
    
    #x, y = zip(*border)
    #plt.scatter(x,y)
    plt.savefig(filename)
    
def view(p,xmin,xmax,ymin,ymax):
    more=1

    border = []
    xList = np.linspace(xmin, xmax, 50)
    for x in xList:
        yList = np.linspace(ymin, ymax, 50)
        for y in yList:
            out = p.output([x,y])
            if out >= 0.5:
                border.append([x,y,1]) 
            else:
                border.append([x,y,0])

    border = np.array(border)
    solutionX = border[:, [0,1]]
    solutionT = border[:, [2]]
    #plot_points(solutionX, solutionT)
    plot_zone(solutionX, solutionT)
    #plot_points(X, T,m="o")
    
    #x, y = zip(*border)
    #plt.scatter(x,y)
    #plt.savefig(filename)    

# Loss (cross entropy) error function
def error(t, output):
    return - t*np.log(output) - (1 - t) * np.log(1-output)

# Activation (sigmoid) function
def sigmoid(x):
    return 1 / (1 + np.exp(-x))


class Perceptron:
    def __init__(self, w=[], activation = None, alpha = 0.01, label = None):
        self.label = label
        self.w = np.array(w)
        self.x = np.zeros(self.w.shape[0])
        self.activation = activation
        self.alpha = alpha

    def output(self,x):
        self.h = self.activation(self.zh(x))
        return self.h

    def zh(self,x):
        #add 'fake' 1 value to x list for bias correspondence
        self.x = np.append(x,1)
        self.net = np.dot(self.x, self.w)
        return self.net

    def update(self,de_dw):
        self.w = self.w - de_dw * self.alpha

#gradients of weights
def gradients(net, t, x):
    t = t[0]
    x1 = x[0]
    x2 = x[1]    
    
    h1 = net.h1
    h2 = net.h2
    h3 = net.h3
    out = net.out
    w1 = net.p4.w[0]
    w2 = net.p4.w[1]     
    w3 = net.p4.w[2]     
    
    dE_dw1 = h1*(out-t)
    dE_dw2 = h2*(out-t)
    dE_dw3 = h3*(out-t)
    dE_dw11 = x1*h1*w1*(1-h1)*(out-t)
    dE_dw12 = x1*h2*w2*(1-h2)*(out-t)
    dE_dw13 = x1*h3*w3*(1-h3)*(out-t)
    dE_dw21 = x2*h1*w1*(1-h1)*(out-t)
    dE_dw22 = x2*h2*w2*(1-h2)*(out-t)
    dE_dw23 = x2*h3*w3*(1-h3)*(out-t)
    dE_db2 = out-t
    dE_db1 = (out-t)*(w1*h1*(1-h1)+w2*h2*(1-h2)+w3*h3*(1-h3))
    
    return [dE_dw11,dE_dw12,dE_dw13,dE_dw21,dE_dw22,dE_dw23,dE_dw1,dE_dw2,dE_dw3,dE_db1,dE_db2]

class Net:
    def __init__(self,alpha = 0.01):
        self.alpha = alpha
        self.p1 = Perceptron([0,0,0],sigmoid,alpha,1)
        self.p2 = Perceptron([0,0,0],sigmoid,alpha,1)
        self.p3 = Perceptron([0,0,0],sigmoid,alpha,1)
        self.p4 = Perceptron([0,0,0,0],sigmoid,alpha,1)

    #process output
    def output(self,x):
        self.h1 = self.p1.output(x)
        self.h2 = self.p2.output(x)
        self.h3 = self.p3.output(x)
        self.out = self.p4.output(np.array([self.h1, self.h2, self.h3]))
        return self.out

    #adjust weights
    def backward(self,g):
        dw11= g[0]
        dw12= g[1]
        dw13= g[2]
        dw21= g[3]
        dw22= g[4]
        dw23= g[5]
        dw1 = g[6]
        dw2 = g[7]
        dw3 = g[8]        
        db1 = g[9]
        db2 = g[10]
        self.p1.update(np.array([dw11,dw21,db1]))
        self.p2.update(np.array([dw12,dw22,db1]))
        self.p3.update(np.array([dw13,dw23,db1]))
        self.p4.update(np.array([dw1,dw2,dw3,db2]))
        
    def train(self,epochs):    
        self.errList = []
    
        for i in range(epochs):
            totalError = 0
            for x, t in zip(X, T):

                out = net.output(x)

                #compute error
                e = error(t,out)
                totalError += e

                de_dw = gradients(net, t, x)
                net.backward(de_dw)

            self.errList.append(totalError)
        
        return self.errList

data = np.array([
    [1,10,1],
    [3,10,0],
    [1.8,2.0,0],
    [0,6,0],
    [-1,-1,1],
    [-2,10,1],
])

X = data[:, [0,1]]
T = data[:, [2]]

datamin = np.min(X, axis=0)
datamax = np.max(X, axis=0)
xmin, ymin = datamin
xmax, ymax = datamax

#iterations
epochs = 4100

#learn rate
alpha = 0.02

#weights
w11= -1
w21= 1

w12= 0
w22= 0

w13= -1
w23= 1

w1 = 1
w2 = 0
w3 = 10

#bias
b1 = 1
b2 = 1

net = Net(alpha)
net.p1.w = [w11,w21,b1]
net.p2.w = [w12,w22,b1]
net.p3.w = [w13,w23,b1]
net.p4.w = [w1,w2,w3,b2]
errList = net.train(epochs)


view(net.p1,xmin,xmax,ymin,ymax)
plt.show()
view(net.p2,xmin,xmax,ymin,ymax)
plt.show()
view(net.p3,xmin,xmax,ymin,ymax)
plt.show()
#b(net.p4,xmin,xmax,ymin,ymax)
#plt.show()

print("final state")
print(net.p1.w)
print(net.p2.w)
print(net.p3.w)
print(net.p4.w)

boundary(net,xmin,xmax,ymin,ymax,X,T,'boundary3.png')

pd.DataFrame(errList, columns=['error']).plot()
plt.savefig("errors3.png")
plt.show()

print("final loss", errList[len(errList)-1])

final state
[-1.68388785  0.64169818 -2.7192972 ]
[ 0.86386984  1.64971577 -2.7192972 ]
[-5.07230654 -0.70116144 -2.7192972 ]
[ 5.25978682 -2.42719886  9.7495752  -2.15953298]

final loss [1.27064093]

#iterations
epochs = 3000

#learn rate
alpha = 0.01

#weights
w11= -1
w21= 1

w12= 1
w22= 1

w13= 1
w23= 1

w1 = 10
w2 = 0.1
w3 = 10

#bias
b1 = 1
b2 = 1

net = Net(alpha)
net.p1.w = [w11,w21,b1]
net.p2.w = [w12,w22,b1]
net.p3.w = [w13,w23,b1]
net.p4.w = [w1,w2,w3,b2]
errList = net.train(epochs)

view(net.p1,xmin,xmax,ymin,ymax)
plt.show()
view(net.p2,xmin,xmax,ymin,ymax)
plt.show()
view(net.p3,xmin,xmax,ymin,ymax)
plt.show()

boundary(net,xmin,xmax,ymin,ymax,X,T,'boundary4.png')

pd.DataFrame(errList, columns=['error']).plot()
plt.savefig("errors4.png")
plt.show()

print("final loss", errList[len(errList)-1])

final loss [0.40816724]

Corciu Mihai - 2021 - mc200520052005@yahoo.com