Back to Topics

from IPython.display import Image
Image(filename = "nn2.png", width=200)

Gradients:

${ zh }_{ 1 }={ w }_{ 1 }{ x }_{ 1 }+{ w }_{ 3 }{ x }_{ 2 }+{ b }_{ 1 }\\ { zh }_{ 2 }={ w }_{ 2 }{ x }_{ 1 }+{ w }_{ 4 }{ x }_{ 2 }+{ b }_{ 1 }\\ { h }_{ 1 }=\sigma ({ zh }_{ 1 })\\ { h }_{ 2 }=\sigma ({ zh }_{ 2 })\\ { z }_{ 0 }={ w }_{ 5 }{ h }_{ 1 }+{ w }_{ 6 }{ h }_{ 2 }+{ b }_{ 2 }\\ out=\sigma ({ z }_{ 0 })\\ \\ E=-t\cdot ln(out)-(1-t)\cdot ln(1-out)$

$\frac { dE }{ { dw }_{ 1 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ { dw }_{ 1 } } -(1-t)\frac { 1 }{ 1-out } (-1)\frac { { d }_{ out } }{ { dw }_{ 1 } } \\ =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ { dw }_{ 1 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ { dw }_{ 1 } } \\ \frac { { d }_{ out } }{ { dw }_{ 1 } } =\frac { { d }_{ out } }{ { dz }_{ 0 } } \frac { { dz }_{ 0 } }{ { dh }_{ 1 } } \frac { { dh }_{ 1 } }{ { dzh }_{ 1 } } \frac { { dzh }_{ 1 } }{ { dw }_{ 1 } } =out(1-out){ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 }){ x }_{ 1 } \\ \frac { { dE } }{ { dw }_{ 1 } } =-t\frac { 1 }{ out } out(1-out){ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 }){ x }_{ 1 }+\\ (1-t)\frac { 1 }{ 1-out } out(1-out){ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 }){ x }_{ 1 }=\\ -t(1-out){ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 }){ x }_{ 1 }+(1-t)out{ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 }){ x }_{ 1 }=\\ ={ w }_{ 5 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })(-t+t\cdot out+out-t\cdot out)=\\ ={ w }_{ 5 }{ x }_{ 1 }{ h }_{ 1 }(1-{ h }_{ 1 })(out-t)$

The same:

$ \frac { dE }{ { dw }_{ 2 } } ={ w }_{ 6 }{ x }_{ 1 }{ h }_{ 2 }(1-{ h }_{ 2 })(out-t)\\ \frac { dE }{ { dw }_{ 3 } } ={ w }_{ 5 }{ x }_{ 2 }{ h }_{ 1 }(1-{ h }_{ 1 })(out-t)\\ \frac { dE }{ { dw }_{ 4 } } ={ w }_{ 6 }{ x }_{ 2 }{ h }_{ 2 }(1-{ h }_{ 2 })(out-t) $

$ \frac { dE }{ { db }_{ 1 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ db_{ 1 } } -(1-t)\frac { 1 }{ 1-out } (-1)\frac { { d }_{ out } }{ { db }_{ 1 } } =\\ =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ db_{ 1 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ db_{ 1 } } \\ \frac { { d }_{ out } }{ { db }_{ 1 } } =\frac { { d }_{ out } }{ { dz }_{ 0 } } \frac { { dz }_{ 0 } }{ { db }_{ 1 } } =\frac { { d }_{ out } }{ { dz }_{ 0 } } (\frac { d }{ { db }_{ 1 } } { w }_{ 5 }{ h }_{ 1 }+\frac { d }{ { db }_{ 1 } } { w }_{ 6 }{ h }_{ 2 })=\\ =\frac { { d }_{ out } }{ { dz }_{ 0 } } ({ w }_{ 5 }\frac { d }{ { db }_{ 1 } } { h }_{ 1 }+{ w }_{ 6 }\frac { d }{ { db }_{ 1 } } { h }_{ 2 })=\\ =\frac { { d }_{ out } }{ { dz }_{ 0 } } ({ w }_{ 5 }\frac { { dh }_{ 1 } }{ { dzh }_{ 1 } } \frac { { dzh }_{ 1 } }{ { db }_{ 1 } } +{ w }_{ 6 }\frac { { dh }_{ 2 } }{ { dzh }_{ 2 } } \frac { { dzh }_{ 2 } }{ { db }_{ 1 } } )=\\ =out(1-out)({ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 6 }{ h }_{ 2 }(1-{ h }_{ 2 }))\\ \\ \frac { dE }{ { db }_{ 1 } } =-t\frac { 1 }{ out } out(1-out)({ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 6 }{ h }_{ 2 }(1-{ h }_{ 2 }))+\\ (1-t)\frac { 1 }{ 1-out } out(1-out)({ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 6 }{ h }_{ 2 }(1-{ h }_{ 2 }))=\\ =(...)(-t+t\cdot out+out-t\cdot out)=\\ =(out-t)({ w }_{ 5 }{ h }_{ 1 }(1-{ h }_{ 1 })+{ w }_{ 6 }{ h }_{ 2 }(1-{ h }_{ 2 })) $

$ \frac { dE }{ { dw }_{ 5 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ dw_{ 5 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ dw_{ 5 } } \\ \frac { { d }_{ out } }{ dw_{ 5 } } =\frac { { d }_{ out } }{ { dz }_{ 0 } } \frac { { dz }_{ 0 } }{ { dw }_{ 5 } } =out(1-out){ h }_{ 1 }\\ \frac { dE }{ { dw }_{ 5 } } =-t\frac { 1 }{ out } out(1-out){ h }_{ 1 }+(1-t)\frac { 1 }{ 1-out } out(1-out){ h }_{ 1 }=\\ =-t(1-out)\cdot { h }_{ 1 }+(1-t)out{ \cdot h }_{ 1 }={ h }_{ 1 }(out-t)\\ Similarly:\\ \frac { dE }{ { dw }_{ 6 } } ={ h }_{ 2 }(out-t)\\ $

$ \frac { dE }{ { db }_{ 2 } } =-t\frac { 1 }{ out } \frac { { d }_{ out } }{ db_{ 2 } } +(1-t)\frac { 1 }{ 1-out } \frac { { d }_{ out } }{ db_{ 2 } } \\ \frac { { d }_{ out } }{ db_{ 2 } } =\frac { { d }_{ out } }{ { dz }_{ 0 } } \frac { { dz }_{ o } }{ { db }_{ 2 } } =out(1-out)\\ \frac { dE }{ { db }_{ 2 } } =-t\frac { 1 }{ out } out(1-out)+(1-t)\frac { 1 }{ 1-out } out(1-out)=\\ =-t(1-out)+(1-t)out=out-t $

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

def plot_points(X, y, m="_"):
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'blue', edgecolor = 'k',zorder=2, marker=m)
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'red', edgecolor = 'k',zorder=2, marker=m )

def plot_zone(X, y):
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'blue', edgecolor = 'k',zorder=2, marker="+")
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'red', edgecolor = 'k',zorder=2, marker="_" )    
    
def boundary(net,xmin,xmax,ymin,ymax,X,T,filename='boundary.png'):
    more=1
    #plt.xlim(xmin-more,xmax+more)
    #plt.ylim(ymin-more,ymax+more)
    
    #samples
    #x = np.linspace(xmin, xmax, 200)
    #stepsize 
    #x = np.arange(xmin, xmax, 0.1)

    border = []
    xList = np.linspace(xmin, xmax, 50)
    for x in xList:
        yList = np.linspace(ymin, ymax, 50)
        for y in yList:
            out = net.output([x,y])
            if out >= 0.5:
                border.append([x,y,1]) 
            else:
                border.append([x,y,0])

    border = np.array(border)
    solutionX = border[:, [0,1]]
    solutionT = border[:, [2]]
    #plot_points(solutionX, solutionT)
    plot_zone(solutionX, solutionT)
    plot_points(X, T,m="o")
    
    #x, y = zip(*border)
    #plt.scatter(x,y)
    plt.title("Solution boundary")
    plt.savefig(filename)
    
def view(p,xmin,xmax,ymin,ymax):
    more=1

    border = []
    xList = np.linspace(xmin, xmax, 50)
    for x in xList:
        yList = np.linspace(ymin, ymax, 50)
        for y in yList:
            out = p.output([x,y])
            if out >= 0.5:
                border.append([x,y,1]) 
            else:
                border.append([x,y,0])

    border = np.array(border)
    solutionX = border[:, [0,1]]
    solutionT = border[:, [2]]
    #plot_points(solutionX, solutionT)
    plot_zone(solutionX, solutionT)
    #plot_points(X, T,m="o")
    
    #x, y = zip(*border)
    #plt.scatter(x,y)
    #plt.savefig(filename)        

# Loss (cross entropy) error function
def error(t, output):
    return - t*np.log(output) - (1 - t) * np.log(1-output)

# Activation (sigmoid) function
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

class Perceptron:
    def __init__(self, w=[], activation = None, alpha = 0.01, label = None):
        self.label = label
        self.w = np.array(w)
        self.x = np.zeros(self.w.shape[0])
        self.activation = activation
        self.alpha = alpha

    def output(self,x):
        self.h = self.activation(self.zh(x))
        return self.h

    def zh(self,x):
        #add 'fake' 1 value to x list for bias correspondence
        self.x = np.append(x,1)
        self.net = np.dot(self.x, self.w)
        return self.net

    def update(self,de_dw):
        self.w = self.w - de_dw * self.alpha

#gradients of weights
def gradients(net, t, x):
    t = t[0]
    x1 = x[0]
    x2 = x[1]    
    
    h1 = net.h1
    h2 = net.h2
    out = net.out
    w5 = net.p3.w[0]
    w6 = net.p3.w[1]    
    
    dE_dw1 = w5*x1*h1*(1-h1)*(out-t)
    dE_dw2 = w6*x1*h1*(1-h2)*(out-t) 
    dE_dw3 = w5*x2*h1*(1-h1)*(out-t)
    dE_dw4 = w6*x2*h1*(1-h2)*(out-t)
    dE_db1 = (out-t)*(w5*h1*(1-h1) +w6*h2*(1-h2))
    dE_dw5 = h1*(out-t)
    dE_dw6 = h2*(out-t)
    dE_db2 = out-t
    return [dE_dw1,dE_dw2,dE_dw3,dE_dw4,dE_dw5,dE_dw6,dE_db1,dE_db2]

class Net:
    def __init__(self,alpha = 0.01):
        self.alpha = alpha
        self.p1 = Perceptron([0,0,0],sigmoid,alpha,1)
        self.p2 = Perceptron([0,0,0],sigmoid,alpha,1)
        self.p3 = Perceptron([0,0,0],sigmoid,alpha,1)

    #forward = process output
    def output(self,x):
        self.h1 = self.p1.output(x)
        self.h2 = self.p2.output(x)
        self.out = self.p3.output(np.array([self.h1, self.h2]))
        return self.out

    #adjust weights
    def backward(self,g):
        dw1 = g[0]
        dw2 = g[1]
        dw3 = g[2]
        dw4 = g[3]
        dw5 = g[4]
        dw6 = g[5]
        db1 = g[6]
        db2 = g[7]
        self.p1.update(np.array([dw1,dw3,db1]))
        self.p2.update(np.array([dw2,dw4,db1]))
        self.p3.update(np.array([dw5,dw6,db2]))
        
    def train(self,epochs):    
        self.errList = []
    
        for i in range(epochs):
            totalError = 0
            for x, t in zip(X, T):

                out = net.output(x)

                #compute error
                e = error(t,out)
                totalError += e

                de_dw = gradients(net, t, x)
                net.backward(de_dw)

            self.errList.append(totalError)
        
        return self.errList

data = np.array([
    [1,10,1],
    [3,10,0],
    [1.8,2.0,0],
    [0,6,0],
    [-1,-1,1],
    [-2,10,1],
])

X = data[:, [0,1]]
T = data[:, [2]]

datamin = np.min(X, axis=0)
datamax = np.max(X, axis=0)
xmin, ymin = datamin
xmax, ymax = datamax

#iterations
epochs = 1000

#learn rate
alpha = 0.01

#weights
w1 = 1
w2 = 1
w3 = 1
w4 = 1
w5 = 1
w6 = 10 

#bias
b1 = 1
b2 = 1

net = Net(alpha)
net.p1.w = [w1,w3,b1]
net.p2.w = [w2,w4,b1]
net.p3.w = [w5,w6,b2]
errList = net.train(epochs)

view(net.p1,xmin,xmax,ymin,ymax)
plt.title("perceptron 1")
plt.show()
view(net.p2,xmin,xmax,ymin,ymax)
plt.title("perceptron 2")
plt.show()

boundary(net,xmin,xmax,ymin,ymax,X,T,'boundary1.png')

pd.DataFrame(errList, columns=['error']).plot()
plt.savefig("errors1.png")
plt.show()

print("final loss", errList[len(errList)-1])

final loss [1.96336383]

#iterations
epochs = 10000

#learn rate
alpha = 0.01

#weights
w1 = 1
w2 = 1
w3 = 1
w4 = 1
w5 = 1
w6 = -10

#bias
b1 = 1
b2 = 1

net = Net(alpha)
net.p1.w = [w1,w3,b1]
net.p2.w = [w2,w4,b1]
net.p3.w = [w5,w6,b2]
errList = net.train(epochs)

view(net.p1,xmin,xmax,ymin,ymax)
plt.title("perceptron 1")
plt.show()
view(net.p2,xmin,xmax,ymin,ymax)
plt.title("perceptron 2")
plt.show()

print("final state")
print(net.p1.w)
print(net.p2.w)
print(net.p3.w)

#for x, t in zip(X, T):
#    out = net.output(x)
#    print(x,out,t)

boundary(net,xmin,xmax,ymin,ymax,X,T,'boundary2.png')    
          
pd.DataFrame(errList, columns=['error']).plot()
plt.savefig("errors2.png")
plt.show()

print("final loss", errList[len(errList)-1])

final state
[-1.47127436  0.96721716 -6.78865103]
[ 1.98064678  2.19530033 -6.78865103]
[11.24433614 -9.65260675  3.47993887]

final loss [0.17809965]

Corciu Mihai - 2021 - mc200520052005@yahoo.com