Back to Topics

2 inputs, 3 Neurons, 3 output

Class + Explicit network definition

from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o3_sigmoid_01a.png", width=500)

from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o3_sigmoid_01.png", width=500)

Prerequisites

import torch
from torch import nn
from torch import optim
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#predictive
torch.manual_seed(1);

device = torch.device('cpu')
if torch.cuda.is_available():
    device = torch.device('cuda')  
    
#sigmoid explicit
def sigmoid(x):
    return 1/(1+torch.exp(-x))

#sigmoid pre-defined
activation = torch.nn.Sigmoid()

# Loss (Binary Cross Entropy) error function, explicit def
def bce_err(output, target):
    return -target * torch.log(output) - (1-target) * torch.log(1-output)

#sigmoid + BCELoss (Binary Cross Entropy)
criterion = torch.nn.BCEWithLogitsLoss()

def state(interval,i,loss,out,y):
    if(i%interval == 0):
        print("\n========== Epoch", i,"==========")
        print("loss=",loss)
    
        #to numpy
        out1 = out.data.numpy()
        y3 = y.numpy()
        #all values less than 0.5 to < 0
        out2 = out1-0.5
        # <0.5 turned to False, >= 0.5 turned to True
        out3 = (out2>=0)
        #boolean to decimal 0/1
        out4=out3*1   
        #boolean results
        rez = (out4 == y3)        
        #procent of True's(exact classification: y_hat=y, prediction = label/target)
        accuracy = np.min(np.mean(rez,axis=0))
        print("accuracy=",accuracy)              

#training points
data = np.array([
    [0.1,0.1,0,0,0],
    [0.2,0.5,1,0,0],
    [-0.5,-0.3,0,1,0],
    [0.9,0.1,0,0,1],
    [0.5,-0.4,1,1,0],
    [0.3,0.3,0,1,1],
    [-0.8,0.2,1,0,1],
    [0.7,0.4,1,1,1],    
])          

#df = pd.read_csv('test.csv', header=None)
#df = pd.read_csv('data2.csv', header=None)
#data = df.to_numpy()        

Class + Explicit network definition

torch.manual_seed(1);

#features
x = torch.from_numpy(data[:, [0,1]]).double()
#target/labels
y = torch.from_numpy(data[:, 2:5]).double()

class NN:
    
    def __init__(self, n_input, n_hidden, n_output):

        # Weights for inputs to hidden layer
        self.w1 = torch.randn(n_input, n_hidden, dtype=torch.double, requires_grad=True)
        # and bias terms for hidden and output layers
        self.b1 = torch.randn(1, n_hidden, dtype=torch.double, requires_grad=True)
        self.activation = torch.nn.Sigmoid()
        
    def forward(self,x):
        o = self.activation(torch.mm(x,self.w1)+(self.b1))
        return o
    
net = NN(2,3,3)
print(net.w1)
print(net.b1)
print(net.forward(x)) 

tensor([[ 0.6614,  0.2669,  0.0617],
        [ 0.6213, -0.4519, -0.1661]], dtype=torch.float64, requires_grad=True)
tensor([[-1.5228,  0.3817, -1.0276]], dtype=torch.float64, requires_grad=True)
tensor([[0.1987, 0.5898, 0.2615],
        [0.2535, 0.5521, 0.2501],
        [0.1151, 0.5948, 0.2673],
        [0.2962, 0.6403, 0.2712],
        [0.1914, 0.6673, 0.2829],
        [0.2427, 0.5808, 0.2575],
        [0.1270, 0.5194, 0.2478],
        [0.3076, 0.5957, 0.2591]], dtype=torch.float64,
       grad_fn=<SigmoidBackward>)

#learn rate
alpha = 0.1
#iterations
epochs = 5000
#display state
fv = 500

lossHistory = []

for i in range(epochs):
    
    #forward: output/prediction
    out = net.forward(x)
    loss = torch.mean(bce_err(out,y))
         
    #backward: compute gradients
    loss.backward()   

    #update weights
    with torch.no_grad():
        net.w1 -= alpha * net.w1.grad
        net.b1 -= alpha * net.b1.grad

        # Manually zero the gradients after updating weights
        net.w1.grad.zero_()
        net.b1.grad.zero_()    

    lossHistory.append(loss)    
        
    state(fv,i,loss,out,y)

========== Epoch 0 ==========
loss= tensor(0.8049, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.5

========== Epoch 500 ==========
loss= tensor(0.6446, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.5

========== Epoch 1000 ==========
loss= tensor(0.6285, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 1500 ==========
loss= tensor(0.6208, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 2000 ==========
loss= tensor(0.6166, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 2500 ==========
loss= tensor(0.6142, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 3000 ==========
loss= tensor(0.6127, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 3500 ==========
loss= tensor(0.6119, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 4000 ==========
loss= tensor(0.6113, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

========== Epoch 4500 ==========
loss= tensor(0.6110, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.625

#loss evolution
graph_x = np.arange(0, epochs)
graph_y = lossHistory
plt.plot(graph_x, graph_y)
plt.show()

Setting custom weights

net.w1 = torch.tensor( [[1, -1, 1],
                        [1, -1, 1]], dtype=torch.float64, requires_grad=True)
net.b1 = torch.tensor( [[1, -1, 1]], dtype=torch.float64, requires_grad=True)

print(x)
print(net.w1)
print(net.b1)
out = net.forward(x)
print("---\n",out,"\n---")

tensor([[ 0.1000,  0.1000],
        [ 0.2000,  0.5000],
        [-0.5000, -0.3000],
        [ 0.9000,  0.1000],
        [ 0.5000, -0.4000],
        [ 0.3000,  0.3000],
        [-0.8000,  0.2000],
        [ 0.7000,  0.4000]], dtype=torch.float64)
tensor([[ 1., -1.,  1.],
        [ 1., -1.,  1.]], dtype=torch.float64, requires_grad=True)
tensor([[ 1., -1.,  1.]], dtype=torch.float64, requires_grad=True)
---
 tensor([[0.7685, 0.2315, 0.7685],
        [0.8455, 0.1545, 0.8455],
        [0.5498, 0.4502, 0.5498],
        [0.8808, 0.1192, 0.8808],
        [0.7503, 0.2497, 0.7503],
        [0.8320, 0.1680, 0.8320],
        [0.5987, 0.4013, 0.5987],
        [0.8909, 0.1091, 0.8909]], dtype=torch.float64,
       grad_fn=<SigmoidBackward>) 
---

net.w1 = torch.tensor( [[1, 1, 1],
                        [1, 1, 1]], dtype=torch.float64, requires_grad=True)
net.b1 = torch.tensor( [[1, 1, 1]], dtype=torch.float64, requires_grad=True)

print(x)
print(net.w1)
print(net.b1)
out = net.forward(x)
print("---\n",out,"\n---")

tensor([[ 0.1000,  0.1000],
        [ 0.2000,  0.5000],
        [-0.5000, -0.3000],
        [ 0.9000,  0.1000],
        [ 0.5000, -0.4000],
        [ 0.3000,  0.3000],
        [-0.8000,  0.2000],
        [ 0.7000,  0.4000]], dtype=torch.float64)
tensor([[1., 1., 1.],
        [1., 1., 1.]], dtype=torch.float64, requires_grad=True)
tensor([[1., 1., 1.]], dtype=torch.float64, requires_grad=True)
---
 tensor([[0.7685, 0.7685, 0.7685],
        [0.8455, 0.8455, 0.8455],
        [0.5498, 0.5498, 0.5498],
        [0.8808, 0.8808, 0.8808],
        [0.7503, 0.7503, 0.7503],
        [0.8320, 0.8320, 0.8320],
        [0.5987, 0.5987, 0.5987],
        [0.8909, 0.8909, 0.8909]], dtype=torch.float64,
       grad_fn=<SigmoidBackward>) 
---

Corciu Mihai - 2021 - mc200520052005@yahoo.com