
from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o1_sigmoid_01a.png", width=500)


from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o1_sigmoid_01.png", width=500)


import torch
from torch import nn
from torch import optim
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from draw import display_solution

#predictive
torch.manual_seed(1);

device = torch.device('cpu')
if torch.cuda.is_available():
    device = torch.device('cuda')  
    
#sigmoid explicit
def sigmoid(x):
    return 1/(1+torch.exp(-x))

#sigmoid pre-defined
activation = torch.nn.Sigmoid()

# Loss (Binary Cross Entropy) error function, explicit def
def bce_err(output, target):
    return -target * torch.log(output) - (1-target) * torch.log(1-output)

#sigmoid + BCELoss (Binary Cross Entropy)
criterion = torch.nn.BCEWithLogitsLoss()

def state(interval,i,loss,out,y):
    if(i%interval == 0):
        print("\n========== Epoch", i,"==========")
        print("loss=",loss)
        accuracy = np.mean( ((out > 0.5)==y).numpy() )
        print("accuracy=",accuracy)
        
#array data points: x1, x2
data = np.array([
    [1,10,1],
    [3,10,0],
    [1.8,2.0,0],
    [-1,-1,1],
    [-2,10,1],
])         

#df = pd.read_csv('test.csv', header=None)
#df = pd.read_csv('data2.csv', header=None)
#data = df.to_numpy()


#features
x = torch.from_numpy(data[:, [0,1]]).double()
#target/labels
y = torch.from_numpy(data[:, [2]]).double()

# Define the size of each layer in our network
n_input = 2    # Number of input units, must match number of input features
n_hidden = 1   # Number of hidden units 
n_output = 1   # Number of output units

# Weights for inputs to hidden layer
w = torch.randn(n_input, n_hidden, dtype=torch.double, requires_grad=True)

# and bias terms for hidden and output layers
b = torch.randn(1, n_hidden, dtype=torch.double, requires_grad=True)

print(x)
print(w)
print(b)
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
print(out)

tensor([[ 1.0000, 10.0000],
        [ 3.0000, 10.0000],
        [ 1.8000,  2.0000],
        [-1.0000, -1.0000],
        [-2.0000, 10.0000]], dtype=torch.float64)
tensor([[0.6614],
        [0.2669]], dtype=torch.float64, requires_grad=True)
tensor([[0.0617]], dtype=torch.float64, requires_grad=True)
tensor([[0.9675],
        [0.9911],
        [0.8564],
        [0.2960],
        [0.8035]], dtype=torch.float64, grad_fn=<SigmoidBackward>)


#learn rate
alpha = 0.1
#iterations
epochs = 20
#display state
fv = 5

lossHistory = []
predictionHistory = []

#optional manually set weights
#w = torch.tensor([[1],[1]], dtype=torch.float64, requires_grad=True)
#b = torch.tensor([[1]], dtype=torch.float64, requires_grad=True)

weights = w.data.numpy()
bias = b.data.numpy()
weights = weights.reshape(1,weights.shape[0])
predictionHistory.append([weights[0][0],weights[0][1],bias[0][0]])

for i in range(epochs):
    
    #forward: output/prediction
    out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))  
    loss = torch.mean(bce_err(out,y))
    
    #alternative
    #out = torch.mm(X, W.view(2,1))+b
    #loss = criterion(out, y)  
      
    #backward: compute gradients
    loss.backward()   

    #update weights (disable gradients when update weights)
    with torch.no_grad():
        w -= alpha * w.grad
        b -= alpha * b.grad

        # Manually zero the gradients after updating weights
        w.grad.zero_()
        b.grad.zero_()    

    lossHistory.append(loss)    
        
    weights = w.data.numpy()
    bias = b.data.numpy()
    weights = weights.reshape(1,weights.shape[0])
    predictionHistory.append([weights[0][0],weights[0][1],bias[0][0]])

    state(fv,i,loss,out,y)

display_solution(data,[weights[0][0],weights[0][1],bias[0][0]],predictionHistory,lossHistory,epochs,ylim=None)

========== Epoch 0 ==========
loss= tensor(1.6268, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.4

========== Epoch 5 ==========
loss= tensor(0.7871, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.4

========== Epoch 10 ==========
loss= tensor(0.5315, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 1.0

========== Epoch 15 ==========
loss= tensor(0.4087, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 1.0

(100,)
(100,)


w = torch.tensor([[1],[-1]], dtype=torch.float64)
b = torch.tensor([[-1]], dtype=torch.float64)

print(x)
print(w,b)
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
print("---\n",out,"\n---")

tensor([[ 1.0000, 10.0000],
        [ 3.0000, 10.0000],
        [ 1.8000,  2.0000],
        [-1.0000, -1.0000],
        [-2.0000, 10.0000]], dtype=torch.float64)
tensor([[ 1.],
        [-1.]], dtype=torch.float64) tensor([[-1.]], dtype=torch.float64)
---
 tensor([[4.5398e-05],
        [3.3535e-04],
        [2.3148e-01],
        [2.6894e-01],
        [2.2603e-06]], dtype=torch.float64) 
---


w = torch.tensor([[1],[1]], dtype=torch.float64)
b = torch.tensor([[1]], dtype=torch.float64)

print(x)
print(w,b)
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
print("---\n",out,"\n---")

tensor([[ 1.0000, 10.0000],
        [ 3.0000, 10.0000],
        [ 1.8000,  2.0000],
        [-1.0000, -1.0000],
        [-2.0000, 10.0000]], dtype=torch.float64)
tensor([[1.],
        [1.]], dtype=torch.float64) tensor([[1.]], dtype=torch.float64)
---
 tensor([[1.0000],
        [1.0000],
        [0.9918],
        [0.2689],
        [0.9999]], dtype=torch.float64) 
---

2 inputs, 1 output¶

Explicit network definition¶

Prerequisites¶

Explicit network definition¶

Setting custom weights¶