Tutorial 2: PyTorch basics¶

Outline¶

Datasets
Models
Losses
Optimizers

Official resources: * Deep Learning with PyTorch: a 60 Minute Blitz * PyTorch documentation

Here no details, mostly visualization, with linear regression.

Imports¶

[ ]:

import numpy as np
import torch
import matplotlib.pyplot as plt
from matplotlib import colors
plt.rcParams.update({'font.size': 16})

[ ]:

x = torch.rand(20, 5)
x
((x.norm(dim=1) - 1).abs() < 1e-10).float()

tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])

Datasets¶

A raw dataset¶

The data are obtained from the model

\[x \mapsto y:=Ax + b\]

with \(x\in\mathbb R^{input\_dim}\) and \(y\in\mathbb R^{output\_dim}\).

[ ]:

input_dim = 1
output_dim = 1

A = 2 * np.random.rand(output_dim, input_dim) - 1
b = 2 * np.random.rand(output_dim) - 1

true_model = lambda x: A @ x + b

We get a training set by randomly sampling and noisy observations \begin{align*} &x_i \sim U([-1, 1])\\ &y_i = A x_i + b + \nu_i \end{align*}

[ ]:

n_train = 1000
noise_level = 0.04

# Generate a random set of n_train samples
X_train = np.random.rand(n_train, input_dim)
y_train = np.array([true_model(x) for x in X_train])

# Add some noise
y_train += noise_level * np.random.standard_normal(size=y_train.shape)

[ ]:

if input_dim == output_dim == 1:
    fig = plt.figure()
    fig.clf()
    ax = fig.gca()
    ax.plot(X_train, y_train, '.')
    ax.grid(True)
    ax.set_xlabel('X_train')
    ax.set_ylabel('y_train')

../../../_images/ipynbs_colabs_pyg_tutorial_project_Tutorial02_14_0.png

PyTorch `Dataset`¶

[ ]:

#%% Dataset to manage vector to vector data
class VectorialDataset(torch.utils.data.Dataset):
    def __init__(self, input_data, output_data):
        super(VectorialDataset, self).__init__()
        self.input_data = torch.tensor(input_data.astype('f'))
        self.output_data = torch.tensor(output_data.astype('f'))

    def __len__(self):
        return self.input_data.shape[0]

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        sample = (self.input_data[idx, :],
                  self.output_data[idx, :])
        return sample

[ ]:

training_set = VectorialDataset(input_data=X_train, output_data=y_train)

[ ]:

len(training_set)

[ ]:

training_set[10:12]

(tensor([[0.9776],
         [0.5333]]), tensor([[0.8377],
         [0.8677]]))

Here almost useless, but think about e.g. images.

PyTorch `DataLoader`¶

[ ]:

batch_size = 120
train_loader = torch.utils.data.DataLoader(training_set,
                                           batch_size=batch_size,
                                           shuffle=True)

The role of batch_size:

[ ]:

for idx, batch in enumerate(train_loader):
    print('Batch n. %2d: input size=%s, output size=%s' % (idx+1, batch[0].shape, batch[1].shape))

Batch n.  1: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  2: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  3: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  4: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  5: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  6: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  7: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  8: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  9: input size=torch.Size([40, 1]), output size=torch.Size([40, 1])

[ ]:

8 * 120

The role of shuffle:

[ ]:

first_batch = []

for epoch in range(2):
    for idx, batch in enumerate(train_loader):
        if idx == 0:
            first_batch.append(batch)

np.c_[X_train[:batch_size], first_batch[0][0].numpy(), first_batch[1][0].numpy()]

array([[0.974855  , 0.97102839, 0.88224959],
       [0.20088936, 0.50331622, 0.82345295],
       [0.3588191 , 0.93372488, 0.82671511],
       [0.31635773, 0.1723303 , 0.21061797],
       [0.26437699, 0.83520246, 0.54161078],
       [0.26334294, 0.94008124, 0.98018283],
       [0.70451306, 0.19887626, 0.96016997],
       [0.62485536, 0.42722419, 0.33184233],
       [0.81680064, 0.3787725 , 0.34424016],
       [0.26399209, 0.74914658, 0.96811068],
       [0.97757121, 0.67563111, 0.33945435],
       [0.533324  , 0.76730102, 0.5155586 ],
       [0.43001335, 0.45874023, 0.45199746],
       [0.24557755, 0.99072611, 0.59223241],
       [0.0575559 , 0.51748616, 0.08978312],
       [0.14739718, 0.8914445 , 0.81213194],
       [0.41385026, 0.93936813, 0.25697485],
       [0.1030207 , 0.54127955, 0.38445267],
       [0.02066877, 0.44041118, 0.93936813],
       [0.58362063, 0.99943674, 0.43276   ],
       [0.06981526, 0.66758424, 0.32515258],
       [0.14331137, 0.05650147, 0.42637974],
       [0.42193655, 0.83311635, 0.74914658],
       [0.84811793, 0.33808348, 0.35586321],
       [0.0189641 , 0.90636659, 0.52024269],
       [0.73704536, 0.29632658, 0.26971427],
       [0.27161635, 0.69540745, 0.03069129],
       [0.48832314, 0.77608848, 0.40542439],
       [0.51526818, 0.4440622 , 0.88237053],
       [0.16132186, 0.62918919, 0.19699152],
       [0.5155586 , 0.34730271, 0.14666671],
       [0.58478156, 0.23124492, 0.43424493],
       [0.98785734, 0.73400754, 0.76431674],
       [0.82934937, 0.28136384, 0.54069471],
       [0.78133812, 0.49659568, 0.20088936],
       [0.63356466, 0.95077932, 0.19162408],
       [0.41486208, 0.92843264, 0.91998875],
       [0.16991878, 0.2285466 , 0.14739718],
       [0.79862486, 0.19108887, 0.54440981],
       [0.11348485, 0.27566925, 0.02066877],
       [0.53058885, 0.81712794, 0.13077131],
       [0.46555561, 0.71535236, 0.73704535],
       [0.90537262, 0.63851529, 0.09674653],
       [0.07209757, 0.35959649, 0.86732358],
       [0.4429715 , 0.48319075, 0.35832527],
       [0.84571465, 0.53246367, 0.16232426],
       [0.30218019, 0.65447605, 0.73164684],
       [0.19282706, 0.42574444, 0.76095849],
       [0.64629518, 0.19162408, 0.62918919],
       [0.48141778, 0.67841244, 0.12691697],
       [0.48284637, 0.38445267, 0.90531546],
       [0.10448172, 0.17072287, 0.44554165],
       [0.68613395, 0.45841208, 0.54127955],
       [0.23854881, 0.77972329, 0.48141778],
       [0.98891487, 0.3227919 , 0.15311468],
       [0.10262068, 0.08978312, 0.80341679],
       [0.66323465, 0.32799962, 0.4803564 ],
       [0.43883588, 0.98925787, 0.0189641 ],
       [0.71445548, 0.94664162, 0.27177146],
       [0.41307755, 0.81503284, 0.79137987],
       [0.45199746, 0.05631886, 0.49857965],
       [0.332212  , 0.06867729, 0.92075837],
       [0.87890637, 0.22722761, 0.88264573],
       [0.67880036, 0.11701425, 0.5830515 ],
       [0.39524466, 0.51557332, 0.18987247],
       [0.65663108, 0.42868343, 0.51748616],
       [0.81712795, 0.70451307, 0.16033462],
       [0.99243442, 0.62319732, 0.35287657],
       [0.70124457, 0.51199895, 0.26517075],
       [0.49627951, 0.38222787, 0.75223863],
       [0.09674652, 0.65588737, 0.51843774],
       [0.51244907, 0.51188725, 0.93014342],
       [0.04139904, 0.41324618, 0.3827664 ],
       [0.83661566, 0.73678124, 0.67634982],
       [0.0897793 , 0.332212  , 0.70207703],
       [0.57658073, 0.40861318, 0.70911026],
       [0.41032992, 0.93449783, 0.88450426],
       [0.35357984, 0.56973714, 0.34730271],
       [0.13408873, 0.23664314, 0.84811795],
       [0.59377689, 0.06405374, 0.4395273 ],
       [0.7760885 , 0.18300015, 0.04894073],
       [0.78109434, 0.93848354, 0.71502668],
       [0.65355077, 0.85074848, 0.7340827 ],
       [0.99585638, 0.42193654, 0.66309381],
       [0.48319076, 0.76793796, 0.56760997],
       [0.35287656, 0.68209291, 0.28590071],
       [0.88385744, 0.2764878 , 0.64954144],
       [0.06001577, 0.22370355, 0.71445549],
       [0.11574473, 0.66783738, 0.48827842],
       [0.54127957, 0.18701187, 0.61755055],
       [0.94209682, 0.67117596, 0.75304455],
       [0.78124028, 0.47662666, 0.41636717],
       [0.17324309, 0.99298179, 0.07569684],
       [0.47196952, 0.03098176, 0.06405374],
       [0.8122213 , 0.76838279, 0.43839231],
       [0.23124491, 0.28843415, 0.87987804],
       [0.79924142, 0.7632069 , 0.88009071],
       [0.51787832, 0.35483232, 0.80817389],
       [0.40861316, 0.84813797, 0.24848798],
       [0.03098176, 0.15202665, 0.64543229],
       [0.62636482, 0.55481392, 0.06780223],
       [0.76686785, 0.88124359, 0.78733301],
       [0.43031192, 0.66070163, 0.56004781],
       [0.27619917, 0.5830515 , 0.49622932],
       [0.48364738, 0.005606  , 0.49360588],
       [0.16253059, 0.36158645, 0.49659568],
       [0.36552527, 0.40570346, 0.28195608],
       [0.42530521, 0.35403642, 0.11147705],
       [0.63091337, 0.56905544, 0.27202424],
       [0.21403415, 0.78509218, 0.19753647],
       [0.54069473, 0.26046693, 0.88385743],
       [0.10491601, 0.36747354, 0.35959649],
       [0.97806818, 0.83590823, 0.74486321],
       [0.17453117, 0.80944741, 0.98761004],
       [0.38019614, 0.39897215, 0.530092  ],
       [0.13077131, 0.30865127, 0.13302526],
       [0.40778304, 0.22105536, 0.65663105],
       [0.83590825, 0.29269406, 0.94683015],
       [0.43338311, 0.68397522, 0.97806817],
       [0.19753647, 0.79862487, 0.61909825]])

Models¶

We implement a linear model

\[x \mapsto model(x):=A x + b\]

with \(A\in \mathbb{R}^{input\_dim\times output\_dim}\), \(b\in\mathbb{R}^{output\_dim}\).

[ ]:

import torch.nn as nn
import torch

#%% Linear layer
class LinearModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(LinearModel, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim

        self.linear = nn.Linear(self.input_dim, self.output_dim, bias=True)

    def forward(self, x):
        out = self.linear(x)
        return out

    def reset(self):
        self.linear.reset_parameters()

[ ]:

model = LinearModel(input_dim, output_dim)

Model inspection¶

[ ]:

print(model)

LinearModel(
  (linear): Linear(in_features=1, out_features=1, bias=True)
)

See pytorch.org/docs/stable/nn.html for many other layers.

[ ]:

list(model.parameters())

[Parameter containing:
 tensor([[-0.2594]], requires_grad=True), Parameter containing:
 tensor([0.2894], requires_grad=True)]

[ ]:

model.linear.weight

Parameter containing:
tensor([[-0.2594]], requires_grad=True)

[ ]:

model.linear.bias

Parameter containing:
tensor([0.2894], requires_grad=True)

The `forward` method¶

[ ]:

x = torch.randn(5, input_dim)
model.forward(x)

tensor([[-0.0766],
        [ 0.5903],
        [ 0.3571],
        [ 0.4407],
        [ 0.2370]], grad_fn=<AddmmBackward0>)

[ ]:

[model.linear.weight @ xx + model.linear.bias for xx in x]

[tensor([-0.0766], grad_fn=<AddBackward0>),
 tensor([0.5903], grad_fn=<AddBackward0>),
 tensor([0.3571], grad_fn=<AddBackward0>),
 tensor([0.4407], grad_fn=<AddBackward0>),
 tensor([0.2370], grad_fn=<AddBackward0>)]

[ ]:

if input_dim == output_dim == 1:
    fig = plt.figure()
    fig.clf()
    ax = fig.gca()
    ax.plot(training_set.input_data, training_set.output_data, '.')
    ax.plot(training_set.input_data, model.forward(training_set.input_data).detach().numpy(), '.')
    ax.grid(True)
    ax.set_xlabel('X_train')
    ax.legend(['y_train', 'model(X_train)'])

../../../_images/ipynbs_colabs_pyg_tutorial_project_Tutorial02_41_0.png

Losses¶

The MSE loss¶

The MSE loss is

\[L(y, y') = \|y-y'\|_2^2\]

[ ]:

import torch.nn as nn
loss_fun = nn.MSELoss(reduction='mean')

More at pytorch.org/docs/stable/nn.html#loss-functions

[ ]:

x = torch.tensor(np.array([1, 2, 1]).astype('f'))
z = torch.tensor(np.array([0, 0, 0]).astype('f'))
loss_fun(x, z)

tensor(2.)

Giving a score to the model (parameters) given the training set¶

The loss for one data pair \((x_i, y_i)\) is

\[L(model(x_i), y_i)\]

The cumulative (average) loss on the training set is

\[L(X_{train}, y_{train}):=\frac{1}{n_{train}} \sum_{i=1}^{n_{train}} L(model(x_i), y_i)\]

[ ]:

if input_dim == output_dim == 1:

    state_dict = model.state_dict()

    ww, bb = np.meshgrid(np.linspace(-2, 2, 30), np.linspace(-2, 2, 30))

    loss_values = 0 * ww
    for i in range(ww.shape[0]):
        for j in range(ww.shape[1]):
            state_dict['linear.weight'] = torch.tensor([[ww[i, j]]])
            state_dict['linear.bias'] = torch.tensor([bb[i, j]])
            model.load_state_dict(state_dict)
            loss_values[i, j] = loss_fun(model.forward(training_set.input_data),  training_set.output_data)

    fig = plt.figure(figsize=(10, 8))
    fig.clf()
    ax = fig.gca()
    levels = np.logspace(np.log(np.min(loss_values)), np.log(np.max(loss_values)), 20)
    c=ax.contourf(ww, bb, loss_values, levels=levels, norm=colors.LogNorm())
    plt.colorbar(c)
    ax.plot(A[0], b, 'r*', markersize=10)
    ax.set_ylabel('bias')
    ax.set_xlabel('weight')
    ax.legend(['(A, b)'])

    ax.grid(True)

../../../_images/ipynbs_colabs_pyg_tutorial_project_Tutorial02_50_0.png

Optimization¶

Goal: minimize the loss given the data:

\[\min\limits_{par \in model} L(X_{train}, y_{train}) =\min\limits_{par \in model} \frac{1}{n_{train}} \sum_{i=1}^{n_{train}} L(x_i, model(x_i))\]

In this case \(par =\{A, b\}\).

Iterative gradient based optimization: \begin{align*} par^{(0)} &= par_0\\ par^{(k+1)} &= par^{(k)} - \eta^{(k)} \nabla_{par} L(X_{train}, y_{train}) \end{align*}

with learning rate \(\eta^{(k)}\).

Gradients¶

The model is

\[model(x) = A x + b\]

With \(input\_dim = output\_dim = 1\) we have \(A:=a\in\mathbb{R},\ b\in\mathbb{R}\).

For a data pair \((x, y)\) the loss is \begin{align*} L(x, y) &= \|model(x)-y\|_2^2\\ &= (model(x)-y)^2\\ &= (a x + b - y)^2\\ &= a^2 x^2 + 2 a b x - 2 a xy + b^2 -2 b y + y^2. \end{align*}

We can compute \begin{align*} \nabla_a L(x, y) &= 2 a x^2 + 2 b x - 2 xy =2 x (a x + b - y)\\ \nabla_b L(x, y) &= 2 a x + 2 b - 2 y = 2 (ax + b - y). \end{align*}

[ ]:

x = torch.randn(1, input_dim)
y =  torch.randn(1, output_dim)

model.zero_grad()
loss = loss_fun(model.forward(x),  y)
loss.backward()

[ ]:

if input_dim == output_dim == 1:
    print(model.linear.weight.grad)
    print(2 * x * (model.linear.weight * x + model.linear.bias - y))

    print(model.linear.bias.grad)
    print(2 * (model.linear.weight * x + model.linear.bias - y))

tensor([[-1.3322]])
tensor([[-1.3322]], grad_fn=<MulBackward0>)
tensor([5.2172])
tensor([[5.2172]], grad_fn=<MulBackward0>)

Handmade optimization¶

[ ]:

if input_dim == output_dim == 1:

    num_iter = 200
    lr = 0.5 # 0.01

    train_hist = {}
    train_hist['weight'] = []
    train_hist['bias'] = []

    model.reset()
    state_dict = model.state_dict()

    for _ in range(num_iter):

        model.zero_grad()
        loss = loss_fun(model.forward(training_set.input_data), training_set.output_data)
        loss.backward()

        w = model.linear.weight.item()
        b = model.linear.bias.item()

        dw = model.linear.weight.grad.item()
        db = model.linear.bias.grad.item()

        state_dict['linear.weight'] += torch.tensor([-lr * dw])
        state_dict['linear.bias'] += torch.tensor([-lr * db])
        model.load_state_dict(state_dict)

        train_hist['weight'].append(w)
        train_hist['bias'].append(b)

    for label in train_hist:
        train_hist[label] = np.array(train_hist[label])

[ ]:

if input_dim == output_dim == 1:
    fig = plt.figure(figsize=(8, 8))
    fig.clf()
    ax = fig.gca()
    levels = np.logspace(np.log(np.min(loss_values)), np.log(np.max(loss_values)), 20)
    ax.contourf(ww, bb, loss_values, levels=levels, norm=colors.LogNorm())
    ax.set_xlabel('weight')
    ax.set_ylabel('bias')
    ax.grid(True)
    ax.set_xlim(-2, 2)
    ax.set_ylim(-2, 2)

    ax.plot(train_hist['weight'], train_hist['bias'], '.-b')
    ax.plot(A[0], b, 'r*', markersize=10)

    #ax.legend(['optim', '(A, b)'])

../../../_images/ipynbs_colabs_pyg_tutorial_project_Tutorial02_60_0.png

Stochastic GD¶

Avoid loading the full training set
Avoid evaluating the model on the full training set

At every step, compute the loss on a batch of data \((X^{(k)}, y^{(k)})\sim (X_{train}, y_{train})\):

\begin{align*} L(X^{(k)}, y^{(k)}) := \sum_{(x, y) \in (X^{(k)}, y^{(k)})} L(x, model(x)). \end{align*}

and then update with this approximated gradient: \begin{align*} par^{(k+1)} &= par^{(k)} - \eta^{(k)} \nabla_{par} L(X^{(k)}, y^{(k)}). \end{align*}

PyTorch `optim`¶

[ ]:

lr = 0.1
weight_decay = 5e-4
optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

More at pytorch.org/docs/stable/optim.html.

[ ]:

n_epochs = 100

[ ]:

train_hist = {}
train_hist['loss'] = []

if input_dim == output_dim == 1:
    train_hist['weight'] = []
    train_hist['bias'] = []

# Initialize training
model.reset()
model.train()

for epoch in range(n_epochs):
    for idx, batch in enumerate(train_loader):
        optimizer.zero_grad()
        loss = loss_fun(model.forward(batch[0]),  batch[1])
        loss.backward()
        optimizer.step()

        train_hist['loss'].append(loss.item())
        if input_dim == output_dim == 1:
            train_hist['weight'].append(model.linear.weight.item())
            train_hist['bias'].append(model.linear.bias.item())

        print('[Epoch %4d/%4d] [Batch %4d/%4d] Loss: % 2.2e' % (epoch + 1, n_epochs,
                                                                idx + 1, len(train_loader),
                                                                loss.item()))

model.eval()

[Epoch    1/ 100] [Batch    1/   9] Loss:  4.36e+00
[Epoch    1/ 100] [Batch    2/   9] Loss:  3.77e+00
[Epoch    1/ 100] [Batch    3/   9] Loss:  3.12e+00
[Epoch    1/ 100] [Batch    4/   9] Loss:  2.61e+00
[Epoch    1/ 100] [Batch    5/   9] Loss:  2.13e+00
[Epoch    1/ 100] [Batch    6/   9] Loss:  1.77e+00
[Epoch    1/ 100] [Batch    7/   9] Loss:  1.39e+00
[Epoch    1/ 100] [Batch    8/   9] Loss:  1.06e+00
[Epoch    1/ 100] [Batch    9/   9] Loss:  8.17e-01
[Epoch    2/ 100] [Batch    1/   9] Loss:  5.72e-01
[Epoch    2/ 100] [Batch    2/   9] Loss:  4.09e-01
[Epoch    2/ 100] [Batch    3/   9] Loss:  2.64e-01
[Epoch    2/ 100] [Batch    4/   9] Loss:  1.70e-01
[Epoch    2/ 100] [Batch    5/   9] Loss:  7.79e-02
[Epoch    2/ 100] [Batch    6/   9] Loss:  4.52e-02
[Epoch    2/ 100] [Batch    7/   9] Loss:  3.23e-02
[Epoch    2/ 100] [Batch    8/   9] Loss:  2.99e-02
[Epoch    2/ 100] [Batch    9/   9] Loss:  4.17e-02
[Epoch    3/ 100] [Batch    1/   9] Loss:  5.58e-02
[Epoch    3/ 100] [Batch    2/   9] Loss:  9.14e-02
[Epoch    3/ 100] [Batch    3/   9] Loss:  1.20e-01
[Epoch    3/ 100] [Batch    4/   9] Loss:  1.77e-01
[Epoch    3/ 100] [Batch    5/   9] Loss:  1.93e-01
[Epoch    3/ 100] [Batch    6/   9] Loss:  2.05e-01
[Epoch    3/ 100] [Batch    7/   9] Loss:  2.15e-01
[Epoch    3/ 100] [Batch    8/   9] Loss:  1.98e-01
[Epoch    3/ 100] [Batch    9/   9] Loss:  2.22e-01
[Epoch    4/ 100] [Batch    1/   9] Loss:  2.24e-01
[Epoch    4/ 100] [Batch    2/   9] Loss:  1.84e-01
[Epoch    4/ 100] [Batch    3/   9] Loss:  1.38e-01
[Epoch    4/ 100] [Batch    4/   9] Loss:  1.46e-01
[Epoch    4/ 100] [Batch    5/   9] Loss:  1.18e-01
[Epoch    4/ 100] [Batch    6/   9] Loss:  1.01e-01
[Epoch    4/ 100] [Batch    7/   9] Loss:  6.84e-02
[Epoch    4/ 100] [Batch    8/   9] Loss:  6.57e-02
[Epoch    4/ 100] [Batch    9/   9] Loss:  5.43e-02
[Epoch    5/ 100] [Batch    1/   9] Loss:  3.76e-02
[Epoch    5/ 100] [Batch    2/   9] Loss:  2.32e-02
[Epoch    5/ 100] [Batch    3/   9] Loss:  1.82e-02
[Epoch    5/ 100] [Batch    4/   9] Loss:  1.70e-02
[Epoch    5/ 100] [Batch    5/   9] Loss:  1.38e-02
[Epoch    5/ 100] [Batch    6/   9] Loss:  1.66e-02
[Epoch    5/ 100] [Batch    7/   9] Loss:  1.87e-02
[Epoch    5/ 100] [Batch    8/   9] Loss:  2.39e-02
[Epoch    5/ 100] [Batch    9/   9] Loss:  2.11e-02
[Epoch    6/ 100] [Batch    1/   9] Loss:  2.29e-02
[Epoch    6/ 100] [Batch    2/   9] Loss:  2.94e-02
[Epoch    6/ 100] [Batch    3/   9] Loss:  2.93e-02
[Epoch    6/ 100] [Batch    4/   9] Loss:  3.26e-02
[Epoch    6/ 100] [Batch    5/   9] Loss:  3.15e-02
[Epoch    6/ 100] [Batch    6/   9] Loss:  3.25e-02
[Epoch    6/ 100] [Batch    7/   9] Loss:  2.89e-02
[Epoch    6/ 100] [Batch    8/   9] Loss:  2.36e-02
[Epoch    6/ 100] [Batch    9/   9] Loss:  2.61e-02
[Epoch    7/ 100] [Batch    1/   9] Loss:  2.02e-02
[Epoch    7/ 100] [Batch    2/   9] Loss:  1.84e-02
[Epoch    7/ 100] [Batch    3/   9] Loss:  1.67e-02
[Epoch    7/ 100] [Batch    4/   9] Loss:  1.19e-02
[Epoch    7/ 100] [Batch    5/   9] Loss:  1.22e-02
[Epoch    7/ 100] [Batch    6/   9] Loss:  1.00e-02
[Epoch    7/ 100] [Batch    7/   9] Loss:  9.60e-03
[Epoch    7/ 100] [Batch    8/   9] Loss:  8.42e-03
[Epoch    7/ 100] [Batch    9/   9] Loss:  8.33e-03
[Epoch    8/ 100] [Batch    1/   9] Loss:  8.87e-03
[Epoch    8/ 100] [Batch    2/   9] Loss:  9.08e-03
[Epoch    8/ 100] [Batch    3/   9] Loss:  8.53e-03
[Epoch    8/ 100] [Batch    4/   9] Loss:  7.64e-03
[Epoch    8/ 100] [Batch    5/   9] Loss:  1.00e-02
[Epoch    8/ 100] [Batch    6/   9] Loss:  7.82e-03
[Epoch    8/ 100] [Batch    7/   9] Loss:  9.74e-03
[Epoch    8/ 100] [Batch    8/   9] Loss:  7.11e-03
[Epoch    8/ 100] [Batch    9/   9] Loss:  8.81e-03
[Epoch    9/ 100] [Batch    1/   9] Loss:  7.99e-03
[Epoch    9/ 100] [Batch    2/   9] Loss:  6.81e-03
[Epoch    9/ 100] [Batch    3/   9] Loss:  7.64e-03
[Epoch    9/ 100] [Batch    4/   9] Loss:  5.91e-03
[Epoch    9/ 100] [Batch    5/   9] Loss:  6.59e-03
[Epoch    9/ 100] [Batch    6/   9] Loss:  5.74e-03
[Epoch    9/ 100] [Batch    7/   9] Loss:  5.04e-03
[Epoch    9/ 100] [Batch    8/   9] Loss:  5.53e-03
[Epoch    9/ 100] [Batch    9/   9] Loss:  2.84e-03
[Epoch   10/ 100] [Batch    1/   9] Loss:  3.95e-03
[Epoch   10/ 100] [Batch    2/   9] Loss:  4.69e-03
[Epoch   10/ 100] [Batch    3/   9] Loss:  4.29e-03
[Epoch   10/ 100] [Batch    4/   9] Loss:  4.70e-03
[Epoch   10/ 100] [Batch    5/   9] Loss:  4.75e-03
[Epoch   10/ 100] [Batch    6/   9] Loss:  3.88e-03
[Epoch   10/ 100] [Batch    7/   9] Loss:  4.09e-03
[Epoch   10/ 100] [Batch    8/   9] Loss:  4.17e-03
[Epoch   10/ 100] [Batch    9/   9] Loss:  4.34e-03
[Epoch   11/ 100] [Batch    1/   9] Loss:  3.10e-03
[Epoch   11/ 100] [Batch    2/   9] Loss:  4.02e-03
[Epoch   11/ 100] [Batch    3/   9] Loss:  3.82e-03
[Epoch   11/ 100] [Batch    4/   9] Loss:  3.14e-03
[Epoch   11/ 100] [Batch    5/   9] Loss:  3.04e-03
[Epoch   11/ 100] [Batch    6/   9] Loss:  3.75e-03
[Epoch   11/ 100] [Batch    7/   9] Loss:  3.40e-03
[Epoch   11/ 100] [Batch    8/   9] Loss:  3.04e-03
[Epoch   11/ 100] [Batch    9/   9] Loss:  2.07e-03
[Epoch   12/ 100] [Batch    1/   9] Loss:  2.87e-03
[Epoch   12/ 100] [Batch    2/   9] Loss:  3.41e-03
[Epoch   12/ 100] [Batch    3/   9] Loss:  2.66e-03
[Epoch   12/ 100] [Batch    4/   9] Loss:  2.34e-03
[Epoch   12/ 100] [Batch    5/   9] Loss:  2.19e-03
[Epoch   12/ 100] [Batch    6/   9] Loss:  2.70e-03
[Epoch   12/ 100] [Batch    7/   9] Loss:  2.31e-03
[Epoch   12/ 100] [Batch    8/   9] Loss:  2.30e-03
[Epoch   12/ 100] [Batch    9/   9] Loss:  2.55e-03
[Epoch   13/ 100] [Batch    1/   9] Loss:  2.30e-03
[Epoch   13/ 100] [Batch    2/   9] Loss:  2.16e-03
[Epoch   13/ 100] [Batch    3/   9] Loss:  1.48e-03
[Epoch   13/ 100] [Batch    4/   9] Loss:  2.16e-03
[Epoch   13/ 100] [Batch    5/   9] Loss:  2.47e-03
[Epoch   13/ 100] [Batch    6/   9] Loss:  2.45e-03
[Epoch   13/ 100] [Batch    7/   9] Loss:  2.23e-03
[Epoch   13/ 100] [Batch    8/   9] Loss:  2.49e-03
[Epoch   13/ 100] [Batch    9/   9] Loss:  1.93e-03
[Epoch   14/ 100] [Batch    1/   9] Loss:  2.30e-03
[Epoch   14/ 100] [Batch    2/   9] Loss:  1.79e-03
[Epoch   14/ 100] [Batch    3/   9] Loss:  1.86e-03
[Epoch   14/ 100] [Batch    4/   9] Loss:  1.75e-03
[Epoch   14/ 100] [Batch    5/   9] Loss:  2.04e-03
[Epoch   14/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   14/ 100] [Batch    7/   9] Loss:  1.63e-03
[Epoch   14/ 100] [Batch    8/   9] Loss:  2.14e-03
[Epoch   14/ 100] [Batch    9/   9] Loss:  2.73e-03
[Epoch   15/ 100] [Batch    1/   9] Loss:  1.77e-03
[Epoch   15/ 100] [Batch    2/   9] Loss:  1.96e-03
[Epoch   15/ 100] [Batch    3/   9] Loss:  1.96e-03
[Epoch   15/ 100] [Batch    4/   9] Loss:  1.94e-03
[Epoch   15/ 100] [Batch    5/   9] Loss:  2.00e-03
[Epoch   15/ 100] [Batch    6/   9] Loss:  1.52e-03
[Epoch   15/ 100] [Batch    7/   9] Loss:  1.70e-03
[Epoch   15/ 100] [Batch    8/   9] Loss:  1.56e-03
[Epoch   15/ 100] [Batch    9/   9] Loss:  1.12e-03
[Epoch   16/ 100] [Batch    1/   9] Loss:  2.10e-03
[Epoch   16/ 100] [Batch    2/   9] Loss:  1.43e-03
[Epoch   16/ 100] [Batch    3/   9] Loss:  1.95e-03
[Epoch   16/ 100] [Batch    4/   9] Loss:  1.79e-03
[Epoch   16/ 100] [Batch    5/   9] Loss:  1.60e-03
[Epoch   16/ 100] [Batch    6/   9] Loss:  1.80e-03
[Epoch   16/ 100] [Batch    7/   9] Loss:  1.43e-03
[Epoch   16/ 100] [Batch    8/   9] Loss:  1.43e-03
[Epoch   16/ 100] [Batch    9/   9] Loss:  1.30e-03
[Epoch   17/ 100] [Batch    1/   9] Loss:  1.38e-03
[Epoch   17/ 100] [Batch    2/   9] Loss:  1.83e-03
[Epoch   17/ 100] [Batch    3/   9] Loss:  1.72e-03
[Epoch   17/ 100] [Batch    4/   9] Loss:  1.82e-03
[Epoch   17/ 100] [Batch    5/   9] Loss:  1.95e-03
[Epoch   17/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   17/ 100] [Batch    7/   9] Loss:  1.39e-03
[Epoch   17/ 100] [Batch    8/   9] Loss:  1.49e-03
[Epoch   17/ 100] [Batch    9/   9] Loss:  1.29e-03
[Epoch   18/ 100] [Batch    1/   9] Loss:  1.53e-03
[Epoch   18/ 100] [Batch    2/   9] Loss:  2.22e-03
[Epoch   18/ 100] [Batch    3/   9] Loss:  1.67e-03
[Epoch   18/ 100] [Batch    4/   9] Loss:  1.16e-03
[Epoch   18/ 100] [Batch    5/   9] Loss:  1.73e-03
[Epoch   18/ 100] [Batch    6/   9] Loss:  1.35e-03
[Epoch   18/ 100] [Batch    7/   9] Loss:  1.44e-03
[Epoch   18/ 100] [Batch    8/   9] Loss:  1.62e-03
[Epoch   18/ 100] [Batch    9/   9] Loss:  1.65e-03
[Epoch   19/ 100] [Batch    1/   9] Loss:  1.74e-03
[Epoch   19/ 100] [Batch    2/   9] Loss:  1.66e-03
[Epoch   19/ 100] [Batch    3/   9] Loss:  1.89e-03
[Epoch   19/ 100] [Batch    4/   9] Loss:  1.28e-03
[Epoch   19/ 100] [Batch    5/   9] Loss:  1.58e-03
[Epoch   19/ 100] [Batch    6/   9] Loss:  1.21e-03
[Epoch   19/ 100] [Batch    7/   9] Loss:  1.83e-03
[Epoch   19/ 100] [Batch    8/   9] Loss:  1.46e-03
[Epoch   19/ 100] [Batch    9/   9] Loss:  1.43e-03
[Epoch   20/ 100] [Batch    1/   9] Loss:  1.63e-03
[Epoch   20/ 100] [Batch    2/   9] Loss:  1.61e-03
[Epoch   20/ 100] [Batch    3/   9] Loss:  1.55e-03
[Epoch   20/ 100] [Batch    4/   9] Loss:  1.43e-03
[Epoch   20/ 100] [Batch    5/   9] Loss:  1.38e-03
[Epoch   20/ 100] [Batch    6/   9] Loss:  1.75e-03
[Epoch   20/ 100] [Batch    7/   9] Loss:  1.64e-03
[Epoch   20/ 100] [Batch    8/   9] Loss:  1.54e-03
[Epoch   20/ 100] [Batch    9/   9] Loss:  1.62e-03
[Epoch   21/ 100] [Batch    1/   9] Loss:  1.45e-03
[Epoch   21/ 100] [Batch    2/   9] Loss:  1.75e-03
[Epoch   21/ 100] [Batch    3/   9] Loss:  1.14e-03
[Epoch   21/ 100] [Batch    4/   9] Loss:  1.67e-03
[Epoch   21/ 100] [Batch    5/   9] Loss:  1.66e-03
[Epoch   21/ 100] [Batch    6/   9] Loss:  1.75e-03
[Epoch   21/ 100] [Batch    7/   9] Loss:  1.37e-03
[Epoch   21/ 100] [Batch    8/   9] Loss:  1.69e-03
[Epoch   21/ 100] [Batch    9/   9] Loss:  1.56e-03
[Epoch   22/ 100] [Batch    1/   9] Loss:  1.40e-03
[Epoch   22/ 100] [Batch    2/   9] Loss:  1.38e-03
[Epoch   22/ 100] [Batch    3/   9] Loss:  1.46e-03
[Epoch   22/ 100] [Batch    4/   9] Loss:  1.85e-03
[Epoch   22/ 100] [Batch    5/   9] Loss:  1.52e-03
[Epoch   22/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   22/ 100] [Batch    7/   9] Loss:  1.83e-03
[Epoch   22/ 100] [Batch    8/   9] Loss:  1.61e-03
[Epoch   22/ 100] [Batch    9/   9] Loss:  1.47e-03
[Epoch   23/ 100] [Batch    1/   9] Loss:  1.78e-03
[Epoch   23/ 100] [Batch    2/   9] Loss:  1.49e-03
[Epoch   23/ 100] [Batch    3/   9] Loss:  1.47e-03
[Epoch   23/ 100] [Batch    4/   9] Loss:  1.82e-03
[Epoch   23/ 100] [Batch    5/   9] Loss:  1.59e-03
[Epoch   23/ 100] [Batch    6/   9] Loss:  1.61e-03
[Epoch   23/ 100] [Batch    7/   9] Loss:  1.53e-03
[Epoch   23/ 100] [Batch    8/   9] Loss:  1.33e-03
[Epoch   23/ 100] [Batch    9/   9] Loss:  1.15e-03
[Epoch   24/ 100] [Batch    1/   9] Loss:  1.51e-03
[Epoch   24/ 100] [Batch    2/   9] Loss:  1.24e-03
[Epoch   24/ 100] [Batch    3/   9] Loss:  1.74e-03
[Epoch   24/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   24/ 100] [Batch    5/   9] Loss:  1.55e-03
[Epoch   24/ 100] [Batch    6/   9] Loss:  1.72e-03
[Epoch   24/ 100] [Batch    7/   9] Loss:  1.72e-03
[Epoch   24/ 100] [Batch    8/   9] Loss:  1.43e-03
[Epoch   24/ 100] [Batch    9/   9] Loss:  1.50e-03
[Epoch   25/ 100] [Batch    1/   9] Loss:  1.44e-03
[Epoch   25/ 100] [Batch    2/   9] Loss:  1.66e-03
[Epoch   25/ 100] [Batch    3/   9] Loss:  1.90e-03
[Epoch   25/ 100] [Batch    4/   9] Loss:  1.53e-03
[Epoch   25/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   25/ 100] [Batch    6/   9] Loss:  1.28e-03
[Epoch   25/ 100] [Batch    7/   9] Loss:  1.36e-03
[Epoch   25/ 100] [Batch    8/   9] Loss:  1.84e-03
[Epoch   25/ 100] [Batch    9/   9] Loss:  1.69e-03
[Epoch   26/ 100] [Batch    1/   9] Loss:  1.66e-03
[Epoch   26/ 100] [Batch    2/   9] Loss:  1.39e-03
[Epoch   26/ 100] [Batch    3/   9] Loss:  1.60e-03
[Epoch   26/ 100] [Batch    4/   9] Loss:  1.26e-03
[Epoch   26/ 100] [Batch    5/   9] Loss:  1.88e-03
[Epoch   26/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   26/ 100] [Batch    7/   9] Loss:  1.72e-03
[Epoch   26/ 100] [Batch    8/   9] Loss:  1.53e-03
[Epoch   26/ 100] [Batch    9/   9] Loss:  1.48e-03
[Epoch   27/ 100] [Batch    1/   9] Loss:  1.46e-03
[Epoch   27/ 100] [Batch    2/   9] Loss:  1.64e-03
[Epoch   27/ 100] [Batch    3/   9] Loss:  1.37e-03
[Epoch   27/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   27/ 100] [Batch    5/   9] Loss:  1.82e-03
[Epoch   27/ 100] [Batch    6/   9] Loss:  1.45e-03
[Epoch   27/ 100] [Batch    7/   9] Loss:  1.49e-03
[Epoch   27/ 100] [Batch    8/   9] Loss:  1.75e-03
[Epoch   27/ 100] [Batch    9/   9] Loss:  1.33e-03
[Epoch   28/ 100] [Batch    1/   9] Loss:  1.52e-03
[Epoch   28/ 100] [Batch    2/   9] Loss:  1.83e-03
[Epoch   28/ 100] [Batch    3/   9] Loss:  1.40e-03
[Epoch   28/ 100] [Batch    4/   9] Loss:  1.46e-03
[Epoch   28/ 100] [Batch    5/   9] Loss:  1.66e-03
[Epoch   28/ 100] [Batch    6/   9] Loss:  1.63e-03
[Epoch   28/ 100] [Batch    7/   9] Loss:  1.40e-03
[Epoch   28/ 100] [Batch    8/   9] Loss:  1.63e-03
[Epoch   28/ 100] [Batch    9/   9] Loss:  1.53e-03
[Epoch   29/ 100] [Batch    1/   9] Loss:  1.53e-03
[Epoch   29/ 100] [Batch    2/   9] Loss:  1.60e-03
[Epoch   29/ 100] [Batch    3/   9] Loss:  1.24e-03
[Epoch   29/ 100] [Batch    4/   9] Loss:  1.81e-03
[Epoch   29/ 100] [Batch    5/   9] Loss:  1.74e-03
[Epoch   29/ 100] [Batch    6/   9] Loss:  1.55e-03
[Epoch   29/ 100] [Batch    7/   9] Loss:  1.56e-03
[Epoch   29/ 100] [Batch    8/   9] Loss:  1.41e-03
[Epoch   29/ 100] [Batch    9/   9] Loss:  1.81e-03
[Epoch   30/ 100] [Batch    1/   9] Loss:  1.25e-03
[Epoch   30/ 100] [Batch    2/   9] Loss:  1.33e-03
[Epoch   30/ 100] [Batch    3/   9] Loss:  1.57e-03
[Epoch   30/ 100] [Batch    4/   9] Loss:  1.76e-03
[Epoch   30/ 100] [Batch    5/   9] Loss:  1.65e-03
[Epoch   30/ 100] [Batch    6/   9] Loss:  1.60e-03
[Epoch   30/ 100] [Batch    7/   9] Loss:  1.52e-03
[Epoch   30/ 100] [Batch    8/   9] Loss:  1.70e-03
[Epoch   30/ 100] [Batch    9/   9] Loss:  1.85e-03
[Epoch   31/ 100] [Batch    1/   9] Loss:  1.50e-03
[Epoch   31/ 100] [Batch    2/   9] Loss:  1.36e-03
[Epoch   31/ 100] [Batch    3/   9] Loss:  1.55e-03
[Epoch   31/ 100] [Batch    4/   9] Loss:  1.98e-03
[Epoch   31/ 100] [Batch    5/   9] Loss:  1.51e-03
[Epoch   31/ 100] [Batch    6/   9] Loss:  1.43e-03
[Epoch   31/ 100] [Batch    7/   9] Loss:  1.30e-03
[Epoch   31/ 100] [Batch    8/   9] Loss:  1.80e-03
[Epoch   31/ 100] [Batch    9/   9] Loss:  1.69e-03
[Epoch   32/ 100] [Batch    1/   9] Loss:  1.38e-03
[Epoch   32/ 100] [Batch    2/   9] Loss:  1.83e-03
[Epoch   32/ 100] [Batch    3/   9] Loss:  1.30e-03
[Epoch   32/ 100] [Batch    4/   9] Loss:  1.84e-03
[Epoch   32/ 100] [Batch    5/   9] Loss:  1.34e-03
[Epoch   32/ 100] [Batch    6/   9] Loss:  1.61e-03
[Epoch   32/ 100] [Batch    7/   9] Loss:  1.70e-03
[Epoch   32/ 100] [Batch    8/   9] Loss:  1.54e-03
[Epoch   32/ 100] [Batch    9/   9] Loss:  1.44e-03
[Epoch   33/ 100] [Batch    1/   9] Loss:  1.35e-03
[Epoch   33/ 100] [Batch    2/   9] Loss:  1.88e-03
[Epoch   33/ 100] [Batch    3/   9] Loss:  1.45e-03
[Epoch   33/ 100] [Batch    4/   9] Loss:  1.81e-03
[Epoch   33/ 100] [Batch    5/   9] Loss:  1.47e-03
[Epoch   33/ 100] [Batch    6/   9] Loss:  1.40e-03
[Epoch   33/ 100] [Batch    7/   9] Loss:  1.44e-03
[Epoch   33/ 100] [Batch    8/   9] Loss:  1.78e-03
[Epoch   33/ 100] [Batch    9/   9] Loss:  1.27e-03
[Epoch   34/ 100] [Batch    1/   9] Loss:  1.50e-03
[Epoch   34/ 100] [Batch    2/   9] Loss:  1.60e-03
[Epoch   34/ 100] [Batch    3/   9] Loss:  1.63e-03
[Epoch   34/ 100] [Batch    4/   9] Loss:  1.41e-03
[Epoch   34/ 100] [Batch    5/   9] Loss:  1.51e-03
[Epoch   34/ 100] [Batch    6/   9] Loss:  1.77e-03
[Epoch   34/ 100] [Batch    7/   9] Loss:  1.51e-03
[Epoch   34/ 100] [Batch    8/   9] Loss:  1.69e-03
[Epoch   34/ 100] [Batch    9/   9] Loss:  1.13e-03
[Epoch   35/ 100] [Batch    1/   9] Loss:  1.41e-03
[Epoch   35/ 100] [Batch    2/   9] Loss:  1.75e-03
[Epoch   35/ 100] [Batch    3/   9] Loss:  1.37e-03
[Epoch   35/ 100] [Batch    4/   9] Loss:  1.35e-03
[Epoch   35/ 100] [Batch    5/   9] Loss:  1.37e-03
[Epoch   35/ 100] [Batch    6/   9] Loss:  2.06e-03
[Epoch   35/ 100] [Batch    7/   9] Loss:  1.75e-03
[Epoch   35/ 100] [Batch    8/   9] Loss:  1.42e-03
[Epoch   35/ 100] [Batch    9/   9] Loss:  1.58e-03
[Epoch   36/ 100] [Batch    1/   9] Loss:  1.77e-03
[Epoch   36/ 100] [Batch    2/   9] Loss:  1.13e-03
[Epoch   36/ 100] [Batch    3/   9] Loss:  1.85e-03
[Epoch   36/ 100] [Batch    4/   9] Loss:  1.77e-03
[Epoch   36/ 100] [Batch    5/   9] Loss:  1.40e-03
[Epoch   36/ 100] [Batch    6/   9] Loss:  1.57e-03
[Epoch   36/ 100] [Batch    7/   9] Loss:  1.56e-03
[Epoch   36/ 100] [Batch    8/   9] Loss:  1.51e-03
[Epoch   36/ 100] [Batch    9/   9] Loss:  1.32e-03
[Epoch   37/ 100] [Batch    1/   9] Loss:  1.31e-03
[Epoch   37/ 100] [Batch    2/   9] Loss:  1.58e-03
[Epoch   37/ 100] [Batch    3/   9] Loss:  1.51e-03
[Epoch   37/ 100] [Batch    4/   9] Loss:  1.68e-03
[Epoch   37/ 100] [Batch    5/   9] Loss:  1.71e-03
[Epoch   37/ 100] [Batch    6/   9] Loss:  1.29e-03
[Epoch   37/ 100] [Batch    7/   9] Loss:  1.80e-03
[Epoch   37/ 100] [Batch    8/   9] Loss:  1.52e-03
[Epoch   37/ 100] [Batch    9/   9] Loss:  1.74e-03
[Epoch   38/ 100] [Batch    1/   9] Loss:  1.50e-03
[Epoch   38/ 100] [Batch    2/   9] Loss:  1.65e-03
[Epoch   38/ 100] [Batch    3/   9] Loss:  1.42e-03
[Epoch   38/ 100] [Batch    4/   9] Loss:  1.90e-03
[Epoch   38/ 100] [Batch    5/   9] Loss:  1.37e-03
[Epoch   38/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   38/ 100] [Batch    7/   9] Loss:  1.41e-03
[Epoch   38/ 100] [Batch    8/   9] Loss:  1.38e-03
[Epoch   38/ 100] [Batch    9/   9] Loss:  2.07e-03
[Epoch   39/ 100] [Batch    1/   9] Loss:  1.69e-03
[Epoch   39/ 100] [Batch    2/   9] Loss:  1.50e-03
[Epoch   39/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   39/ 100] [Batch    4/   9] Loss:  1.72e-03
[Epoch   39/ 100] [Batch    5/   9] Loss:  1.24e-03
[Epoch   39/ 100] [Batch    6/   9] Loss:  1.64e-03
[Epoch   39/ 100] [Batch    7/   9] Loss:  1.33e-03
[Epoch   39/ 100] [Batch    8/   9] Loss:  1.77e-03
[Epoch   39/ 100] [Batch    9/   9] Loss:  1.32e-03
[Epoch   40/ 100] [Batch    1/   9] Loss:  1.30e-03
[Epoch   40/ 100] [Batch    2/   9] Loss:  1.58e-03
[Epoch   40/ 100] [Batch    3/   9] Loss:  1.46e-03
[Epoch   40/ 100] [Batch    4/   9] Loss:  1.39e-03
[Epoch   40/ 100] [Batch    5/   9] Loss:  1.92e-03
[Epoch   40/ 100] [Batch    6/   9] Loss:  1.67e-03
[Epoch   40/ 100] [Batch    7/   9] Loss:  1.81e-03
[Epoch   40/ 100] [Batch    8/   9] Loss:  1.41e-03
[Epoch   40/ 100] [Batch    9/   9] Loss:  1.41e-03
[Epoch   41/ 100] [Batch    1/   9] Loss:  1.49e-03
[Epoch   41/ 100] [Batch    2/   9] Loss:  1.63e-03
[Epoch   41/ 100] [Batch    3/   9] Loss:  1.25e-03
[Epoch   41/ 100] [Batch    4/   9] Loss:  1.58e-03
[Epoch   41/ 100] [Batch    5/   9] Loss:  1.65e-03
[Epoch   41/ 100] [Batch    6/   9] Loss:  1.70e-03
[Epoch   41/ 100] [Batch    7/   9] Loss:  1.38e-03
[Epoch   41/ 100] [Batch    8/   9] Loss:  1.76e-03
[Epoch   41/ 100] [Batch    9/   9] Loss:  1.63e-03
[Epoch   42/ 100] [Batch    1/   9] Loss:  1.67e-03
[Epoch   42/ 100] [Batch    2/   9] Loss:  1.35e-03
[Epoch   42/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   42/ 100] [Batch    4/   9] Loss:  1.60e-03
[Epoch   42/ 100] [Batch    5/   9] Loss:  1.25e-03
[Epoch   42/ 100] [Batch    6/   9] Loss:  1.93e-03
[Epoch   42/ 100] [Batch    7/   9] Loss:  1.27e-03
[Epoch   42/ 100] [Batch    8/   9] Loss:  1.81e-03
[Epoch   42/ 100] [Batch    9/   9] Loss:  1.35e-03
[Epoch   43/ 100] [Batch    1/   9] Loss:  1.50e-03
[Epoch   43/ 100] [Batch    2/   9] Loss:  1.20e-03
[Epoch   43/ 100] [Batch    3/   9] Loss:  1.42e-03
[Epoch   43/ 100] [Batch    4/   9] Loss:  1.78e-03
[Epoch   43/ 100] [Batch    5/   9] Loss:  1.57e-03
[Epoch   43/ 100] [Batch    6/   9] Loss:  2.03e-03
[Epoch   43/ 100] [Batch    7/   9] Loss:  1.35e-03
[Epoch   43/ 100] [Batch    8/   9] Loss:  1.76e-03
[Epoch   43/ 100] [Batch    9/   9] Loss:  1.12e-03
[Epoch   44/ 100] [Batch    1/   9] Loss:  1.39e-03
[Epoch   44/ 100] [Batch    2/   9] Loss:  1.49e-03
[Epoch   44/ 100] [Batch    3/   9] Loss:  1.84e-03
[Epoch   44/ 100] [Batch    4/   9] Loss:  1.60e-03
[Epoch   44/ 100] [Batch    5/   9] Loss:  1.51e-03
[Epoch   44/ 100] [Batch    6/   9] Loss:  1.60e-03
[Epoch   44/ 100] [Batch    7/   9] Loss:  1.63e-03
[Epoch   44/ 100] [Batch    8/   9] Loss:  1.42e-03
[Epoch   44/ 100] [Batch    9/   9] Loss:  1.59e-03
[Epoch   45/ 100] [Batch    1/   9] Loss:  1.88e-03
[Epoch   45/ 100] [Batch    2/   9] Loss:  1.29e-03
[Epoch   45/ 100] [Batch    3/   9] Loss:  1.49e-03
[Epoch   45/ 100] [Batch    4/   9] Loss:  1.18e-03
[Epoch   45/ 100] [Batch    5/   9] Loss:  1.90e-03
[Epoch   45/ 100] [Batch    6/   9] Loss:  1.53e-03
[Epoch   45/ 100] [Batch    7/   9] Loss:  1.68e-03
[Epoch   45/ 100] [Batch    8/   9] Loss:  1.70e-03
[Epoch   45/ 100] [Batch    9/   9] Loss:  1.01e-03
[Epoch   46/ 100] [Batch    1/   9] Loss:  1.63e-03
[Epoch   46/ 100] [Batch    2/   9] Loss:  1.41e-03
[Epoch   46/ 100] [Batch    3/   9] Loss:  1.57e-03
[Epoch   46/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   46/ 100] [Batch    5/   9] Loss:  1.48e-03
[Epoch   46/ 100] [Batch    6/   9] Loss:  1.43e-03
[Epoch   46/ 100] [Batch    7/   9] Loss:  1.57e-03
[Epoch   46/ 100] [Batch    8/   9] Loss:  1.72e-03
[Epoch   46/ 100] [Batch    9/   9] Loss:  1.79e-03
[Epoch   47/ 100] [Batch    1/   9] Loss:  1.75e-03
[Epoch   47/ 100] [Batch    2/   9] Loss:  1.43e-03
[Epoch   47/ 100] [Batch    3/   9] Loss:  1.63e-03
[Epoch   47/ 100] [Batch    4/   9] Loss:  1.47e-03
[Epoch   47/ 100] [Batch    5/   9] Loss:  1.63e-03
[Epoch   47/ 100] [Batch    6/   9] Loss:  1.72e-03
[Epoch   47/ 100] [Batch    7/   9] Loss:  1.61e-03
[Epoch   47/ 100] [Batch    8/   9] Loss:  1.41e-03
[Epoch   47/ 100] [Batch    9/   9] Loss:  1.01e-03
[Epoch   48/ 100] [Batch    1/   9] Loss:  1.39e-03
[Epoch   48/ 100] [Batch    2/   9] Loss:  1.89e-03
[Epoch   48/ 100] [Batch    3/   9] Loss:  1.47e-03
[Epoch   48/ 100] [Batch    4/   9] Loss:  1.69e-03
[Epoch   48/ 100] [Batch    5/   9] Loss:  1.67e-03
[Epoch   48/ 100] [Batch    6/   9] Loss:  1.38e-03
[Epoch   48/ 100] [Batch    7/   9] Loss:  1.49e-03
[Epoch   48/ 100] [Batch    8/   9] Loss:  1.63e-03
[Epoch   48/ 100] [Batch    9/   9] Loss:  1.38e-03
[Epoch   49/ 100] [Batch    1/   9] Loss:  1.79e-03
[Epoch   49/ 100] [Batch    2/   9] Loss:  1.69e-03
[Epoch   49/ 100] [Batch    3/   9] Loss:  1.46e-03
[Epoch   49/ 100] [Batch    4/   9] Loss:  1.91e-03
[Epoch   49/ 100] [Batch    5/   9] Loss:  1.69e-03
[Epoch   49/ 100] [Batch    6/   9] Loss:  1.28e-03
[Epoch   49/ 100] [Batch    7/   9] Loss:  1.37e-03
[Epoch   49/ 100] [Batch    8/   9] Loss:  1.50e-03
[Epoch   49/ 100] [Batch    9/   9] Loss:  1.12e-03
[Epoch   50/ 100] [Batch    1/   9] Loss:  1.58e-03
[Epoch   50/ 100] [Batch    2/   9] Loss:  1.47e-03
[Epoch   50/ 100] [Batch    3/   9] Loss:  1.21e-03
[Epoch   50/ 100] [Batch    4/   9] Loss:  1.83e-03
[Epoch   50/ 100] [Batch    5/   9] Loss:  1.72e-03
[Epoch   50/ 100] [Batch    6/   9] Loss:  1.56e-03
[Epoch   50/ 100] [Batch    7/   9] Loss:  1.48e-03
[Epoch   50/ 100] [Batch    8/   9] Loss:  1.71e-03
[Epoch   50/ 100] [Batch    9/   9] Loss:  1.25e-03
[Epoch   51/ 100] [Batch    1/   9] Loss:  1.69e-03
[Epoch   51/ 100] [Batch    2/   9] Loss:  1.81e-03
[Epoch   51/ 100] [Batch    3/   9] Loss:  1.55e-03
[Epoch   51/ 100] [Batch    4/   9] Loss:  1.77e-03
[Epoch   51/ 100] [Batch    5/   9] Loss:  1.61e-03
[Epoch   51/ 100] [Batch    6/   9] Loss:  1.15e-03
[Epoch   51/ 100] [Batch    7/   9] Loss:  1.62e-03
[Epoch   51/ 100] [Batch    8/   9] Loss:  1.37e-03
[Epoch   51/ 100] [Batch    9/   9] Loss:  1.66e-03
[Epoch   52/ 100] [Batch    1/   9] Loss:  1.62e-03
[Epoch   52/ 100] [Batch    2/   9] Loss:  1.49e-03
[Epoch   52/ 100] [Batch    3/   9] Loss:  1.59e-03
[Epoch   52/ 100] [Batch    4/   9] Loss:  1.62e-03
[Epoch   52/ 100] [Batch    5/   9] Loss:  1.28e-03
[Epoch   52/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   52/ 100] [Batch    7/   9] Loss:  1.69e-03
[Epoch   52/ 100] [Batch    8/   9] Loss:  1.73e-03
[Epoch   52/ 100] [Batch    9/   9] Loss:  1.32e-03
[Epoch   53/ 100] [Batch    1/   9] Loss:  1.86e-03
[Epoch   53/ 100] [Batch    2/   9] Loss:  1.62e-03
[Epoch   53/ 100] [Batch    3/   9] Loss:  1.34e-03
[Epoch   53/ 100] [Batch    4/   9] Loss:  1.45e-03
[Epoch   53/ 100] [Batch    5/   9] Loss:  1.92e-03
[Epoch   53/ 100] [Batch    6/   9] Loss:  1.33e-03
[Epoch   53/ 100] [Batch    7/   9] Loss:  1.40e-03
[Epoch   53/ 100] [Batch    8/   9] Loss:  1.62e-03
[Epoch   53/ 100] [Batch    9/   9] Loss:  1.55e-03
[Epoch   54/ 100] [Batch    1/   9] Loss:  1.46e-03
[Epoch   54/ 100] [Batch    2/   9] Loss:  1.57e-03
[Epoch   54/ 100] [Batch    3/   9] Loss:  1.76e-03
[Epoch   54/ 100] [Batch    4/   9] Loss:  1.60e-03
[Epoch   54/ 100] [Batch    5/   9] Loss:  1.07e-03
[Epoch   54/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   54/ 100] [Batch    7/   9] Loss:  1.72e-03
[Epoch   54/ 100] [Batch    8/   9] Loss:  1.54e-03
[Epoch   54/ 100] [Batch    9/   9] Loss:  2.13e-03
[Epoch   55/ 100] [Batch    1/   9] Loss:  1.80e-03
[Epoch   55/ 100] [Batch    2/   9] Loss:  1.18e-03
[Epoch   55/ 100] [Batch    3/   9] Loss:  1.59e-03
[Epoch   55/ 100] [Batch    4/   9] Loss:  1.38e-03
[Epoch   55/ 100] [Batch    5/   9] Loss:  1.60e-03
[Epoch   55/ 100] [Batch    6/   9] Loss:  1.83e-03
[Epoch   55/ 100] [Batch    7/   9] Loss:  1.62e-03
[Epoch   55/ 100] [Batch    8/   9] Loss:  1.45e-03
[Epoch   55/ 100] [Batch    9/   9] Loss:  1.67e-03
[Epoch   56/ 100] [Batch    1/   9] Loss:  1.66e-03
[Epoch   56/ 100] [Batch    2/   9] Loss:  1.42e-03
[Epoch   56/ 100] [Batch    3/   9] Loss:  1.42e-03
[Epoch   56/ 100] [Batch    4/   9] Loss:  1.82e-03
[Epoch   56/ 100] [Batch    5/   9] Loss:  1.43e-03
[Epoch   56/ 100] [Batch    6/   9] Loss:  1.40e-03
[Epoch   56/ 100] [Batch    7/   9] Loss:  1.68e-03
[Epoch   56/ 100] [Batch    8/   9] Loss:  1.79e-03
[Epoch   56/ 100] [Batch    9/   9] Loss:  1.23e-03
[Epoch   57/ 100] [Batch    1/   9] Loss:  1.37e-03
[Epoch   57/ 100] [Batch    2/   9] Loss:  1.67e-03
[Epoch   57/ 100] [Batch    3/   9] Loss:  1.80e-03
[Epoch   57/ 100] [Batch    4/   9] Loss:  1.63e-03
[Epoch   57/ 100] [Batch    5/   9] Loss:  1.42e-03
[Epoch   57/ 100] [Batch    6/   9] Loss:  1.33e-03
[Epoch   57/ 100] [Batch    7/   9] Loss:  1.71e-03
[Epoch   57/ 100] [Batch    8/   9] Loss:  1.28e-03
[Epoch   57/ 100] [Batch    9/   9] Loss:  2.89e-03
[Epoch   58/ 100] [Batch    1/   9] Loss:  1.53e-03
[Epoch   58/ 100] [Batch    2/   9] Loss:  1.34e-03
[Epoch   58/ 100] [Batch    3/   9] Loss:  1.68e-03
[Epoch   58/ 100] [Batch    4/   9] Loss:  1.45e-03
[Epoch   58/ 100] [Batch    5/   9] Loss:  1.52e-03
[Epoch   58/ 100] [Batch    6/   9] Loss:  1.60e-03
[Epoch   58/ 100] [Batch    7/   9] Loss:  1.70e-03
[Epoch   58/ 100] [Batch    8/   9] Loss:  1.45e-03
[Epoch   58/ 100] [Batch    9/   9] Loss:  2.61e-03
[Epoch   59/ 100] [Batch    1/   9] Loss:  1.45e-03
[Epoch   59/ 100] [Batch    2/   9] Loss:  1.14e-03
[Epoch   59/ 100] [Batch    3/   9] Loss:  1.58e-03
[Epoch   59/ 100] [Batch    4/   9] Loss:  2.05e-03
[Epoch   59/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   59/ 100] [Batch    6/   9] Loss:  1.46e-03
[Epoch   59/ 100] [Batch    7/   9] Loss:  1.99e-03
[Epoch   59/ 100] [Batch    8/   9] Loss:  1.55e-03
[Epoch   59/ 100] [Batch    9/   9] Loss:  1.49e-03
[Epoch   60/ 100] [Batch    1/   9] Loss:  1.28e-03
[Epoch   60/ 100] [Batch    2/   9] Loss:  1.63e-03
[Epoch   60/ 100] [Batch    3/   9] Loss:  1.51e-03
[Epoch   60/ 100] [Batch    4/   9] Loss:  1.62e-03
[Epoch   60/ 100] [Batch    5/   9] Loss:  1.99e-03
[Epoch   60/ 100] [Batch    6/   9] Loss:  1.75e-03
[Epoch   60/ 100] [Batch    7/   9] Loss:  1.34e-03
[Epoch   60/ 100] [Batch    8/   9] Loss:  1.37e-03
[Epoch   60/ 100] [Batch    9/   9] Loss:  1.70e-03
[Epoch   61/ 100] [Batch    1/   9] Loss:  1.42e-03
[Epoch   61/ 100] [Batch    2/   9] Loss:  1.68e-03
[Epoch   61/ 100] [Batch    3/   9] Loss:  1.46e-03
[Epoch   61/ 100] [Batch    4/   9] Loss:  1.76e-03
[Epoch   61/ 100] [Batch    5/   9] Loss:  1.64e-03
[Epoch   61/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   61/ 100] [Batch    7/   9] Loss:  1.62e-03
[Epoch   61/ 100] [Batch    8/   9] Loss:  1.59e-03
[Epoch   61/ 100] [Batch    9/   9] Loss:  1.19e-03
[Epoch   62/ 100] [Batch    1/   9] Loss:  1.71e-03
[Epoch   62/ 100] [Batch    2/   9] Loss:  1.38e-03
[Epoch   62/ 100] [Batch    3/   9] Loss:  1.58e-03
[Epoch   62/ 100] [Batch    4/   9] Loss:  1.40e-03
[Epoch   62/ 100] [Batch    5/   9] Loss:  1.76e-03
[Epoch   62/ 100] [Batch    6/   9] Loss:  1.63e-03
[Epoch   62/ 100] [Batch    7/   9] Loss:  1.51e-03
[Epoch   62/ 100] [Batch    8/   9] Loss:  1.53e-03
[Epoch   62/ 100] [Batch    9/   9] Loss:  1.54e-03
[Epoch   63/ 100] [Batch    1/   9] Loss:  1.47e-03
[Epoch   63/ 100] [Batch    2/   9] Loss:  1.33e-03
[Epoch   63/ 100] [Batch    3/   9] Loss:  1.57e-03
[Epoch   63/ 100] [Batch    4/   9] Loss:  1.49e-03
[Epoch   63/ 100] [Batch    5/   9] Loss:  1.66e-03
[Epoch   63/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   63/ 100] [Batch    7/   9] Loss:  1.87e-03
[Epoch   63/ 100] [Batch    8/   9] Loss:  1.57e-03
[Epoch   63/ 100] [Batch    9/   9] Loss:  1.23e-03
[Epoch   64/ 100] [Batch    1/   9] Loss:  1.51e-03
[Epoch   64/ 100] [Batch    2/   9] Loss:  1.46e-03
[Epoch   64/ 100] [Batch    3/   9] Loss:  1.51e-03
[Epoch   64/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   64/ 100] [Batch    5/   9] Loss:  1.40e-03
[Epoch   64/ 100] [Batch    6/   9] Loss:  1.87e-03
[Epoch   64/ 100] [Batch    7/   9] Loss:  1.59e-03
[Epoch   64/ 100] [Batch    8/   9] Loss:  1.62e-03
[Epoch   64/ 100] [Batch    9/   9] Loss:  1.46e-03
[Epoch   65/ 100] [Batch    1/   9] Loss:  1.54e-03
[Epoch   65/ 100] [Batch    2/   9] Loss:  1.85e-03
[Epoch   65/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   65/ 100] [Batch    4/   9] Loss:  1.58e-03
[Epoch   65/ 100] [Batch    5/   9] Loss:  1.20e-03
[Epoch   65/ 100] [Batch    6/   9] Loss:  1.41e-03
[Epoch   65/ 100] [Batch    7/   9] Loss:  1.54e-03
[Epoch   65/ 100] [Batch    8/   9] Loss:  1.61e-03
[Epoch   65/ 100] [Batch    9/   9] Loss:  1.96e-03
[Epoch   66/ 100] [Batch    1/   9] Loss:  1.59e-03
[Epoch   66/ 100] [Batch    2/   9] Loss:  1.51e-03
[Epoch   66/ 100] [Batch    3/   9] Loss:  1.85e-03
[Epoch   66/ 100] [Batch    4/   9] Loss:  1.87e-03
[Epoch   66/ 100] [Batch    5/   9] Loss:  1.38e-03
[Epoch   66/ 100] [Batch    6/   9] Loss:  1.83e-03
[Epoch   66/ 100] [Batch    7/   9] Loss:  1.13e-03
[Epoch   66/ 100] [Batch    8/   9] Loss:  1.53e-03
[Epoch   66/ 100] [Batch    9/   9] Loss:  1.08e-03
[Epoch   67/ 100] [Batch    1/   9] Loss:  1.58e-03
[Epoch   67/ 100] [Batch    2/   9] Loss:  1.63e-03
[Epoch   67/ 100] [Batch    3/   9] Loss:  1.48e-03
[Epoch   67/ 100] [Batch    4/   9] Loss:  1.70e-03
[Epoch   67/ 100] [Batch    5/   9] Loss:  1.42e-03
[Epoch   67/ 100] [Batch    6/   9] Loss:  1.64e-03
[Epoch   67/ 100] [Batch    7/   9] Loss:  1.30e-03
[Epoch   67/ 100] [Batch    8/   9] Loss:  1.84e-03
[Epoch   67/ 100] [Batch    9/   9] Loss:  1.35e-03
[Epoch   68/ 100] [Batch    1/   9] Loss:  1.20e-03
[Epoch   68/ 100] [Batch    2/   9] Loss:  1.39e-03
[Epoch   68/ 100] [Batch    3/   9] Loss:  1.68e-03
[Epoch   68/ 100] [Batch    4/   9] Loss:  1.62e-03
[Epoch   68/ 100] [Batch    5/   9] Loss:  1.66e-03
[Epoch   68/ 100] [Batch    6/   9] Loss:  2.08e-03
[Epoch   68/ 100] [Batch    7/   9] Loss:  1.69e-03
[Epoch   68/ 100] [Batch    8/   9] Loss:  1.50e-03
[Epoch   68/ 100] [Batch    9/   9] Loss:  7.45e-04
[Epoch   69/ 100] [Batch    1/   9] Loss:  1.44e-03
[Epoch   69/ 100] [Batch    2/   9] Loss:  1.59e-03
[Epoch   69/ 100] [Batch    3/   9] Loss:  1.40e-03
[Epoch   69/ 100] [Batch    4/   9] Loss:  1.82e-03
[Epoch   69/ 100] [Batch    5/   9] Loss:  1.41e-03
[Epoch   69/ 100] [Batch    6/   9] Loss:  1.64e-03
[Epoch   69/ 100] [Batch    7/   9] Loss:  1.86e-03
[Epoch   69/ 100] [Batch    8/   9] Loss:  1.44e-03
[Epoch   69/ 100] [Batch    9/   9] Loss:  1.15e-03
[Epoch   70/ 100] [Batch    1/   9] Loss:  1.51e-03
[Epoch   70/ 100] [Batch    2/   9] Loss:  1.37e-03
[Epoch   70/ 100] [Batch    3/   9] Loss:  1.53e-03
[Epoch   70/ 100] [Batch    4/   9] Loss:  1.69e-03
[Epoch   70/ 100] [Batch    5/   9] Loss:  1.94e-03
[Epoch   70/ 100] [Batch    6/   9] Loss:  1.69e-03
[Epoch   70/ 100] [Batch    7/   9] Loss:  1.51e-03
[Epoch   70/ 100] [Batch    8/   9] Loss:  1.47e-03
[Epoch   70/ 100] [Batch    9/   9] Loss:  8.61e-04
[Epoch   71/ 100] [Batch    1/   9] Loss:  1.63e-03
[Epoch   71/ 100] [Batch    2/   9] Loss:  1.59e-03
[Epoch   71/ 100] [Batch    3/   9] Loss:  1.65e-03
[Epoch   71/ 100] [Batch    4/   9] Loss:  1.85e-03
[Epoch   71/ 100] [Batch    5/   9] Loss:  1.30e-03
[Epoch   71/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   71/ 100] [Batch    7/   9] Loss:  1.57e-03
[Epoch   71/ 100] [Batch    8/   9] Loss:  1.63e-03
[Epoch   71/ 100] [Batch    9/   9] Loss:  1.42e-03
[Epoch   72/ 100] [Batch    1/   9] Loss:  1.77e-03
[Epoch   72/ 100] [Batch    2/   9] Loss:  1.47e-03
[Epoch   72/ 100] [Batch    3/   9] Loss:  1.65e-03
[Epoch   72/ 100] [Batch    4/   9] Loss:  1.58e-03
[Epoch   72/ 100] [Batch    5/   9] Loss:  1.90e-03
[Epoch   72/ 100] [Batch    6/   9] Loss:  1.38e-03
[Epoch   72/ 100] [Batch    7/   9] Loss:  1.31e-03
[Epoch   72/ 100] [Batch    8/   9] Loss:  1.70e-03
[Epoch   72/ 100] [Batch    9/   9] Loss:  1.25e-03
[Epoch   73/ 100] [Batch    1/   9] Loss:  1.54e-03
[Epoch   73/ 100] [Batch    2/   9] Loss:  1.43e-03
[Epoch   73/ 100] [Batch    3/   9] Loss:  1.25e-03
[Epoch   73/ 100] [Batch    4/   9] Loss:  1.71e-03
[Epoch   73/ 100] [Batch    5/   9] Loss:  1.61e-03
[Epoch   73/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   73/ 100] [Batch    7/   9] Loss:  1.49e-03
[Epoch   73/ 100] [Batch    8/   9] Loss:  2.03e-03
[Epoch   73/ 100] [Batch    9/   9] Loss:  1.44e-03
[Epoch   74/ 100] [Batch    1/   9] Loss:  2.06e-03
[Epoch   74/ 100] [Batch    2/   9] Loss:  1.47e-03
[Epoch   74/ 100] [Batch    3/   9] Loss:  1.68e-03
[Epoch   74/ 100] [Batch    4/   9] Loss:  1.45e-03
[Epoch   74/ 100] [Batch    5/   9] Loss:  1.62e-03
[Epoch   74/ 100] [Batch    6/   9] Loss:  1.57e-03
[Epoch   74/ 100] [Batch    7/   9] Loss:  1.50e-03
[Epoch   74/ 100] [Batch    8/   9] Loss:  1.21e-03
[Epoch   74/ 100] [Batch    9/   9] Loss:  1.35e-03
[Epoch   75/ 100] [Batch    1/   9] Loss:  1.54e-03
[Epoch   75/ 100] [Batch    2/   9] Loss:  1.56e-03
[Epoch   75/ 100] [Batch    3/   9] Loss:  1.86e-03
[Epoch   75/ 100] [Batch    4/   9] Loss:  1.27e-03
[Epoch   75/ 100] [Batch    5/   9] Loss:  1.64e-03
[Epoch   75/ 100] [Batch    6/   9] Loss:  1.88e-03
[Epoch   75/ 100] [Batch    7/   9] Loss:  1.30e-03
[Epoch   75/ 100] [Batch    8/   9] Loss:  1.43e-03
[Epoch   75/ 100] [Batch    9/   9] Loss:  1.57e-03
[Epoch   76/ 100] [Batch    1/   9] Loss:  1.25e-03
[Epoch   76/ 100] [Batch    2/   9] Loss:  1.68e-03
[Epoch   76/ 100] [Batch    3/   9] Loss:  1.29e-03
[Epoch   76/ 100] [Batch    4/   9] Loss:  1.85e-03
[Epoch   76/ 100] [Batch    5/   9] Loss:  1.74e-03
[Epoch   76/ 100] [Batch    6/   9] Loss:  1.29e-03
[Epoch   76/ 100] [Batch    7/   9] Loss:  1.70e-03
[Epoch   76/ 100] [Batch    8/   9] Loss:  1.67e-03
[Epoch   76/ 100] [Batch    9/   9] Loss:  1.61e-03
[Epoch   77/ 100] [Batch    1/   9] Loss:  1.87e-03
[Epoch   77/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   77/ 100] [Batch    3/   9] Loss:  1.40e-03
[Epoch   77/ 100] [Batch    4/   9] Loss:  1.61e-03
[Epoch   77/ 100] [Batch    5/   9] Loss:  1.26e-03
[Epoch   77/ 100] [Batch    6/   9] Loss:  1.43e-03
[Epoch   77/ 100] [Batch    7/   9] Loss:  1.56e-03
[Epoch   77/ 100] [Batch    8/   9] Loss:  1.47e-03
[Epoch   77/ 100] [Batch    9/   9] Loss:  2.52e-03
[Epoch   78/ 100] [Batch    1/   9] Loss:  1.59e-03
[Epoch   78/ 100] [Batch    2/   9] Loss:  1.20e-03
[Epoch   78/ 100] [Batch    3/   9] Loss:  1.74e-03
[Epoch   78/ 100] [Batch    4/   9] Loss:  1.60e-03
[Epoch   78/ 100] [Batch    5/   9] Loss:  1.87e-03
[Epoch   78/ 100] [Batch    6/   9] Loss:  1.83e-03
[Epoch   78/ 100] [Batch    7/   9] Loss:  1.54e-03
[Epoch   78/ 100] [Batch    8/   9] Loss:  1.07e-03
[Epoch   78/ 100] [Batch    9/   9] Loss:  1.79e-03
[Epoch   79/ 100] [Batch    1/   9] Loss:  1.47e-03
[Epoch   79/ 100] [Batch    2/   9] Loss:  1.44e-03
[Epoch   79/ 100] [Batch    3/   9] Loss:  1.24e-03
[Epoch   79/ 100] [Batch    4/   9] Loss:  1.95e-03
[Epoch   79/ 100] [Batch    5/   9] Loss:  1.43e-03
[Epoch   79/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   79/ 100] [Batch    7/   9] Loss:  1.63e-03
[Epoch   79/ 100] [Batch    8/   9] Loss:  1.85e-03
[Epoch   79/ 100] [Batch    9/   9] Loss:  1.70e-03
[Epoch   80/ 100] [Batch    1/   9] Loss:  1.44e-03
[Epoch   80/ 100] [Batch    2/   9] Loss:  1.66e-03
[Epoch   80/ 100] [Batch    3/   9] Loss:  1.14e-03
[Epoch   80/ 100] [Batch    4/   9] Loss:  1.44e-03
[Epoch   80/ 100] [Batch    5/   9] Loss:  1.49e-03
[Epoch   80/ 100] [Batch    6/   9] Loss:  1.70e-03
[Epoch   80/ 100] [Batch    7/   9] Loss:  1.62e-03
[Epoch   80/ 100] [Batch    8/   9] Loss:  1.91e-03
[Epoch   80/ 100] [Batch    9/   9] Loss:  1.78e-03
[Epoch   81/ 100] [Batch    1/   9] Loss:  1.41e-03
[Epoch   81/ 100] [Batch    2/   9] Loss:  1.74e-03
[Epoch   81/ 100] [Batch    3/   9] Loss:  1.51e-03
[Epoch   81/ 100] [Batch    4/   9] Loss:  1.58e-03
[Epoch   81/ 100] [Batch    5/   9] Loss:  1.94e-03
[Epoch   81/ 100] [Batch    6/   9] Loss:  1.57e-03
[Epoch   81/ 100] [Batch    7/   9] Loss:  1.54e-03
[Epoch   81/ 100] [Batch    8/   9] Loss:  1.26e-03
[Epoch   81/ 100] [Batch    9/   9] Loss:  1.47e-03
[Epoch   82/ 100] [Batch    1/   9] Loss:  1.97e-03
[Epoch   82/ 100] [Batch    2/   9] Loss:  1.89e-03
[Epoch   82/ 100] [Batch    3/   9] Loss:  1.63e-03
[Epoch   82/ 100] [Batch    4/   9] Loss:  1.15e-03
[Epoch   82/ 100] [Batch    5/   9] Loss:  1.58e-03
[Epoch   82/ 100] [Batch    6/   9] Loss:  1.66e-03
[Epoch   82/ 100] [Batch    7/   9] Loss:  1.34e-03
[Epoch   82/ 100] [Batch    8/   9] Loss:  1.40e-03
[Epoch   82/ 100] [Batch    9/   9] Loss:  1.29e-03
[Epoch   83/ 100] [Batch    1/   9] Loss:  1.37e-03
[Epoch   83/ 100] [Batch    2/   9] Loss:  1.67e-03
[Epoch   83/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   83/ 100] [Batch    4/   9] Loss:  1.86e-03
[Epoch   83/ 100] [Batch    5/   9] Loss:  1.49e-03
[Epoch   83/ 100] [Batch    6/   9] Loss:  1.33e-03
[Epoch   83/ 100] [Batch    7/   9] Loss:  1.57e-03
[Epoch   83/ 100] [Batch    8/   9] Loss:  1.68e-03
[Epoch   83/ 100] [Batch    9/   9] Loss:  1.23e-03
[Epoch   84/ 100] [Batch    1/   9] Loss:  1.42e-03
[Epoch   84/ 100] [Batch    2/   9] Loss:  1.43e-03
[Epoch   84/ 100] [Batch    3/   9] Loss:  1.45e-03
[Epoch   84/ 100] [Batch    4/   9] Loss:  1.52e-03
[Epoch   84/ 100] [Batch    5/   9] Loss:  1.74e-03
[Epoch   84/ 100] [Batch    6/   9] Loss:  1.88e-03
[Epoch   84/ 100] [Batch    7/   9] Loss:  1.41e-03
[Epoch   84/ 100] [Batch    8/   9] Loss:  1.78e-03
[Epoch   84/ 100] [Batch    9/   9] Loss:  1.31e-03
[Epoch   85/ 100] [Batch    1/   9] Loss:  1.40e-03
[Epoch   85/ 100] [Batch    2/   9] Loss:  1.78e-03
[Epoch   85/ 100] [Batch    3/   9] Loss:  1.49e-03
[Epoch   85/ 100] [Batch    4/   9] Loss:  1.90e-03
[Epoch   85/ 100] [Batch    5/   9] Loss:  1.28e-03
[Epoch   85/ 100] [Batch    6/   9] Loss:  1.82e-03
[Epoch   85/ 100] [Batch    7/   9] Loss:  1.59e-03
[Epoch   85/ 100] [Batch    8/   9] Loss:  1.52e-03
[Epoch   85/ 100] [Batch    9/   9] Loss:  1.35e-03
[Epoch   86/ 100] [Batch    1/   9] Loss:  1.42e-03
[Epoch   86/ 100] [Batch    2/   9] Loss:  1.56e-03
[Epoch   86/ 100] [Batch    3/   9] Loss:  1.45e-03
[Epoch   86/ 100] [Batch    4/   9] Loss:  1.31e-03
[Epoch   86/ 100] [Batch    5/   9] Loss:  1.41e-03
[Epoch   86/ 100] [Batch    6/   9] Loss:  1.53e-03
[Epoch   86/ 100] [Batch    7/   9] Loss:  1.72e-03
[Epoch   86/ 100] [Batch    8/   9] Loss:  1.78e-03
[Epoch   86/ 100] [Batch    9/   9] Loss:  2.50e-03
[Epoch   87/ 100] [Batch    1/   9] Loss:  1.65e-03
[Epoch   87/ 100] [Batch    2/   9] Loss:  1.44e-03
[Epoch   87/ 100] [Batch    3/   9] Loss:  1.99e-03
[Epoch   87/ 100] [Batch    4/   9] Loss:  1.74e-03
[Epoch   87/ 100] [Batch    5/   9] Loss:  1.42e-03
[Epoch   87/ 100] [Batch    6/   9] Loss:  1.45e-03
[Epoch   87/ 100] [Batch    7/   9] Loss:  1.63e-03
[Epoch   87/ 100] [Batch    8/   9] Loss:  1.40e-03
[Epoch   87/ 100] [Batch    9/   9] Loss:  1.77e-03
[Epoch   88/ 100] [Batch    1/   9] Loss:  1.40e-03
[Epoch   88/ 100] [Batch    2/   9] Loss:  1.56e-03
[Epoch   88/ 100] [Batch    3/   9] Loss:  1.64e-03
[Epoch   88/ 100] [Batch    4/   9] Loss:  1.70e-03
[Epoch   88/ 100] [Batch    5/   9] Loss:  1.36e-03
[Epoch   88/ 100] [Batch    6/   9] Loss:  1.61e-03
[Epoch   88/ 100] [Batch    7/   9] Loss:  1.76e-03
[Epoch   88/ 100] [Batch    8/   9] Loss:  1.57e-03
[Epoch   88/ 100] [Batch    9/   9] Loss:  1.46e-03
[Epoch   89/ 100] [Batch    1/   9] Loss:  1.43e-03
[Epoch   89/ 100] [Batch    2/   9] Loss:  1.83e-03
[Epoch   89/ 100] [Batch    3/   9] Loss:  1.32e-03
[Epoch   89/ 100] [Batch    4/   9] Loss:  1.63e-03
[Epoch   89/ 100] [Batch    5/   9] Loss:  1.46e-03
[Epoch   89/ 100] [Batch    6/   9] Loss:  1.80e-03
[Epoch   89/ 100] [Batch    7/   9] Loss:  1.63e-03
[Epoch   89/ 100] [Batch    8/   9] Loss:  1.56e-03
[Epoch   89/ 100] [Batch    9/   9] Loss:  1.18e-03
[Epoch   90/ 100] [Batch    1/   9] Loss:  1.57e-03
[Epoch   90/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   90/ 100] [Batch    3/   9] Loss:  1.62e-03
[Epoch   90/ 100] [Batch    4/   9] Loss:  1.85e-03
[Epoch   90/ 100] [Batch    5/   9] Loss:  1.37e-03
[Epoch   90/ 100] [Batch    6/   9] Loss:  1.74e-03
[Epoch   90/ 100] [Batch    7/   9] Loss:  1.34e-03
[Epoch   90/ 100] [Batch    8/   9] Loss:  1.40e-03
[Epoch   90/ 100] [Batch    9/   9] Loss:  1.70e-03
[Epoch   91/ 100] [Batch    1/   9] Loss:  1.50e-03
[Epoch   91/ 100] [Batch    2/   9] Loss:  1.47e-03
[Epoch   91/ 100] [Batch    3/   9] Loss:  1.76e-03
[Epoch   91/ 100] [Batch    4/   9] Loss:  1.54e-03
[Epoch   91/ 100] [Batch    5/   9] Loss:  1.26e-03
[Epoch   91/ 100] [Batch    6/   9] Loss:  1.38e-03
[Epoch   91/ 100] [Batch    7/   9] Loss:  1.78e-03
[Epoch   91/ 100] [Batch    8/   9] Loss:  1.89e-03
[Epoch   91/ 100] [Batch    9/   9] Loss:  1.27e-03
[Epoch   92/ 100] [Batch    1/   9] Loss:  1.71e-03
[Epoch   92/ 100] [Batch    2/   9] Loss:  1.41e-03
[Epoch   92/ 100] [Batch    3/   9] Loss:  1.84e-03
[Epoch   92/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   92/ 100] [Batch    5/   9] Loss:  1.37e-03
[Epoch   92/ 100] [Batch    6/   9] Loss:  1.52e-03
[Epoch   92/ 100] [Batch    7/   9] Loss:  1.46e-03
[Epoch   92/ 100] [Batch    8/   9] Loss:  1.63e-03
[Epoch   92/ 100] [Batch    9/   9] Loss:  1.52e-03
[Epoch   93/ 100] [Batch    1/   9] Loss:  1.43e-03
[Epoch   93/ 100] [Batch    2/   9] Loss:  1.62e-03
[Epoch   93/ 100] [Batch    3/   9] Loss:  1.35e-03
[Epoch   93/ 100] [Batch    4/   9] Loss:  1.77e-03
[Epoch   93/ 100] [Batch    5/   9] Loss:  1.86e-03
[Epoch   93/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   93/ 100] [Batch    7/   9] Loss:  1.43e-03
[Epoch   93/ 100] [Batch    8/   9] Loss:  1.30e-03
[Epoch   93/ 100] [Batch    9/   9] Loss:  1.97e-03
[Epoch   94/ 100] [Batch    1/   9] Loss:  1.64e-03
[Epoch   94/ 100] [Batch    2/   9] Loss:  1.67e-03
[Epoch   94/ 100] [Batch    3/   9] Loss:  1.33e-03
[Epoch   94/ 100] [Batch    4/   9] Loss:  1.35e-03
[Epoch   94/ 100] [Batch    5/   9] Loss:  1.22e-03
[Epoch   94/ 100] [Batch    6/   9] Loss:  1.67e-03
[Epoch   94/ 100] [Batch    7/   9] Loss:  1.32e-03
[Epoch   94/ 100] [Batch    8/   9] Loss:  2.25e-03
[Epoch   94/ 100] [Batch    9/   9] Loss:  1.63e-03
[Epoch   95/ 100] [Batch    1/   9] Loss:  1.45e-03
[Epoch   95/ 100] [Batch    2/   9] Loss:  1.61e-03
[Epoch   95/ 100] [Batch    3/   9] Loss:  1.42e-03
[Epoch   95/ 100] [Batch    4/   9] Loss:  1.63e-03
[Epoch   95/ 100] [Batch    5/   9] Loss:  1.46e-03
[Epoch   95/ 100] [Batch    6/   9] Loss:  1.63e-03
[Epoch   95/ 100] [Batch    7/   9] Loss:  1.50e-03
[Epoch   95/ 100] [Batch    8/   9] Loss:  1.82e-03
[Epoch   95/ 100] [Batch    9/   9] Loss:  1.63e-03
[Epoch   96/ 100] [Batch    1/   9] Loss:  2.01e-03
[Epoch   96/ 100] [Batch    2/   9] Loss:  1.58e-03
[Epoch   96/ 100] [Batch    3/   9] Loss:  1.28e-03
[Epoch   96/ 100] [Batch    4/   9] Loss:  1.30e-03
[Epoch   96/ 100] [Batch    5/   9] Loss:  1.47e-03
[Epoch   96/ 100] [Batch    6/   9] Loss:  1.91e-03
[Epoch   96/ 100] [Batch    7/   9] Loss:  1.39e-03
[Epoch   96/ 100] [Batch    8/   9] Loss:  1.63e-03
[Epoch   96/ 100] [Batch    9/   9] Loss:  1.54e-03
[Epoch   97/ 100] [Batch    1/   9] Loss:  1.69e-03
[Epoch   97/ 100] [Batch    2/   9] Loss:  1.39e-03
[Epoch   97/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   97/ 100] [Batch    4/   9] Loss:  1.57e-03
[Epoch   97/ 100] [Batch    5/   9] Loss:  1.53e-03
[Epoch   97/ 100] [Batch    6/   9] Loss:  1.78e-03
[Epoch   97/ 100] [Batch    7/   9] Loss:  1.51e-03
[Epoch   97/ 100] [Batch    8/   9] Loss:  1.56e-03
[Epoch   97/ 100] [Batch    9/   9] Loss:  1.04e-03
[Epoch   98/ 100] [Batch    1/   9] Loss:  1.97e-03
[Epoch   98/ 100] [Batch    2/   9] Loss:  1.26e-03
[Epoch   98/ 100] [Batch    3/   9] Loss:  1.59e-03
[Epoch   98/ 100] [Batch    4/   9] Loss:  1.48e-03
[Epoch   98/ 100] [Batch    5/   9] Loss:  1.69e-03
[Epoch   98/ 100] [Batch    6/   9] Loss:  1.55e-03
[Epoch   98/ 100] [Batch    7/   9] Loss:  1.50e-03
[Epoch   98/ 100] [Batch    8/   9] Loss:  1.57e-03
[Epoch   98/ 100] [Batch    9/   9] Loss:  1.51e-03
[Epoch   99/ 100] [Batch    1/   9] Loss:  1.41e-03
[Epoch   99/ 100] [Batch    2/   9] Loss:  1.47e-03
[Epoch   99/ 100] [Batch    3/   9] Loss:  1.79e-03
[Epoch   99/ 100] [Batch    4/   9] Loss:  1.81e-03
[Epoch   99/ 100] [Batch    5/   9] Loss:  1.28e-03
[Epoch   99/ 100] [Batch    6/   9] Loss:  1.95e-03
[Epoch   99/ 100] [Batch    7/   9] Loss:  1.60e-03
[Epoch   99/ 100] [Batch    8/   9] Loss:  1.27e-03
[Epoch   99/ 100] [Batch    9/   9] Loss:  1.40e-03
[Epoch  100/ 100] [Batch    1/   9] Loss:  1.79e-03
[Epoch  100/ 100] [Batch    2/   9] Loss:  1.91e-03
[Epoch  100/ 100] [Batch    3/   9] Loss:  1.53e-03
[Epoch  100/ 100] [Batch    4/   9] Loss:  1.64e-03
[Epoch  100/ 100] [Batch    5/   9] Loss:  1.82e-03
[Epoch  100/ 100] [Batch    6/   9] Loss:  1.38e-03
[Epoch  100/ 100] [Batch    7/   9] Loss:  1.26e-03
[Epoch  100/ 100] [Batch    8/   9] Loss:  1.38e-03
[Epoch  100/ 100] [Batch    9/   9] Loss:  1.28e-03

LinearModel(
  (linear): Linear(in_features=1, out_features=1, bias=True)
)

[ ]:

if input_dim == output_dim == 1:
    n_test = 500
    X_test = np.random.rand(n_test, input_dim)
    y_pred = []

    state_dict = model.state_dict()

    for idx in range(len(train_hist['weight'])):
        state_dict['linear.weight'] = torch.tensor([[train_hist['weight'][idx]]])
        state_dict['linear.bias'] = torch.tensor([train_hist['bias'][idx]])
        model.load_state_dict(state_dict)

        y_pred.append(model.forward(torch.tensor(X_test.astype('f'))).detach().numpy())

[ ]:

if input_dim == output_dim == 1:
    fig = plt.figure(figsize=(15, 5))
    fig.clf()

    ax = fig.add_subplot(1, 3, 1)
    levels = np.logspace(np.log(np.min(loss_values)), np.log(np.max(loss_values)), 20)
    ax.contourf(ww, bb, loss_values, levels=levels, norm=colors.LogNorm())
    ax.plot(train_hist['weight'], train_hist['bias'], '.-b')
    ax.plot(A[0], b, 'r*', markersize=10)
    ax.set_xlabel('weight')
    ax.set_ylabel('bias')
    #ax.legend(['optim', '(A, b)'])
    ax.grid(True)
    ax.set_xlim(-2, 2)
    ax.set_ylim(-2, 2)

    ax = fig.add_subplot(1, 3, 2)
    ax.loglog(np.abs(train_hist['loss']))
    ax.set_xlabel('Iter')
    ax.set_ylabel('Loss')
    ax.grid(True)

    ax = fig.add_subplot(1, 3, 3)
    ax.plot(X_train, y_train, '.')
    a=ax.plot(X_test, y_pred[0], '-', alpha=0.1)
    for y in y_pred[1:]:
        ax.plot(X_test, y, '-', alpha=0.1, color=a[0].get_color())
    ax.plot(X_test, y_pred[-1], 'k')
    ax.grid(True)

    fig.tight_layout()

else:
    fig = plt.figure()
    fig.clf()
    ax = fig.gca()
    ax.loglog(np.abs(train_hist['loss']))
    ax.set_xlabel('Iter')
    ax.set_ylabel('Loss')
    ax.grid(True)

../../../_images/ipynbs_colabs_pyg_tutorial_project_Tutorial02_70_0.png