3.23. PCA

In [1]:
from conx import *
import random
Using Theano backend.
conx, version 3.5.4

3.23.1. Non-Linearly Separable

In [2]:
import math
In [3]:
def distance(x1, y1, x2, y2):
    return math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2)
In [4]:
negatives = []
while len(negatives) < 500:
    x = random.random()
    y = random.random()
    d = distance(x, y, 0.5, 0.5)
    if d > 0.375 and d < 0.5:
        negatives.append([x, y])
positives = []
while len(positives) < 500:
    x = random.random()
    y = random.random()
    d = distance(x, y, 0.5, 0.5)
    if d < 0.25:
        positives.append([x, y])
In [5]:
symbols = {
    "Positive": "bo",
    "Negative": "ro"
}
In [6]:
scatter([
         ["Positive", positives],
         ["Negative", negatives],
        ],
    symbols=symbols,
    height=8.0, width=8.0)
_images/PCA_7_0.png
In [7]:
net = Network("Non-Linearly Separable", 2, 5, 1, activation="sigmoid")
net.compile(error="mae", optimizer="adam")
In [8]:
net
Out[8]:
Non-Linearly SeparableLayer: output (output) shape = (1,) Keras class = Dense activation = sigmoidoutputWeights from hidden to output output/kernel has shape (5, 1) output/bias has shape (1,)Layer: hidden (hidden) shape = (5,) Keras class = Dense activation = sigmoidhiddenWeights from input to hidden hidden/kernel has shape (2, 5) hidden/bias has shape (5,)Layer: input (input) shape = (2,) Keras class = Inputinput
In [9]:
ds = Dataset()
In [10]:
ds.load([(p, [ 1.0], "Positive") for p in positives] +
        [(n, [ 0.0], "Negative") for n in negatives])
In [11]:
ds.shuffle()
In [12]:
ds.split(.1)
In [13]:
net.set_dataset(ds)
In [14]:
net.test(tolerance=0.4)
========================================================
Testing validation dataset with tolerance 0.4...
Total count: 900
      correct: 447
      incorrect: 453
Total percentage correct: 0.49666666666666665
In [15]:
net.dashboard()
In [16]:
symbols = {
    "Positive (correct)": "w+",
    "Positive (wrong)": "k+",
    "Negative (correct)": "r_",
    "Negative (wrong)": "k_",
}
In [17]:
net.plot_activation_map(scatter=net.test(interactive=False), symbols=symbols, title="Before Training")
_images/PCA_18_0.png

You may want to either net.reset() or net.retrain() if the following cell doesn’t complete with 100% accuracy. Calling net.reset() may be needed if the network has landed in a local maxima; net.retrain() may be necessary if the network just needs additional training.

In [18]:
if net.saved():
    net.load()
    net.plot_results()
else:
    net.train(epochs=10000, accuracy=1.0, report_rate=50,
              tolerance=0.4, batch_size=len(net.dataset.train_inputs),
              plot=True, record=100)
    net.save()
_images/PCA_20_0.png
In [19]:
net.plot_activation_map(scatter=net.test(interactive=False), symbols=symbols, title="After Training")
_images/PCA_21_0.png
In [20]:
states = [net.propagate_to("hidden", input) for input in net.dataset.inputs]
pca = PCA(states)
In [21]:
symbols = {
    "Positive (correct)": "b+",
    "Positive (wrong)": "k+",
    "Negative (correct)": "r_",
    "Negative (wrong)": "k_",
}
In [22]:
pb = net.playback(lambda net, epoch: scatter(**pca.transform_network_bank(net, "hidden", test=True),
                                             symbols=symbols,
                                             title="Epoch %s" % epoch, interactive=False))
pb
_images/PCA_24_1.svg
In [23]:
movie = net.movie(lambda net, epoch: scatter(**pca.transform_network_bank(net, "hidden", test=True),
                                             symbols=symbols,
                                             format="pil",
                                             title="Epoch %s" % epoch, interactive=False), step=1)
movie
Out[23]:
In [24]:
pb.goto("end")
In [25]:
def function(x, y):
    outputs = net.propagate([x, y])
    return outputs[0]
In [26]:
heatmap(function)
_images/PCA_28_0.png
In [27]:
scatter(**pca.transform_network_bank(net, "hidden", test=True),
        symbols=symbols)
_images/PCA_29_0.png
In [28]:
matrix = [[-1 for i in range(50)] for j in range(50)]
for y in frange(0, 1, .01):
    for x in frange(0, 1, .01):
        hiddens = net.propagate_to("hidden", [x, y])
        vector = pca.transform_one(hiddens, scale=True)
        try:
            matrix[int(vector[1] * 50)][int(vector[0] * 50)] = net.propagate([x, y])[0]
        except:
            pass
In [29]:
heatmap(matrix)
_images/PCA_31_0.png