3.2. XOR Multiple Inputs/Targets

This notebook explores networks with multiple input and output banks.

In [39]:
import conx as cx
In [2]:
net = cx.Network("XOR Network", 2, 3, 1, activation="sigmoid")
In [3]:
net.compile(loss='mean_squared_error', optimizer='sgd', lr=0.3, momentum=0.9)
In [4]:
XOR = [
    ([0, 0], [0], "0"),
    ([0, 1], [1], "1"),
    ([1, 0], [1], "1"),
    ([1, 1], [0], "0")
]
In [5]:
net.dataset.load(XOR)
net.dataset.summary()
_________________________________________________________________
XOR Network Dataset:
Patterns    Shape                 Range
=================================================================
inputs      (2,)                  (0.0, 1.0)
targets     (1,)                  (0.0, 1.0)
=================================================================
Total patterns: 4
   Training patterns: 4
   Testing patterns: 0
_________________________________________________________________
In [6]:
dash = net.dashboard()
dash
In [7]:
net.propagate([0, 1])
Out[7]:
[0.3950902223587036]
In [8]:
net.train(epochs=1000, accuracy=1, report_rate=25, record=True)
_images/XOR_8_0.svg
========================================================
       |  Training |  Training
Epochs |     Error |  Accuracy
------ | --------- | ---------
#  644 |   0.00666 |   1.00000
In [9]:
zeros = net.dataset.inputs.select(lambda i,ds: ds.labels[i] == "0")
ones = net.dataset.inputs.select(lambda i,ds: ds.labels[i] == "1")

net.playback(lambda net,epoch: (
    net.plot_activation_map(scatter=[["0", zeros], ["1", ones]],
                            symbols={"0": "ko", "1": "k+"},
                            title="Epoch %s" % epoch,
                            format="image"),
    net.plot('all', end=epoch+1, format="image")))
_images/XOR_9_1.png
_images/XOR_9_2.png

We need to remember to reset the network’s weights to the last epoch of training, so that subsequent interactions with the network reflect its learned knowledge.

In [10]:
net.set_weights_from_history(-1)
In [11]:
states = [net.propagate_to("hidden", pattern) for pattern in net.dataset.inputs]
pca = cx.PCA(states)
In [12]:
symbols = {
    "0 (correct)": "bo",
    "0 (wrong)": "bx",
    "1 (correct)": "ro",
    "1 (wrong)": "rx",
}
net.playback(lambda net,epoch: cx.scatter(**pca.transform_network_bank(net, "hidden"),
                                          symbols=symbols,
                                          format='svg'))
_images/XOR_13_1.svg
In [13]:
net.set_weights_from_history(-1)
In [14]:
net.propagate_to("input", [0, 1])
Out[14]:
[0.0, 1.0]
In [15]:
net.propagate([0.5, 0.5])
Out[15]:
[0.23022033274173737]
In [16]:
net.propagate_to("hidden", [1, 0])
Out[16]:
[0.08819267153739929, 0.03618663176894188, 0.010262570343911648]
In [17]:
net.propagate_to("output", [1, 1])
Out[17]:
[0.09953280538320541]
In [18]:
net.propagate_to("input", [0.25, 0.25])
Out[18]:
[0.25, 0.25]
In [19]:
net.propagate_from("input", [1.0, 1.0])
Out[19]:
[0.099532805]
In [20]:
net.propagate_from("hidden", [1.0, 0.0, -1.0])
Out[20]:
[0.00016305932]
In [21]:
net2 = cx.Network("XOR2 Network")

net2.add(cx.Layer("input1", 1),
         cx.Layer("input2", 1),
         cx.Layer("hidden1", 10, activation="sigmoid"),
         cx.Layer("hidden2", 10, activation="sigmoid"),
         cx.Layer("shared-hidden", 5, activation="sigmoid"),
         cx.Layer("output1", 1, activation="sigmoid"),
         cx.Layer("output2", 1, activation="sigmoid"))

net2.connect("input1", "hidden1")
net2.connect("input2", "hidden2")
net2.connect("hidden1", "shared-hidden")
net2.connect("hidden2", "shared-hidden")
net2.connect("shared-hidden", "output1")
net2.connect("shared-hidden", "output2")
In [22]:
net2.picture()
Out[22]:
Layer: output1 (output) output range: (0, 1) shape = (1,) Keras class = Dense activation = sigmoidoutput1Layer: output2 (output) output range: (0, 1) shape = (1,) Keras class = Dense activation = sigmoidoutput2Weights from shared-hidden to output1Weights from shared-hidden to output2Layer: shared-hidden (hidden) output range: (0, 1) shape = (5,) Keras class = Dense activation = sigmoidshared-hiddenWeights from hidden1 to shared-hiddenLayer: hidden1 (hidden) output range: (0, 1) shape = (10,) Keras class = Dense activation = sigmoidhidden1Weights from hidden2 to shared-hiddenLayer: hidden2 (hidden) output range: (0, 1) shape = (10,) Keras class = Dense activation = sigmoidhidden2Weights from input1 to hidden1Layer: input1 (input) output range: (-Infinity, +Infinity) shape = (1,) Keras class = Inputinput1Weights from input2 to hidden2Layer: input2 (input) output range: (-Infinity, +Infinity) shape = (1,) Keras class = Inputinput2XOR2 Network
In [23]:
net2.layers[2].incoming_connections
Out[23]:
[<Layer name='input1', shape=(1,), act='None'>]
In [24]:
net2.compile(loss='mean_squared_error', optimizer='SGD', lr=0.3, momentum=0.9)
In [25]:
net2.config["hspace"] = 200
dash = net2.dashboard()
dash
In [26]:
net2.propagate_to("hidden1", [[1], [1]])
Out[26]:
[0.41164860129356384,
 0.6558945775032043,
 0.5304865837097168,
 0.6376801133155823,
 0.6549503207206726,
 0.6577186584472656,
 0.3398575186729431,
 0.3786383271217346,
 0.375860333442688,
 0.6729684472084045]
In [27]:
net2.propagate([[1], [1]])
Out[27]:
[[0.5289972424507141], [0.5216334462165833]]
In [28]:
XOR2 = [
    ([[0],[0]], [[0],[0]]),
    ([[0],[1]], [[1],[1]]),
    ([[1],[0]], [[1],[1]]),
    ([[1],[1]], [[0],[0]])
]
In [29]:
net2.dataset.load(XOR2)
In [30]:
net2.get_weights("hidden2")
Out[30]:
[[[0.10360556840896606,
   0.007417023181915283,
   0.7205445170402527,
   -0.46124929189682007,
   -0.3977455794811249,
   -0.3434865474700928,
   -0.5686421394348145,
   0.5728721022605896,
   0.5689478516578674,
   -0.7266545295715332]],
 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]
In [31]:
net2.propagate([[1], [1]])
Out[31]:
[[0.5289972424507141], [0.5216334462165833]]
In [32]:
import time
net2.reset()
for i in range(20):
    (epoch_count, results) = net2.train(epochs=100, verbose=0, report_rate=25)
    for index in range(4):
        net2.propagate(XOR2[index][0])
        time.sleep(0.1)
_images/XOR_33_0.svg
In [33]:
net2.reset()
net2.train(epochs=2000, accuracy=1.0, report_rate=25)
_images/XOR_34_0.svg
========================================================
       |  Training |   output1 |   output2
Epochs |     Error |       acc |       acc
------ | --------- | --------- | ---------
#  915 |   0.01573 |   1.00000 |   1.00000
In [34]:
net2.propagate_from("shared-hidden", [0.0] * 5)
Out[34]:
[[0.831886], [0.80715334]]
In [35]:
net2.propagate_to("hidden1", [[1], [1]])
Out[35]:
[0.12185605615377426,
 0.1576666235923767,
 0.7544072270393372,
 0.48832881450653076,
 0.15166418254375458,
 0.6500229835510254,
 0.9043633937835693,
 0.1985720694065094,
 0.7953702211380005,
 0.3377590775489807]
In [36]:
net2.dataset.slice(2)
In [37]:
net2.train(epochs=2000, accuracy=1.0, report_rate=25)
No training required: accuracy already to desired value
Training dataset status:
       |  Training |   output1 |   output2
Epochs |     Error |       acc |       acc
------ | --------- | --------- | ---------
#  915 |   0.01573 |   1.00000 |   1.00000
In [38]:
net2.plot('all')
_images/XOR_39_0.png