# LSTMΒΆ

In [1]:

from conx import Network, LSTMLayer, Layer
import numpy as np
import matplotlib.pyplot as plt

Using Theano backend.

In [2]:

%matplotlib inline

In [3]:

# since we are using stateful rnn tsteps can be set to 1
tsteps = 1
batch_size = 25
epochs = 1
# number of elements ahead that are used to make the prediction

In [4]:

def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
"""Generates an absolute cosine time series with the amplitude
exponentially decreasing
Arguments:
amp: amplitude of the cosine function
period: period of the cosine function
x0: initial x of the time series
xn: final x of the time series
step: step of the time series discretization
k: exponential rate
"""
cos = np.zeros(((xn - x0) * step, 1, 1))
for i in range(len(cos)):
idx = x0 + i * step
cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
return cos

In [5]:

print('Generating Data...')
cos = gen_cosine_amp()
print('Input shape:', cos.shape)

expected_output = np.zeros((len(cos), 1))
for i in range(len(cos) - lahead):
expected_output[i, 0] = np.mean(cos[i + 1:i + lahead + 1])

print('Output shape:', expected_output.shape)

Generating Data...
Input shape: (50000, 1, 1)
Output shape: (50000, 1)

In [6]:

print('Creating Model...')
net = Network("LSTM")
net.add(Layer("input", (1, 1), batch_shape=(25, 1, 1)))
50,
batch_size=25,
return_sequences=True,
stateful=True))
50,
return_sequences=False,
stateful=True))
net.connect()
net.compile(loss='mse', optimizer='rmsprop')

Creating Model...

In [7]:

net

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
theano/scan_module/scan_perform.pyx in theano.scan_module.scan_perform.perform (/home/dblank/.theano/compiledir_Linux-4.10--generic-x86_64-with-Ubuntu-17.04-zesty-x86_64-3.5.3-64/scan_perform/mod.cpp:4490)()

ValueError: Shape mismatch: x has 25 rows but z has 1 rows

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.5/dist-packages/theano/compile/function_module.py in __call__(self, *args, **kwargs)
883             outputs =\
--> 884                 self.fn() if output_subset is None else\
885                 self.fn(output_subset=output_subset)

/usr/local/lib/python3.5/dist-packages/theano/scan_module/scan_op.py in rval(p, i, o, n, allow_gc)
988                  allow_gc=allow_gc):
--> 989             r = p(n, [x[0] for x in i], o)
990             for o in node.outputs:

/usr/local/lib/python3.5/dist-packages/theano/scan_module/scan_op.py in p(node, args, outs)
977                                                 outs,
--> 978                                                 self, node)
979         except (ImportError, theano.gof.cmodule.MissingGXX):

theano/scan_module/scan_perform.pyx in theano.scan_module.scan_perform.perform (/home/dblank/.theano/compiledir_Linux-4.10--generic-x86_64-with-Ubuntu-17.04-zesty-x86_64-3.5.3-64/scan_perform/mod.cpp:4606)()

/usr/local/lib/python3.5/dist-packages/theano/gof/link.py in raise_with_op(node, thunk, exc_info, storage_map)
324         pass
--> 325     reraise(exc_type, exc_value, exc_trace)
326

/usr/lib/python3/dist-packages/six.py in reraise(tp, value, tb)
684         if value.__traceback__ is not tb:
--> 685             raise value.with_traceback(tb)
686         raise value

theano/scan_module/scan_perform.pyx in theano.scan_module.scan_perform.perform (/home/dblank/.theano/compiledir_Linux-4.10--generic-x86_64-with-Ubuntu-17.04-zesty-x86_64-3.5.3-64/scan_perform/mod.cpp:4490)()

ValueError: Shape mismatch: x has 25 rows but z has 1 rows
Apply node that caused the error: Gemm{no_inplace}(Subtensor{::, int64::}.0, TensorConstant{0.20000000298023224}, lstm1/variable[t-1], <TensorType(float32, matrix)>, TensorConstant{0.20000000298023224})
Toposort index: 5
Inputs types: [TensorType(float32, matrix), TensorType(float32, scalar), TensorType(float32, matrix), TensorType(float32, matrix), TensorType(float32, scalar)]
Inputs shapes: [(1, 50), (), (25, 50), (50, 50), ()]
Inputs strides: [(800, 4), (), (200, 4), (800, 4), ()]
Inputs values: ['not shown', array(0.20000000298023224, dtype=float32), 'not shown', 'not shown', array(0.20000000298023224, dtype=float32)]
Outputs clients: [[Elemwise{Composite{(clip((i0 + i1), i2, i3) * tanh(i4))}}(TensorConstant{(1, 1) of 0.5}, Gemm{no_inplace}.0, TensorConstant{(1, 1) of 0}, TensorConstant{(1, 1) of 1}, Elemwise{Composite{((clip((i0 + i1), i2, i3) * i4) + (clip((i5 + i6), i2, i3) * tanh(i7)))}}.0)]]

HINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag 'optimizer=fast_compile'. If that does not work, Theano optimizations can be disabled with 'optimizer=None'.
HINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.5/dist-packages/IPython/core/formatters.py in __call__(self, obj)
334             method = get_real_method(obj, self.print_method)
335             if method is not None:
--> 336                 return method()
337             return None
338         else:

/usr/local/lib/python3.5/dist-packages/conx/network.py in _repr_svg_(self)
192     def _repr_svg_(self):
193         if all([layer.model for layer in self.layers]):
--> 194             return self.build_svg()
195         else:
196             return None

/usr/local/lib/python3.5/dist-packages/conx/network.py in build_svg(self, inputs, class_id, opts)
982                             in_layer = [layer for layer in self.layers if layer.kind() == "input"][0]
983                             v = in_layer.make_dummy_vector()
--> 984                     image = self.propagate_to_image(layer_name, v)
985                 else: # no propagate
986                     # get image based on ontputs

/usr/local/lib/python3.5/dist-packages/conx/network.py in propagate_to_image(self, layer_name, input, batch_size)
735             if self.num_input_layers == 1:
736                 input = input[0]
--> 737         outputs = self.propagate_to(layer_name, input, batch_size)
738         array = np.array(outputs)
739         image = self[layer_name].make_image(array, self.config)

/usr/local/lib/python3.5/dist-packages/conx/network.py in propagate_to(self, layer_name, inputs, batch_size, visualize)
707                 inputs = inputs[0]
708         if self.num_input_layers == 1:
--> 709             outputs = self[layer_name].model.predict(np.array([inputs]), batch_size=batch_size)
710         else:
711             # get just inputs for this layer, in order:

/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in predict(self, x, batch_size, verbose)
1515         f = self.predict_function
1516         return self._predict_loop(f, ins,
-> 1517                                   batch_size=batch_size, verbose=verbose)
1518
1519     def train_on_batch(self, x, y,

/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in _predict_loop(self, f, ins, batch_size, verbose)
1139                 ins_batch = _slice_arrays(ins, batch_ids)
1140
-> 1141             batch_outs = f(ins_batch)
1142             if not isinstance(batch_outs, list):
1143                 batch_outs = [batch_outs]

/usr/local/lib/python3.5/dist-packages/keras/backend/theano_backend.py in __call__(self, inputs)
1195     def __call__(self, inputs):
1196         assert isinstance(inputs, (list, tuple))
-> 1197         return self.function(*inputs)
1198
1199

/usr/local/lib/python3.5/dist-packages/theano/compile/function_module.py in __call__(self, *args, **kwargs)
896                     node=self.fn.nodes[self.fn.position_of_error],
897                     thunk=thunk,
--> 898                     storage_map=getattr(self.fn, 'storage_map', None))
899             else:
900                 # old-style linkers raise their own exceptions

/usr/local/lib/python3.5/dist-packages/theano/gof/link.py in raise_with_op(node, thunk, exc_info, storage_map)
323         # extra long error message in that case.
324         pass
--> 325     reraise(exc_type, exc_value, exc_trace)
326
327

/usr/lib/python3/dist-packages/six.py in reraise(tp, value, tb)
683             value = tp()
684         if value.__traceback__ is not tb:
--> 685             raise value.with_traceback(tb)
686         raise value
687

/usr/local/lib/python3.5/dist-packages/theano/compile/function_module.py in __call__(self, *args, **kwargs)
882         try:
883             outputs =\
--> 884                 self.fn() if output_subset is None else\
885                 self.fn(output_subset=output_subset)
886         except Exception:

/usr/local/lib/python3.5/dist-packages/theano/scan_module/scan_op.py in rval(p, i, o, n, allow_gc)
987         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,
988                  allow_gc=allow_gc):
--> 989             r = p(n, [x[0] for x in i], o)
990             for o in node.outputs:
991                 compute_map[o][0] = True

/usr/local/lib/python3.5/dist-packages/theano/scan_module/scan_op.py in p(node, args, outs)
976                                                 args,
977                                                 outs,
--> 978                                                 self, node)
979         except (ImportError, theano.gof.cmodule.MissingGXX):
980             p = self.execute

theano/scan_module/scan_perform.pyx in theano.scan_module.scan_perform.perform (/home/dblank/.theano/compiledir_Linux-4.10--generic-x86_64-with-Ubuntu-17.04-zesty-x86_64-3.5.3-64/scan_perform/mod.cpp:4606)()

/usr/local/lib/python3.5/dist-packages/theano/gof/link.py in raise_with_op(node, thunk, exc_info, storage_map)
323         # extra long error message in that case.
324         pass
--> 325     reraise(exc_type, exc_value, exc_trace)
326
327

/usr/lib/python3/dist-packages/six.py in reraise(tp, value, tb)
683             value = tp()
684         if value.__traceback__ is not tb:
--> 685             raise value.with_traceback(tb)
686         raise value
687

theano/scan_module/scan_perform.pyx in theano.scan_module.scan_perform.perform (/home/dblank/.theano/compiledir_Linux-4.10--generic-x86_64-with-Ubuntu-17.04-zesty-x86_64-3.5.3-64/scan_perform/mod.cpp:4490)()

ValueError: Shape mismatch: x has 25 rows but z has 1 rows
Apply node that caused the error: Gemm{no_inplace}(Subtensor{::, int64::}.0, TensorConstant{0.20000000298023224}, lstm1/variable[t-1], <TensorType(float32, matrix)>, TensorConstant{0.20000000298023224})
Toposort index: 5
Inputs types: [TensorType(float32, matrix), TensorType(float32, scalar), TensorType(float32, matrix), TensorType(float32, matrix), TensorType(float32, scalar)]
Inputs shapes: [(1, 50), (), (25, 50), (50, 50), ()]
Inputs strides: [(800, 4), (), (200, 4), (800, 4), ()]
Inputs values: ['not shown', array(0.20000000298023224, dtype=float32), 'not shown', 'not shown', array(0.20000000298023224, dtype=float32)]
Outputs clients: [[Elemwise{Composite{(clip((i0 + i1), i2, i3) * tanh(i4))}}(TensorConstant{(1, 1) of 0.5}, Gemm{no_inplace}.0, TensorConstant{(1, 1) of 0}, TensorConstant{(1, 1) of 1}, Elemwise{Composite{((clip((i0 + i1), i2, i3) * i4) + (clip((i5 + i6), i2, i3) * tanh(i7)))}}.0)]]

HINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag 'optimizer=fast_compile'. If that does not work, Theano optimizations can be disabled with 'optimizer=None'.
HINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.
Apply node that caused the error: forall_inplace,cpu,scan_fn}(TensorConstant{1}, InplaceDimShuffle{1,0,2}.0, IncSubtensor{InplaceSet;:int64:}.0, IncSubtensor{InplaceSet;:int64:}.0, TensorConstant{1}, Subtensor{::, int64:int64:}.0, Subtensor{::, :int64:}.0, Subtensor{::, int64:int64:}.0, Subtensor{::, int64::}.0)
Toposort index: 77
Inputs types: [TensorType(int64, scalar), TensorType(float32, (True, False, False)), TensorType(float32, 3D), TensorType(float32, 3D), TensorType(int64, scalar), TensorType(float32, matrix), TensorType(float32, matrix), TensorType(float32, matrix), TensorType(float32, matrix)]
Inputs shapes: [(), (1, 1, 200), (2, 25, 50), (2, 25, 50), (), (50, 50), (50, 50), (50, 50), (50, 50)]
Inputs strides: [(), (800, 800, 4), (5000, 200, 4), (5000, 200, 4), (), (800, 4), (800, 4), (800, 4), (800, 4)]
Inputs values: [array(1), 'not shown', 'not shown', 'not shown', array(1), 'not shown', 'not shown', 'not shown', 'not shown']
Outputs clients: [[Subtensor{int64}(forall_inplace,cpu,scan_fn}.0, Constant{1})], [Subtensor{int64}(forall_inplace,cpu,scan_fn}.1, Constant{1})], [InplaceDimShuffle{1,0,2}(forall_inplace,cpu,scan_fn}.2)]]

HINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag 'optimizer=fast_compile'. If that does not work, Theano optimizations can be disabled with 'optimizer=None'.
HINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.

Out[7]:

<Network name='LSTM' (compiled)>

In [9]:

net.dataset.load_direct(cos, expected_output)

In [10]:

print('Training')
for i in range(epochs):
print('Epoch', i, '/', epochs)

# Note that the last state for sample i in a batch will
# be used as initial state for sample i in the next batch.
# Thus we are simultaneously training on batch_size series with
# lower resolution than the original series contained in cos.
# Each of these series are offset by one step and can be
# extracted with cos[i::batch_size].

net.train(batch_size=batch_size,
epochs=1,
shuffle=False)
net.model.reset_states()

Training
Epoch 0 / 1
Training...
Epoch 1/1
50000/50000 [==============================] - 5s - loss: 337.9365 - acc: 0.0000e+00
========================================================================
Epoch #    1 | train error 337.93646 | train accuracy 0.00000 | validate% 0.03124

In [11]:

print('Predicting')
predicted_output = net.model.predict(cos, batch_size=batch_size)

Predicting

In [12]:

print('Plotting Results')
plt.subplot(2, 1, 1)
plt.plot(expected_output)
plt.title('Expected')
plt.subplot(2, 1, 2)
plt.plot(predicted_output)
plt.title('Predicted')
plt.show()

Plotting Results