use biases of 1.0 for the forget gate
This commit is contained in:
1 parent
1b05d91e23
commit
0a45fbe230
2 files changed
+40
-18
No files matched your search
+16
-10
@@ -27,18 +27,24 @@ function LSTM.lstm(input_size, rnn_size, n, dropout)
|
||||
input_size_L = rnn_size
|
||||
end
|
||||
-- evaluate the input sums at once for efficiency
|
||||
local i2h = nn.Linear(input_size_L, 4 * rnn_size)(x)
|
||||
local h2h = nn.Linear(rnn_size, 4 * rnn_size)(prev_h)
|
||||
local i2h = nn.Linear(input_size_L, 4 * rnn_size)(x):annotate{name='i2h_'..L}
|
||||
local h2h = nn.Linear(rnn_size, 4 * rnn_size)(prev_h):annotate{name='h2h_'..L}
|
||||
local all_input_sums = nn.CAddTable()({i2h, h2h})
|
||||
-- decode the gates
|
||||
local sigmoid_chunk = nn.Narrow(2, 1, 3 * rnn_size)(all_input_sums)
|
||||
sigmoid_chunk = nn.Sigmoid()(sigmoid_chunk)
|
||||
local in_gate = nn.Narrow(2, 1, rnn_size)(sigmoid_chunk)
|
||||
local forget_gate = nn.Narrow(2, rnn_size + 1, rnn_size)(sigmoid_chunk)
|
||||
local out_gate = nn.Narrow(2, 2 * rnn_size + 1, rnn_size)(sigmoid_chunk)
|
||||
-- local sigmoid_chunk = nn.Narrow(2, 1, 3 * rnn_size)(all_input_sums)
|
||||
-- sigmoid_chunk = nn.Sigmoid()(sigmoid_chunk)
|
||||
-- local in_gate = nn.Narrow(2, 1, rnn_size)(sigmoid_chunk)
|
||||
-- local forget_gate = nn.Narrow(2, rnn_size + 1, rnn_size)(sigmoid_chunk)
|
||||
-- local out_gate = nn.Narrow(2, 2 * rnn_size + 1, rnn_size)(sigmoid_chunk)
|
||||
local reshaped = nn.Reshape(4, rnn_size)(all_input_sums)
|
||||
local n1, n2, n3, n4 = nn.SplitTable(2)(reshaped):split(4)
|
||||
local in_gate = nn.Sigmoid()(n1)
|
||||
local forget_gate = nn.Sigmoid()(n2)
|
||||
local out_gate = nn.Sigmoid()(n3)
|
||||
-- decode the write inputs
|
||||
local in_transform = nn.Narrow(2, 3 * rnn_size + 1, rnn_size)(all_input_sums)
|
||||
in_transform = nn.Tanh()(in_transform)
|
||||
-- local in_transform = nn.Narrow(2, 3 * rnn_size + 1, rnn_size)(all_input_sums)
|
||||
-- in_transform = nn.Tanh()(in_transform)
|
||||
local in_transform = nn.Tanh()(n4)
|
||||
-- perform the LSTM update
|
||||
local next_c = nn.CAddTable()({
|
||||
nn.CMulTable()({forget_gate, prev_c}),
|
||||
@@ -54,7 +60,7 @@ function LSTM.lstm(input_size, rnn_size, n, dropout)
|
||||
-- set up the decoder
|
||||
local top_h = outputs[#outputs]
|
||||
if dropout > 0 then top_h = nn.Dropout(dropout)(top_h) end
|
||||
local proj = nn.Linear(rnn_size, input_size)(top_h)
|
||||
local proj = nn.Linear(rnn_size, input_size)(top_h):annotate{name='decoder'}
|
||||
local logsoft = nn.LogSoftMax()(proj)
|
||||
table.insert(outputs, logsoft)
|
||||
|
||||
|
||||
Reference in new issue
Block a user