fitst commit, support Chinese and min_freq filtering

This commit is contained in:
Jeff Zhang committed 2015-07-07 12:09:34 +08:00
commit c874651412
14 files changed
+41232

No files matched your search

+235
View File
@@ -0,0 +1,235 @@
-- Modified from https://github.com/oxford-cs-ml-2015/practical6
-- the modification included support for train/val/test splits
local CharSplitLMMinibatchLoader = {}
CharSplitLMMinibatchLoader.__index = CharSplitLMMinibatchLoader
function CharSplitLMMinibatchLoader.create(data_dir, batch_size, seq_length, split_fractions, min_freq)
-- split_fractions is e.g. {0.9, 0.05, 0.05}
local self = {}
setmetatable(self, CharSplitLMMinibatchLoader)
local input_file = path.join(data_dir, 'input.txt')
local vocab_file = path.join(data_dir, 'vocab.t7')
local tensor_file = path.join(data_dir, 'data.t7')
-- fetch file attributes to determine if we need to rerun preprocessing
local run_prepro = false
if not (path.exists(vocab_file) or path.exists(tensor_file)) then
-- prepro files do not exist, generate them
print('vocab.t7 and data.t7 do not exist. Running preprocessing...')
run_prepro = true
else
-- check if the input file was modified since last time we
-- ran the prepro. if so, we have to rerun the preprocessing
local input_attr = lfs.attributes(input_file)
local vocab_attr = lfs.attributes(vocab_file)
local tensor_attr = lfs.attributes(tensor_file)
if input_attr.modification > vocab_attr.modification or input_attr.modification > tensor_attr.modification then
print('vocab.t7 or data.t7 detected as stale. Re-running preprocessing...')
run_prepro = true
end
end
if run_prepro then
-- construct a tensor with all the data, and vocab file
print('one-time setup: preprocessing input text file ' .. input_file .. '...')
CharSplitLMMinibatchLoader.text_to_tensor(input_file, vocab_file, tensor_file, min_freq)
end
print('loading data files...')
local data = torch.load(tensor_file)
self.vocab_mapping = torch.load(vocab_file)
-- cut off the end so that it divides evenly
local len = data:size(1)
if len % (batch_size * seq_length) ~= 0 then
print('cutting off end of data so that the batches/sequences divide evenly')
data = data:sub(1, batch_size * seq_length
* math.floor(len / (batch_size * seq_length)))
end
-- count vocab
self.vocab_size = 0
for _ in pairs(self.vocab_mapping) do
self.vocab_size = self.vocab_size + 1
end
-- self.batches is a table of tensors
print('reshaping tensor...')
self.batch_size = batch_size
self.seq_length = seq_length
local ydata = data:clone()
ydata:sub(1,-2):copy(data:sub(2,-1))
ydata[-1] = data[1]
self.x_batches = data:view(batch_size, -1):split(seq_length, 2) -- #rows = #batches
self.nbatches = #self.x_batches
self.y_batches = ydata:view(batch_size, -1):split(seq_length, 2) -- #rows = #batches
assert(#self.x_batches == #self.y_batches)
-- lets try to be helpful here
if self.nbatches < 50 then
print('WARNING: less than 50 batches in the data in total? Looks like very small dataset. You probably want to use smaller batch_size and/or seq_length.')
end
-- perform safety checks on split_fractions
assert(split_fractions[1] >= 0 and split_fractions[1] <= 1, 'bad split fraction ' .. split_fractions[1] .. ' for train, not between 0 and 1')
assert(split_fractions[2] >= 0 and split_fractions[2] <= 1, 'bad split fraction ' .. split_fractions[2] .. ' for val, not between 0 and 1')
assert(split_fractions[3] >= 0 and split_fractions[3] <= 1, 'bad split fraction ' .. split_fractions[3] .. ' for test, not between 0 and 1')
if split_fractions[3] == 0 then
-- catch a common special case where the user might not want a test set
self.ntrain = math.floor(self.nbatches * split_fractions[1])
self.nval = self.nbatches - self.ntrain
self.ntest = 0
else
-- divide data to train/val and allocate rest to test
self.ntrain = math.floor(self.nbatches * split_fractions[1])
self.nval = math.floor(self.nbatches * split_fractions[2])
self.ntest = self.nbatches - self.nval - self.ntrain -- the rest goes to test (to ensure this adds up exactly)
end
self.split_sizes = {self.ntrain, self.nval, self.ntest}
self.batch_ix = {0,0,0}
print(string.format('data load done. Number of data batches in train: %d, val: %d, test: %d', self.ntrain, self.nval, self.ntest))
collectgarbage()
return self
end
function CharSplitLMMinibatchLoader:reset_batch_pointer(split_index, batch_index)
batch_index = batch_index or 0
self.batch_ix[split_index] = batch_index
end
function CharSplitLMMinibatchLoader:next_batch(split_index)
if self.split_sizes[split_index] == 0 then
-- perform a check here to make sure the user isn't screwing something up
local split_names = {'train', 'val', 'test'}
print('ERROR. Code requested a batch for split ' .. split_names[split_index] .. ', but this split has no data.')
os.exit() -- crash violently
end
-- split_index is integer: 1 = train, 2 = val, 3 = test
self.batch_ix[split_index] = self.batch_ix[split_index] + 1
if self.batch_ix[split_index] > self.split_sizes[split_index] then
self.batch_ix[split_index] = 1 -- cycle around to beginning
end
-- pull out the correct next batch
local ix = self.batch_ix[split_index]
if split_index == 2 then ix = ix + self.ntrain end -- offset by train set size
if split_index == 3 then ix = ix + self.ntrain + self.nval end -- offset by train + val
return self.x_batches[ix], self.y_batches[ix]
end
-- chinese vocab
function get_vocab(str, min_freq)
local len = #str
local left = 0
local arr = {0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}
local unordered = {}
local start = 1
local wordLen = 0
g_total_chars = 0
while len ~= left do
local tmp = string.byte(str, start)
local i = #arr
while arr[i] do
if tmp >= arr[i] then
break
end
i = i - 1
end
wordLen = i + wordLen
local tmpString = string.sub(str, start, wordLen)
start = start + i
left = left + i
if not unordered[tmpString] then
unordered[tmpString] = 1
else
unordered[tmpString] = unordered[tmpString] + 1 end
g_total_chars = g_total_chars + 1
end
final_res = {}
for char_val, char_cnt in pairs(unordered) do
if char_cnt >= min_freq then
final_res[char_val] = true
end
end
return final_res
end
-- change raw data to tokens
function get_data(str, vocab_mapping)
-- can not use torch.ByteTensor because it support mo more than 256
-- local data = torch.ByteTensor(g_total_chars) -- store it into 1D first, then rearrange
local data = torch.ShortTensor(g_total_chars)
local len = #str
local left = 0
local arr = {0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}
local start = 1
local wordLen = 0
local count = 1
while len ~= left do
local tmp = string.byte(str, start)
local i = #arr
while arr[i] do
if tmp >= arr[i] then
break
end
i = i - 1
end
wordLen = i + wordLen
local tmpString = string.sub(str, start, wordLen)
start = start + i
left = left + i
if vocab_mapping[tmpString] then
data[count] = vocab_mapping[tmpString]
else
data[count] = vocab_mapping['UNKNOW']
end
count = count + 1
end
return data
end
-- *** STATIC method ***
function CharSplitLMMinibatchLoader.text_to_tensor(in_textfile, out_vocabfile, out_tensorfile, min_freq)
local timer = torch.Timer()
print('loading text file...')
local f = torch.DiskFile(in_textfile)
local rawdata = f:readString('*a') -- NOTE: this reads the whole file at once
f:close()
-- create vocabulary if it doesn't exist yet
print('creating vocabulary mapping...')
-- record all characters to a set
local unordered = get_vocab(rawdata, min_freq)
-- sort into a table (i.e. keys become 1..N)
local ordered = {}
for char in pairs(unordered) do ordered[#ordered + 1] = char end
table.sort(ordered)
-- invert `ordered` to create the char->int mapping
local vocab_mapping = {}
count_vocab = 0
for i, char in ipairs(ordered) do
vocab_mapping[char] = i
count_vocab = count_vocab + 1
end
vocab_mapping['UNKNOW'] = count_vocab + 1
-- construct a tensor with all the data
print('putting data into tensor, it takes a lot of time...')
local data = get_data(rawdata, vocab_mapping)
-- save output preprocessed files
print('saving ' .. out_vocabfile)
torch.save(out_vocabfile, vocab_mapping)
print('saving ' .. out_tensorfile)
torch.save(out_tensorfile, data)
end
return CharSplitLMMinibatchLoader
+20
View File
@@ -0,0 +1,20 @@
local OneHot, parent = torch.class('OneHot', 'nn.Module')
function OneHot:__init(outputSize)
parent.__init(self)
self.outputSize = outputSize
-- We'll construct one-hot encodings by using the index method to
-- reshuffle the rows of an identity matrix. To avoid recreating
-- it every iteration we'll cache it.
self._eye = torch.eye(outputSize)
end
function OneHot:updateOutput(input)
self.output:resize(input:size(1), self.outputSize):zero()
if self._eye == nil then self._eye = torch.eye(self.outputSize) end
self._eye = self._eye:float()
local longInput = input:long()
self.output:copy(self._eye:index(1, longInput))
return self.output
end
+13
View File
@@ -0,0 +1,13 @@
-- misc utilities
function clone_list(tensor_list, zero_too)
-- utility function. todo: move away to some utils file?
-- takes a list of tensors and returns a list of cloned tensors
local out = {}
for k,v in pairs(tensor_list) do
out[k] = v:clone()
if zero_too then out[k]:zero() end
end
return out
end
+162
View File
@@ -0,0 +1,162 @@
-- adapted from https://github.com/wojciechz/learning_to_execute
-- utilities for combining/flattening parameters in a model
-- the code in this script is more general than it needs to be, which is
-- why it is kind of a large
require 'torch'
local model_utils = {}
function model_utils.combine_all_parameters(...)
--[[ like module:getParameters, but operates on many modules ]]--
-- get parameters
local networks = {...}
local parameters = {}
local gradParameters = {}
for i = 1, #networks do
local net_params, net_grads = networks[i]:parameters()
if net_params then
for _, p in pairs(net_params) do
parameters[#parameters + 1] = p
end
for _, g in pairs(net_grads) do
gradParameters[#gradParameters + 1] = g
end
end
end
local function storageInSet(set, storage)
local storageAndOffset = set[torch.pointer(storage)]
if storageAndOffset == nil then
return nil
end
local _, offset = unpack(storageAndOffset)
return offset
end
-- this function flattens arbitrary lists of parameters,
-- even complex shared ones
local function flatten(parameters)
if not parameters or #parameters == 0 then
return torch.Tensor()
end
local Tensor = parameters[1].new
local storages = {}
local nParameters = 0
for k = 1,#parameters do
local storage = parameters[k]:storage()
if not storageInSet(storages, storage) then
storages[torch.pointer(storage)] = {storage, nParameters}
nParameters = nParameters + storage:size()
end
end
local flatParameters = Tensor(nParameters):fill(1)
local flatStorage = flatParameters:storage()
for k = 1,#parameters do
local storageOffset = storageInSet(storages, parameters[k]:storage())
parameters[k]:set(flatStorage,
storageOffset + parameters[k]:storageOffset(),
parameters[k]:size(),
parameters[k]:stride())
parameters[k]:zero()
end
local maskParameters= flatParameters:float():clone()
local cumSumOfHoles = flatParameters:float():cumsum(1)
local nUsedParameters = nParameters - cumSumOfHoles[#cumSumOfHoles]
local flatUsedParameters = Tensor(nUsedParameters)
local flatUsedStorage = flatUsedParameters:storage()
for k = 1,#parameters do
local offset = cumSumOfHoles[parameters[k]:storageOffset()]
parameters[k]:set(flatUsedStorage,
parameters[k]:storageOffset() - offset,
parameters[k]:size(),
parameters[k]:stride())
end
for _, storageAndOffset in pairs(storages) do
local k, v = unpack(storageAndOffset)
flatParameters[{{v+1,v+k:size()}}]:copy(Tensor():set(k))
end
if cumSumOfHoles:sum() == 0 then
flatUsedParameters:copy(flatParameters)
else
local counter = 0
for k = 1,flatParameters:nElement() do
if maskParameters[k] == 0 then
counter = counter + 1
flatUsedParameters[counter] = flatParameters[counter+cumSumOfHoles[k]]
end
end
assert (counter == nUsedParameters)
end
return flatUsedParameters
end
-- flatten parameters and gradients
local flatParameters = flatten(parameters)
local flatGradParameters = flatten(gradParameters)
-- return new flat vector that contains all discrete parameters
return flatParameters, flatGradParameters
end
function model_utils.clone_many_times(net, T)
local clones = {}
local params, gradParams
if net.parameters then
params, gradParams = net:parameters()
if params == nil then
params = {}
end
end
local paramsNoGrad
if net.parametersNoGrad then
paramsNoGrad = net:parametersNoGrad()
end
local mem = torch.MemoryFile("w"):binary()
mem:writeObject(net)
for t = 1, T do
-- We need to use a new reader for each clone.
-- We don't want to use the pointers to already read objects.
local reader = torch.MemoryFile(mem:storage(), "r"):binary()
local clone = reader:readObject()
reader:close()
if net.parameters then
local cloneParams, cloneGradParams = clone:parameters()
local cloneParamsNoGrad
for i = 1, #params do
cloneParams[i]:set(params[i])
cloneGradParams[i]:set(gradParams[i])
end
if paramsNoGrad then
cloneParamsNoGrad = clone:parametersNoGrad()
for i =1,#paramsNoGrad do
cloneParamsNoGrad[i]:set(paramsNoGrad[i])
end
end
end
clones[t] = clone
collectgarbage()
end
mem:close()
return clones
end
return model_utils