class NN
Constants
- VERSION
Attributes
activation[RW]
batch_size[RW]
betas[RW]
biases[RW]
dropout_ratio[RW]
gammas[RW]
learning_rate[RW]
momentum[RW]
training[R]
weight_decay[RW]
weights[RW]
Public Class Methods
load(file_name)
click to toggle source
# File lib/nn.rb, line 46 def self.load(file_name) Marshal.load(File.binread(file_name)) end
load_json(file_name)
click to toggle source
# File lib/nn.rb, line 50 def self.load_json(file_name) json = JSON.parse(File.read(file_name)) nn = self.new(json["num_nodes"], learning_rate: json["learning_rate"], batch_size: json["batch_size"], activation: json["activation"].map(&:to_sym), momentum: json["momentum"], weight_decay: json["weight_decay"], use_dropout: json["use_dropout"], dropout_ratio: json["dropout_ratio"], use_batch_norm: json["use_batch_norm"], ) nn.weights = json["weights"].map{|weight| SFloat.cast(weight)} nn.biases = json["biases"].map{|bias| SFloat.cast(bias)} if json["use_batch_norm"] nn.gammas = json["gammas"].map{|gamma| SFloat.cast(gamma)} nn.betas = json["betas"].map{|beta| SFloat.cast(beta)} end nn end
new(num_nodes, learning_rate: 0.01, batch_size: 1, activation: %i(relu identity), momentum: 0, weight_decay: 0, use_dropout: false, dropout_ratio: 0.5, use_batch_norm: false)
click to toggle source
# File lib/nn.rb, line 21 def initialize(num_nodes, learning_rate: 0.01, batch_size: 1, activation: %i(relu identity), momentum: 0, weight_decay: 0, use_dropout: false, dropout_ratio: 0.5, use_batch_norm: false) SFloat.srand(rand(2 ** 64)) @num_nodes = num_nodes @learning_rate = learning_rate @batch_size = batch_size @activation = activation @momentum = momentum @weight_decay = weight_decay @use_dropout = use_dropout @dropout_ratio = dropout_ratio @use_batch_norm = use_batch_norm init_weight_and_bias init_gamma_and_beta if @use_batch_norm @training = true init_layers end
Public Instance Methods
accurate(x_test, y_test, tolerance = 0.5, &block)
click to toggle source
# File lib/nn.rb, line 93 def accurate(x_test, y_test, tolerance = 0.5, &block) correct = 0 num_test_data = x_test.is_a?(SFloat) ? x_test.shape[0] : x_test.length (num_test_data.to_f / @batch_size).ceil.times do |i| x = SFloat.zeros(@batch_size, @num_nodes.first) y = SFloat.zeros(@batch_size, @num_nodes.last) @batch_size.times do |j| k = i * @batch_size + j break if k >= num_test_data if x_test.is_a?(SFloat) x[j, true] = x_test[k, true] y[j, true] = y_test[k, true] else x[j, true] = SFloat.cast(x_test[k]) y[j, true] = SFloat.cast(y_test[k]) end end x, y = block.call(x, y) if block out = forward(x, false) @batch_size.times do |j| vout = out[j, true] vy = y[j, true] case @activation[1] when :identity correct += 1 unless (NMath.sqrt((vout - vy) ** 2) < tolerance).to_a.include?(0) when :softmax correct += 1 if vout.max_index == vy.max_index end end end correct.to_f / num_test_data end
learn(x_train, y_train, &block)
click to toggle source
# File lib/nn.rb, line 126 def learn(x_train, y_train, &block) if x_train.is_a?(SFloat) indexes = (0...x_train.shape[0]).to_a.sample(@batch_size) x = x_train[indexes, true] y = y_train[indexes, true] else indexes = (0...x_train.length).to_a.sample(@batch_size) x = SFloat.zeros(@batch_size, @num_nodes.first) y = SFloat.zeros(@batch_size, @num_nodes.last) @batch_size.times do |i| x[i, true] = SFloat.cast(x_train[indexes[i]]) y[i, true] = SFloat.cast(y_train[indexes[i]]) end end x, y = block.call(x, y) if block forward(x) backward(y) update_weight_and_bias update_gamma_and_beta if @use_batch_norm @layers[-1].loss(y) end
run(x)
click to toggle source
# File lib/nn.rb, line 148 def run(x) if x.is_a?(Array) forward(SFloat.cast(x), false).to_a else forward(x, false) end end
save(file_name)
click to toggle source
# File lib/nn.rb, line 156 def save(file_name) File.binwrite(file_name, Marshal.dump(self)) end
save_json(file_name)
click to toggle source
# File lib/nn.rb, line 160 def save_json(file_name) json = { "version" => VERSION, "num_nodes" => @num_nodes, "learning_rate" => @learning_rate, "batch_size" => @batch_size, "activation" => @activation, "momentum" => @momentum, "weight_decay" => @weight_decay, "use_dropout" => @use_dropout, "dropout_ratio" => @dropout_ratio, "use_batch_norm" => @use_batch_norm, "weights" => @weights.map(&:to_a), "biases" => @biases.map(&:to_a), } if @use_batch_norm json_batch_norm = { "gammas" => @gammas, "betas" => @betas } json.merge!(json_batch_norm) end File.write(file_name, JSON.dump(json)) end
test(x_test, y_test, tolerance = 0.5, &block)
click to toggle source
# File lib/nn.rb, line 87 def test(x_test, y_test, tolerance = 0.5, &block) acc = accurate(x_test, y_test, tolerance, &block) puts "accurate: #{acc}" acc end
train(x_train, y_train, epochs, func = nil, &block)
click to toggle source
# File lib/nn.rb, line 71 def train(x_train, y_train, epochs, func = nil, &block) num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length (1..epochs).each do |epoch| loss = nil (num_train_data.to_f / @batch_size).ceil.times do loss = learn(x_train, y_train, &func) if loss.nan? puts "loss is nan" return end end puts "epoch #{epoch}/#{epochs} loss: #{loss}" block.call(epoch) if block end end
Private Instance Methods
backward(y)
click to toggle source
# File lib/nn.rb, line 242 def backward(y) dout = @layers[-1].backward(y) @layers[0...-1].reverse.each do |layer| dout = layer.backward(dout) end end
forward(x, training = true)
click to toggle source
# File lib/nn.rb, line 234 def forward(x, training = true) @training = training @layers.each do |layer| x = layer.forward(x) end x end
init_gamma_and_beta()
click to toggle source
# File lib/nn.rb, line 205 def init_gamma_and_beta @gammas = Array.new(@num_nodes.length - 2, 1) @betas = Array.new(@num_nodes.length - 2, 0) @gamma_amounts = Array.new(@num_nodes.length - 2, 0) @beta_amounts = Array.new(@num_nodes.length - 2, 0) end
init_layers()
click to toggle source
# File lib/nn.rb, line 212 def init_layers @layers = [] @num_nodes[0...-2].each_index do |i| @layers << Affine.new(self, i) @layers << BatchNorm.new(self, i) if @use_batch_norm @layers << case @activation[0] when :sigmoid Sigmoid.new when :relu ReLU.new end @layers << Dropout.new(self) if @use_dropout end @layers << Affine.new(self, -1) @layers << case @activation[1] when :identity Identity.new(self) when :softmax Softmax.new(self) end end
init_weight_and_bias()
click to toggle source
# File lib/nn.rb, line 187 def init_weight_and_bias @weights = Array.new(@num_nodes.length - 1) @biases = Array.new(@num_nodes.length - 1) @weight_amounts = Array.new(@num_nodes.length - 1, 0) @bias_amounts = Array.new(@num_nodes.length - 1, 0) @num_nodes[0...-1].each_index do |i| weight = SFloat.new(@num_nodes[i], @num_nodes[i + 1]).rand_norm bias = SFloat.new(@num_nodes[i + 1]).rand_norm if @activation[0] == :relu @weights[i] = weight / Math.sqrt(@num_nodes[i]) * Math.sqrt(2) @biases[i] = bias / Math.sqrt(@num_nodes[i]) * Math.sqrt(2) else @weights[i] = weight / Math.sqrt(@num_nodes[i]) @biases[i] = bias / Math.sqrt(@num_nodes[i]) end end end
update_gamma_and_beta()
click to toggle source
# File lib/nn.rb, line 264 def update_gamma_and_beta @layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i| gamma_amount = layer.d_gamma * @learning_rate beta_amount = layer.d_beta * @learning_rate if @momentum > 0 gamma_amount += @momentum * @gamma_amounts[i] @gamma_amounts[i] = gamma_amount beta_amount += @momentum * @beta_amounts[i] @beta_amounts[i] = beta_amount end @gammas[i] -= gamma_amount @betas[i] -= gamma_amount end end
update_weight_and_bias()
click to toggle source
# File lib/nn.rb, line 249 def update_weight_and_bias @layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i| weight_amount = layer.d_weight * @learning_rate bias_amount = layer.d_bias * @learning_rate if @momentum > 0 weight_amount += @momentum * @weight_amounts[i] @weight_amounts[i] = weight_amount bias_amount += @momentum * @bias_amounts[i] @bias_amounts[i] = bias_amount end @weights[i] -= weight_amount @biases[i] -= bias_amount end end