class NluAdapter::Metrics

A utility class to calculate classification matrics @see scikit-learn.org/stable/modules/model_evaluation.html#classification-metrics

Public Class Methods

new(actual, predicted, class_labels = []) click to toggle source

Constructor @param actual [Array] an array of actual results (true values) @param predicted [Array] an array of predicted results (predicted values) @param class_labels [Array] an array of class_labels for the results,

this is required if actual & predicted values are numeric
# File lib/nlu_adapter/metrics.rb, line 16
def initialize(actual, predicted, class_labels = [])
        @actual = actual
        @predicted = predicted
        @class_labels = class_labels

        @actual.map!{ |a| a.is_a?(String)? a.intern : a }
        @predicted.map!{ |p| p.is_a?(String)? p.intern : p }
        @class_labels.map!{ |l| l.is_a?(String)? l.intern : l }
end

Public Instance Methods

accuracy() click to toggle source

Caclulate the accuracy @return [Float] accuracy @see en.wikipedia.org/wiki/Accuracy_and_precision

# File lib/nlu_adapter/metrics.rb, line 30
def accuracy
        if @actual.size != @predicted.size
                #todo: throw error
                puts "actual & predicted array are not of same size"
                return
        end

        total = @actual.size
        correct = 0
        @actual.each_with_index do |v, i|
                correct+=1 if @predicted[i] == v
        end
        (correct.fdiv(total) * 100).round(4)
end
class_totals() click to toggle source

Get totals of actual values per class @return [Hash] Hash of class totals

# File lib/nlu_adapter/metrics.rb, line 98
def class_totals
        return @class_totals
end
classification_report() click to toggle source

Generate classification report @return [Hash] precision, recall and totals for each class name as key

# File lib/nlu_adapter/metrics.rb, line 161
def classification_report
        confusion_matrix if !@m
        report = {}
        @class_labels.each do |label|
                report[label] = {
                        precision: precision(label),
                        recall: recall(label),
                        class_total: class_totals[label]
                }
        end

        return report
end
confusion_matrix() click to toggle source

Get the confusion matrix @return [Matrix] confusion matrix @see en.wikipedia.org/wiki/Confusion_matrix

# File lib/nlu_adapter/metrics.rb, line 49
def confusion_matrix
        class_labels = @class_labels
        actual = @actual
        predicted = @predicted

        #if no class_labels, convert to numeric values, and extract the class_labels
        if @class_labels.empty?
                class_labels = (@actual + @predicted).sort.uniq.sort
                class_labels.map!{|c| c.intern}
                @actual.each_with_index do |v, i|
                        actual[i] = class_labels.index(v)
                end

                @predicted.each_with_index do |v, i|
                        predicted[i] = class_labels.index(v)
                end
        else
                #check if any string passed in actual/predicted
                i = actual.select { |a| a.is_a?(Symbol) }.size
                j = predicted.select { |p| p.is_a?(Symbol) }.size

                if i > 0 || j > 0
                        #todo: fix it OR throw error
                        puts "actual, predicted & class_labels array having string values not implemented yet"
                        return

                end
        end

        m = Matrix.zero(class_labels.size)

        @class_totals = Hash[class_labels.collect { |c| [c, 0] }]
        actual.each_with_index do |vi, i|
                vj = predicted[i]
                m[vi, vj] = m[vi, vj] + 1
                @class_totals[class_labels[vi]] += 1
        end

        @class_labels = class_labels
        @actual = actual
        @predicted = predicted

        @m = m
        return m
end
fn(class_name) click to toggle source

Get false negative @param class_name [String] class name @return [Integer] false negative value for class_name

# File lib/nlu_adapter/metrics.rb, line 128
def fn(class_name)
        i = @class_labels.index(class_name.intern)
        return nil if i == nil || @m == nil || @m.empty?
        fn = @m.row(i).sum - tp(class_name)
        return fn
end
fp(class_name) click to toggle source

Get false positive @param class_name [String] class name @return [Integer] false positive value for class_name

# File lib/nlu_adapter/metrics.rb, line 117
def fp(class_name)
        i = @class_labels.index(class_name.intern)
        return nil if i == nil || @m == nil || @m.empty?
        fp = @m.column(i).sum - tp(class_name)
        return fp
end
precision(class_name) click to toggle source

Get the precision for given class @param class_name [String] class name @return [Float] precision rounded off to 4 decimal places

# File lib/nlu_adapter/metrics.rb, line 139
def precision(class_name)
        confusion_matrix if !@m
        return 0.00 if tp(class_name) == 0
        precision = tp(class_name).fdiv((tp(class_name) + fp(class_name))).round(4)
        return precision
end
recall(class_name) click to toggle source

Get the recall for given class @param class_name [String] class name @return [Float] recall rounded off to 4 decimal places

# File lib/nlu_adapter/metrics.rb, line 150
def recall(class_name)
        confusion_matrix if !@m
        return 0.00 if tp(class_name) == 0
        recall = tp(class_name).fdiv((tp(class_name) + fn(class_name))).round(4)

        return recall
end
tp(class_name) click to toggle source

Get total positives @param class_name [String] class name @return [Integer] total positive value for class_name

# File lib/nlu_adapter/metrics.rb, line 106
def tp(class_name)
        i = @class_labels.index(class_name.intern)
        return nil if i == nil || @m == nil || @m.empty?
        tp = @m[i, i]
        return tp
end