class Milkode::DocumentTable

Attributes

table[R]

Public Class Methods

define_schema() click to toggle source
# File lib/milkode/database/document_table.rb, line 14
    def self.define_schema
      begin
        Groonga::Schema.define do |schema|
          schema.create_table("documents", :type => :hash) do |table|          
            table.string("path")
            table.string("package")
            table.string("restpath")
            table.text("content")
            table.time("timestamp")
            table.string("suffix")
          end

          schema.create_table("terms",
                              :type => :patricia_trie,
                              :key_normalize => true,
                              :default_tokenizer => "TokenBigramSplitSymbolAlphaDigit") do |table|
            table.index("documents.path", :with_position => true)
            table.index("documents.package", :with_position => true)
            table.index("documents.restpath", :with_position => true)
            table.index("documents.content", :with_position => true)
            table.index("documents.suffix", :with_position => true)
          end
        end
      rescue Groonga::Schema::ColumnCreationWithDifferentOptions
        puts <<EOF
WARNING: Milkode database is old. (Renewal at 1.4.0)
Can't get the new features. Please execute rebuild command.

  $ milk rebuild --all

EOF

        Groonga::Schema.define do |schema|
          schema.create_table("documents", :type => :hash) do |table|          
            table.string("path")
            table.string("package")
            table.string("restpath")
            table.text("content")
            table.time("timestamp")
            table.text("suffix")
          end

          schema.create_table("terms",
                              :type => :patricia_trie,
                              :key_normalize => true,
                              :default_tokenizer => "TokenBigramSplitSymbolAlphaDigit") do |table|
            table.index("documents.path", :with_position => true)
            table.index("documents.package", :with_position => true)
            table.index("documents.restpath", :with_position => true)
            table.index("documents.content", :with_position => true)
            table.index("documents.suffix", :with_position => true)
          end
        end
      end
    end
drilldown(result, column, num = nil) click to toggle source
# File lib/milkode/database/document_table.rb, line 345
def self.drilldown(result, column, num = nil)
  drilled = result.group(column).map {|record| [record.n_sub_records, record.key]}.sort_by {|a| a[0]}.reverse
  num ? drilled[0, num] : drilled
end
new(table) click to toggle source
# File lib/milkode/database/document_table.rb, line 107
def initialize(table)
  @table = table
end

Public Instance Methods

add(package_dir, restpath, package_name = nil) click to toggle source

指定ファイルをテーブルに追加

@param package_dir パッケージディレクトリ -> '/path/to/Package' @param restpath パッケージディレクトリ以下のパス名 -> 'src/Foo.hpp' @param package_name パッケージ名(未指定の場合は Fie.basename(package_dir) )

@retval :newfile 新規追加 @retval :update 更新 @retval nil タイムスタンプ比較により更新無し

# File lib/milkode/database/document_table.rb, line 125
def add(package_dir, restpath, package_name = nil)
  filename  = File.join(package_dir, restpath)           # フルパスの作成
  filename  = File.expand_path(filename)                 # 絶対パスに変換
  path      = Util.filename_to_utf8(filename)           # データベースに格納する時のファイル名はutf8
  package   = package_name || File.basename(package_dir)
  package   = Util.filename_to_utf8(package)
  restpath  = Util.filename_to_utf8(restpath)
  suffix    = File.extname(path).sub('.', "")
  timestamp = Util.truncate_nsec(File.mtime(filename)) # OSへの問い合わせは変換前のファイル名で

  record = @table[path]

  unless record
    # 新規追加
    @table.add(path, 
               :path => path,
               :package => package,
               :restpath => restpath,
               :content => load_content(filename),
               :timestamp => timestamp,
               :suffix => suffix)
    return :newfile
  else
    if (record.timestamp < timestamp)
      # 更新
      record.package   = package
      record.restpath = restpath
      record.content   = load_content(filename)
      record.timestamp = timestamp
      record.suffix    = suffix
      return :update
    else
      # タイムスタンプ比較により更新無し
      return nil
    end
  end
end
cleanup() { |r| ... } click to toggle source

実体の存在しないデータを削除

# File lib/milkode/database/document_table.rb, line 198
def cleanup
  self.each do |r|
    unless File.exist? r.path
      yield r if block_given?
      # p r.restpath
      remove(r.path)
    end
  end
end
cleanup_package_name(package_name, ignore_checker = nil) { |r| ... } click to toggle source

指定されたパッケージのクリーンアップ

# File lib/milkode/database/document_table.rb, line 373
def cleanup_package_name(package_name, ignore_checker = nil)
  # クリーンアップ対象のファイルを検索
  result = @table.select { |record| record.package == package_name }

  # 存在しない&無視ファイルの削除
  result.each do |r|
    if !File.exist?(r.path) || (ignore_checker && ignore_checker.ignore?("/#{r.restpath}"))
      yield r if block_given?
      # p r.restpath
      remove(r.path)
    end
  end
end
dump() click to toggle source
# File lib/milkode/database/document_table.rb, line 393
def dump
  self.each do |r|
    p [r.path, r.package, r.restpath, r.content, r.timestamp, r.suffix]
  end
end
each() { |r| ... } click to toggle source
# File lib/milkode/database/document_table.rb, line 387
def each
  @table.select.each do |r|
    yield r
  end
end
find_shortpath(shortpath) click to toggle source

shortpathの一致するレコードを取得

# File lib/milkode/database/document_table.rb, line 176
def find_shortpath(shortpath)
  package, restpath = Util.divide_shortpath(shortpath)
  result = @table.select { |record| (record.package == package) & (record.restpath == restpath) }
  return result.records[0]
end
find_shortpath_below(shortpath) click to toggle source

指定パス以下のファイルを全て取得

# File lib/milkode/database/document_table.rb, line 183
def find_shortpath_below(shortpath)
  if (shortpath.nil? || shortpath.empty?)
    @table.select.records
  else
    package, restpath = Util.divide_shortpath(shortpath)

    if (restpath.nil? || restpath.empty?)
      @table.select { |record| record.package == package }.to_a
    else
      @table.select { |record| (record.package == package) & (record.restpath =~ restpath)}.to_a
    end
  end
end
package_records(name) click to toggle source
# File lib/milkode/database/document_table.rb, line 403
def package_records(name)
  search(:strict_packages => [name])
end
remove(name) click to toggle source
# File lib/milkode/database/document_table.rb, line 163
def remove(name)
  @table[name].delete
end
remove_all(&block) click to toggle source
# File lib/milkode/database/document_table.rb, line 171
def remove_all(&block)
  remove_records(@table.select, &block)
end
remove_match_path(path, &block) click to toggle source
# File lib/milkode/database/document_table.rb, line 167
def remove_match_path(path, &block)
  remove_records(search(:paths => [path]), &block)
end
remove_records(records) { |record| ... } click to toggle source

レコードをまとめて削除する

過去の方法
  検索結果にマッチしたレコード等をまとめて削除
  削除前にインデックスを削除し、削除後にインデックスを再度追加してい
  大量のレコードを削除する場合に高速に動作する

現在の方法
  上記の方法がかえって遅くなったので元に戻す
  普通に速くなった気がする
# File lib/milkode/database/document_table.rb, line 80
def remove_records(records, &block)
  # Groonga::Schema.define do |schema|
  #   schema.change_table("terms") do |table|
  #     table.remove_index("documents.path")
  #     table.remove_index("documents.package")
  #     table.remove_index("documents.restpath")
  #     table.remove_index("documents.content")
  #     table.remove_index("documents.suffix")
  #   end
  # end

  records.each do |record|
    yield record if block
    record.key.delete
  end

  # Groonga::Schema.define do |schema|
  #   schema.change_table("terms") do |table|
  #     table.index("documents.path", :with_position => true)
  #     table.index("documents.package", :with_position => true)
  #     table.index("documents.restpath", :with_position => true)
  #     table.index("documents.content", :with_position => true)
  #     table.index("documents.suffix", :with_position => true)
  #   end
  # end
end
search_with_match(options) click to toggle source

詳細検索

@param options 検索オプション、ハッシュで指定

:patterns => マッチする行
:keywords => 検索キーワード
:paths    => ファイルパス(AND)
:packages => パッケージ名(OR)
:strict_packages => 厳密なパッケージ名(OR)
:restpaths => 短縮パス(AND)
:suffixs  => 拡張子
:offset   => オフセット(default = 0)
:limit    => 表示リミット(default = -1)
# File lib/milkode/database/document_table.rb, line 220
def search_with_match(options)
  patterns = options[:patterns] || []
  keywords = options[:keywords] || []
  packages = options[:packages] || []
  strict_packages = options[:strict_packages] || []
  paths    = options[:paths]    || []
  restpaths = options[:restpaths]    || []
  suffixs  = options[:suffixs]  || []
  fpath_or_packages = options[:fpath_or_packages] || []
  offset   = options[:offset]   || 0
  limit    = options[:limit]    || -1
  
  result = @table.select do |record|
    expression = nil

    # マッチする行
    patterns.each do |word|
      sub_expression = record.content =~ word
      if expression.nil?
        expression = sub_expression
      else
        expression &= sub_expression
      end
    end
    
    # キーワード(絞り込むための手がかり)
    keywords.each do |word|
      sub_expression = record.content =~ word
      sub_expression |= record.restpath =~ word
      sub_expression |= record.package =~ word
      if expression.nil?
        expression = sub_expression
      else
        expression &= sub_expression
      end
    end
    
    # パッケージ(OR)
    pe = package_expression(record, packages) 
    if (pe)
      if expression.nil?
        expression = pe
      else
        expression &= pe
      end
    end
    
    # 厳密なパッケージ(OR)
    pe = strict_packages_expression(record, strict_packages) 
    if (pe)
      if expression.nil?
        expression = pe
      else
        expression &= pe
      end
    end
    
    # ファイルパス
    paths.each do |word|
      sub_expression = record.path =~ word
      if expression.nil?
        expression = sub_expression
      else
        expression &= sub_expression
      end
    end

    # 短縮パス
    restpaths.each do |word|
      sub_expression = record.restpath =~ word
      if expression.nil?
        expression = sub_expression
      else
        expression &= sub_expression
      end
    end

    # 拡張子(OR)
    se = suffix_expression(record, suffixs) 
    if (se)
      if expression.nil?
        expression = se
      else
        expression &= se
      end
    end
    
    # ファイル名かパッケージ名
    fpath_or_packages.each do |word|
      sub_expression = record.restpath =~ word
      sub_expression |= record.package =~ word
      if expression.nil?
        expression = sub_expression
      else
        expression &= sub_expression
      end
    end
    
    # 検索式
    expression
  end

  # スコアとタイムスタンプでソート
  # records = result.sort([{:key => "_score", :order => "descending"},
  #                       {:key => "timestamp", :order => "descending"}],
  #                      :offset => offset,
  #                      :limit => limit)
  
  # ファイル名でソート
  records = Util.groonga_table_sort(result,
                                    [{:key => "package", :order => "ascending"},
                                     {:key => "restpath", :order => "ascending"}],
                                    :offset => offset,
                                    :limit => limit)

  # 検索結果のレコード(limitの影響を受ける), 総マッチ数(limitの影響を受けない), result(Groonga::Hash)
  return records, result.size, result
end
select_all_sort_by_shortpath(offset = 0, limit = -1) click to toggle source
# File lib/milkode/database/document_table.rb, line 350
def select_all_sort_by_shortpath(offset = 0, limit = -1)
  result = @table.select

  # @todo ここが速度低下の原因?と思ったけど、ここは全て選択の部分だけか・・・

  # 2010/10/29 ongaeshi
  # 本当はこのようにgroongaAPIでソートしたいのだが上手くいかなかった
  #       # ファイル名順にソート
  #       records = table.sort([{:key => "shortpath", :order => "descending"}],
  #                            :offset => offset,
  #                            :limit => limit)

  # ソート
  if (limit != -1)
    records = result.records.sort_by{|record| DocumentRecord::shortpath(record).downcase }[offset, limit]
  else
    records = result.records.sort_by{|record| DocumentRecord::shortpath(record).downcase }[offset..limit]
  end

  return records, result.size
end
size() click to toggle source
# File lib/milkode/database/document_table.rb, line 111
def size
  @table.size
end
to_a() click to toggle source
# File lib/milkode/database/document_table.rb, line 399
def to_a
  @table.to_a
end

Private Instance Methods

load_content(filename) click to toggle source
# File lib/milkode/database/document_table.rb, line 409
def load_content(filename)
  Util.load_content($stdout, filename)
end
package_expression(record, packages) click to toggle source
# File lib/milkode/database/document_table.rb, line 413
def package_expression(record, packages)
  sub = nil
  
  packages.each do |word|
    e = record.package =~ word
    if sub.nil?
      sub = e
    else
      sub |= e
    end
  end

  sub
end
strict_packages_expression(record, packages) click to toggle source
# File lib/milkode/database/document_table.rb, line 428
def strict_packages_expression(record, packages)
  sub = nil
  
  packages.each do |word|
    e = record.package == word
    if sub.nil?
      sub = e
    else
      sub |= e
    end
  end

  sub
end
suffix_expression(record, suffixs) click to toggle source
# File lib/milkode/database/document_table.rb, line 443
def suffix_expression(record, suffixs)
  sub = nil
  
  suffixs.each do |word|
    e = record.suffix == word
    if sub.nil?
      sub = e
    else
      sub |= e
    end
  end

  sub
end