class InventoryRefresh::InventoryCollection

For more usage examples please follow spec examples in

@example storing Vm model data into the DB

@ems = ManageIQ::Providers::BaseManager.first
puts @ems.vms.collect(&:ems_ref) # => []

# Init InventoryCollection
vms_inventory_collection = ::InventoryRefresh::InventoryCollection.new(
  :model_class => ManageIQ::Providers::CloudManager::Vm, :parent => @ems, :association => :vms
)

# Fill InventoryCollection with data
# Starting with no vms, lets add vm1 and vm2
vms_inventory_collection.build(:ems_ref => "vm1", :name => "vm1")
vms_inventory_collection.build(:ems_ref => "vm2", :name => "vm2")

# Save InventoryCollection to the db
InventoryRefresh::SaveInventory.save_inventory(@ems, [vms_inventory_collection])

# The result in the DB is that vm1 and vm2 were created
puts @ems.vms.collect(&:ems_ref) # => ["vm1", "vm2"]

@example In another refresh, vm1 does not exist anymore and vm3 was added

# Init InventoryCollection
vms_inventory_collection = ::InventoryRefresh::InventoryCollection.new(
  :model_class => ManageIQ::Providers::CloudManager::Vm, :parent => @ems, :association => :vms
)

# Fill InventoryCollection with data
vms_inventory_collection.build(:ems_ref => "vm2", :name => "vm2")
vms_inventory_collection.build(:ems_ref => "vm3", :name => "vm3")

# Save InventoryCollection to the db
InventoryRefresh::SaveInventory.save_inventory(@ems, [vms_inventory_collection])

# The result in the DB is that vm1 was deleted, vm2 was updated and vm3 was created
puts @ems.vms.collect(&:ems_ref) # => ["vm2", "vm3"]

Attributes

arel[R]
assert_graph_integrity[R]
association[R]
attributes_blacklist[RW]
attributes_whitelist[RW]
batch_extra_attributes[R]
check_changed[R]
complete[R]
create_only[R]
created_records[R]
custom_reconnect_block[R]
custom_save_block[R]
data_collection_finalized[RW]

@return [Boolean] A true value marks that we collected all the data of the InventoryCollection,

meaning we also collected all the references.
data_storage[RW]

@return [InventoryRefresh::InventoryCollection::DataStorage] An InventoryCollection encapsulating all data with

indexes
default_values[R]
deleted_records[R]
dependees[RW]

@return [Set] A set of InventoryCollection objects that depends on this InventoryCollection object.

dependency_attributes[R]
internal_attributes[R]
inventory_object_attributes[R]
manager_ref[R]
manager_ref_allowed_nil[R]
model_class[R]
name[R]
parent[R]
references_storage[R]
retention_strategy[R]
saved[RW]

@return [Boolean] true if this collection is already saved into the DB. E.g. InventoryCollections with

DB only strategy are marked as saved. This causes InventoryCollection not being a dependency for any other
InventoryCollection, since it is already persisted into the DB.
saver_strategy[R]
secondary_refs[R]
strategy[R]
transitive_dependency_attributes[R]
unconnected_edges[R]
update_only[R]
updated_records[R]
use_ar_object[R]

Public Class Methods

new(properties = {}) click to toggle source

@param [Hash] properties - see init methods for params description

# File lib/inventory_refresh/inventory_collection.rb, line 126
def initialize(properties = {})
  init_basic_properties(properties[:association],
                        properties[:model_class],
                        properties[:name],
                        properties[:parent])

  init_flags(properties[:complete],
             properties[:create_only],
             properties[:check_changed],
             properties[:update_only],
             properties[:use_ar_object],
             properties[:assert_graph_integrity])

  init_strategies(properties[:strategy],
                  properties[:retention_strategy])

  init_references(properties[:manager_ref],
                  properties[:manager_ref_allowed_nil],
                  properties[:secondary_refs])

  init_ic_relations(properties[:dependency_attributes])

  init_arels(properties[:arel])

  init_custom_procs(properties[:custom_save_block],
                    properties[:custom_reconnect_block])

  init_model_attributes(properties[:attributes_blacklist],
                        properties[:attributes_whitelist],
                        properties[:inventory_object_attributes],
                        properties[:batch_extra_attributes])

  init_data(properties[:default_values])

  init_storages

  init_changed_records_stats
end

Public Instance Methods

all_column_names() click to toggle source

@return [Array<Symbol>] Array of all column names on the InventoryCollection

# File lib/inventory_refresh/inventory_collection.rb, line 286
def all_column_names
  @all_column_names ||= model_class.columns.map { |x| x.name.to_sym }
end
base_class_name() click to toggle source

@return [String] Base class name of the model_class of this InventoryCollection

# File lib/inventory_refresh/inventory_collection.rb, line 437
def base_class_name
  return "" unless model_class

  @base_class_name ||= model_class.base_class.name
end
base_columns() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 281
def base_columns
  @base_columns ||= (unique_index_columns + internal_columns + not_null_columns).uniq
end
batch_size() click to toggle source

@return [Integer] default batch size for talking to the DB

# File lib/inventory_refresh/inventory_collection.rb, line 461
def batch_size
  1000
end
batch_size_pure_sql() click to toggle source

@return [Integer] default batch size for talking to the DB if not using ApplicationRecord objects

# File lib/inventory_refresh/inventory_collection.rb, line 466
def batch_size_pure_sql
  10_000
end
blacklist_attributes!(attributes) click to toggle source

Add passed attributes to blacklist. The manager_ref attributes cannot be blacklisted, otherwise we will not be able to identify the inventory_object. We do not automatically remove attributes causing fixed dependencies, so beware that without them, you won't be able to create the record.

@param attributes [Array<Symbol>] Attributes we want to blacklist @return [Array<Symbol>] All blacklisted attributes

# File lib/inventory_refresh/inventory_collection.rb, line 402
def blacklist_attributes!(attributes)
  self.attributes_blacklist += attributes - (fixed_attributes + internal_attributes)
end
build_multi_selection_condition(hashes, keys = unique_index_keys) click to toggle source

Builds a multiselection conditions like (table1.a = a1 AND table2.b = b1) OR (table1.a = a2 AND table2.b = b2)

@param hashes [Array<Hash>] data we want to use for the query @param keys [Array<Symbol>] keys of attributes involved @return [String] A condition usable in .where of an ActiveRecord relation

# File lib/inventory_refresh/inventory_collection.rb, line 475
def build_multi_selection_condition(hashes, keys = unique_index_keys)
  arel_table = model_class.arel_table
  # We do pure SQL OR, since Arel is nesting every .or into another parentheses, otherwise this would be just
  # inject(:or) instead of to_sql with .join(" OR ")
  hashes.map { |hash| "(#{keys.map { |key| arel_table[key].eq(hash[key]) }.inject(:and).to_sql})" }.join(" OR ")
end
clone() click to toggle source

@return [InventoryCollection] a shallow copy of InventoryCollection, the copy will share data_storage of the

original collection, otherwise we would be copying a lot of records in memory.
# File lib/inventory_refresh/inventory_collection.rb, line 418
def clone
  cloned = self.class.new(:model_class           => model_class,
                          :manager_ref           => manager_ref,
                          :association           => association,
                          :parent                => parent,
                          :arel                  => arel,
                          :strategy              => strategy,
                          :custom_save_block     => custom_save_block,
                          # We want cloned IC to be update only, since this is used for cycle resolution
                          :update_only           => true,
                          # Dependency attributes need to be a hard copy, since those will differ for each
                          # InventoryCollection
                          :dependency_attributes => dependency_attributes.clone)

  cloned.data_storage = data_storage
  cloned
end
db_collection_for_comparison() click to toggle source

Returns iterator for the passed references and a query

@return [InventoryRefresh::ApplicationRecordIterator] Iterator for the references and query

# File lib/inventory_refresh/inventory_collection.rb, line 485
def db_collection_for_comparison
  InventoryRefresh::ApplicationRecordIterator.new(:inventory_collection => self)
end
db_collection_for_comparison_for(references) click to toggle source

Builds an ActiveRecord::Relation that can fetch all the references from the DB

@param references [Hash{String => InventoryRefresh::InventoryCollection::Reference}] passed references @return [ActiveRecord::Relation] relation that can fetch all the references from the DB

# File lib/inventory_refresh/inventory_collection.rb, line 515
def db_collection_for_comparison_for(references)
  query = full_collection_for_comparison.where(targeted_selection_for(references))
  if pure_sql_record_fetching?
    return get_connection.query(query.select(*select_keys).to_sql)
  end

  query
end
dependencies() click to toggle source

@return [Array<InventoryRefresh::InventoryCollection>] all unique non saved dependencies

# File lib/inventory_refresh/inventory_collection.rb, line 379
def dependencies
  filtered_dependency_attributes.values.map(&:to_a).flatten.uniq.reject(&:saved?)
end
dependency_attributes_for(inventory_collections) click to toggle source

Returns what attributes are causing a dependencies to certain InventoryCollection objects.

@param inventory_collections [Array<InventoryRefresh::InventoryCollection>] @return [Array<InventoryRefresh::InventoryCollection>] attributes causing the dependencies to certain

InventoryCollection objects
# File lib/inventory_refresh/inventory_collection.rb, line 388
def dependency_attributes_for(inventory_collections)
  attributes = Set.new
  inventory_collections.each do |inventory_collection|
    attributes += filtered_dependency_attributes.select { |_key, value| value.include?(inventory_collection) }.keys
  end
  attributes
end
filtered_dependency_attributes() click to toggle source

List attributes causing a dependency and filters them by attributes_blacklist and attributes_whitelist

@return [Hash{Symbol => Set}] attributes causing a dependency and filtered by blacklist and whitelist

# File lib/inventory_refresh/inventory_collection.rb, line 326
def filtered_dependency_attributes
  filtered_attributes = dependency_attributes

  if attributes_blacklist.present?
    filtered_attributes = filtered_attributes.reject { |key, _value| attributes_blacklist.include?(key) }
  end

  if attributes_whitelist.present?
    filtered_attributes = filtered_attributes.select { |key, _value| attributes_whitelist.include?(key) }
  end

  filtered_attributes
end
fixed_attributes() click to toggle source

Attributes that are needed to be able to save the record, i.e. attributes that are part of the unique index and attributes with presence validation or NOT NULL constraint

@return [Array<Symbol>] attributes that are needed for saving of the record

# File lib/inventory_refresh/inventory_collection.rb, line 344
def fixed_attributes
  not_null_attributes = []

  if model_class
    # Attrs having presence validator
    presence_validators = model_class.validators.detect { |x| x.kind_of?(ActiveRecord::Validations::PresenceValidator) }
    not_null_attributes += presence_validators.attributes if presence_validators.present?

    # Column names having NOT NULL constraint
    non_null_constraints = model_class.columns_hash.values.reject(&:null).map(&:name) - [model_class.primary_key]
    not_null_attributes += non_null_constraints.map(&:to_sym)

    # Column names having NOT NULL constraint transformed to relation names
    not_null_attributes += non_null_constraints.map {|x| foreign_key_to_association_mapping[x]}.compact
  end
  # Attributes that has to be always on the entity, so attributes making unique index of the record + attributes
  # that have presence validation

  fixed_attributes = manager_ref
  fixed_attributes += not_null_attributes.uniq
  fixed_attributes
end
fixed_dependencies() click to toggle source

Returns fixed dependencies, which are the ones we can't move, because we wouldn't be able to save the data

@returns [Set<InventoryRefresh::InventoryCollection>] all unique non saved fixed dependencies

# File lib/inventory_refresh/inventory_collection.rb, line 370
def fixed_dependencies
  fixed_attrs = fixed_attributes

  filtered_dependency_attributes.each_with_object(Set.new) do |(key, value), fixed_deps|
    fixed_deps.merge(value) if fixed_attrs.include?(key)
  end.reject(&:saved?)
end
full_collection_for_comparison() click to toggle source

@return [ActiveRecord::Relation] relation that can fetch all the references from the DB

# File lib/inventory_refresh/inventory_collection.rb, line 525
def full_collection_for_comparison
  return arel unless arel.nil?
  rel = parent.send(association)
  rel
end
get_connection() click to toggle source

@return [ActiveRecord::ConnectionAdapters::AbstractAdapter] ActiveRecord connection

# File lib/inventory_refresh/inventory_collection.rb, line 503
def get_connection
  ActiveRecord::Base.connection
end
inspect() click to toggle source

@return [String] a concise form of the InventoryCollection for easy logging

# File lib/inventory_refresh/inventory_collection.rb, line 456
def inspect
  to_s
end
internal_columns() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 252
def internal_columns
  return @internal_columns if @internal_columns

  @internal_columns = [] + internal_timestamp_columns
  @internal_columns << :type if supports_sti?
  @internal_columns += [resource_version_column,
                        :resource_timestamps_max,
                        :resource_timestamps,
                        :resource_timestamp,
                        :resource_counters_max,
                        :resource_counters,
                        :resource_counter].collect do |col|
    col if supports_column?(col)
  end.compact
end
internal_timestamp_columns() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 268
def internal_timestamp_columns
  return @internal_timestamp_columns if @internal_timestamp_columns

  @internal_timestamp_columns = %i(created_at created_on updated_at updated_on).collect do |timestamp_col|
    timestamp_col if supports_column?(timestamp_col)
  end.compact
end
inventory_object?(value) click to toggle source

@param value [Object] Object we want to test @return [Boolean] true is value is kind of InventoryRefresh::InventoryObject

# File lib/inventory_refresh/inventory_collection.rb, line 292
def inventory_object?(value)
  value.kind_of?(::InventoryRefresh::InventoryObject)
end
inventory_object_lazy?(value) click to toggle source

@param value [Object] Object we want to test @return [Boolean] true is value is kind of InventoryRefresh::InventoryObjectLazy

# File lib/inventory_refresh/inventory_collection.rb, line 298
def inventory_object_lazy?(value)
  value.kind_of?(::InventoryRefresh::InventoryObjectLazy)
end
manager_ref_to_cols() click to toggle source

Convert manager_ref list of attributes to list of DB columns

@return [Array<String>] Converted manager_ref list of attributes to list of DB columns

# File lib/inventory_refresh/inventory_collection.rb, line 315
def manager_ref_to_cols
  # TODO(lsmola) this should contain the polymorphic _type, otherwise the IC with polymorphic unique key will get
  # conflicts
  manager_ref.map do |ref|
    association_to_foreign_key_mapping[ref] || ref
  end
end
new_inventory_object(hash) click to toggle source

Creates InventoryRefresh::InventoryObject object from passed hash data

@param hash [Hash] Object data @return [InventoryRefresh::InventoryObject] Instantiated InventoryRefresh::InventoryObject

# File lib/inventory_refresh/inventory_collection.rb, line 535
def new_inventory_object(hash)
  manager_ref.each do |x|
    # TODO(lsmola) with some effort, we can do this, but it's complex
    raise "A lazy_find with a :key can't be a part of the manager_uuid" if inventory_object_lazy?(hash[x]) && hash[x].key
  end

  inventory_object_class.new(self, hash)
end
not_null_columns() click to toggle source

@return [Array] Array of column names that have not null constraint

# File lib/inventory_refresh/inventory_collection.rb, line 277
def not_null_columns
  @not_null_constraint_columns ||= model_class.columns.reject(&:null).map { |x| x.name.to_sym } - [model_class.primary_key.to_sym]
end
object_index_with_keys(keys, record) click to toggle source

Builds string uuid from passed Object and keys

@param keys [Array<Symbol>] Indexes into the Hash data @param record [ApplicationRecord] ActiveRecord record @return [String] Concatenated values on keys from data

# File lib/inventory_refresh/inventory_collection.rb, line 307
def object_index_with_keys(keys, record)
  # TODO(lsmola) remove, last usage is in k8s reconnect logic
  build_stringified_reference_for_record(record, keys)
end
pure_sql_record_fetching?() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 507
def pure_sql_record_fetching?
  !use_ar_object?
end
resource_version_column() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 248
def resource_version_column
  :resource_version
end
select_keys() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 498
def select_keys
  @select_keys ||= [@model_class.primary_key] + manager_ref_to_cols.map(&:to_s) + internal_columns.map(&:to_s)
end
store_created_records(records) click to toggle source

Caches what records were created, for later use, e.g. post provision behavior

@param records [Array<ApplicationRecord, Hash>] list of stored records

# File lib/inventory_refresh/inventory_collection.rb, line 175
def store_created_records(records)
  @created_records.concat(records_identities(records))
end
store_deleted_records(records) click to toggle source

Caches what records were deleted/soft-deleted, for later use, e.g. post provision behavior

@param records [Array<ApplicationRecord, Hash>] list of stored records

# File lib/inventory_refresh/inventory_collection.rb, line 189
def store_deleted_records(records)
  @deleted_records.concat(records_identities(records))
end
store_unconnected_edges(inventory_object, inventory_object_key, inventory_object_lazy) click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 165
def store_unconnected_edges(inventory_object, inventory_object_key, inventory_object_lazy)
  (@unconnected_edges ||= []) <<
    InventoryRefresh::InventoryCollection::UnconnectedEdge.new(
      inventory_object, inventory_object_key, inventory_object_lazy
    )
end
store_updated_records(records) click to toggle source

Caches what records were updated, for later use, e.g. post provision behavior

@param records [Array<ApplicationRecord, Hash>] list of stored records

# File lib/inventory_refresh/inventory_collection.rb, line 182
def store_updated_records(records)
  @updated_records.concat(records_identities(records))
end
targeted_selection_for(references) click to toggle source

Builds a multiselection conditions like (table1.a = a1 AND table2.b = b1) OR (table1.a = a2 AND table2.b = b2) for passed references

@param references [Hash{String => InventoryRefresh::InventoryCollection::Reference}] passed references @return [String] A condition usable in .where of an ActiveRecord relation

# File lib/inventory_refresh/inventory_collection.rb, line 494
def targeted_selection_for(references)
  build_multi_selection_condition(references.map(&:second))
end
to_s() click to toggle source

@return [String] a concise form of the inventoryCollection for easy logging

# File lib/inventory_refresh/inventory_collection.rb, line 444
def to_s
  whitelist = ", whitelist: [#{attributes_whitelist.to_a.join(", ")}]" if attributes_whitelist.present?
  blacklist = ", blacklist: [#{attributes_blacklist.to_a.join(", ")}]" if attributes_blacklist.present?

  strategy_name = ", strategy: #{strategy}" if strategy

  name = model_class || association

  "InventoryCollection:<#{name}>#{whitelist}#{blacklist}#{strategy_name}"
end
uniq_keys_candidates(keys) click to toggle source

Find candidates for unique key. Candidate must cover all columns we are passing as keys.

@param keys [Array<Symbol>] @raise [Exception] if the unique index for the columns was not found @return [Array<ActiveRecord::ConnectionAdapters::IndexDefinition>] Array of unique indexes fitting the keys

# File lib/inventory_refresh/inventory_collection.rb, line 236
def uniq_keys_candidates(keys)
  # Find all uniq indexes that that are covering our keys
  uniq_key_candidates = unique_indexes.each_with_object([]) { |i, obj| obj << i if (keys - i.columns.map(&:to_sym)).empty? }

  if unique_indexes.blank? || uniq_key_candidates.blank?
    raise "#{self} and its table #{model_class.table_name} must have a unique index defined "\
            "covering columns #{keys} to be able to use saver_strategy :concurrent_safe_batch."
  end

  uniq_key_candidates
end
unique_index_columns() click to toggle source

@return [Array<Symbol>] all columns that are part of the best fit unique index

# File lib/inventory_refresh/inventory_collection.rb, line 194
def unique_index_columns
  return @unique_index_columns if @unique_index_columns

  @unique_index_columns = unique_index_for(unique_index_keys).columns.map(&:to_sym)
end
unique_index_for(keys) click to toggle source

Finds an index that fits the list of columns (keys) the best

@param keys [Array<Symbol>] @raise [Exception] if the unique index for the columns was not found @return [ActiveRecord::ConnectionAdapters::IndexDefinition] unique index fitting the keys

# File lib/inventory_refresh/inventory_collection.rb, line 223
def unique_index_for(keys)
  @unique_index_for_keys_cache ||= {}
  @unique_index_for_keys_cache[keys] if @unique_index_for_keys_cache[keys]

  # Take the uniq key having the least number of columns
  @unique_index_for_keys_cache[keys] = uniq_keys_candidates(keys).min_by { |x| x.columns.count }
end
unique_index_keys() click to toggle source
# File lib/inventory_refresh/inventory_collection.rb, line 200
def unique_index_keys
  @unique_index_keys ||= manager_ref_to_cols.map(&:to_sym)
end
unique_indexes() click to toggle source

@return [Array<ActiveRecord::ConnectionAdapters::IndexDefinition>] array of all unique indexes known to model

# File lib/inventory_refresh/inventory_collection.rb, line 205
def unique_indexes
  @unique_indexes_cache if @unique_indexes_cache

  @unique_indexes_cache = model_class.connection.indexes(model_class.table_name).select(&:unique)

  if @unique_indexes_cache.blank?
    raise "#{self} and its table #{model_class.table_name} must have a unique index defined, to"\
            " be able to use saver_strategy :concurrent_safe_batch."
  end

  @unique_indexes_cache
end
whitelist_attributes!(attributes) click to toggle source

Add passed attributes to whitelist. The manager_ref attributes always needs to be in the white list, otherwise we will not be able to identify theinventory_object. We do not automatically add attributes causing fixed dependencies, so beware that without them, you won't be able to create the record.

@param attributes [Array<Symbol>] Attributes we want to whitelist @return [Array<Symbol>] All whitelisted attributes

# File lib/inventory_refresh/inventory_collection.rb, line 412
def whitelist_attributes!(attributes)
  self.attributes_whitelist += attributes + (fixed_attributes + internal_attributes)
end

Private Instance Methods

inventory_object_class() click to toggle source

Creates dynamically a subclass of InventoryRefresh::InventoryObject, that will be used per InventoryCollection object. This approach is needed because we want different InventoryObject's getters&setters for each InventoryCollection.

@return [InventoryRefresh::InventoryObject] new isolated subclass of InventoryRefresh::InventoryObject

# File lib/inventory_refresh/inventory_collection.rb, line 553
def inventory_object_class
  @inventory_object_class ||= begin
    klass = Class.new(::InventoryRefresh::InventoryObject)
    klass.add_attributes(inventory_object_attributes) if inventory_object_attributes
    klass
  end
end
record_identity(record) click to toggle source

Returns a hash with a simple record identity

@param record [ApplicationRecord, Hash] list of stored records @return [Hash{Symbol => Bigint}] record identity

# File lib/inventory_refresh/inventory_collection.rb, line 574
def record_identity(record)
  identity = record.try(:[], :id) || record.try(:[], "id") || record.try(:id)
  raise "Cannot obtain identity of the #{record}" if identity.blank?
  {
    :id => identity
  }
end
records_identities(records) click to toggle source

Returns array of records identities

@param records [Array<ApplicationRecord>, Array] list of stored records @return [Array<Hash>] array of records identities

# File lib/inventory_refresh/inventory_collection.rb, line 565
def records_identities(records)
  records = [records] unless records.respond_to?(:map)
  records.map { |record| record_identity(record) }
end