class Blufin::YmlSchemaValidator
Constants
- AMOUNT
- APP
- CHILD_OF
- CHILD_TYPE
- COMMON
- CONFIG
- CONFIG_INTERNAL
- CONFIG_OAUTH
- CURRENCY
- DATETIME_TYPES
- DESCRIPTION
- DESCRIPTION_HITS
- DESCRIPTION_TEXT
- DESCRIPTION_TYPE
- ENCRYPTED
- ERROR_MULTIPLE_FILES
- FKEY
- FLAG
- FLAG_AUTO_INCREMENT
- FLAG_INDEX
- FLAG_NULLABLE
- FLAG_PRIMARY_KEY
- FLAG_UNIQUE
- ID
- INT_TYPES
- LINK
- MAX_TABLE_CHARACTERS
- MOCK
- PATH_TO_CONFIG_YML
- REGEX_DECIMAL
- REGEX_ENUM
- REGEX_ENUM_CUSTOM
- REGEX_ENUM_SYSTEM
- REGEX_VARCHAR
- REQUIRED
- REQUIRED_IF
- REQUIRED_IF_ENUM
- RESERVED_WORDS
- RESOURCE_SCHEMA
- RESOURCE_TABLE
- RESOURCE_TYPE_OBJECT
- RESOURCE_TYPE_OBJECT_LINK
- RESOURCE_TYPE_OBJECT_LIST
- RESOURCE_TYPE_PARENT
- STRUCTURE
- TEXT_TYPES
- TRANSIENT
- TYPE
- TYPE_BOOLEAN
- TYPE_DATE
- TYPE_DATETIME
- TYPE_DATETIME_INSERT
- TYPE_DATETIME_UPDATE
- TYPE_DECIMAL
- TYPE_ENUM
- TYPE_ENUM_CUSTOM
- TYPE_ENUM_SYSTEM
- TYPE_INT
- TYPE_INT_AUTO
- TYPE_INT_BIG
- TYPE_INT_SMALL
- TYPE_INT_TINY
- TYPE_TEXT
- TYPE_TEXT_LONG
- TYPE_VARCHAR
- VALID_SCHEMAS
- VALID_SCHEMAS_GENERATE
- VALID_SCHEMAS_REGEX
Public Class Methods
Initialize the class. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 207 def initialize(site, error_handler) begin @site = Blufin::SiteResolver::validate_site(site) @error_handler = error_handler @schema_data = {} @schema_descriptions = {} @schema_fks = {} # app.person.id (IS REFERENCED BY) => [app.company.primary_person_id:INT, app.invoice.person_id:INT] @schema_fks_dependencies = {} # app.purchase (HAS 'FKEY' FIELDS TO) => [app.person, app.user, app.company] @schema_fks_placeholders = {} # app.product (HAS THE FOLLOWING PLACEHOLDERS) => [app.company[link], app.product_variation[]] @schema_fks_links = {} # app.address (IS LINKED TO FROM) => [app.company, app.person] @schema_resources = {} @primary_key_count = nil @yml_schema_outputter = Blufin::YmlOutputter.new(@site) @yml_enum_scanner = Blufin::ScannerJavaEnums.new(@site) # Contains more descriptive error output. @errors_array = [] # Validate "config:" section. This must be done before looping files because we will need some of the date from it. @schema_config = {} validate_config_section config_files = Blufin::Files.get_files_in_dir(PATH_TO_CONFIG_YML) schema_files = Blufin::Files.get_files_in_dir("#{Blufin::SiteResolver::get_site_location(@site)}/#{Blufin::Site::PATH_TO_YML_API_SCHEMA}") # Loop through the array of files. (config_files + schema_files).each { |file| scan_file(file) } validate_table_duplicates validate_foreign_keys validate_foreign_keys_placeholders validate_required_schema_definitions build_resources build_fks_dependencies validate_resources validate_http_methods @yml_schema_outputter.set_schema_fks_links(@schema_fks_links) rescue Exception => e Blufin::Terminal::print_exception(e) end end
Private Class Methods
Returns true if the column name matches regex for OBJECT, OBJECT_LIST or OBJECT_LINK @return bool
# File lib/core/yml/schema/yml_schema_validator.rb, line 1844 def self.column_is_object(column_name) column_name =~ /^(#{VALID_SCHEMAS_REGEX})\.[\w]+/ || column_name =~ /^(#{VALID_SCHEMAS_REGEX})\.[\w]+\[\]$/ || column_name =~ /^(#{VALID_SCHEMAS_REGEX})\.[\w]+\[#{LINK}\]$/ end
Public Instance Methods
Use this to extract flags OUTSIDE the validator. @return Blufin::YmlSchemaFlags
# File lib/core/yml/schema/yml_schema_validator.rb, line 317 def extract_flags(flags) raise RuntimeError, 'Input cannot be nil in YmlSchemaValidator::extract_flags()' if flags.nil? || flags.strip == '' Blufin::YmlCommon::extract_flags(flags)[0] end
@return Array
# File lib/core/yml/schema/yml_schema_validator.rb, line 266 def get_errors_array @errors_array end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 276 def get_schema_config @schema_config end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 271 def get_schema_data @schema_data end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 281 def get_schema_descriptions @schema_descriptions end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 286 def get_schema_fks @schema_fks end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 296 def get_schema_fks_dependencies @schema_fks_dependencies end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 301 def get_schema_fks_links @schema_fks_links end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 291 def get_schema_fks_placeholders @schema_fks_placeholders end
@return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 306 def get_schema_resources @schema_resources end
@return Object
# File lib/core/yml/schema/yml_schema_validator.rb, line 311 def get_yml_schema_outputter @yml_schema_outputter end
Private Instance Methods
Add a 'section:schema' error. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1775 def add_error(error_file, schema, table, column, error_message, error_detail) add_error_abstract(error_file, schema, table, column, error_message, error_detail) end
The abstract method for adding errors. @return
# File lib/core/yml/schema/yml_schema_validator.rb, line 1787 def add_error_abstract(error_file, schema, table, column, error_message, error_detail, multi_line_content = nil, section = 'schema') error_object = Blufin::SchemaError.new error_object.multi_line_content = multi_line_content error_object.section = section if schema.nil? && table.nil? && column.nil? error_object.schema_address = "\xe2\x80\x94" else schema_address = [] schema_address << (schema.nil? ? nil : schema) schema_address << (table.nil? ? nil : table) schema_address << (column.nil? ? nil : column) schema_address.compact! error_object.schema_address = schema_address end if error_file.nil? if !table.nil? error_object.file = "#{table}.yml" else error_object.file = "\xe2\x80\x94" end else if error_file == ERROR_MULTIPLE_FILES error_object.file = ERROR_MULTIPLE_FILES else file_parts = error_file.split('/') error_object.file = file_parts[file_parts.length - 1] end end error_object.error_message = (error_message.nil? ? "\xe2\x80\x94" : error_message) error_object.error_detail = (error_detail.nil? ? "\xe2\x80\x94" : error_detail) @error_handler.add_error_schema(error_object) end
Add a 'section:schema' error with multi-line content). @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1781 def add_error_with_multi_content(error_file, schema, table, column, error_message, error_detail, multi_line_content) add_error_abstract(error_file, schema, table, column, error_message, error_detail, multi_line_content) end
Add to the global @schema_data Array. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1834 def add_schema_data(schema, table, column, data) @schema_data[schema] = {} if @schema_data[schema].nil? @schema_data[schema][table] = {} if @schema_data[schema][table].nil? @schema_data[schema][table][column] = data end
Adds 'dependencies' to end-point array (and validates there are no duplicates in a table). @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1733 def build_fks_dependencies @schema_resources.each do |key, data| dependencies = [] @schema_fks.each do |sfk_key, sfk_data| sfk_data.each do |sfk_data_inner| sfk_split = sfk_data_inner.split('.') sfk = "#{sfk_split[0]}.#{sfk_split[1]}" if key == sfk sfk_key_split = sfk_key.split('.') dependency = "#{sfk_key_split[0]}.#{sfk_key_split[1]}" dependencies << dependency unless dependencies.include?(dependency) end end end @schema_fks_dependencies[key] = dependencies end end
Build the initial end-point hash but more work is done to it after. @return Hash
# File lib/core/yml/schema/yml_schema_validator.rb, line 1722 def build_resource_hash_start(type, schema, table, end_point) { :type => type, :schema => schema, :table => table, :resource => end_point } end
Builds the final end-point hash that is used to generate all the end-points, etc. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1537 def build_resources schema_table_array = [] @schema_data.each do |schema, schema_data| unless schema_data.nil? schema_data.each do |table_name, table_data| 1 == 1 if table_data # Stop IDE complaining. unless table_name.nil? schema_table_array << "#{schema}.#{table_name}" end end end end # List of tables which have "parent-tables". tables_with_parents = {} # Start populating @schema_end_points with tables that are CHILD or CHILD_MULTI. # This also generates the 'resource' string: IE -> sale/shipment/component @schema_fks.each do |key, value| key_split = key.split('.') key_schema = key_split[0] key_table = key_split[1] value.each do |fk_source| fks_split = fk_source.split(':') fks_split = fks_split[0].split('.') fks_schema = fks_split[0] fks_table = fks_split[1] # Checks if [sale_ebay] has [sale_] in it... if fks_table =~ /\A#{key_table}_/ # Turns [sale_ebay] into [ebay]... child_table = fks_table.gsub(/\A#{key_table}_/, '') # Turns [sale_ebay] into [sale]... parent_table = fks_table.gsub(/_#{child_table}\z/, '') # Figures out what the end-point source is (to put in resource hash)... end_point_source = (tables_with_parents.keys.include?(parent_table)) ? tables_with_parents[parent_table][:resource] : key_table.gsub('_', '-') placeholder_key = "#{fks_schema}.#{parent_table}" if schema_table_array.include?(placeholder_key) placeholder_type = nil placeholder_matches = [] if !@schema_fks_placeholders[placeholder_key].nil? && @schema_fks_placeholders[placeholder_key].any? @schema_fks_placeholders[placeholder_key].each do |placeholder| schema_table = "#{fks_schema}.#{fks_table}" if placeholder =~ /\A#{schema_table}\z/ placeholder_matches << placeholder placeholder_type = RESOURCE_TYPE_OBJECT elsif placeholder =~ /\A#{schema_table}\[\]\z/ placeholder_matches << placeholder placeholder_type = RESOURCE_TYPE_OBJECT_LIST end end end # Something like: sale/shipment/component ... resource_string = "#{end_point_source}/#{child_table.gsub('_', '-')}" add_error_with_multi_content(nil, key_schema, key_table, nil, "More than one type of placeholder found within #{key_schema}.#{key_table} for: #{resource_string}", 'Can only have 1!', placeholder_matches) if placeholder_matches.length > 1 # Adds [sale_ebay] to an Array of tables which have "parent-tables". tables_with_parents[fks_table] = build_resource_hash_start(placeholder_type, fks_schema, fks_table, resource_string) unless placeholder_type.nil? end end end end # Add LINKED and PARENT tables to @schema_end_points. @schema_data.each do |schema, table_data| table_data.each do |table, data| 1 == 1 if data # Suppresses IDE error. st = "#{schema}.#{table}" if tables_with_parents.keys.include?(table) @schema_resources[st] = tables_with_parents[table].dup else if !@schema_fks_placeholders.nil? && @schema_fks_placeholders.any? @schema_fks_placeholders.each do |key, data| if !data.nil? && data.any? data.each do |ph| if ph =~ /#{schema}.#{table}\[#{LINK}\]/ unless @schema_resources[st].nil? if @schema_resources[st][:type] != RESOURCE_TYPE_OBJECT_LINK raise RuntimeError, "This statement should never be reached. It means that table '#{st}' was expected to be be 'LINKED' but was also defined as: #{@schema_resources[st][:type]}" end end @schema_resources[st] = build_resource_hash_start(RESOURCE_TYPE_OBJECT_LINK, schema, table, table.gsub('_', '-')) ph_clean = remove_placeholder_trailing_braces(ph) # Whilst we're looping, might as well create the @schema_fks_links HASH. @schema_fks_links[ph_clean] = [] if @schema_fks_links[ph_clean].nil? @schema_fks_links[ph_clean] << key end end end end end @schema_resources[st] = build_resource_hash_start(RESOURCE_TYPE_PARENT, schema, table, table.gsub('_', '-')) if @schema_resources[st].nil? end end end # Add :tree & :depth key to @schema_end_points. @schema_resources.each do |key, data| tree = [] item_last = nil data_resource_split = data[:resource].split('/') data_resource_split.each_with_index do |fragment, idx| item_frag = fragment.gsub('-', '_') if idx > 0 @schema_resources[key][:parent] = item_last if idx == (data_resource_split.length - 1) item_last = "#{tree[idx - 1].gsub('-', '_')}_#{item_frag}" tree << item_last else @schema_resources[key][:parent] = nil item_last = item_frag tree << item_last end end @schema_resources[key][:tree] = tree @schema_resources[key][:depth] = data_resource_split.length end # Add :dependents key to @schema_end_points. @schema_resources.each do |key, data| deps = data[:tree].dup deps.pop if deps.any? deps.each do |dep| key = "#{data[:schema]}.#{dep}" @schema_resources[key][:dependents] = [] if @schema_resources[key][:dependents].nil? @schema_resources[key][:dependents] << data[:table] end end end # Build @resource hash (for YmlSchemaOutputter) @schema_resources.each { |key, data| @yml_schema_outputter.add_resource(key, data) } # Check that placeholders are correct (IE: Something this is a CHILD is not also LINKED somewhere else). if !@schema_fks_placeholders.nil? && @schema_fks_placeholders.any? placeholder_type_error = false placeholder_type_error_view = [] @schema_fks_placeholders.each do |key, data| if !data.nil? && data.any? data.each do |ph| # Remove '[]' or '[link]' for multi-nested placeholders. schema_table = remove_placeholder_trailing_braces(ph) unless @schema_resources[schema_table].nil? if ph =~ /\A#{schema_table}\z/ placeholder_type = RESOURCE_TYPE_OBJECT elsif ph =~ /\A#{schema_table}\[\]\z/ placeholder_type = RESOURCE_TYPE_OBJECT_LIST elsif ph =~ /\A#{schema_table}\[#{LINK}\]\z/ placeholder_type = RESOURCE_TYPE_OBJECT_LINK else raise RuntimeError, "#{schema_table} \xe2\x86\x92 All tables should match at least one placeholder 'type'." end if placeholder_type != @schema_resources[schema_table][:type] placeholder_type_error = true key_split = key.split('.') add_error(nil, key_split[0], key_split[1], nil, "Attempted to define as '#{placeholder_type}' but was already defined as '#{@schema_resources[schema_table][:type]}'.", ph) placeholder_type_error_view << "\x1B[38;5;196m#{key} \xe2\x86\x92 #{ph} \xe2\x86\x92 #{@schema_resources[schema_table][:type].upcase} \xe2\x80\x94 ERROR!\x1B[0m" else placeholder_type_error_view << "\x1B[38;5;84m#{key}\x1B[0m \xe2\x86\x92 #{ph} \xe2\x80\x94 #{@schema_resources[schema_table][:type].upcase} \xe2\x80\x94 OK!" end end end end end # Add errors (if any). @errors_array.push({"Cannot mix placeholder 'types'." => placeholder_type_error_view}) if placeholder_type_error end @schema_config.each do |schema, schema_data| schema_data.each do |table, table_data| begin @schema_resources["#{schema}.#{table}"][:methods_internal] = {} @schema_resources["#{schema}.#{table}"][:methods_oauth] = {} unless @schema_resources["#{schema}.#{table}"].nil? table_data[CONFIG_INTERNAL].each { |key, value| @schema_resources["#{schema}.#{table}"][:methods_internal][key] = value } if table_data.has_key?(CONFIG_INTERNAL) table_data[CONFIG_OAUTH].each { |key, value| @schema_resources["#{schema}.#{table}"][:methods_oauth][key] = value } if table_data.has_key?(CONFIG_OAUTH) end rescue end end end end
@return void noinspection RubyUnusedLocalVariable
# File lib/core/yml/schema/yml_schema_validator.rb, line 446 def check_file_lines_manually(file, schema, table) file_is_valid = true field_name = nil field_type = nil initial_comments_found = false previous_was_comment = false previous_was_blank_line = false line_count = 0 line_contents = [] validating_config = false validated_config = false validating_schema = false validated_schema = false lines = Blufin::Files::read_file(file) lines.each_with_index do |line, idx| line_count = line_count + 1 if line =~ /\Aconfig:/ if validated_config add_error(file, schema, table, nil, 'Found more than one "config:" section.', "Multiple \"config:\" sections found.") file_is_valid = false end validating_config = true validating_schema = false validated_config = true next elsif line =~ /\Aschema:/ if validated_schema add_error(file, schema, table, nil, 'Found more than one "schema:" section.', "Multiple \"schema:\" sections found.") file_is_valid = false end validating_config = false validating_schema = true validated_schema = true next end if validating_schema if line =~ /\A#.?/ || line =~ /\A\s{2}#.?/ || line =~ /\A\s{4}#.?/ previous_was_comment = true previous_was_blank_line = false elsif line == "\n" initial_comments_found = true if previous_was_comment && !initial_comments_found if previous_was_blank_line add_error(file, schema, table, nil, 'More than one blank (or possibly invalid) line found.', "Line: #{line_count}") file_is_valid = false end previous_was_blank_line = true elsif line =~ /\A\s{2}[a-z_]+:/ line_contents = validate_line_content(line, line_contents, file, schema, table, line_count) field_name = line.gsub(':', '').strip field_type = nil previous_was_blank_line = false elsif line =~ /\A\s{2}[a-z_.]+\[\]:/ line_contents = validate_line_content(line, line_contents, file, schema, table, line_count) field_name = line.gsub(':', '').strip field_type = nil previous_was_blank_line = false elsif line =~ /\A\s{2}[a-z_.]+\[#{LINK}\]:/ line_contents = validate_line_content(line, line_contents, file, schema, table, line_count) field_name = line.gsub(':', '').strip field_type = nil previous_was_blank_line = false elsif line =~ /\A\s{2}[a-z_.]+:/ line_contents = validate_line_content(line, line_contents, file, schema, table, line_count) field_name = line.gsub(':', '').strip field_type = nil previous_was_blank_line = false elsif line =~ /\A\s{4}[a-z_]+:/ if line =~ /\A\s{4}type:/ field_type = line.dup.split(':') field_type = field_type[1].strip end if previous_was_blank_line add_error(file, schema, table, nil, 'More than one blank (or possibly invalid) line found.', "Line: #{line_count}") file_is_valid = false end previous_was_blank_line = false else add_error_with_multi_content(file, schema, table, nil, 'Something is wrong with this line, perhaps check format, key-capitalization or whitespace?', "Line: #{line_count}", [line]) file_is_valid = false end if idx == (lines.length - 1) && line[-1, 2] == "\n" add_error(file, schema, table, nil, 'Cannot have blank, trailing lines in YML file.', "Line: #{line_count}") file_is_valid = false end return unless line_contents end end # Make sure that each file has a "config:" section. unless validated_config add_error(file, schema, table, nil, '"config:" section is missing.', "No \"config:\" section found.") file_is_valid = false end # Make sure that each file has a "schema:" section. unless validated_schema add_error(file, schema, table, nil, '"schema:" section is missing.', "No \"schema:\" section found.") file_is_valid = false end file_is_valid end
Returns TRUE if string contains an uppercase letter. @return boolean
# File lib/core/yml/schema/yml_schema_validator.rb, line 1195 def contains_uppercase_letter_or_number(string) string =~ /[A-Z]/ || string =~/[0-9]/ end
Validates column data and extracts data to global array. Adds errors if they exist. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 576 def extract_and_validate_column(file, schema, table, column_name, column_data) define_count = 0 define_order = {} @column_count = @column_count + 1 @type = (!column_data.nil? && column_data.has_key?(TYPE)) ? column_data[TYPE] : nil @types << @type unless @type.nil? @booleans_finished = true if @type != TYPE_BOOLEAN && column_name.downcase != ID # Cannot have both required && required_if properties. add_error(file, schema, table, column_name, "Cannot have both #{REQUIRED} and #{REQUIRED_IF} properties.", column_name) if !column_data.nil? && !column_data[REQUIRED].nil? && !column_data[REQUIRED_IF].nil? # Make sure column_name is lowercase & doesn't contain numbers. if contains_uppercase_letter_or_number(column_name) add_error(file, schema, table, column_name, 'Column name must be lowercase & not contain numbers.', column_name) return end # Make sure the column isn't a Java reserved word (or phrase/word that has special meaning within our stack). reserved_words = RESERVED_WORDS reserved_words << 'mock parent_id' if reserved_words.include?(column_name) add_error(file, schema, table, column_name, 'Column name cannot be a Java reserved word or phrase/word used by our stack.', column_name) return end # If this is a foreign key placeholder IE: app.ebay_aliases[]: or app.order_ebay: ... if Blufin::YmlSchemaValidator::column_is_object(column_name) fkp_key = "#{schema}.#{table}" @schema_fks_placeholders[fkp_key] = [] if @schema_fks_placeholders[fkp_key].nil? @schema_fks_placeholders[fkp_key] << column_name column_data = {} unless column_data.is_a?(Hash) column_data.each do |key, value| case key when DESCRIPTION @type = 'FAKE-TYPE' # Hacky fix validate_description(file, schema, table, column_name, value) when TYPE add_error(file, schema, table, column_name, "Placeholders cannot have #{TYPE} property.", "Found: #{value}") next when FLAG add_error(file, schema, table, column_name, "Placeholders cannot have #{FLAG} property.", "Found: #{value}") next when FKEY add_error(file, schema, table, column_name, "Placeholders cannot have #{FKEY} property.", "Found: #{value}") next when REQUIRED validate_required(file, schema, table, column_name, value) when REQUIRED_IF validate_required_if(file, schema, table, column_name, value) when ENCRYPTED add_error(file, schema, table, column_name, "Placeholders cannot have #{ENCRYPTED} property.", "Found: #{value}") next else add_error(file, schema, table, column_name, 'Invalid key.', key) end end # Add data to @schema_data (with type, which doesn't exist in the YML file(s) as it's inferred). if column_name =~ /^(#{VALID_SCHEMAS_REGEX})\.[\w]+$/ column_data[TYPE] = Blufin::ScannerJavaEmbeddedObjects::OBJECT elsif column_name =~ /^(#{VALID_SCHEMAS_REGEX})\.[\w]+\[\]$/ column_data[TYPE] = Blufin::ScannerJavaEmbeddedObjects::OBJECT_LIST elsif column_name =~ /^(#{VALID_SCHEMAS_REGEX})\.[\w]+\[#{LINK}\]$/ column_data[TYPE] = Blufin::ScannerJavaEmbeddedObjects::OBJECT_LINK else raise RuntimeError, "Unable to determine object type: #{column_name}" end if [Blufin::ScannerJavaEmbeddedObjects::OBJECT_LIST, Blufin::ScannerJavaEmbeddedObjects::OBJECT_LINK].include?(column_data[TYPE]) add_error(file, schema, table, column_name, "#{column_data[TYPE]} fields cannot have a #{REQUIRED} property.", column_name) if column_data.has_key?(REQUIRED) add_error(file, schema, table, column_name, "#{column_data[TYPE]} fields cannot have a #{REQUIRED_IF} property.", column_name) if column_data.has_key?(REQUIRED_IF) return if column_data.has_key?(REQUIRED) || column_data.has_key?(REQUIRED_IF) end add_schema_data(schema, table, column_name, column_data) return else # Add data to @schema_data add_schema_data(schema, table, column_name, column_data) end # Make sure the column has data. if Blufin::YmlCommon::is_empty(column_data) raise RuntimeError, "Expected Hash, instead got: #{column_data.class}" unless column_data.is_a?(Hash) add_error(file, schema, table, column_name, 'Column has no defining data.', column_name) return end # Make sure there are only 1 of each key (although YAML parser should never allow this). if column_data.keys != column_data.keys.uniq add_error(file, schema, table, column_name, 'Found duplicate key.', nil) return end # Make sure there is a type. All columns need it. if column_data[TYPE].nil? add_error(file, schema, table, column_name, "No #{TYPE} found. All columns need to have a #{TYPE}.", nil) return end # IDs are validated separately & uniquely. if column_name.downcase == ID validate_id_column(file, schema, table, column_name, column_data) return end # Fields containing the word 'currency' are validated separately & uniquely. if column_name.downcase =~ /#{CURRENCY}/ validate_currency_code_column(file, schema, table, column_name, column_data) return end # Fields named 'amount' are validated separately & uniquely. if column_name.downcase =~ /\A#{AMOUNT}\z/ validate_amount_column(file, schema, table, column_name, column_data) return end @flags = nil @foreign_key = false @column_name = column_name column_data.each do |key, value| key_invalid = false case key when DESCRIPTION validate_description(file, schema, table, column_name, value) when TYPE validate_type(file, schema, table, column_name, value) when FLAG @flags = extract_flags_for_validator(file, schema, table, column_name, value) validate_flags(file, schema, table, column_name) when FKEY @foreign_key = true validate_foreign_key(file, schema, table, column_name, value) when REQUIRED validate_required(file, schema, table, column_name, value) when REQUIRED_IF validate_required_if(file, schema, table, column_name, value) when ENCRYPTED validate_encrypted(file, schema, table, column_name, value) else key_invalid = true add_error(file, schema, table, column_name, 'Invalid key', key) end unless key_invalid define_count = define_count + 1 define_order[key] = define_count end end validate_definition_order(file, schema, table, column_name, [TYPE, FLAG, FKEY, ENCRYPTED, DESCRIPTION], define_order, 'Keys') end
Extracts a YmlSchemaFlags
object, throws error if a flag is unsupported. @return Blufin::YmlSchemaFlags
# File lib/core/yml/schema/yml_schema_validator.rb, line 1171 def extract_flags_for_validator(file, schema, table, column, flags) # Check for excessive spaces. add_error(file, schema, table, column, "Too many spaces between 'flags'.", nil) if flags =~ /\s{2,}/ if flags == '' || flags.nil? add_error(file, schema, table, column, 'Flags cannot be empty or "NULL". Perhaps you meant "NULLABLE"?', nil) return end flag_result = Blufin::YmlCommon::extract_flags(flags) if flag_result[1].any? flag_result[1].each do |error| add_error(file, schema, table, column, error[0], error[1]) end end flag_result[0] end
Validates a FK string. @return boolean
# File lib/core/yml/schema/yml_schema_validator.rb, line 1201 def fk_is_valid(fk) fk =~ /\A(#{VALID_SCHEMAS_REGEX})\.[a-z_]+\.[a-z_]+\z/ end
Removed trailing [] or [link] from placeholders strings. @return String
# File lib/core/yml/schema/yml_schema_validator.rb, line 1828 def remove_placeholder_trailing_braces(placeholder) (placeholder =~ /\A[a-z_.]+\[#{LINK}\]/) ? placeholder.gsub("[#{LINK}]", '') : placeholder.gsub('[]', '') end
Scans a file – validates & extracts data. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 326 def scan_file(file) schema, table, valid = validate_file(file) # Make sure the table isn't a Java reserved word (or phrase/word that has special meaning within our stack). if RESERVED_WORDS.include?(table) add_error(file, schema, table, nil, 'Table name cannot be a Java reserved word or phrase/word used by our stack.', table) return end # Make sure there are no uppercase letters in schema + table.. add_error(file, nil, nil, nil, 'Folder contains uppercase letters and/or numbers.', schema) if contains_uppercase_letter_or_number(schema) add_error(file, nil, nil, nil, 'File contains uppercase letters and/or numbers.', table) if contains_uppercase_letter_or_number(table) # Make sure that the table name is no longer than 35 characters. add_error(file, nil, nil, nil, "Table name is too long. Maximum allowed characters is #{MAX_TABLE_CHARACTERS}.", "#{table.length} characters") if table.length > MAX_TABLE_CHARACTERS # If file is invalid, return NULL. return unless valid # Do a manual check for excessive blank lines, etc. return unless check_file_lines_manually(file, schema, table) # Get the YML data from the file. begin all_data = YAML.load_file(File.expand_path(file)) rescue Exception => e add_error(file, nil, nil, nil, 'Unable to parse file \xe2\x80\x94 invalid YML.', e.message) return end data = all_data['schema'] # Make sure the file has data. if Blufin::YmlCommon::is_empty(data) add_error(file, nil, nil, nil, 'File is empty.', nil) return end # Make sure that the first column in every table is ID. add_error(file, schema, table, data.keys.first, "First column in table must be: #{ID.upcase}", data.keys.first) unless data.keys.first == ID @primary_key_count = 0 @column_count = 0 @types = [] @transient_fields = [] @data = data @booleans_finished = false columns_with_name = [] # Loop the columns. data.each do |column, column_data| extract_and_validate_column(file, schema, table, column, column_data) columns_with_name << column if column =~ /name/ end # Make sure if there is ONLY 1 "name" field that it matches the table -- IE: user (table) -> user_name OR user_[.*]_name if columns_with_name.length == 1 column_name = columns_with_name[0] unless column_name =~ /#{table}_[a-z_]*name/ add_error(file, schema, table, column_name, "Name field should match regex \xe2\x86\x92 #{table}_name OR #{table}_[a-z_]name", "Found: #{column_name}") end end types_more_than_one = [] @types.detect { |e| types_more_than_one << [e, @types.count(e)] if @types.count(e) > 1 } # Make sure there are ONLY ONE of certain types of column -- INT_AUTO being 1 of them. [TYPE_DATETIME_INSERT, TYPE_DATETIME_UPDATE, TYPE_INT_AUTO].each do |allowed_only_once| types_more_than_one.each do |found_more_than_once| if found_more_than_once[0] == allowed_only_once add_error(file, schema, table, nil, "Can only have one #{allowed_only_once} field per table.", "Found: #{found_more_than_once[1]} times") end end end # Make sure that column names don't conflict with transients. if @transient_fields.length > 0 @transient_fields.each do |transient_field| if @schema_data[schema][table].keys.include?(transient_field) unless @schema_data[schema][table][transient_field][TRANSIENT] @error_handler.add_error(Blufin::YmlErrorHandler::FIELD_NAME_TRANSIENT_CONFLICT, "#{schema}/#{table}.yml", 'schema', transient_field, transient_field) end end end end add_error(file, schema, table, nil, "No #{FLAG_PRIMARY_KEY} found.", "# of PKs: #{@primary_key_count}") if @primary_key_count < 1 add_error(file, schema, table, nil, "More than 1 #{FLAG_PRIMARY_KEY} found.", "# of PKs: #{@primary_key_count}") if @primary_key_count > 1 end
Validates any column called 'amount'. This column is special and must be validated separately. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 821 def validate_amount_column(file, schema, table, column_name, column_data) # Make sure 'amount' column is DECIMAL(13,2) if column_data[TYPE].nil? || column_data[TYPE] != "#{TYPE_DECIMAL}(13,2)" add_error(file, schema, table, column_name, "#{AMOUNT.upcase} column preceding #{CURRENCY.upcase} must be of type: #{TYPE_DECIMAL}(13,2)", column_data[TYPE]) return end # Make sure there is no 'description', 'fkey' or 'required_if'. add_error(file, schema, table, column_name, "#{AMOUNT.upcase} column cannot have: #{FKEY}", column_data[FKEY]) unless column_data[FKEY].nil? add_error(file, schema, table, column_name, "#{AMOUNT.upcase} column cannot have: #{REQUIRED} (this is inferred).", column_data[REQUIRED]) unless column_data[REQUIRED].nil? add_error(file, schema, table, column_name, "#{AMOUNT.upcase} column cannot have: #{REQUIRED_IF}", column_data[REQUIRED_IF]) unless column_data[REQUIRED_IF].nil? end
Validate config: section. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1514 def validate_config_section result = validate(@site, %W(#{Blufin::Site::PATH_TO_YML_API_SCHEMA}/#{APP} #{Blufin::Site::PATH_TO_YML_API_SCHEMA}/#{COMMON}), STRUCTURE, @error_handler) result.each do |key, data| key_split = key.split('/') schema = key_split[key_split.length - 2] table = key_split[key_split.length - 1].gsub(/(\.)[A-Za-z0-9]+\z/, '') @schema_config[schema] = {} if @schema_config[schema].nil? @schema_config[schema][table] = data['config'].nil? ? {} : data['config'] end result = validate(@site, %W(#{PATH_TO_CONFIG_YML}), STRUCTURE, @error_handler, true) result.each do |key, data| if data.is_a?(Hash) && !data[CONFIG].nil? key_split = key.split('/') schema = key_split[key_split.length - 2] table = key_split[key_split.length - 1].gsub(/(\.)[A-Za-z0-9]+\z/, '') @schema_config[schema] = {} if @schema_config[schema].nil? @schema_config[schema][table] = data[CONFIG].nil? ? {} : data[CONFIG] end end end
Validate any column with the word 'currency' in it. This column is special and must be validated separately. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 807 def validate_currency_code_column(file, schema, table, column_name, column_data) # Make sure the type is ENUM_SYSTEM('Money') add_error(file, schema, table, column_name, "#{CURRENCY.upcase} columns must be of type: #{TYPE_ENUM_SYSTEM}('Money')", column_data[TYPE]) unless column_data[TYPE] == "#{TYPE_ENUM_SYSTEM}('Money')" && !column_data[TYPE].nil? # Make sure there is no 'fkey' or 'required_if'. add_error(file, schema, table, column_name, "#{CURRENCY.upcase} column cannot have: #{FKEY}", column_data[FKEY]) unless column_data[FKEY].nil? add_error(file, schema, table, column_name, "#{CURRENCY.upcase} column cannot have: #{REQUIRED} (this is inferred).", column_data[REQUIRED]) unless column_data[REQUIRED].nil? add_error(file, schema, table, column_name, "#{CURRENCY.upcase} column cannot have: #{REQUIRED_IF}", column_data[REQUIRED_IF]) unless column_data[REQUIRED_IF].nil? end
Validates the order in which keys are defined for a column. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1156 def validate_definition_order(file, schema, table, column_name, correct_order, definition_order, entities) found_count = 0 correct_order.each do |current_key| unless definition_order[current_key].nil? found_count = found_count + 1 unless definition_order[current_key] == found_count add_error(file, schema, table, column_name, "#{entities} order must be: #{correct_order.join(',')}", definition_order.keys.join(',')) return end end end end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 837 def validate_description(file, schema, table, column_name, value) if Blufin::YmlCommon::is_empty(value) add_error(file, schema, table, column_name, 'No description text found.', "Found: #{value.nil? || value.strip == '' ? "\x1B[38;5;196mNothing\x1B[0m" : value}") return end if @type.nil? add_error(file, schema, table, column_name, "#{TYPE} must be defined before #{DESCRIPTION}, cannot validate #{DESCRIPTION}.", nil) return end if @schema_descriptions[column_name].nil? @schema_descriptions[column_name] = [ { DESCRIPTION_TEXT => value, DESCRIPTION_TYPE => @type, DESCRIPTION_HITS => 1 } ] else hash_matched = false new_array_of_hashes = [] @schema_descriptions[column_name].each do |hash| if hash[DESCRIPTION_TEXT] == value && hash[DESCRIPTION_TYPE] == @type new_array_of_hashes << { DESCRIPTION_TEXT => value, DESCRIPTION_TYPE => @type, DESCRIPTION_HITS => hash[DESCRIPTION_HITS] + 1 } hash_matched = true break else new_array_of_hashes << hash end end unless hash_matched new_array_of_hashes << { DESCRIPTION_TEXT => value, DESCRIPTION_TYPE => @type, DESCRIPTION_HITS => 1 } end @schema_descriptions[column_name] = new_array_of_hashes end end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1144 def validate_encrypted(file, schema, table, column_name, value) # Make sure the value is lowercase 'true' (and nothing else) add_error(file, schema, table, column_name, "#{ENCRYPTED} value must be lowercase 'true' or omitted.", "Found: #{value}") unless value == true && !!value == value # Make sure the type is TEXT add_error(file, schema, table, column_name, "#{ENCRYPTED} field must have type: #{TYPE_TEXT}.", @type) unless @type == TYPE_TEXT end
Validates the file and extracts 'schema' + 'table' data. @return List
# File lib/core/yml/schema/yml_schema_validator.rb, line 421 def validate_file(file) valid = true # Make sure file exists. unless Blufin::Files.file_exists(file) add_error(file, nil, nil, nil, 'File not found!', nil) valid = false end # Make sure this is a '.yml.' file. unless Blufin::YmlCommon.is_yml_file(file) add_error(file, nil, nil, nil, 'File must have extension: .yml', nil) valid = false end file_parts = file.split('/') schema = file_parts[file_parts.length - 2] table = File.basename(file, '.yml') # Make sure the schema is valid. unless VALID_SCHEMAS.include? schema add_error(file, schema, nil, nil, "Invalid schema. Must be one of: #{VALID_SCHEMAS.join(', ')}", schema) valid = false end return schema, table, valid end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 977 def validate_flags(file, schema, table, column_name) return if @flags.nil? if @type.nil? add_error(file, schema, table, column_name, "#{TYPE} must be defined before #{FLAG}, cannot validate #{FLAG}.", nil) return end raise RuntimeError, "Flags must be of type: Blufin::YmlSchemaFlags. You passed: #{@flags.class}" unless @flags.is_a? Blufin::YmlSchemaFlags if @flags.auto_increment add_error(file, schema, table, column_name, "Only #{ID.upcase} fields can have #{FLAG_AUTO_INCREMENT} flags.", @flags.flags_raw) return end # Not validating AUTO_INCREMENT flag because that gets handles differently. validate_definition_order(file, schema, table, column_name, [FLAG_PRIMARY_KEY, FLAG_INDEX, FLAG_UNIQUE, FLAG_NULLABLE], @flags.definition_order, 'Flags') if @flags.primary_key @primary_key_count = @primary_key_count + 1 add_error(file, schema, table, column_name, "#{FLAG_PRIMARY_KEY} cannot also be #{FLAG_INDEX}.", @flags.flags_raw) if @flags.index end if @flags.primary_key add_error(file, schema, table, column_name, "#{@type} cannot have a #{FLAG_PRIMARY_KEY} flag.", @flags.flags_raw) unless @type == TYPE_INT && column_name == ID end if @flags.nullable && [TYPE_BOOLEAN, TYPE_DATETIME_INSERT, TYPE_DATETIME_UPDATE].include?(@type) add_error(file, schema, table, column_name, "#{@type} cannot have an #{FLAG_NULLABLE} flag.", @flags.flags_raw) end if @flags.unique && [TYPE_BOOLEAN, TYPE_DATETIME_INSERT, TYPE_DATETIME_UPDATE].include?(@type) add_error(file, schema, table, column_name, "#{@type} cannot have an #{FLAG_UNIQUE} flag.", @flags.flags_raw) end if @flags.unique && [TYPE_TEXT, TYPE_TEXT_LONG, TYPE_BOOLEAN].include?(@type) add_error(file, schema, table, column_name, "#{@type} cannot have an #{FLAG_UNIQUE} flag.", @flags.flags_raw) end # ENUMs can only have INDEX, INDEX UNIQUE, or no flags -- nothing else. if (@type =~ REGEX_ENUM || @type =~ REGEX_ENUM_CUSTOM || @type =~ REGEX_ENUM_SYSTEM) && @flags != nil unless @flags.flags_raw == "#{FLAG_INDEX} #{FLAG_UNIQUE}" || @flags.flags_raw == "#{FLAG_INDEX}" add_error(file, schema, table, column_name, "#{TYPE_ENUM}s can only have NO flags, '#{FLAG_INDEX}' or '#{FLAG_INDEX} #{FLAG_UNIQUE}' flags \xe2\x80\x94 nothing else.", @flags.flags_raw) end end # Certain flags require others... [ [[FLAG_UNIQUE, @flags.unique], [FLAG_INDEX, @flags.index]] ].each do |required_combination| if required_combination[0][1] && required_combination[1][1].nil? add_error(file, schema, table, column_name, "If flag: #{required_combination[0][0]} is set then flag: #{required_combination[1][0]} must also be set.", @flags.flags_raw) end end # Certain flag combinations are invalid... [ [[FLAG_UNIQUE, @flags.unique], [FLAG_NULLABLE, @flags.nullable]] ].each do |invalid_combination| if invalid_combination[0][1] && invalid_combination[1][1] add_error(file, schema, table, column_name, "Cannot have flags: #{invalid_combination[0][0]} & #{invalid_combination[1][0]} together.", @flags.flags_raw) end end end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1045 def validate_foreign_key(file, schema, table, column_name, target_column) if @type.nil? add_error(file, schema, table, column_name, "#{TYPE} must be defined before #{FKEY}, cannot validate #{FKEY}.", nil) return end # Make sure the column matches regex format. unless fk_is_valid(target_column) add_error(file, schema, table, column_name, 'Foreign key target column not in correct format: {schema}.{table}.{column}', target_column) return end target_column_split = target_column.split('.') unless target_column_split[0] == schema add_error(file, schema, table, column_name, 'Cannot create cross-schema foreign keys.', "#{FKEY}: #{target_column}") end unless target_column_split[2] == ID add_error(file, schema, table, column_name, "Foreign keys can only be to an #{ID.upcase}", "#{FKEY}: #{target_column}") end unless @type == TYPE_INT add_error(file, schema, table, column_name, "Foreign key source columns must be of type #{TYPE_INT}, not:", @type) end referencing_column = "#{schema}.#{table}.#{column_name}" @schema_fks[target_column] = [] if @schema_fks[target_column].nil? @schema_fks[target_column] << "#{referencing_column}:#{@type}" if target_column_split[2] == ID @transient_fields << target_column_split[1] # Add transient to @schema_data add_schema_data(schema, table, column_name.gsub(/_#{ID}$/, ''), { TYPE => Blufin::ScannerJavaEmbeddedObjects::OBJECT, TRANSIENT => [target_column_split[0], target_column_split[1]] }) else Blufin::Terminal::output("Cannot create @Transient object for #{schema}.#{table}.#{column_name} because field name doesn't end in '#{ID}", Blufin::Terminal::MSG_WARNING) end end
Make sure that FKs are correct. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1222 def validate_foreign_keys @schema_fks.each do |fk, fk_data| # Make sure the column matches regex format. unless fk_is_valid(fk) add_error(fk, nil, nil, nil, 'Foreign key target column not in correct format: app.table.column', fk) return end fk_exploded = fk.split('.') expected_reference_end = "#{fk_exploded[1]}_#{fk_exploded[2]}" # Make sure referenced schema.table.column actually exists. unless @schema_data[fk_exploded[0]] != nil && @schema_data[fk_exploded[0]][fk_exploded[1]] != nil && @schema_data[fk_exploded[0]][fk_exploded[1]][fk_exploded[2]] != nil add_error_with_multi_content(nil, fk_exploded[0], nil, nil, "Referenced #{fk} doesn't exist, referenced by below:", "#{FKEY}: #{fk}", fk_data) next end # Make sure the target column has a type and that it's an TINYINT, INT, INT_AUTO, ENUM or VARCHAR. if @schema_data[fk_exploded[0]][fk_exploded[1]][fk_exploded[2]][TYPE].nil? add_error(nil, fk_exploded[0], fk_exploded[1], fk_exploded[2], "#{FKEY} column has no type.", nil) next else fk_type = @schema_data[fk_exploded[0]][fk_exploded[1]][fk_exploded[2]][TYPE] unless [TYPE_INT_TINY, TYPE_INT, TYPE_INT_AUTO].include?(fk_type) || fk_type =~ /\A(#{TYPE_VARCHAR}|#{TYPE_ENUM}|#{TYPE_ENUM_CUSTOM}|#{TYPE_ENUM_SYSTEM})\(/ # TODO, CAN WE FK STRINGS - TEST THIS OUT IN YML?? add_error(nil, fk_exploded[0], fk_exploded[1], fk_exploded[2], "Column is FK'd so must be: #{[TYPE_INT_TINY, TYPE_INT, TYPE_INT_AUTO, TYPE_ENUM, TYPE_ENUM_CUSTOM, TYPE_ENUM_SYSTEM].join(', ')} or #{TYPE_VARCHAR}.", fk_type) end end fk_data.each do |referencing_column| referencing_column = referencing_column.split(':') # colon is correct, we're splitting -- > app.message_ebay.message_id:INT referencing_column_type = referencing_column[1] referencing_column = referencing_column[0] referencing_column_parent = "#{fk_exploded[0]}.#{fk_exploded[1]}" rc_exploded = referencing_column.split('.') placeholder = nil unless @schema_fks_placeholders["#{fk_exploded[0]}.#{fk_exploded[1]}"].nil? @schema_fks_placeholders["#{fk_exploded[0]}.#{fk_exploded[1]}"].each do |n| if "#{rc_exploded[0]}.#{rc_exploded[1]}" == n.gsub(/\[(link)?\]$/, '') placeholder = n break end end end # Make sure that foreign keys with a placeholder are never null OR required_if. unless placeholder.nil? referencing_column_data = @schema_data[rc_exploded[0]][rc_exploded[1]][rc_exploded[2]] if referencing_column_data[FLAG].is_a?(String) flag_result = Blufin::YmlCommon::extract_flags(referencing_column_data[FLAG]) add_error(nil, rc_exploded[0], rc_exploded[1], rc_exploded[2], "FK cannot have a #{FLAG_NULLABLE} #{FLAG} because it has a placeholder in: #{referencing_column_parent}", placeholder) if flag_result[0].nullable add_error(nil, rc_exploded[0], rc_exploded[1], rc_exploded[2], "FK cannot have a #{REQUIRED_IF} property because it has a placeholder in: #{referencing_column_parent}", placeholder) if referencing_column_data.has_key?(REQUIRED_IF) end end # Make sure the reference name matches.. IE app.ebay_user.id is referenced by ebay_user_id unless rc_exploded[2] =~ /[a-z_]*#{expected_reference_end}/ add_error(nil, rc_exploded[0], rc_exploded[1], rc_exploded[2], "FK naming convention not respected, expected field to end in: #{expected_reference_end}", "Got: #{rc_exploded[2]}") end # Make sure the column matches regex format. unless fk_is_valid(referencing_column) add_error(nil, rc_exploded[0], rc_exploded[1], rc_exploded[2], 'Foreign key source column not in correct format: app.table.column', referencing_column) return end # Make sure the type matches that of the parent column. if referencing_column_type != fk_type unless referencing_column_type == TYPE_INT && fk_type == TYPE_INT_AUTO add_error(nil, rc_exploded[0], rc_exploded[1], rc_exploded[2], "FK type mismatch. Expected: #{fk_type}", "got: #{referencing_column_type}") end end unless @schema_data[rc_exploded[0]] != nil && @schema_data[rc_exploded[0]][rc_exploded[1]] != nil && @schema_data[rc_exploded[0]][rc_exploded[1]][rc_exploded[2]] != nil raise RuntimeError, "#{referencing_column} @schema_data not found." end end end end
Validate all he Foreign Key placeholders. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1309 def validate_foreign_keys_placeholders if !@schema_fks_placeholders.nil? && @schema_fks_placeholders.any? placeholder_error = false placeholder_error_view = [] placeholder_circular_error = false placeholder_circular_dependencies = [] # Make sure that FK place-holders are correct. @schema_fks_placeholders.each do |parent, placeholders| parent_split = parent.split('.') parent_schema = parent_split[0] parent_table = parent_split[1] # Loop Placeholders. placeholders.each do |placeholder| placeholder_type = RESOURCE_TYPE_OBJECT placeholder_type = RESOURCE_TYPE_OBJECT_LIST if placeholder =~ /\A[a-z_.]+\[\]/ placeholder_type = RESOURCE_TYPE_OBJECT_LINK if placeholder =~ /\A[a-z_.]+\[#{LINK}\]/ # Remove '[]' or '[link]' for multi-nested placeholders. placeholder_dup = remove_placeholder_trailing_braces(placeholder) child = placeholder_dup.split('.') child_schema = child[0] child_table = child[1] if @schema_data[child_schema] != nil && @schema_data[child_schema][child_table] != nil if placeholder =~ /\A[a-z_.]+\[#{LINK}\]/ # TODO - FINISH (OR REMOVE) else # Add error if CHILD TABLE doesn't match PARENT TABLE (IE: person_customer → ebay_user = NO MATCH) if child_table =~ /\A#{parent_table}_/ placeholder_error_view << " \x1B[38;5;240m#{parent_table} \xe2\x86\x92 #{child_table}\x1B[0m" else placeholder_error = true placeholder_error_view << " \x1B[38;5;196m#{parent_table} \xe2\x86\x92 #{child_table}\x1B[0m" next end fk_found = false @schema_data[child_schema][child_table].each do |referenced_table_data| # Skip Placeholders (in referenced table) next if referenced_table_data[0] =~ /\A(#{VALID_SCHEMAS_REGEX})\.[a-z_]+\[\]\z/ || referenced_table_data[0] =~ /\A(#{VALID_SCHEMAS_REGEX})\.[a-z_]+\z/ unless referenced_table_data[1][FKEY].nil? if "#{parent}.#{ID}" == referenced_table_data[1][FKEY] fk_found = true # If OBJECT or OBJECT_LIST, add CHILD_OF and CHILD_TYPE properties (that will end up in metadata). if [RESOURCE_TYPE_OBJECT, RESOURCE_TYPE_OBJECT_LIST].include?(placeholder_type) @schema_data[child_schema][child_table][referenced_table_data[0]][CHILD_OF] = "#{parent_table}" @schema_data[child_schema][child_table][referenced_table_data[0]][CHILD_TYPE] = "DataType.#{placeholder_type}" end end end end # Add error if FK not found. add_error(nil, parent_schema, parent_table, nil, "Unreferenced placeholder. #{child_schema}.#{child_table} doesn't have FK to #{parent}", "#{placeholder}:") unless fk_found end else add_error(nil, parent_schema, parent_table, nil, "Placeholder reference doesn't exist, no such table.", "#{placeholder}:") end # Checks for circular dependencies. unless @schema_fks_placeholders[placeholder_dup].nil? @schema_fks_placeholders[placeholder_dup].each do |check_value| check_value = remove_placeholder_trailing_braces(check_value) if check_value == parent placeholder_circular_error = true placeholder_circular_dependencies << " \x1B[38;5;196m#{check_value} \xe2\x86\x92 #{placeholder_dup} \xe2\x86\x92 #{parent}\x1B[0m" else placeholder_circular_dependencies << " \x1B[38;5;240m#{check_value} \xe2\x86\x92 #{placeholder_dup} \xe2\x86\x92 #{parent}\x1B[0m" end end end end end @errors_array.push({"Incorrect Placeholders! Child tables must follow convention: \x1B[38;5;220mparent \xe2\x86\x92 parent_child\x1B[0m" => placeholder_error_view}) if placeholder_error @errors_array.push({"You have circular dependencies within your placeholders: \x1B[38;5;220mparent \xe2\x86\x92 child \xe2\x86\x92 parent\x1B[0m" => placeholder_circular_dependencies}) if placeholder_circular_error end end
Checks HTTP methods are correct. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1491 def validate_http_methods # Make sure that REQUIRED and REQUIRED_IF objects don't have a POST. @schema_resources.each do |key, resource| schema = resource[:schema] table = resource[:table] parent = resource[:parent] methods_internal = resource[:methods_internal].keys methods_oauth = resource[:methods_oauth].keys if resource[:type] == RESOURCE_TYPE_OBJECT parent_definition_data = @schema_data[schema][parent]["#{schema}.#{table}"] raise RuntimeError, 'parent_definition_data not found.' if parent_definition_data.nil? || parent_definition_data == '' if parent_definition_data.has_key?(REQUIRED) || parent_definition_data.has_key?(REQUIRED_IF) @error_handler.add_error(Blufin::YmlErrorHandler::API_METHOD_INVALID_FOR_REQUIRED_OBJECT, "#{schema}/#{table}.yml", 'config', 'internal', methods_internal.join(', ')) if methods_internal.include?(Blufin::YmlConfigValidator::POST) || methods_internal.include?(Blufin::YmlConfigValidator::DELETE) @error_handler.add_error(Blufin::YmlErrorHandler::API_METHOD_INVALID_FOR_REQUIRED_OBJECT, "#{schema}/#{table}.yml", 'config', 'oauth', methods_internal.join(', ')) if methods_oauth.include?(Blufin::YmlConfigValidator::POST) || methods_oauth.include?(Blufin::YmlConfigValidator::DELETE) end end end end
Validate the ID
column. This column is special and must be validated separately. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 748 def validate_id_column(file, schema, table, column_name, column_data) # Make sure ID is first column. add_error(file, schema, table, column_name, 'ID column must be first column in table.', "Is actually ##{@column_count}") unless @column_count == 1 # Check for invalid keys. invalid_keys = Blufin::YmlCommon.validate_keys(column_data.keys, [TYPE, FLAG]) add_error(file, schema, table, column_name, 'ID column has invalid keys.', invalid_keys.join(',')) unless invalid_keys.nil? # Make sure the type is INT_AUTO add_error(file, schema, table, column_name, "ID column is missing type which must be: #{TYPE_INT_AUTO}", nil) if column_data[TYPE].nil? add_error(file, schema, table, column_name, "ID columns must be of type: #{TYPE_INT_AUTO}", column_data[TYPE]) unless column_data[TYPE] == TYPE_INT_AUTO && !column_data[TYPE].nil? flag_error_message = "ID column must have the following flags: #{[FLAG_PRIMARY_KEY, "#{FLAG_AUTO_INCREMENT} (optional)"].join(', ')}" # Make sure the flags are: (optional) AUTO_INCREMENT(??) & PRIMARY_KEY if !column_data[FLAG].nil? flags = extract_flags_for_validator(file, schema, table, column_name, column_data[FLAG]) # Make sure the 1st flag is either: AUTO_INCREMENT or there is no 1st flag. if flags.auto_increment add_error(file, schema, table, column_name, "ID column first flag can only be: #{FLAG_AUTO_INCREMENT}(??) and/or PRIMARY KEY", flags.flags_raw) unless flags.auto_increment && flags.auto_increment_amount != nil && flags.auto_increment_sort_order == 1 pk_order = 2 pk_text = 'second' else pk_order = 1 pk_text = 'first' end # Make sure the 2nd flag is: PRIMARY_KEY add_error(file, schema, table, column_name, "#{ID.upcase} column #{pk_text} flag must be: #{FLAG_PRIMARY_KEY}", nil) unless flags.primary_key == true && flags.primary_key_sort_order == pk_order @primary_key_count = @primary_key_count + 1 if flags.primary_key # Make sure there are no other flags. add_error(file, schema, table, column_name, "#{ID.upcase} column should not have flag: #{FLAG_INDEX} as this is already implied.", flags.flags_raw) if flags.index add_error(file, schema, table, column_name, "#{ID.upcase} column cannot have flag: #{FLAG_NULLABLE}", flags.flags_raw) if flags.nullable add_error(file, schema, table, column_name, "#{ID.upcase} column should not have flag: #{FLAG_UNIQUE} as this is already implied.", flags.flags_raw) if flags.unique if flags.auto_increment @yml_schema_outputter.add_auto_increment_indexed_table(schema, table, flags.auto_increment_amount) else @yml_schema_outputter.add_auto_increment_table(schema, table) end else add_error(file, schema, table, column_name, flag_error_message, nil) end # Make sure there is no 'description', 'fkey' or 'required_if'. add_error(file, schema, table, column_name, "#{ID.upcase} column cannot have: #{FKEY}", column_data[FKEY]) unless column_data[FKEY].nil? add_error(file, schema, table, column_name, "#{ID.upcase} column cannot have: #{DESCRIPTION}", column_data[DESCRIPTION]) unless column_data[DESCRIPTION].nil? add_error(file, schema, table, column_name, "#{ID.upcase} column cannot have: #{REQUIRED} (this is inferred).", column_data[REQUIRED]) unless column_data[REQUIRED].nil? add_error(file, schema, table, column_name, "#{ID.upcase} column cannot have: #{REQUIRED_IF}", column_data[REQUIRED_IF]) unless column_data[REQUIRED_IF].nil? end
Makes sure when checking lines manually, that we don't have an identical definition (as these won't show up in hashes). @return Array
# File lib/core/yml/schema/yml_schema_validator.rb, line 564 def validate_line_content(line, line_contents, file, schema, table, line_count) if line_contents.include?(line) add_error(file, schema, table, nil, "Duplicate definition found: #{line}", "Line: #{line_count}") false else line_contents << line line_contents end end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1090 def validate_required(file, schema, table, column_name, value) # Make sure this is an OBJECT. unless column_name =~ /\A(#{VALID_SCHEMAS_REGEX})\.[a-z_]+\z/ add_error(file, schema, table, column_name, "Column cannot have a #{REQUIRED} property because it is not a container.", column_name) return end # Make sure the value is lowercase 'true' (and nothing else) add_error(file, schema, table, column_name, "#{REQUIRED} value must be lowercase 'true' or omitted.", "Found: #{value}") unless value == true && !!value == value end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1106 def validate_required_if(file, schema, table, column_name, value) # Make sure value is in format: XX=YY unless value =~ /\A[a-z_]+=[a-zA-Z_]+\z/ add_error(file, schema, table, column_name, "#{REQUIRED_IF} value must be in form of XX=YY where XX is an ENUM field.", "Found: #{value}") return end value_split = value.split('=') enum_field = value_split[0] enum_value = value_split[1] # Make sure the ENUM field exists. if @data[enum_field].nil? add_error(file, schema, table, column_name, "#{REQUIRED_IF} references non-existent ENUM field.", "#{table}.#{enum_field}") return end enum_string = @data[enum_field][TYPE] # Make sure the ENUM field is actually an ENUM field. unless enum_string =~ Blufin::YmlSchemaValidator::REGEX_ENUM || enum_string =~ Blufin::YmlSchemaValidator::REGEX_ENUM_CUSTOM || enum_string =~ REGEX_ENUM_SYSTEM add_error(file, schema, table, column_name, "#{REQUIRED_IF} references a field which is not a valid ENUM.", "#{table}.#{enum_field}") return end # Make sure the ENUM value exists. add_error(file, schema, table, column_name, "#{REQUIRED_IF} uses a value not found in enum field: #{enum_string}", "#{enum_value}") unless Blufin::YmlCommon::enum_value_extractor(enum_string, @site).include?(enum_value) unless column_name =~ /\A(#{VALID_SCHEMAS_REGEX})\.[a-z_]+\z/ || column_name =~ /\A(#{VALID_SCHEMAS_REGEX})\.[a-z_]+\[(link)?\]\z/ # Make sure that it has a nullable flag. add_error(file, schema, table, column_name, "Columns with a #{REQUIRED_IF} property need to have a #{FLAG_NULLABLE} flag because they might be empty.", nil) unless @flags && @flags.nullable end end
Checks if there are any hard-coded schema definitions required and validates they exist (correctly). @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1389 def validate_required_schema_definitions begin embedded_data = Blufin::SiteEmbedded::get_data auth_level = Blufin::SiteAuth::get_auth_level auth_level_objects = Blufin::SiteAuth::AUTHENTICATION_LEVELS[auth_level] auth_level_objects.each do |auth_level_object| embedded_object = embedded_data[auth_level_object] expected_schema = embedded_object[:schema] expected_table = embedded_object[:table] expected_data = embedded_object[:data] # Must check here if embedded objects actually exist. if @schema_data[expected_schema].nil? || @schema_data[expected_schema][expected_table].nil? @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_OBJECT_NO_TABLE, nil, nil, nil, "#{expected_schema}.#{expected_table}") next end actual_data = @schema_data[expected_schema][expected_table] expected_data.each do |field, data| field_type = data[:type] field_key = field field_detail = "#{expected_schema}/#{expected_table}.yml - #{field_key}" field_is_object = false case field_type when Blufin::ScannerJavaEmbeddedObjects::OBJECT field_key = "#{expected_schema}.#{field}" field_is_object = true when Blufin::ScannerJavaEmbeddedObjects::OBJECT_LIST field_key = "#{expected_schema}.#{field}[]" field_is_object = true when Blufin::ScannerJavaEmbeddedObjects::OBJECT_LINK field_key = "#{expected_schema}.#{field}[link]" field_is_object = true when Blufin::YmlSchemaValidator::TYPE_ENUM_SYSTEM field_type = "#{data[:type]}('#{data[:type_java]}')" else end # Check field exists. unless actual_data.keys.include?(field_key) # If field doesn't exist. @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_FIELD_MISSING, "#{expected_schema}/#{expected_table}.yml", nil, nil, "#{field_key} \xe2\x86\x92 #{data.inspect}}") next end unless field_is_object actual_type = actual_data[field_key]['type'] actual_flag = actual_data[field_key]['flag'] actual_fkey = actual_data[field_key]['fkey'] # Check type. @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_FIELD_WRONG_TYPE, field_detail, nil, nil, Blufin::YmlErrorHandler::error_expected_actual(data[:type], actual_type)) unless actual_type == field_type # Check flags. if data[:flag].nil? @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_FLAGS_INVALID, field_detail, nil, nil, Blufin::YmlErrorHandler::error_expected_actual('[NO FLAGS]', actual_flag)) unless actual_flag.nil? else unless actual_flag == data[:flag].join(' ') @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_FLAGS_INVALID, field_detail, nil, nil, Blufin::YmlErrorHandler::error_expected_actual(data[:flag].join(' '), actual_flag)) end end # Check fkey. if data[:fkey].nil? @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_FKEY_INVALID, field_detail, nil, nil, Blufin::YmlErrorHandler::error_expected_actual('[NO FKEY]', actual_fkey)) unless actual_fkey.nil? else @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_FKEY_INVALID, field_detail, nil, nil, Blufin::YmlErrorHandler::error_expected_actual(data[:fkey], actual_fkey)) unless actual_fkey == data[:fkey] end # Check encrypted. if data[:encrypted] @yml_error_handler.add_error(Blufin::YmlErrorHandler::EMBEDDED_MUST_BE_ENCRYPTED, field_detail, nil, nil, 'Currently not encrypted.') if data[:encrypted] && (actual_data[field_key]['encrypted'].nil? || !actual_data[field_key]['encrypted']) end end end end rescue Exception => e # TODO - Was originally (intentionally) left blank but now re-activated even though it throws a "Double" Exception. - 6/6/17. # Hits here when config/config.yml has errors but needs to be handled better. # Maybe work on this once this exception hits again and you re-read this comment :) - 3/17/19. Blufin::Terminal::print_exception(e) end end
Validates @schema_resources. Checks that all tables which are “Embedded” have all HTTP methods enabled -> GET, POST, PATCH, DELETE @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1754 def validate_resources embedded_data = Blufin::SiteEmbedded::get_data auth_level = Blufin::SiteAuth::get_auth_level auth_level_objects = Blufin::SiteAuth::AUTHENTICATION_LEVELS[auth_level] raise 'This error occurred because you forgot to run Blufin::SiteAuth::init() somewhere...' if auth_level_objects.nil? auth_level_objects.each do |auth_level_object| schema = embedded_data[auth_level_object][:schema] table = embedded_data[auth_level_object][:table] return if @schema_resources["#{schema}.#{table}"].nil? http_methods = @schema_resources["#{schema}.#{table}"][:methods_internal].keys # Validate HTTP methods depending on schema. if schema == CONFIG @error_handler.add_error(Blufin::YmlErrorHandler::API_ENDPOINTS_NOT_ALLOWED, "#{schema}/#{table}.yml", 'config', 'internal', http_methods.join(', ')) if http_methods.include?(Blufin::YmlConfigValidator::POST) && http_methods.include?(Blufin::YmlConfigValidator::PATCH) && http_methods.include?(Blufin::YmlConfigValidator::DELETE) else @error_handler.add_error(Blufin::YmlErrorHandler::API_ENDPOINTS_MISSING, "#{schema}/#{table}.yml", 'config', 'internal', nil) unless http_methods.include?(Blufin::YmlConfigValidator::GET) && http_methods.include?(Blufin::YmlConfigValidator::POST) && http_methods.include?(Blufin::YmlConfigValidator::PATCH) && http_methods.include?(Blufin::YmlConfigValidator::DELETE) end end end
Make sure there are no duplicate endpoint names in the various schemas, IE: app.sale & common.sale would be considered duplicates. @return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 1207 def validate_table_duplicates table_names = {} @schema_data.each do |schema, schema_data| schema_data.each do |key, value| if table_names.keys.include?(key) add_error(nil, nil, nil, nil, "Duplicate table in separate schemas: #{schema}.#{key}", "Already exists: #{table_names[key]}.#{key}") else table_names[key] = schema end end end end
@return void
# File lib/core/yml/schema/yml_schema_validator.rb, line 886 def validate_type(file, schema, table, column_name, type) if type == TYPE_INT_AUTO add_error(file, schema, table, column_name, "Type: #{TYPE_INT_AUTO} should only be used for #{ID.upcase} fields.", type) return end if type == TYPE_BOOLEAN && @booleans_finished add_error(file, schema, table, column_name, "Type: #{TYPE_BOOLEAN} must be at the very top of file. Only #{ID.upcase} can come before it.", "Position: #{@column_count}") return end types_to_skip = [ TYPE_BOOLEAN, TYPE_INT, TYPE_INT_TINY, TYPE_INT_SMALL, TYPE_INT_BIG, TYPE_TEXT, TYPE_TEXT_LONG ] unless types_to_skip.include?(type) if type =~ REGEX_ENUM begin enum_values = Blufin::YmlCommon::enum_value_extractor(type, @site) rescue add_error(file, schema, table, column_name, "#{TYPE_ENUM} is invalid somehow. Please check your syntax.", type) return end if enum_values.include?(nil) || enum_values.include?('') add_error(file, schema, table, column_name, "#{TYPE_ENUM} contains nil (or blank) values.", type) else add_error(file, schema, table, column_name, "#{TYPE_ENUM} contains duplicate values.", type) unless enum_values.map(&:upcase).uniq.length == enum_values.length # Make sure that system-generated enums are all capital + underscores. enum_values.each do |enum_value| add_error(file, schema, table, column_name, "#{type} can only contain capital letters & underscores.", enum_value) unless enum_value =~ /\A[A-Z_]+\z/ end end elsif type =~ REGEX_ENUM_CUSTOM begin enum_values = @yml_enum_scanner.get_enum_custom_values_for(Blufin::YmlCommon::enum_name_extractor(@type)) add_error(file, schema, table, column_name, "#{type} has no values defined in Java counterpart.", type) unless enum_values.any? rescue add_error(file, schema, table, column_name, "#{type} has no Java counterpart.", type) end elsif type =~ REGEX_ENUM_SYSTEM begin enum_values = @yml_enum_scanner.get_enum_system_values_for(Blufin::YmlCommon::enum_name_extractor(@type)) add_error(file, schema, table, column_name, "#{type} has no values defined in Java counterpart.", type) unless enum_values.any? rescue add_error(file, schema, table, column_name, "#{type} has no Java counterpart.", type) end elsif type =~ REGEX_VARCHAR varchar_amount = Blufin::Strings::extract_using_regex(type, /\(\d+\)\z/, %w{( )}) add_error(file, schema, table, column_name, "Invalid #{TYPE_VARCHAR} definition, must be integer between 1 - 4096.", type) if varchar_amount.to_i.to_s != varchar_amount || (varchar_amount.to_i > 4096 || varchar_amount.to_i < 1) elsif type =~ REGEX_DECIMAL decimal_m, decimal_d = Blufin::YmlCommon::decimal_extract_values(type) add_error(file, schema, table, column_name, 'Decimal M (1st number) must be between 1 - 65.', type) unless decimal_m.to_i > 0 && decimal_m.to_i < 66 add_error(file, schema, table, column_name, 'Decimal D (2nd number) must be between 1 - 30.', type) unless decimal_d.to_i > 0 && decimal_d.to_i < 31 add_error(file, schema, table, column_name, 'Decimal M >= D -- 1st digit must be equal or larger than 2nd.', type) if (decimal_m.to_i < decimal_d.to_i) elsif [ Blufin::YmlSchemaValidator::TYPE_DATETIME, Blufin::YmlSchemaValidator::TYPE_DATETIME_INSERT, Blufin::YmlSchemaValidator::TYPE_DATETIME_UPDATE ].include?(type) add_error(file, schema, table, column_name, "#{type} fields must end in '_datetime'", column_name) unless column_name =~ /_datetime\z/ elsif type == TYPE_DATE add_error(file, schema, table, column_name, "#{type} fields must end in '_date'", column_name) if column_name !~ /_date\z/ && column_name != 'date' else extra = '' types_to_skip.each do |valid_type| if type.downcase == valid_type.downcase extra = " Perhaps you meant: #{valid_type} (capitalized)" break end end extra = " Perhaps you meant: #{TYPE_DECIMAL}(?,?)" if extra == '' && type.downcase == TYPE_DECIMAL.downcase error_message = "Invalid #{TYPE}.#{extra}" error_message << " #{TYPE_ENUM}s definitions cannot be blank an/or have spaces." if type =~ /\AENUM/ add_error(file, schema, table, column_name, error_message, type) end end # Make sure the INSERT + UPDATE fields are named correctly. add_error(file, schema, table, column_name, "#{TYPE_DATETIME_INSERT} fields must end in: created_datetime", @column_name) if type == TYPE_DATETIME_INSERT && !(@column_name =~ /^([a-z_]+_)?created_datetime$/) add_error(file, schema, table, column_name, "#{TYPE_DATETIME_UPDATE} fields must end in: modified_datetime", @column_name) if type == TYPE_DATETIME_UPDATE && !(@column_name =~ /^([a-z_]+_)?modified_datetime$/) end