class Confiner::Plugins::Gitlab

Constants

MERGE_REQUEST_TITLE
ONLY_QUARANTINE_METADATA
QUARANTINE_METADATA

Public Class Methods

new(options, **args) click to toggle source

GitLab Confiner Plugin @param [Hash] args the arguments for GitLab @option args [String] :private_token the private token for GitLab to connect to @option args [String] :project_id the project id (or name) where the pipelines are fetched from @option args [String] :target_project where failure issues will be searched, and where an MR will be filed @option args [String] :failure_issue_labels what labels to search for when searching for the failure issue (comma separated) @option args [String] :failure_issue_prefix what prefix an issue has in GitLab Issues to search for the failure issue

@option args [Hash] :environment metadata about the environment the tests are running and which will be confined @option :environment [String] :name the name of the environment @option :environment [String] :pattern the pattern of how to quarantine/dequarantine

@option args [Integer] :timeout the timeout that HTTParty will consume (timeout of requests) @option args [Integer] :threshold the failure / pass threshold @option args [String] :endpoint the GitLab API Endpoint (e.g. gitlab.com/api/v4) @option args [String] :pwd the path of the working directory for where the tests are located @option args [String] :ref the default Git ref used when updating @option args [String] :job_pattern the regex pattern to match names of jobs in GitLab (Job = Suite Name)

@example

gitlab = Confiner::Plugins::Gitlab.new({ debug: true }, private_token: 'ABC-123', project_id: 'my-group/my-project', target_project: 'my-group/my-project', failure_issue_labels: 'QA,test', failure_issue_prefix: 'Failure in ', timeout: 10, threshold: 3, endpoint: 'https://gitlab.com/api/v4', pwd: 'qa', ref: 'master')
gitlab.quarantine
gitlab.dequarantine
Calls superclass method Confiner::Plugin::new
# File lib/confiner/plugins/gitlab.rb, line 48
def initialize(options, **args)
  super

  ENV['GITLAB_API_HTTPARTY_OPTIONS'] = ENV.fetch('GITLAB_API_HTTPARTY_OPTIONS') { "{read_timeout: #{timeout}}" }

  raise ArgumentError, 'Missing private_token' if private_token.nil?

  @gitlab_client = ::Gitlab.client(private_token: private_token, endpoint: endpoint)
end

Public Instance Methods

dequarantine() click to toggle source

Dequarantine Action - Automatically Dequarantine tests

# File lib/confiner/plugins/gitlab.rb, line 150
      def dequarantine
        log :gitlab, 'Beginning Dequarantine Process', 2

        @examples = get_examples

        dequarantines = @examples.select(&:passed?).map(&:name).uniq.each_with_object([]) do |passed_example, dequarantines|
          passes = @examples.select { _1.name == passed_example && _1.passed? }
          fails = @examples.select { _1.name == passed_example && _1.failed? }

          if passes.size >= threshold
            next log(:dequarantine, "Detected #{fails.size} failures for #{passed_example}, thus, not de-quarantining", 3) unless fails.size.zero?

            dequarantines << passed_example

            example = @examples.find { _1.name == passed_example }

            log :dequarantine, "Dequarantining #{example} (#{passes.size} >= #{threshold})", 3

            begin
              file_contents = get_example_file_contents(example)
              new_contents, changed_line_no, failure_issue = remove_quarantine_metadata(file_contents, example)

              next log(:warn, <<~MESSAGE.tr("\n", ' '), 4) if file_contents == new_contents
                Unable to dequarantine. This is likely due to the quarantine being applied to the outer context.
                See https://gitlab.com/gitlab-org/quality/confiner/-/issues/3
              MESSAGE

              if @options.dry_run
                log :dry_run, 'Skipping creating branch, committing and filing merge request', 4
              else
                branch = create_branch(failure_issue, 'dequarantine', example)
                commit_changes(branch, <<~COMMIT_MESSAGE, example, new_contents)
                  Dequarantine end-to-end test

                  Dequarantine #{example.name}
                COMMIT_MESSAGE

                create_merge_request('[DEQUARANTINE]',example, branch) do
                  markdown_occurrences = []

                  passes.each do |pass|
                    markdown_occurrences << "1. [#{pass.occurrence[:job]}](#{pass.occurrence[:pipeline_url]})"
                  end

                  meets_or_exceeds = passes.size > threshold ? 'exceeds' : 'meets'

                  <<~MARKDOWN
                    ## What does this MR do?

                    Dequarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_no})

                    This quarantined test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have passed
                    #{passes.size} times consecutively, which #{meets_or_exceeds} the threshold of #{threshold} times.

                    #{markdown_occurrences.join("\n")}

                    > #{failure_issue}

                    <div align="center">
                    (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
                    </div>
                  MARKDOWN
                end
              end

              log :dequarantine, "Done Dequarantining #{passed_example}", 3
            rescue => e
              log :fatal, "There was an issue dequarantining #{example}. Error was #{e.message}\n#{e.backtrace}"
            end
          end
        end

        if dequarantines.any?
          log :dequarantine, "Found #{dequarantines.size} candidates to be dequarantined", 3
        else
          log :dequarantine, 'Found no candidates to be dequarantined', 3
        end

        log :gitlab, 'Done with Dequarantine Process', 2
      end
quarantine() click to toggle source

Quarantine Action - Automatically Quarantine tests

# File lib/confiner/plugins/gitlab.rb, line 59
      def quarantine
        log :gitlab, 'Beginning Quarantine Process', 2

        if environment&.any?
          raise ArgumentError, 'Specify both environment[name] and environment[pattern]' unless environment['name'] && environment['pattern']
        end

        # store the examples from the pipelines
        @examples = get_examples

        quarantines = @examples.select(&:failed?).map(&:name).uniq.each_with_object([]) do |failed_example, quarantines|
          # count the number of failures consecutively for this example

          fails = @examples.select { _1.name == failed_example && _1.failed? }
          if fails.size >= threshold
            quarantines << failed_example

            example = @examples.find { _1.name == failed_example }

            log :quarantine, "Quarantining #{failed_example} (#{fails.size} >= #{threshold})", 3

            # check to see if there is a merge request first
            # if there is no merge request...
            #   - Check for an existing issue
            #   - Check for an existing Quarantine MR
            #   - Add a quarantine tag: `it 'should be quarantined', quarantine: { issue: 'https://issue', type: :investigating }`
            #   - File the merge request

            begin
              # begin the quarantining process
              failure_issue = get_failure_issue_for_example(example)
              file_contents = get_example_file_contents(example)
              new_contents, changed_line_no = add_quarantine_metadata(file_contents, example, failure_issue)

              log(:debug, new_contents, 4) if @options.debug

              if @options.dry_run
                log :dry_run, 'Skipping creating branch, committing and filing merge request', 4
              else
                branch = create_branch(failure_issue, 'quarantine', example)
                commit_changes(branch, <<~COMMIT_MESSAGE, example, new_contents)
                  Quarantine end-to-end test

                  Quarantine #{example.name}
                COMMIT_MESSAGE

                create_merge_request('[QUARANTINE]', example, branch) do
                  markdown_occurrences = []

                  fails.each do |fail|
                    markdown_occurrences << "1. [#{fail.occurrence[:job]}](#{fail.occurrence[:pipeline_url]})"
                  end

                  meets_or_exceeds = fails.size > threshold ? 'exceeds' : 'meets'

                  <<~MARKDOWN
                    ## What does this MR do?

                    Quarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_no})

                    This test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have failed
                    #{fails.size} times, which #{meets_or_exceeds} the threshold of #{threshold} times.

                    #{markdown_occurrences.join("\n")}

                    > #{failure_issue['web_url']}

                    <div align="center">
                    (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
                    </div>
                  MARKDOWN
                end
              end

              log :quarantine, "Done Quarantining #{failed_example}", 3
            rescue => e
              log :fatal, "There was an issue quarantining #{example}. Error was #{e.message}\n#{e.backtrace}"
            end
          end
        end

        if quarantines.any?
          log :quarantine, "Found #{quarantines.size} candidates to be quarantined", 3
        else
          log :quarantine, 'Found no candidates to be quarantined', 3
        end

        log :gitlab, 'Done with Quarantine Process', 2
      end

Private Instance Methods

add_quarantine_metadata(content, example, failure_issue) click to toggle source

Add quarantine metadata to the file content and replace it @param [String] content the content to @param [Example] example the example to find and replace @param [Gitlab::ObjectifiedHash] failure_issue the failure issue @return [Array<String, Integer>] first value holds the new content, the second value holds the line number of the test

# File lib/confiner/plugins/gitlab.rb, line 305
def add_quarantine_metadata(content, example, failure_issue)
  find_example_match_in_contents(content, example) do |line|
    if line.include?(',')
      line[line.index(',')] = if environment['name'] && environment['pattern']
                                ONLY_QUARANTINE_METADATA % { issue_url: failure_issue['web_url'], pattern: environment['pattern'] }
                              else
                                QUARANTINE_METADATA % { issue_url: failure_issue['web_url'] }
                              end << ','
    else
      line[line.rindex(' ')] = if environment['name'] && environment['pattern']
                                 ONLY_QUARANTINE_METADATA % { issue_url: failure_issue['web_url'], pattern: environment['pattern'] }
                               else
                                 QUARANTINE_METADATA % { issue_url: failure_issue['web_url'] }
                               end << ' '
    end

    line
  end
end
commit_changes(branch, message, example, new_content) click to toggle source

Commit changes to a branch @param [Gitlab::ObjectifiedHash] branch the branch to commit to @param [String] message the message to commit @param [Example] example the example @param [Gitlab::ObjectifiedHash] new_content the new content to commit @return [Gitlab::ObjectifiedHash] the commit

# File lib/confiner/plugins/gitlab.rb, line 368
def commit_changes(branch, message, example, new_content)
  commit = @gitlab_client.create_commit(target_project, branch['name'], message, [
    { action: :update, file_path: example.file, content: new_content}
  ])

  log :commit, "Created commit #{commit['id']} (#{commit['web_url']}) on #{branch['name']}", 4

  commit
end
create_branch(failure_issue, name_prefix, example) click to toggle source

Create a branch from the ref @param [Gitlab::ObjectifiedHash, String] failure_issue the existing failure issue fetched from the GitLab API, or the full URL @param [String] name_prefix the prefix to attach to the branch name @param [Confiner::Example] example the example @return [Gitlab::ObjectifiedHash] the new branch

# File lib/confiner/plugins/gitlab.rb, line 351
def create_branch(failure_issue, name_prefix, example)
  issue_id = failure_issue.is_a?(String)? failure_issue.split('/').last : failure_issue['iid'] # split url segment, last segment of path is the issue id
  branch_name = [issue_id, name_prefix, environment['name'] || '', example.name.gsub(/\W/, '-')]

  branch = @gitlab_client.create_branch(target_project, branch_name.join('-'), ref)

  log :branch, "Created branch #{branch['name']} (#{branch['web_url']})", 4

  branch
end
create_merge_request(title_prefix, example, branch, &block) click to toggle source

Create a Merge Request with a given branch @param [String] title_prefix the prefix of the title @param [Example] example the example @param [Gitlab::ObjectifiedHash] branch the branch @return [Gitlab::ObjectifiedHash] the created merge request

# File lib/confiner/plugins/gitlab.rb, line 383
def create_merge_request(title_prefix, example, branch, &block)
  description = block.call

  environment_name = environment['name'] ? "[#{environment['name']}]" : ''
  merge_request = @gitlab_client.create_merge_request(target_project,
    "%{prefix} %{environment_name} %{example_name}" % { prefix: title_prefix, environment_name: environment_name, example_name: example.name },

    source_branch: branch['name'],
    target_branch: ref,
    description: description,
    labels: failure_issue_labels,
    squash: true,
    remove_source_branch: true)

  log :mr, "Created merge request #{merge_request['iid']} (#{merge_request['web_url']})", 4

  merge_request
end
find_example_match_in_contents(content, example) { |line| ... } click to toggle source

Find an example in file contents and transform the line @param [String] content the file contents @param [Confiner::Example] example the name of the example @yield [String] the line of the contents found @return [Array<String, Integer>] first item return is the new file contents. second item is the line number where the match was found

# File lib/confiner/plugins/gitlab.rb, line 409
def find_example_match_in_contents(content, example)
  content_to_return = content.dup

  best_match = -1
  lines = content_to_return.split("\n")
  example.name.split.reverse.each do |word|
    # given a full-descriptive RSpec example name:
    #   Plan Group Iterations creates a group iteration
    # scan these words backwards, incrementing the index until we find the best match
    new_match = content_to_return.index(word) || -1
    best_match = new_match if new_match > best_match
  end

  # the matched line where the example is
  match = content_to_return[best_match..].split("\n").first
  matched_line_no = 0

  lines.each_with_index do |line, line_no|
    if line.match?(match)
      matched_line_no = line_no + 1

      log :rspec, "Found match on line #{example.file}:#{matched_line_no}", 4

      resulting_line = block_given? ? yield(line) : line
      lines[line_no] = resulting_line
    end
  end

  [lines.join("\n") << "\n", matched_line_no]
end
get_example_file_contents(example) click to toggle source

Get the file contents of an example @param [Example] example the example to get the contents of

# File lib/confiner/plugins/gitlab.rb, line 294
def get_example_file_contents(example)
  example.file = example.file.sub('./', File.join(pwd, '/')) unless pwd == '.'

  @gitlab_client.file_contents(target_project, example.file)
end
get_examples(threshold: self.threshold * 2, job_pattern: Regexp.new(self.job_pattern)) click to toggle source

Get examples from pipelines @param [Integer] threshold the amount of pipelines to fetch @note The threshold defaults to default threshold multiplied by two to the amount of pipelines @return [Array<Example>] array of examples

# File lib/confiner/plugins/gitlab.rb, line 248
def get_examples(threshold: self.threshold * 2, job_pattern: Regexp.new(self.job_pattern))
  examples = []

  get_last_n_runs(threshold: threshold).each do |run|
    run.each do |pipeline|
      # fetch the pipeline test report
      @gitlab_client.pipeline_test_report(project_id, pipeline['id'])['test_suites'].each do |suite|
        # skip if the job name does not match the job_pattern argument
        next log(:skip, "Skipping #{suite['name']}", 4) unless suite['name'].match(job_pattern)

        log :suite, "Suite: #{suite['name']} (#{pipeline['web_url']})", 4
        suite['test_cases'].each do |example|
          examples << Example.new(**example, occurrence: { job: suite['name'], pipeline_url: pipeline['web_url'] })
          log :test, example['name'], 5
        end
      end
    end
  end

  examples
end
get_failure_issue_for_example(example) click to toggle source

Query the GitLab Issues API to fetch QA failure issues @param [String,Example] example the name of the example to search for @option example [String] the name of the example to search for @option example [Example] an instance of Example

# File lib/confiner/plugins/gitlab.rb, line 274
def get_failure_issue_for_example(example)
  issues = @gitlab_client.issues(target_project,
    labels: failure_issue_labels,
    state: :opened,
    search: "#{failure_issue_prefix}#{example}",
    in: :title,
    per_page: 1
  )

  if issues.any?
    log :issue, "Found issue #{issues.first['web_url']} for `#{example}`", 4

    issues.first
  else
    log :fatal, "No failure issue exists for `#{example}`. Skipping."
  end
end
get_last_n_runs(threshold:) click to toggle source

Get last n amount of runs @param [Integer] threshold the amount of pipelines to fetch

For instance, if threshold: 3, the amount of pipelines will return 6.  (3x Successful, 3x Failed)

@return [Array<Gitlab::ObjectifiedHash>] an array of pipelines returned from GitLab

# File lib/confiner/plugins/gitlab.rb, line 237
def get_last_n_runs(threshold:)
  pipelines = [] # collection of both passing and failing pipelines

  pipelines << @gitlab_client.pipelines(project_id, per_page: threshold, status: :success, ref: ref)
  pipelines << @gitlab_client.pipelines(project_id, per_page: threshold, status: :failed, ref: ref)
end
remove_quarantine_metadata(content, example) click to toggle source

Add dequarantine metadata to the file content and replace it @param [String] content the content to @param [Example] example the example to find and replace @return [Array<(String, Integer, String)>]

# File lib/confiner/plugins/gitlab.rb, line 329
def remove_quarantine_metadata(content, example)
  issue = +''

  [
    find_example_match_in_contents(content, example) do |line|
      if line.include?('quarantine:')
        issue = line[/issue:( *)?'(.+)',/, 2]# pluck the issue URL from the line before removing
        line.gsub(/,( *)quarantine:( *)?{.+}/, '')
      else
        line
      end
    end,

    issue
  ].flatten
end