Script pagure_events_py
[hide private]
[frames] | no frames]

Source Code for Script script-pagure_events_py

  1  #!/usr/bin/python3 
  2   
  3  import json 
  4  import pprint 
  5  import zmq 
  6  import sys 
  7  import os 
  8  import logging 
  9  import requests 
 10  import re 
 11  import munch 
 12   
 13  sys.path.append( 
 14      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 15  ) 
 16   
 17  from coprs import db, app, models 
 18  from coprs.logic.coprs_logic import CoprsLogic, CoprDirsLogic 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.complex_logic import ComplexLogic 
 21  from coprs.logic.packages_logic import PackagesLogic 
 22  from coprs import helpers 
 23   
 24  from urllib.parse import urlparse 
 25   
 26  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 27   
 28  logging.basicConfig( 
 29      filename='{0}/pagure-events.log'.format(app.config.get('LOG_DIR')), 
 30      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 31      level=logging.DEBUG) 
 32   
 33  log = logging.getLogger(__name__) 
 34  log.addHandler(logging.StreamHandler(sys.stdout)) 
 35   
 36  if os.getenv('PAGURE_EVENTS_TESTONLY'): 
 37      ENDPOINT = 'tcp://stg.pagure.io:9940' 
 38  else: 
 39      ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 40   
 41  log.info("ENDPOINT = {}".format(ENDPOINT)) 
 42   
 43  TOPICS = { 
 44      'io.pagure.prod.pagure.git.receive': 'https://pagure.io/', 
 45      'io.pagure.prod.pagure.pull-request.new': 'https://pagure.io/', 
 46      'io.pagure.prod.pagure.pull-request.comment.added': 'https://pagure.io/', 
 47      'org.fedoraproject.prod.pagure.git.receive': 'https://src.fedoraproject.org/', 
 48      'org.fedoraproject.prod.pagure.pull-request.new': 'https://src.fedoraproject.org/', 
 49      'org.fedoraproject.prod.pagure.pull-request.comment.added': 'https://src.fedoraproject.org/', 
 50      'io.pagure.stg.pagure.git.receive': 'https://stg.pagure.io/', # testing only 
 51      'io.pagure.stg.pagure.pull-request.new': 'https://stg.pagure.io/', # testing only 
 52      'io.pagure.stg.pagure.pull-request.comment.added': 'https://stg.pagure.io/', # testing only 
 53  } 
54 55 56 -class ScmPackage(object):
57 - def __init__(self, db_row):
58 self.pkg_id = db_row.package_id 59 self.copr_id = db_row.copr_id 60 61 self.source_json_dict = json.loads(db_row.source_json) 62 self.clone_url = self.source_json_dict.get('clone_url') or '' 63 self.committish = self.source_json_dict.get('committish') or '' 64 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 65 66 self.package = ComplexLogic.get_package_by_id_safe(self.pkg_id) 67 self.copr = ComplexLogic.get_copr_by_id_safe(self.copr_id)
68
69 - def build(self, source_dict_update, copr_dir, update_callback, 70 scm_object_type, scm_object_id, scm_object_url):
82 83 @classmethod
84 - def get_candidates_for_rebuild(cls, clone_url):
85 if db.engine.url.drivername == 'sqlite': 86 placeholder = '?' 87 true = '1' 88 else: 89 placeholder = '%s' 90 true = 'true' 91 92 rows = db.engine.execute( 93 """ 94 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 95 FROM package JOIN copr_dir ON package.copr_dir_id = copr_dir.id 96 WHERE package.source_type = {0} AND 97 package.webhook_rebuild = {1} AND 98 copr_dir.main = {2} AND 99 package.source_json ILIKE {placeholder} 100 """.format(SCM_SOURCE_TYPE, true, true, placeholder=placeholder), '%'+clone_url+'%' 101 ) 102 return [ScmPackage(row) for row in rows]
103
104 - def is_dir_in_commit(self, raw_commit_text):
105 if not self.subdirectory or not raw_commit_text: 106 return True 107 108 for line in raw_commit_text.split('\n'): 109 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line) 110 if match and match.group(2).lower() == self.subdirectory.strip('./').lower(): 111 return True 112 113 return False
114
115 116 -def event_info_from_pr_update(data, base_url):
117 """ 118 Message handler for updated pull-request opened in pagure. 119 Topic: ``*.pagure.pull-request.comment.added`` 120 """ 121 if data['msg']['pullrequest']['status'] != 'Open': 122 log.info('Pull-request not open, discarding.') 123 return False 124 125 if not data['msg']['pullrequest']['comments']: 126 log.info('This is most odd, we\'re not seeing comments.') 127 return False 128 129 last_comment = data['msg']['pullrequest']['comments'][-1] 130 if not last_comment or last_comment['notification'] is False: 131 log.info('Comment was not a notification, discarding.') 132 return False 133 134 return munch.Munch({ 135 'object_id': data['msg']['pullrequest']['id'], 136 'object_type': 'pull-request', 137 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 138 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 139 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 140 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 141 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 142 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 143 'branch_from': data['msg']['pullrequest']['branch_from'], 144 'branch_to': data['msg']['pullrequest']['branch'], 145 'start_commit': data['msg']['pullrequest']['commit_start'], 146 'end_commit': data['msg']['pullrequest']['commit_stop'], 147 })
148
149 150 -def event_info_from_new_pr(data, base_url):
151 """ 152 Message handler for new pull-request opened in pagure. 153 Topic: ``*.pagure.pull-request.new`` 154 """ 155 return munch.Munch({ 156 'object_id': data['msg']['pullrequest']['id'], 157 'object_type': 'pull-request', 158 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'], 159 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'], 160 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'], 161 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'], 162 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'], 163 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'], 164 'branch_from': data['msg']['pullrequest']['branch_from'], 165 'branch_to': data['msg']['pullrequest']['branch'], 166 'start_commit': data['msg']['pullrequest']['commit_start'], 167 'end_commit': data['msg']['pullrequest']['commit_stop'], 168 })
169
170 171 -def event_info_from_push(data, base_url):
172 """ 173 Message handler for push event in pagure. 174 Topic: ``*.pagure.git.receive`` 175 """ 176 return munch.Munch({ 177 'object_id': data['msg']['end_commit'], 178 'object_type': 'commit', 179 'base_project_url_path': data['msg']['repo']['url_path'], 180 'base_clone_url_path': data['msg']['repo']['fullname'], 181 'base_clone_url': base_url + data['msg']['repo']['fullname'], 182 'project_url_path': data['msg']['repo']['url_path'], 183 'clone_url_path': data['msg']['repo']['fullname'], 184 'clone_url': base_url + data['msg']['repo']['fullname'], 185 'branch_from': data['msg']['branch'], 186 'branch_to': data['msg']['branch'], 187 'start_commit': data['msg']['start_commit'], 188 'end_commit': data['msg']['end_commit'], 189 })
190
191 192 -def git_compare_urls(url1, url2):
193 url1 = re.sub(r'(\.git)?/*$', '', str(url1)) 194 url2 = re.sub(r'(\.git)?/*$', '', str(url2)) 195 o1 = urlparse(url1) 196 o2 = urlparse(url2) 197 return (o1.netloc == o2.netloc and o1.path == o2.path)
198
199 200 -def build_on_fedmsg_loop():
201 log.debug("Setting up poller...") 202 pp = pprint.PrettyPrinter(width=120) 203 204 ctx = zmq.Context() 205 s = ctx.socket(zmq.SUB) 206 s.connect(ENDPOINT) 207 208 for topic in TOPICS: 209 s.setsockopt_string(zmq.SUBSCRIBE, topic) 210 211 poller = zmq.Poller() 212 poller.register(s, zmq.POLLIN) 213 214 while True: 215 log.debug('Polling...') 216 evts = poller.poll(10000) 217 if not evts: 218 continue 219 220 log.debug('Receiving...') 221 _, msg_bytes = s.recv_multipart() 222 msg = msg_bytes.decode('utf-8') 223 224 log.debug('Parsing...') 225 data = json.loads(msg) 226 227 log.info('Got topic: {}'.format(data['topic'])) 228 base_url = TOPICS.get(data['topic']) 229 if not base_url: 230 log.error('Unknown topic {} received. Continuing.') 231 continue 232 233 if re.match(r'^.*.pull-request.new$', data['topic']): 234 event_info = event_info_from_new_pr(data, base_url) 235 elif re.match(r'^.*.pull-request.comment.added$', data['topic']): 236 event_info = event_info_from_pr_update(data, base_url) 237 else: 238 event_info = event_info_from_push(data, base_url) 239 240 log.info('event_info = {}'.format(pp.pformat(event_info))) 241 242 if not event_info: 243 log.info('Received event was discarded. Continuing.') 244 continue 245 246 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url) 247 raw_commit_text = None 248 if candidates: 249 # if start_commit != end_commit 250 # then more than one commit and no means to iterate over :( 251 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.end_commit 252 r = requests.get(raw_commit_url) 253 if r.status_code == requests.codes.ok: 254 raw_commit_text = r.text 255 else: 256 log.error('Bad http status {0} from url {1}'.format(r.status_code, raw_commit_url)) 257 258 for pkg in candidates: 259 log.info('Considering pkg id: {}, source_json: {}'.format(pkg.pkg_id, pkg.source_json_dict)) 260 261 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url) 262 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish)) 263 and pkg.is_dir_in_commit(raw_commit_text)): 264 265 log.info('\t -> rebuilding.') 266 267 if event_info.object_type == 'pull-request': 268 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id) 269 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname) 270 update_callback = 'pagure_flag_pull_request' 271 scm_object_url = os.path.join(base_url, event_info.base_project_url_path, 272 'pull-request', str(event_info.object_id)) 273 else: 274 copr_dir = pkg.copr.main_dir 275 update_callback = 'pagure_flag_commit' 276 scm_object_url = os.path.join(base_url, event_info.base_project_url_path, 277 'c', str(event_info.object_id)) 278 279 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url): 280 update_callback = '' 281 282 source_dict_update = { 283 'clone_url': event_info.clone_url, 284 'committish': event_info.end_commit, 285 } 286 287 build = pkg.build( 288 source_dict_update, 289 copr_dir, 290 update_callback, 291 event_info.object_type, 292 event_info.object_id, 293 scm_object_url 294 ) 295 log.info('\t -> {}'.format(build.to_dict())) 296 db.session.commit() 297 else: 298 log.info('\t -> skipping.')
299 300 301 if __name__ == '__main__': 302 while True: 303 try: 304 build_on_fedmsg_loop() 305 except KeyboardInterrupt: 306 sys.exit(1) 307 except: 308 log.exception('Error in fedmsg loop. Restarting it.') 309