1
2
3 import subprocess
4 import argparse
5 import sys
6 import os
7 import json
8 import time
9 import re
10 import logging
11 from flask_sqlalchemy import SQLAlchemy
12 from sqlalchemy.sql import text
13
14 sys.path.append(
15 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
16 )
17
18 from coprs import db, app, helpers, models
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.coprs_logic import CoprsLogic
21
22 logging.basicConfig(
23 filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")),
24 format='[%(asctime)s][%(levelname)6s]: %(message)s',
25 level=logging.DEBUG)
26 log = logging.getLogger(__name__)
27 log.addHandler(logging.StreamHandler(sys.stdout))
28
29 parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.')
30
31 parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'],
32 help='only check for updates from backend BACKEND, default pypi')
33 parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400,
34 help='ignore updates older than SECONDS, default 86400')
35 parser.add_argument('-v', '--version', action='version', version='1.0',
36 help='print program version and exit')
37
38 args = parser.parse_args()
39
40
42 """
43 Run given command in a subprocess
44 """
45 log.info('Executing: '+' '.join(cmd))
46 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
47 (stdout, stderr) = process.communicate()
48 if process.returncode != 0:
49 log.error(stderr)
50 sys.exit(1)
51 return stdout
52
54 try:
55 data = data_bytes.decode("utf-8")
56 data_json = json.loads(data)
57 except Exception as e:
58 log.info(data)
59 log.exception(str(e))
60 return data_json
61
63 cmd_binary = 'curl'
64 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}'
65 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)]
66 result_json = to_json(run_cmd(get_updates_cmd))
67 messages = result_json['raw_messages']
68 pages = result_json['pages']
69
70 for p in range(2, pages+1):
71 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)]
72 result_json = to_json(run_cmd(get_updates_cmd))
73 messages += result_json['raw_messages']
74
75 return messages
76
86
88 pkg_name_pattern = '(' + '|'.join(updated_packages.keys()) + ')'
89 source_type = helpers.BuildSourceEnum(args.backend.lower())
90 if db.engine.url.drivername == "sqlite":
91 placeholder = '?'
92 true = '1'
93 else:
94 placeholder = '%s'
95 true = 'true'
96 rows = db.engine.execute(
97 """
98 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id
99 FROM package
100 LEFT OUTER JOIN build ON build.package_id = package.id
101 WHERE package.source_type = {placeholder} AND
102 package.source_json ~* '{pkg_name_pattern}' AND
103 package.webhook_rebuild = {true} AND
104 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id));
105 """.format(placeholder=placeholder, pkg_name_pattern=pkg_name_pattern, true=true), source_type
106 )
107 return rows
108
109
113
114 - def build(self, copr, new_update_version):
116
117
123
124 - def build(self, copr, new_updated_version):
128
129
131 try:
132 return {
133 'pypi': PyPIPackage,
134 'rubygems': RubyGemsPackage,
135 }[backend](source_json)
136 except KeyError:
137 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
138
139
141 updated_packages = get_updated_packages(get_updates_messages())
142 log.info('Updated packages according to datagrepper: {0}'.format(updated_packages))
143
144 for row in get_copr_package_info_rows(updated_packages):
145 source_json = json.loads(row.source_json)
146 package = package_from_source(args.backend.lower(), source_json)
147
148 latest_build_version = row.pkg_version
149 log.info('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id))
150 if package.name in updated_packages:
151 new_updated_version = updated_packages[package.name]
152 log.debug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version))
153
154
155 if not latest_build_version or not re.match(new_updated_version, latest_build_version):
156 try:
157 copr = CoprsLogic.get_by_id(row.copr_id)[0]
158 except Exception as e:
159 log.exception(e)
160 continue
161
162 log.info('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}'
163 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id))
164 build = package.build(copr, new_updated_version)
165 db.session.commit()
166 log.info('Launched build id {0}'.format(build.id))
167
168 if __name__ == '__main__':
169 try:
170 main()
171 except Exception as e:
172 log.exception(str(e))
173