1 import math
2 import random
3 import string
4
5 from six import with_metaclass
6 from six.moves.urllib.parse import urljoin, urlparse, parse_qs
7 from textwrap import dedent
8 import re
9
10 import flask
11 import posixpath
12 from flask import url_for
13 from dateutil import parser as dt_parser
14 from netaddr import IPAddress, IPNetwork
15 from redis import StrictRedis
16 from sqlalchemy.types import TypeDecorator, VARCHAR
17 import json
18
19 from copr_common.enums import EnumType
20 from coprs import constants
21 from coprs import app
25 """ Generate a random string used as token to access the API
26 remotely.
27
28 :kwarg: size, the size of the token to generate, defaults to 30
29 chars.
30 :return: a string, the API token for the user.
31 """
32 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
33
34
35 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
36 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
37 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
38 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
43
46 vals = {"nothing": 0, "request": 1, "approved": 2}
47
48 @classmethod
50 return [(n, k) for k, n in cls.vals.items() if n != without]
51
54 vals = {"unset": 0,
55 "link": 1,
56 "upload": 2,
57 "pypi": 5,
58 "rubygems": 6,
59 "scm": 8,
60 "custom": 9,
61 }
62
65 """Represents an immutable structure as a json-encoded string.
66
67 Usage::
68
69 JSONEncodedDict(255)
70
71 """
72
73 impl = VARCHAR
74
76 if value is not None:
77 value = json.dumps(value)
78
79 return value
80
82 if value is not None:
83 value = json.loads(value)
84 return value
85
88 - def __init__(self, query, total_count, page=1,
89 per_page_override=None, urls_count_override=None,
90 additional_params=None):
91
92 self.query = query
93 self.total_count = total_count
94 self.page = page
95 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
96 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
97 self.additional_params = additional_params or dict()
98
99 self._sliced_query = None
100
101 - def page_slice(self, page):
102 return (self.per_page * (page - 1),
103 self.per_page * page)
104
105 @property
107 if not self._sliced_query:
108 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
109 return self._sliced_query
110
111 @property
113 return int(math.ceil(self.total_count / float(self.per_page)))
114
116 if start:
117 if self.page - 1 > self.urls_count // 2:
118 return self.url_for_other_page(request, 1), 1
119 else:
120 if self.page < self.pages - self.urls_count // 2:
121 return self.url_for_other_page(request, self.pages), self.pages
122
123 return None
124
126 left_border = self.page - self.urls_count // 2
127 left_border = 1 if left_border < 1 else left_border
128 right_border = self.page + self.urls_count // 2
129 right_border = self.pages if right_border > self.pages else right_border
130
131 return [(self.url_for_other_page(request, i), i)
132 for i in range(left_border, right_border + 1)]
133
134 - def url_for_other_page(self, request, page):
135 args = request.view_args.copy()
136 args["page"] = page
137 args.update(self.additional_params)
138 return flask.url_for(request.endpoint, **args)
139
142 """
143 Get a git branch name from chroot. Follow the fedora naming standard.
144 """
145 os, version, arch = chroot.rsplit("-", 2)
146 if os == "fedora":
147 if version == "rawhide":
148 return "master"
149 os = "f"
150 elif os == "epel" and int(version) <= 6:
151 os = "el"
152 elif os == "mageia" and version == "cauldron":
153 os = "cauldron"
154 version = ""
155 elif os == "mageia":
156 os = "mga"
157 return "{}{}".format(os, version)
158
162 """
163 Pass in a standard style rpm fullname
164
165 Return a name, version, release, epoch, arch, e.g.::
166 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
167 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
168 """
169
170 if filename[-4:] == '.rpm':
171 filename = filename[:-4]
172
173 archIndex = filename.rfind('.')
174 arch = filename[archIndex+1:]
175
176 relIndex = filename[:archIndex].rfind('-')
177 rel = filename[relIndex+1:archIndex]
178
179 verIndex = filename[:relIndex].rfind('-')
180 ver = filename[verIndex+1:relIndex]
181
182 epochIndex = filename.find(':')
183 if epochIndex == -1:
184 epoch = ''
185 else:
186 epoch = filename[:epochIndex]
187
188 name = filename[epochIndex + 1:verIndex]
189 return name, ver, rel, epoch, arch
190
193 """
194 Parse package name from possibly incomplete nvra string.
195 """
196
197 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
198 return splitFilename(pkg)[0]
199
200
201 result = ""
202 pkg = pkg.replace(".rpm", "").replace(".src", "")
203
204 for delim in ["-", "."]:
205 if delim in pkg:
206 parts = pkg.split(delim)
207 for part in parts:
208 if any(map(lambda x: x.isdigit(), part)):
209 return result[:-1]
210
211 result += part + "-"
212
213 return result[:-1]
214
215 return pkg
216
240
243 """
244 Ensure that url either has http or https protocol according to the
245 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
246 """
247 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
248 return url.replace("http://", "https://")
249 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
250 return url.replace("https://", "http://")
251 else:
252 return url
253
256 """
257 Ensure that url either has http or https protocol according to the
258 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
259 """
260 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
261 return url.replace("http://", "https://")
262 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
263 return url.replace("https://", "http://")
264 else:
265 return url
266
269
271 """
272 Usage:
273
274 SQLAlchObject.to_dict() => returns a flat dict of the object
275 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
276 and will include a flat dict of object foo inside of that
277 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
278 a dict of the object, which will include dict of foo
279 (which will include dict of bar) and dict of spam.
280
281 Options can also contain two special values: __columns_only__
282 and __columns_except__
283
284 If present, the first makes only specified fields appear,
285 the second removes specified fields. Both of these fields
286 must be either strings (only works for one field) or lists
287 (for one and more fields).
288
289 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
290 "__columns_only__": "name"}) =>
291
292 The SQLAlchObject will only put its "name" into the resulting dict,
293 while "foo" all of its fields except "id".
294
295 Options can also specify whether to include foo_id when displaying
296 related foo object (__included_ids__, defaults to True).
297 This doesn"t apply when __columns_only__ is specified.
298 """
299
300 result = {}
301 if options is None:
302 options = {}
303 columns = self.serializable_attributes
304
305 if "__columns_only__" in options:
306 columns = options["__columns_only__"]
307 else:
308 columns = set(columns)
309 if "__columns_except__" in options:
310 columns_except = options["__columns_except__"]
311 if not isinstance(options["__columns_except__"], list):
312 columns_except = [options["__columns_except__"]]
313
314 columns -= set(columns_except)
315
316 if ("__included_ids__" in options and
317 options["__included_ids__"] is False):
318
319 related_objs_ids = [
320 r + "_id" for r, _ in options.items()
321 if not r.startswith("__")]
322
323 columns -= set(related_objs_ids)
324
325 columns = list(columns)
326
327 for column in columns:
328 result[column] = getattr(self, column)
329
330 for related, values in options.items():
331 if hasattr(self, related):
332 result[related] = getattr(self, related).to_dict(values)
333 return result
334
335 @property
337 return map(lambda x: x.name, self.__table__.columns)
338
342 self.host = config.get("REDIS_HOST", "127.0.0.1")
343 self.port = int(config.get("REDIS_PORT", "6379"))
344
346 return StrictRedis(host=self.host, port=self.port)
347
350 """
351 Creates connection to redis, now we use default instance at localhost, no config needed
352 """
353 return StrictRedis()
354
357 """
358 Converts datetime to unixtime
359 :param dt: DateTime instance
360 :rtype: float
361 """
362 return float(dt.strftime('%s'))
363
366 """
367 Converts datetime to unixtime from string
368 :param dt_string: datetime string
369 :rtype: str
370 """
371 return dt_to_unixtime(dt_parser.parse(dt_string))
372
375 """
376 Checks is ip is owned by the builders network
377 :param str ip: IPv4 address
378 :return bool: True
379 """
380 ip_addr = IPAddress(ip)
381 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]):
382 if ip_addr in IPNetwork(subnet):
383 return True
384
385 return False
386
389 if v is None:
390 return False
391 return v.lower() in ("yes", "true", "t", "1")
392
395 """
396 Examine given copr and generate proper URL for the `view`
397
398 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
399 and therefore you should *not* pass them manually.
400
401 Usage:
402 copr_url("coprs_ns.foo", copr)
403 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
404 """
405 if copr.is_a_group_project:
406 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
407 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
408
415
419
420
421 from sqlalchemy.engine.default import DefaultDialect
422 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
423
424
425 PY3 = str is not bytes
426 text = str if PY3 else unicode
427 int_type = int if PY3 else (int, long)
428 str_type = str if PY3 else (str, unicode)
432 """Teach SA how to literalize various things."""
445 return process
446
457
460 """NOTE: This is entirely insecure. DO NOT execute the resulting strings."""
461 import sqlalchemy.orm
462 if isinstance(statement, sqlalchemy.orm.Query):
463 statement = statement.statement
464 return statement.compile(
465 dialect=LiteralDialect(),
466 compile_kwargs={'literal_binds': True},
467 ).string
468
471 app.update_template_context(context)
472 t = app.jinja_env.get_template(template_name)
473 rv = t.stream(context)
474 rv.enable_buffering(2)
475 return rv
476
484
487 """
488 Expands variables and sanitize repo url to be used for mock config
489 """
490 parsed_url = urlparse(repo_url)
491 if parsed_url.scheme == "copr":
492 user = parsed_url.netloc
493 prj = parsed_url.path.split("/")[1]
494 repo_url = "/".join([
495 flask.current_app.config["BACKEND_BASE_URL"],
496 "results", user, prj, chroot
497 ]) + "/"
498
499 repo_url = repo_url.replace("$chroot", chroot)
500 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0])
501 return repo_url
502
505 """
506 :param repo: str repo from Copr/CoprChroot/Build/...
507 :param supported_keys list of supported optional parameters
508 :return: dict of optional parameters parsed from the repo URL
509 """
510 supported_keys = supported_keys or ["priority"]
511 if not repo.startswith("copr://"):
512 return {}
513
514 params = {}
515 qs = parse_qs(urlparse(repo).query)
516 for k, v in qs.items():
517 if k in supported_keys:
518
519
520 value = int(v[0]) if v[0].isnumeric() else v[0]
521 params[k] = value
522 return params
523
526 """ Return dict with proper build config contents """
527 chroot = None
528 for i in copr.copr_chroots:
529 if i.mock_chroot.name == chroot_id:
530 chroot = i
531 if not chroot:
532 return {}
533
534 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
535
536 repos = [{
537 "id": "copr_base",
538 "url": copr.repo_url + "/{}/".format(chroot_id),
539 "name": "Copr repository",
540 }]
541
542 if not copr.auto_createrepo:
543 repos.append({
544 "id": "copr_base_devel",
545 "url": copr.repo_url + "/{}/devel/".format(chroot_id),
546 "name": "Copr buildroot",
547 })
548
549 def get_additional_repo_views(repos_list):
550 repos = []
551 for repo in repos_list:
552 params = parse_repo_params(repo)
553 repo_view = {
554 "id": generate_repo_name(repo),
555 "url": pre_process_repo_url(chroot_id, repo),
556 "name": "Additional repo " + generate_repo_name(repo),
557 }
558 repo_view.update(params)
559 repos.append(repo_view)
560 return repos
561
562 repos.extend(get_additional_repo_views(copr.repos_list))
563 repos.extend(get_additional_repo_views(chroot.repos_list))
564
565 return {
566 'project_id': copr.repo_id,
567 'additional_packages': packages.split(),
568 'repos': repos,
569 'chroot': chroot_id,
570 'use_bootstrap_container': copr.use_bootstrap_container,
571 'with_opts': chroot.with_opts.split(),
572 'without_opts': chroot.without_opts.split(),
573 }
574
582
585 if not url:
586 return None
587
588 return re.sub(r'(\.git)?/*$', '', url)
589
592 if not url:
593 return False
594
595 url = trim_git_url(url)
596 return urlparse(url)
597