Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4   
  5  from six import with_metaclass 
  6  from six.moves.urllib.parse import urljoin 
  7   
  8  import flask 
  9  from flask import url_for 
 10  from dateutil import parser as dt_parser 
 11  from netaddr import IPAddress, IPNetwork 
 12  from redis import StrictRedis 
 13  from sqlalchemy.types import TypeDecorator, VARCHAR 
 14  import json 
 15   
 16  from coprs import constants 
 17  from coprs import app 
18 19 20 -def generate_api_token(size=30):
21 """ Generate a random string used as token to access the API 22 remotely. 23 24 :kwarg: size, the size of the token to generate, defaults to 30 25 chars. 26 :return: a string, the API token for the user. 27 """ 28 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
29 30 31 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 32 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 33 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 34 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
35 36 37 -class CounterStatType(object):
38 REPO_DL = "repo_dl"
39
40 41 -class EnumType(type):
42
43 - def __call__(self, attr):
44 if isinstance(attr, int): 45 for k, v in self.vals.items(): 46 if v == attr: 47 return k 48 raise KeyError("num {0} is not mapped".format(attr)) 49 else: 50 return self.vals[attr]
51
52 53 -class PermissionEnum(with_metaclass(EnumType, object)):
54 vals = {"nothing": 0, "request": 1, "approved": 2} 55 56 @classmethod
57 - def choices_list(cls, without=-1):
58 return [(n, k) for k, n in cls.vals.items() if n != without]
59
60 61 -class ActionTypeEnum(with_metaclass(EnumType, object)):
62 vals = { 63 "delete": 0, 64 "rename": 1, 65 "legal-flag": 2, 66 "createrepo": 3, 67 "update_comps": 4, 68 "gen_gpg_key": 5, 69 "rawhide_to_release": 6, 70 "fork": 7, 71 "update_module_md": 8, 72 "build_module": 9, 73 }
74
75 76 -class BackendResultEnum(with_metaclass(EnumType, object)):
77 vals = {"waiting": 0, "success": 1, "failure": 2}
78
79 80 -class RoleEnum(with_metaclass(EnumType, object)):
81 vals = {"user": 0, "admin": 1}
82
83 84 -class StatusEnum(with_metaclass(EnumType, object)):
85 vals = {"failed": 0, 86 "succeeded": 1, 87 "canceled": 2, 88 "running": 3, 89 "pending": 4, 90 "skipped": 5, # if there was this package built already 91 "starting": 6, # build picked by worker but no VM initialized 92 "importing": 7} # SRPM is being imported to dist-git
93
94 95 -class BuildSourceEnum(with_metaclass(EnumType, object)):
96 vals = {"unset": 0, 97 "srpm_link": 1, # url 98 "srpm_upload": 2, # pkg, tmp 99 "git_and_tito": 3, # git_url, git_dir, git_branch, tito_test 100 "mock_scm": 4, # scm_type, scm_url, spec, scm_branch 101 "pypi": 5, # package_name, version, python_versions 102 "rubygems": 6, # gem_name 103 }
104
105 106 # The same enum is also in distgit's helpers.py 107 -class FailTypeEnum(with_metaclass(EnumType, object)):
108 vals = {"unset": 0, 109 # General errors mixed with errors for SRPM URL/upload: 110 "unknown_error": 1, 111 "build_error": 2, 112 "srpm_import_failed": 3, 113 "srpm_download_failed": 4, 114 "srpm_query_failed": 5, 115 "import_timeout_exceeded": 6, 116 # Git and Tito errors: 117 "tito_general_error": 30, 118 "git_clone_failed": 31, 119 "git_wrong_directory": 32, 120 "git_checkout_error": 33, 121 "srpm_build_error": 34, 122 }
123
124 125 -class JSONEncodedDict(TypeDecorator):
126 """Represents an immutable structure as a json-encoded string. 127 128 Usage:: 129 130 JSONEncodedDict(255) 131 132 """ 133 134 impl = VARCHAR 135
136 - def process_bind_param(self, value, dialect):
137 if value is not None: 138 value = json.dumps(value) 139 140 return value
141
142 - def process_result_value(self, value, dialect):
143 if value is not None: 144 value = json.loads(value) 145 return value
146
147 -class Paginator(object):
148
149 - def __init__(self, query, total_count, page=1, 150 per_page_override=None, urls_count_override=None, 151 additional_params=None):
152 153 self.query = query 154 self.total_count = total_count 155 self.page = page 156 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 157 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 158 self.additional_params = additional_params or dict() 159 160 self._sliced_query = None
161
162 - def page_slice(self, page):
163 return (self.per_page * (page - 1), 164 self.per_page * page)
165 166 @property
167 - def sliced_query(self):
168 if not self._sliced_query: 169 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 170 return self._sliced_query
171 172 @property
173 - def pages(self):
174 return int(math.ceil(self.total_count / float(self.per_page)))
175
176 - def border_url(self, request, start):
177 if start: 178 if self.page - 1 > self.urls_count / 2: 179 return self.url_for_other_page(request, 1), 1 180 else: 181 if self.page < self.pages - self.urls_count / 2: 182 return self.url_for_other_page(request, self.pages), self.pages 183 184 return None
185
186 - def get_urls(self, request):
187 left_border = self.page - self.urls_count / 2 188 left_border = 1 if left_border < 1 else left_border 189 right_border = self.page + self.urls_count / 2 190 right_border = self.pages if right_border > self.pages else right_border 191 192 return [(self.url_for_other_page(request, i), i) 193 for i in range(left_border, right_border + 1)]
194
195 - def url_for_other_page(self, request, page):
196 args = request.view_args.copy() 197 args["page"] = page 198 args.update(self.additional_params) 199 return flask.url_for(request.endpoint, **args)
200
201 202 -def chroot_to_branch(chroot):
203 """ 204 Get a git branch name from chroot. Follow the fedora naming standard. 205 """ 206 os, version, arch = chroot.split("-") 207 if os == "fedora": 208 if version == "rawhide": 209 return "master" 210 os = "f" 211 elif os == "epel" and int(version) <= 6: 212 os = "el" 213 elif os == "mageia" and version == "cauldron": 214 os = "cauldron" 215 version = "" 216 elif os == "mageia": 217 os = "mga" 218 return "{}{}".format(os, version)
219
220 221 -def branch_to_os_version(branch):
222 os = None 223 version = None 224 if branch == "master": 225 os = "fedora" 226 version = "rawhide" 227 elif branch[0] == "f": 228 os = "fedora" 229 version = branch[1:] 230 elif branch[:4] == "epel" or branch[:2] == "el": 231 os = "epel" 232 version = branch[-1:] 233 elif branch[:6] == "custom": 234 os = "custom" 235 version = branch[-1:] 236 elif branch[:3] == "mga": 237 os = "mageia" 238 version = branch[3:] 239 elif branch[:8] == "cauldron": 240 os = "mageia" 241 version = "cauldron" 242 return os, version
243
244 245 -def splitFilename(filename):
246 """ 247 Pass in a standard style rpm fullname 248 249 Return a name, version, release, epoch, arch, e.g.:: 250 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 251 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 252 """ 253 254 if filename[-4:] == '.rpm': 255 filename = filename[:-4] 256 257 archIndex = filename.rfind('.') 258 arch = filename[archIndex+1:] 259 260 relIndex = filename[:archIndex].rfind('-') 261 rel = filename[relIndex+1:archIndex] 262 263 verIndex = filename[:relIndex].rfind('-') 264 ver = filename[verIndex+1:relIndex] 265 266 epochIndex = filename.find(':') 267 if epochIndex == -1: 268 epoch = '' 269 else: 270 epoch = filename[:epochIndex] 271 272 name = filename[epochIndex + 1:verIndex] 273 return name, ver, rel, epoch, arch
274
275 276 -def parse_package_name(pkg):
277 """ 278 Parse package name from possibly incomplete nvra string. 279 """ 280 281 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 282 return splitFilename(pkg)[0] 283 284 # doesn"t seem like valid pkg string, try to guess package name 285 result = "" 286 pkg = pkg.replace(".rpm", "").replace(".src", "") 287 288 for delim in ["-", "."]: 289 if delim in pkg: 290 parts = pkg.split(delim) 291 for part in parts: 292 if any(map(lambda x: x.isdigit(), part)): 293 return result[:-1] 294 295 result += part + "-" 296 297 return result[:-1] 298 299 return pkg
300
301 302 -def generate_repo_url(mock_chroot, url):
303 """ Generates url with build results for .repo file. 304 No checks if copr or mock_chroot exists. 305 """ 306 if mock_chroot.os_release == "fedora": 307 if mock_chroot.os_version != "rawhide": 308 mock_chroot.os_version = "$releasever" 309 310 url = urljoin( 311 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 312 mock_chroot.os_version, "$basearch")) 313 314 return url
315
316 317 -def fix_protocol_for_backend(url):
318 """ 319 Ensure that url either has http or https protocol according to the 320 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 321 """ 322 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 323 return url.replace("http://", "https://") 324 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 325 return url.replace("https://", "http://") 326 else: 327 return url
328
329 330 -def fix_protocol_for_frontend(url):
331 """ 332 Ensure that url either has http or https protocol according to the 333 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 334 """ 335 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 336 return url.replace("http://", "https://") 337 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 338 return url.replace("https://", "http://") 339 else: 340 return url
341
342 343 -class Serializer(object):
344
345 - def to_dict(self, options=None):
346 """ 347 Usage: 348 349 SQLAlchObject.to_dict() => returns a flat dict of the object 350 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 351 and will include a flat dict of object foo inside of that 352 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 353 a dict of the object, which will include dict of foo 354 (which will include dict of bar) and dict of spam. 355 356 Options can also contain two special values: __columns_only__ 357 and __columns_except__ 358 359 If present, the first makes only specified fiels appear, 360 the second removes specified fields. Both of these fields 361 must be either strings (only works for one field) or lists 362 (for one and more fields). 363 364 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 365 "__columns_only__": "name"}) => 366 367 The SQLAlchObject will only put its "name" into the resulting dict, 368 while "foo" all of its fields except "id". 369 370 Options can also specify whether to include foo_id when displaying 371 related foo object (__included_ids__, defaults to True). 372 This doesn"t apply when __columns_only__ is specified. 373 """ 374 375 result = {} 376 if options is None: 377 options = {} 378 columns = self.serializable_attributes 379 380 if "__columns_only__" in options: 381 columns = options["__columns_only__"] 382 else: 383 columns = set(columns) 384 if "__columns_except__" in options: 385 columns_except = options["__columns_except__"] 386 if not isinstance(options["__columns_except__"], list): 387 columns_except = [options["__columns_except__"]] 388 389 columns -= set(columns_except) 390 391 if ("__included_ids__" in options and 392 options["__included_ids__"] is False): 393 394 related_objs_ids = [ 395 r + "_id" for r, _ in options.items() 396 if not r.startswith("__")] 397 398 columns -= set(related_objs_ids) 399 400 columns = list(columns) 401 402 for column in columns: 403 result[column] = getattr(self, column) 404 405 for related, values in options.items(): 406 if hasattr(self, related): 407 result[related] = getattr(self, related).to_dict(values) 408 return result
409 410 @property
411 - def serializable_attributes(self):
412 return map(lambda x: x.name, self.__table__.columns)
413
414 415 -class RedisConnectionProvider(object):
416 - def __init__(self, config):
417 self.host = config.get("REDIS_HOST", "127.0.0.1") 418 self.port = int(config.get("REDIS_PORT", "6379"))
419
420 - def get_connection(self):
421 return StrictRedis(host=self.host, port=self.port)
422
423 424 -def get_redis_connection():
425 """ 426 Creates connection to redis, now we use default instance at localhost, no config needed 427 """ 428 return StrictRedis()
429
430 431 -def dt_to_unixtime(dt):
432 """ 433 Converts datetime to unixtime 434 :param dt: DateTime instance 435 :rtype: float 436 """ 437 return float(dt.strftime('%s'))
438
439 440 -def string_dt_to_unixtime(dt_string):
441 """ 442 Converts datetime to unixtime from string 443 :param dt_string: datetime string 444 :rtype: str 445 """ 446 return dt_to_unixtime(dt_parser.parse(dt_string))
447
448 449 -def is_ip_from_builder_net(ip):
450 """ 451 Checks is ip is owned by the builders network 452 :param str ip: IPv4 address 453 :return bool: True 454 """ 455 ip_addr = IPAddress(ip) 456 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 457 if ip_addr in IPNetwork(subnet): 458 return True 459 460 return False
461
462 463 -def str2bool(v):
464 if v is None: 465 return False 466 return v.lower() in ("yes", "true", "t", "1")
467
468 469 -def copr_url(view, copr, **kwargs):
470 """ 471 Examine given copr and generate proper URL for the `view` 472 473 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 474 and therefore you should *not* pass them manually. 475 476 Usage: 477 copr_url("coprs_ns.foo", copr) 478 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 479 """ 480 if copr.is_a_group_project: 481 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 482 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
483
484 485 -def url_for_copr_view(view, group_view, copr, **kwargs):
486 if copr.is_a_group_project: 487 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 488 else: 489 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
490 491 492 from sqlalchemy.engine.default import DefaultDialect 493 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 494 495 # python2/3 compatible. 496 PY3 = str is not bytes 497 text = str if PY3 else unicode 498 int_type = int if PY3 else (int, long) 499 str_type = str if PY3 else (str, unicode)
500 501 502 -class StringLiteral(String):
503 """Teach SA how to literalize various things."""
504 - def literal_processor(self, dialect):
505 super_processor = super(StringLiteral, self).literal_processor(dialect) 506 507 def process(value): 508 if isinstance(value, int_type): 509 return text(value) 510 if not isinstance(value, str_type): 511 value = text(value) 512 result = super_processor(value) 513 if isinstance(result, bytes): 514 result = result.decode(dialect.encoding) 515 return result
516 return process
517
518 519 -class LiteralDialect(DefaultDialect):
520 colspecs = { 521 # prevent various encoding explosions 522 String: StringLiteral, 523 # teach SA about how to literalize a datetime 524 DateTime: StringLiteral, 525 # don't format py2 long integers to NULL 526 NullType: StringLiteral, 527 }
528
529 530 -def literal_query(statement):
531 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 532 import sqlalchemy.orm 533 if isinstance(statement, sqlalchemy.orm.Query): 534 statement = statement.statement 535 return statement.compile( 536 dialect=LiteralDialect(), 537 compile_kwargs={'literal_binds': True}, 538 ).string
539
540 541 -def stream_template(template_name, **context):
542 app.update_template_context(context) 543 t = app.jinja_env.get_template(template_name) 544 rv = t.stream(context) 545 rv.enable_buffering(2) 546 return rv
547