1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 import requests
10
11 from flask import request
12 from sqlalchemy.sql import text
13 from sqlalchemy import or_
14 from sqlalchemy import and_
15 from sqlalchemy import func
16 from sqlalchemy.orm import joinedload
17 from sqlalchemy.orm.exc import NoResultFound
18 from sqlalchemy.sql import false,true
19 from werkzeug.utils import secure_filename
20 from sqlalchemy import desc, asc, bindparam, Integer, String
21 from collections import defaultdict
22
23 from copr_common.enums import FailTypeEnum, StatusEnum
24 from coprs import app
25 from coprs import db
26 from coprs import exceptions
27 from coprs import models
28 from coprs import helpers
29 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT
30 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, UnrepeatableBuildException
31
32 from coprs.logic import coprs_logic
33 from coprs.logic import users_logic
34 from coprs.logic.actions_logic import ActionsLogic
35 from coprs.models import BuildChroot,Build,Package,MockChroot
36 from .coprs_logic import MockChrootsLogic
37
38 log = app.logger
42 @classmethod
43 - def get(cls, build_id):
45
46 @classmethod
57
58 @classmethod
69
70 @classmethod
89
90 @classmethod
98
99 @classmethod
120
121 @classmethod
123 query = text("""
124 SELECT COUNT(*) as result
125 FROM build_chroot JOIN build on build.id = build_chroot.build_id
126 WHERE
127 build.submitted_on < :end
128 AND (
129 build_chroot.started_on > :start
130 OR (build_chroot.started_on is NULL AND build_chroot.status = :status)
131 -- for currently pending builds we need to filter on status=pending because there might be
132 -- failed builds that have started_on=NULL
133 )
134 AND NOT build.canceled
135 """)
136
137 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending"))
138 return res.first().result
139
140 @classmethod
142 query = text("""
143 SELECT COUNT(*) as result
144 FROM build_chroot
145 WHERE
146 started_on < :end
147 AND (ended_on > :start OR (ended_on is NULL AND status = :status))
148 -- for currently running builds we need to filter on status=running because there might be failed
149 -- builds that have ended_on=NULL
150 """)
151
152 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running"))
153 return res.first().result
154
155 @classmethod
172
173 @classmethod
175 data = [["pending"], ["running"], ["avg running"], ["time"]]
176 params = cls.get_graph_parameters(type)
177 cached_data = cls.get_cached_graph_data(params)
178 data[0].extend(cached_data["pending"])
179 data[1].extend(cached_data["running"])
180
181 for i in range(len(data[0]) - 1, params["steps"]):
182 step_start = params["start"] + i * params["step"]
183 step_end = step_start + params["step"]
184 pending = cls.get_pending_jobs_bucket(step_start, step_end)
185 running = cls.get_running_jobs_bucket(step_start, step_end)
186 data[0].append(pending)
187 data[1].append(running)
188 cls.cache_graph_data(type, time=step_start, pending=pending, running=running)
189
190 running_total = 0
191 for i in range(1, params["steps"] + 1):
192 running_total += data[1][i]
193
194 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1))
195
196 for i in range(params["start"], params["end"], params["step"]):
197 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i)))
198
199 return data
200
201 @classmethod
216
217 @classmethod
233
234 @classmethod
236 if type is "10min":
237
238 step = 600
239 steps = 144
240 elif type is "30min":
241
242 step = 1800
243 steps = 48
244 elif type is "24h":
245
246 step = 86400
247 steps = 90
248
249 end = int(time.time())
250 end = end - (end % step)
251 start = end - (steps * step)
252
253 return {
254 "type": type,
255 "step": step,
256 "steps": steps,
257 "start": start,
258 "end": end,
259 }
260
261 @classmethod
273
274 @classmethod
283
284 @classmethod
300
301 @classmethod
310
311 @classmethod
314
315 @classmethod
318
319 @classmethod
324
325 @classmethod
332
333 @classmethod
335 if db.engine.url.drivername == "sqlite":
336 return
337
338 status_to_order = """
339 CREATE OR REPLACE FUNCTION status_to_order (x integer)
340 RETURNS integer AS $$ BEGIN
341 RETURN CASE WHEN x = 3 THEN 1
342 WHEN x = 6 THEN 2
343 WHEN x = 7 THEN 3
344 WHEN x = 4 THEN 4
345 WHEN x = 0 THEN 5
346 WHEN x = 1 THEN 6
347 WHEN x = 5 THEN 7
348 WHEN x = 2 THEN 8
349 WHEN x = 8 THEN 9
350 WHEN x = 9 THEN 10
351 ELSE x
352 END; END;
353 $$ LANGUAGE plpgsql;
354 """
355
356 order_to_status = """
357 CREATE OR REPLACE FUNCTION order_to_status (x integer)
358 RETURNS integer AS $$ BEGIN
359 RETURN CASE WHEN x = 1 THEN 3
360 WHEN x = 2 THEN 6
361 WHEN x = 3 THEN 7
362 WHEN x = 4 THEN 4
363 WHEN x = 5 THEN 0
364 WHEN x = 6 THEN 1
365 WHEN x = 7 THEN 5
366 WHEN x = 8 THEN 2
367 WHEN x = 9 THEN 8
368 WHEN x = 10 THEN 9
369 ELSE x
370 END; END;
371 $$ LANGUAGE plpgsql;
372 """
373
374 db.engine.connect()
375 db.engine.execute(status_to_order)
376 db.engine.execute(order_to_status)
377
378 @classmethod
380 query_select = """
381 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
382 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
383 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id
384 FROM build
385 LEFT OUTER JOIN package
386 ON build.package_id = package.id
387 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
388 ON statuses.build_id=build.id
389 LEFT OUTER JOIN copr
390 ON copr.id = build.copr_id
391 LEFT OUTER JOIN copr_dir
392 ON build.copr_dir_id = copr_dir.id
393 LEFT OUTER JOIN "user"
394 ON copr.user_id = "user".id
395 LEFT OUTER JOIN "group"
396 ON copr.group_id = "group".id
397 WHERE build.copr_id = :copr_id
398 AND (:dirname = '' OR :dirname = copr_dir.name)
399 GROUP BY
400 build.id;
401 """
402
403 if db.engine.url.drivername == "sqlite":
404 def sqlite_status_to_order(x):
405 if x == 3:
406 return 1
407 elif x == 6:
408 return 2
409 elif x == 7:
410 return 3
411 elif x == 4:
412 return 4
413 elif x == 0:
414 return 5
415 elif x == 1:
416 return 6
417 elif x == 5:
418 return 7
419 elif x == 2:
420 return 8
421 elif x == 8:
422 return 9
423 elif x == 9:
424 return 10
425 return 1000
426
427 def sqlite_order_to_status(x):
428 if x == 1:
429 return 3
430 elif x == 2:
431 return 6
432 elif x == 3:
433 return 7
434 elif x == 4:
435 return 4
436 elif x == 5:
437 return 0
438 elif x == 6:
439 return 1
440 elif x == 7:
441 return 5
442 elif x == 8:
443 return 2
444 elif x == 9:
445 return 8
446 elif x == 10:
447 return 9
448 return 1000
449
450 conn = db.engine.connect()
451 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
452 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
453 statement = text(query_select)
454 statement.bindparams(bindparam("copr_id", Integer))
455 statement.bindparams(bindparam("dirname", String))
456 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname})
457 else:
458 statement = text(query_select)
459 statement.bindparams(bindparam("copr_id", Integer))
460 statement.bindparams(bindparam("dirname", String))
461 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname})
462
463 return result
464
465 @classmethod
468
469 @classmethod
477
478 @classmethod
481
482 @classmethod
485
486 @classmethod
506
507 @classmethod
523
524 @classmethod
525 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
526 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
527 chroot_names=None, **build_options):
528 """
529 :type user: models.User
530 :type copr: models.Copr
531
532 :type chroot_names: List[str]
533
534 :rtype: models.Build
535 """
536 source_type = helpers.BuildSourceEnum("scm")
537 source_json = json.dumps({"type": scm_type,
538 "clone_url": clone_url,
539 "committish": committish,
540 "subdirectory": subdirectory,
541 "spec": spec,
542 "srpm_build_method": srpm_build_method})
543 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
544
545 @classmethod
546 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template,
547 python_versions, chroot_names=None, **build_options):
565
566 @classmethod
579
580 @classmethod
581 - def create_new_from_custom(cls, user, copr,
582 script, script_chroot=None, script_builddeps=None,
583 script_resultdir=None, chroot_names=None, **kwargs):
584 """
585 :type user: models.User
586 :type copr: models.Copr
587 :type script: str
588 :type script_chroot: str
589 :type script_builddeps: str
590 :type script_resultdir: str
591 :type chroot_names: List[str]
592 :rtype: models.Build
593 """
594 source_type = helpers.BuildSourceEnum("custom")
595 source_dict = {
596 'script': script,
597 'chroot': script_chroot,
598 'builddeps': script_builddeps,
599 'resultdir': script_resultdir,
600 }
601
602 return cls.create_new(user, copr, source_type, json.dumps(source_dict),
603 chroot_names, **kwargs)
604
605 @classmethod
606 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
607 chroot_names=None, **build_options):
608 """
609 :type user: models.User
610 :type copr: models.Copr
611 :param f_uploader(file_path): function which stores data at the given `file_path`
612 :return:
613 """
614 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
615 tmp_name = os.path.basename(tmp)
616 filename = secure_filename(orig_filename)
617 file_path = os.path.join(tmp, filename)
618 f_uploader(file_path)
619
620
621 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
622 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
623 tmp_dir=tmp_name,
624 filename=filename)
625
626
627 source_type = helpers.BuildSourceEnum("upload")
628 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
629 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
630
631 try:
632 build = cls.create_new(user, copr, source_type, source_json,
633 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options)
634 except Exception:
635 shutil.rmtree(tmp)
636 raise
637
638 return build
639
640 @classmethod
641 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
642 git_hashes=None, skip_import=False, background=False, batch=None,
643 srpm_url=None, **build_options):
644 """
645 :type user: models.User
646 :type copr: models.Copr
647 :type chroot_names: List[str]
648 :type source_type: int value from helpers.BuildSourceEnum
649 :type source_json: str in json format
650 :type pkgs: str
651 :type git_hashes: dict
652 :type skip_import: bool
653 :type background: bool
654 :type batch: models.Batch
655 :rtype: models.Build
656 """
657 if chroot_names is None:
658 chroots = [c for c in copr.active_chroots]
659 else:
660 chroots = []
661 for chroot in copr.active_chroots:
662 if chroot.name in chroot_names:
663 chroots.append(chroot)
664
665 build = cls.add(
666 user=user,
667 pkgs=pkgs,
668 copr=copr,
669 chroots=chroots,
670 source_type=source_type,
671 source_json=source_json,
672 enable_net=build_options.get("enable_net", copr.build_enable_net),
673 background=background,
674 git_hashes=git_hashes,
675 skip_import=skip_import,
676 batch=batch,
677 srpm_url=srpm_url,
678 )
679
680 if user.proven:
681 if "timeout" in build_options:
682 build.timeout = build_options["timeout"]
683
684 return build
685
686 @classmethod
687 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
688 repos=None, chroots=None, timeout=None, enable_net=True,
689 git_hashes=None, skip_import=False, background=False, batch=None,
690 srpm_url=None):
691
692 if chroots is None:
693 chroots = []
694
695 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
696 copr, "Can't build while there is an operation in progress: {action}")
697 users_logic.UsersLogic.raise_if_cant_build_in_copr(
698 user, copr,
699 "You don't have permissions to build in this copr.")
700
701 if not repos:
702 repos = copr.repos
703
704
705 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
706 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
707 "with bad characters. Forgot to split?")
708
709
710 if not source_type or not source_json:
711 source_type = helpers.BuildSourceEnum("link")
712 source_json = json.dumps({"url":pkgs})
713
714 if skip_import and srpm_url:
715 chroot_status = StatusEnum("pending")
716 source_status = StatusEnum("succeeded")
717 elif srpm_url:
718 chroot_status = StatusEnum("waiting")
719 source_status = StatusEnum("importing")
720 else:
721 chroot_status = StatusEnum("waiting")
722 source_status = StatusEnum("pending")
723
724 build = models.Build(
725 user=user,
726 pkgs=pkgs,
727 copr=copr,
728 repos=repos,
729 source_type=source_type,
730 source_json=source_json,
731 source_status=source_status,
732 submitted_on=int(time.time()),
733 enable_net=bool(enable_net),
734 is_background=bool(background),
735 batch=batch,
736 srpm_url=srpm_url,
737 )
738
739 if timeout:
740 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
741
742 db.session.add(build)
743
744
745
746 if not chroots:
747 chroots = copr.active_chroots
748
749 for chroot in chroots:
750 git_hash = None
751 if git_hashes:
752 git_hash = git_hashes.get(chroot.name)
753 buildchroot = models.BuildChroot(
754 build=build,
755 status=chroot_status,
756 mock_chroot=chroot,
757 git_hash=git_hash,
758 )
759 db.session.add(buildchroot)
760
761 return build
762
763 @classmethod
764 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None,
765 scm_object_type=None, scm_object_id=None, scm_object_url=None):
766
767 source_dict = package.source_json_dict
768 source_dict.update(source_dict_update)
769 source_json = json.dumps(source_dict)
770
771 if not copr_dir:
772 copr_dir = package.copr.main_dir
773
774 build = models.Build(
775 user=None,
776 pkgs=None,
777 package=package,
778 copr=package.copr,
779 repos=package.copr.repos,
780 source_status=StatusEnum("pending"),
781 source_type=package.source_type,
782 source_json=source_json,
783 submitted_on=int(time.time()),
784 enable_net=package.copr.build_enable_net,
785 timeout=DEFAULT_BUILD_TIMEOUT,
786 copr_dir=copr_dir,
787 update_callback=update_callback,
788 scm_object_type=scm_object_type,
789 scm_object_id=scm_object_id,
790 scm_object_url=scm_object_url,
791 )
792 db.session.add(build)
793
794 chroots = package.copr.active_chroots
795 status = StatusEnum("waiting")
796 for chroot in chroots:
797 buildchroot = models.BuildChroot(
798 build=build,
799 status=status,
800 mock_chroot=chroot,
801 git_hash=None
802 )
803 db.session.add(buildchroot)
804
805 cls.process_update_callback(build)
806 return build
807
808
809 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
810
811 @classmethod
823
824
825 @classmethod
827 """
828 Deletes the locally stored data for build purposes. This is typically
829 uploaded srpm file, uploaded spec file or webhook POST content.
830 """
831
832 data = json.loads(build.source_json)
833 if 'tmp' in data:
834 tmp = data["tmp"]
835 storage_path = app.config["STORAGE_DIR"]
836 try:
837 shutil.rmtree(os.path.join(storage_path, tmp))
838 except:
839 pass
840
841
842 @classmethod
844 """
845 :param build:
846 :param upd_dict:
847 example:
848 {
849 "builds":[
850 {
851 "id": 1,
852 "copr_id": 2,
853 "started_on": 1390866440
854 },
855 {
856 "id": 2,
857 "copr_id": 1,
858 "status": 0,
859 "chroot": "fedora-18-x86_64",
860 "result_dir": "baz",
861 "ended_on": 1390866440
862 }]
863 }
864 """
865 log.info("Updating build {} by: {}".format(build.id, upd_dict))
866
867
868 for attr in ["built_packages", "srpm_url"]:
869 value = upd_dict.get(attr, None)
870 if value:
871 setattr(build, attr, value)
872
873
874 if upd_dict.get("task_id") == build.task_id:
875 build.result_dir = upd_dict.get("result_dir", "")
876
877 if upd_dict.get("status") == StatusEnum("succeeded"):
878 new_status = StatusEnum("importing")
879 else:
880 new_status = upd_dict.get("status")
881
882 build.source_status = new_status
883 if new_status == StatusEnum("failed") or \
884 new_status == StatusEnum("skipped"):
885 for ch in build.build_chroots:
886 ch.status = new_status
887 ch.ended_on = upd_dict.get("ended_on") or time.time()
888 db.session.add(ch)
889
890 if new_status == StatusEnum("failed"):
891 build.fail_type = FailTypeEnum("srpm_build_error")
892
893 cls.process_update_callback(build)
894 db.session.add(build)
895 return
896
897 if "chroot" in upd_dict:
898
899 for build_chroot in build.build_chroots:
900 if build_chroot.name == upd_dict["chroot"]:
901 build_chroot.result_dir = upd_dict.get("result_dir", "")
902
903 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
904 build_chroot.status = upd_dict["status"]
905
906 if upd_dict.get("status") in BuildsLogic.terminal_states:
907 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
908
909 if upd_dict.get("status") == StatusEnum("starting"):
910 build_chroot.started_on = upd_dict.get("started_on") or time.time()
911
912 db.session.add(build_chroot)
913
914
915
916 if (build.module
917 and upd_dict.get("status") == StatusEnum("succeeded")
918 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)):
919 ActionsLogic.send_build_module(build.copr, build.module)
920
921 cls.process_update_callback(build)
922 db.session.add(build)
923
924 @classmethod
939
940 @classmethod
942 headers = {
943 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key'))
944 }
945
946 if build.srpm_url:
947 progress = 50
948 else:
949 progress = 10
950
951 state_table = {
952 'failed': ('failure', 0),
953 'succeeded': ('success', 100),
954 'canceled': ('canceled', 0),
955 'running': ('pending', progress),
956 'pending': ('pending', progress),
957 'skipped': ('error', 0),
958 'starting': ('pending', progress),
959 'importing': ('pending', progress),
960 'forked': ('error', 0),
961 'waiting': ('pending', progress),
962 'unknown': ('error', 0),
963 }
964
965 build_url = os.path.join(
966 app.config['PUBLIC_COPR_BASE_URL'],
967 'coprs', build.copr.full_name.replace('@', 'g/'),
968 'build', str(build.id)
969 )
970
971 data = {
972 'username': 'Copr build',
973 'comment': '#{}'.format(build.id),
974 'url': build_url,
975 'status': state_table[build.state][0],
976 'percent': state_table[build.state][1],
977 'uid': str(build.id),
978 }
979
980 log.info('Sending data to Pagure API: %s', pprint.pformat(data))
981 response = requests.post(api_url, data=data, headers=headers)
982 log.info('Pagure API response: %s', response.text)
983
984 @classmethod
1007
1008 @classmethod
1009 - def delete_build(cls, user, build, send_delete_action=True):
1030
1031 @classmethod
1042
1043 @classmethod
1062
1063 @classmethod
1071
1072 @classmethod
1075
1076 @classmethod
1079
1082 @classmethod
1091
1092 @classmethod
1103
1104 @classmethod
1107
1108 @classmethod
1111
1112 @classmethod
1115
1116 @classmethod
1119
1120 @classmethod
1123
1126 @classmethod
1128 query = """
1129 SELECT
1130 package.id as package_id,
1131 package.name AS package_name,
1132 build.id AS build_id,
1133 build_chroot.status AS build_chroot_status,
1134 build.pkg_version AS build_pkg_version,
1135 mock_chroot.id AS mock_chroot_id,
1136 mock_chroot.os_release AS mock_chroot_os_release,
1137 mock_chroot.os_version AS mock_chroot_os_version,
1138 mock_chroot.arch AS mock_chroot_arch
1139 FROM package
1140 JOIN (SELECT
1141 MAX(build.id) AS max_build_id_for_chroot,
1142 build.package_id AS package_id,
1143 build_chroot.mock_chroot_id AS mock_chroot_id
1144 FROM build
1145 JOIN build_chroot
1146 ON build.id = build_chroot.build_id
1147 WHERE build.copr_id = {copr_id}
1148 AND build_chroot.status != 2
1149 GROUP BY build.package_id,
1150 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
1151 ON package.id = max_build_ids_for_a_chroot.package_id
1152 JOIN build
1153 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
1154 JOIN build_chroot
1155 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
1156 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
1157 JOIN mock_chroot
1158 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
1159 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
1160 """.format(copr_id=copr.id)
1161 rows = db.session.execute(query)
1162 return rows
1163