[2024-11-29 02:18:39,755][ INFO][PID:1835408] Marking build as starting [2024-11-29 02:18:39,841][ INFO][PID:1835408] Trying to allocate VM [2024-11-29 02:19:48,118][ INFO][PID:1835408] Allocated host ResallocHost, ticket_id=6123671, hostname=44.202.24.9 [2024-11-29 02:19:48,121][ INFO][PID:1835408] Allocating ssh connection to builder [2024-11-29 02:19:48,124][ INFO][PID:1835408] Checking that builder machine is OK [2024-11-29 02:19:48,643][ INFO][PID:1835408] Running remote command: copr-builder-ready fedora-40-x86_64 [2024-11-29 02:19:49,180][ INFO][PID:1835408] Red Hat subscription not needed for fedora-40-x86_64 Builder is ready to be used [2024-11-29 02:19:49,181][ INFO][PID:1835408] Filling build.info file with builder info [2024-11-29 02:19:49,205][ INFO][PID:1835408] Sending build state back to frontend: { "builds": [ { "timeout": 172800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "rezso", "project_name": "ML", "project_dirname": "ML", "submitter": "rezso", "ended_on": null, "started_on": 1732846789.1993277, "submitted_on": null, "status": 3, "chroot": "fedora-40-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8324975-fedora-40-x86_64", "build_id": 8324975, "package_name": "cutlass", "package_version": "3.6.0-20241009.0.gitcc3c29a8.cu12_6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rezso/ML/cutlass", "git_hash": "5a03c12ed7a917e73aa37a7de99f8da105a49ac0", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rezso/ML--rezso", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/ML/fedora-40-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/CUDA/fedora-40-x86_64/", "id": "copr_rezso_CUDA", "name": "Additional repo copr_rezso_CUDA" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rezso/ML", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rezso/ML", "result_dir": "08324975-cutlass", "built_packages": "", "id": 8324975, "mockchain_macros": { "copr_username": "rezso", "copr_projectname": "ML", "vendor": "Fedora Project COPR (rezso/ML)" } } ] } [2024-11-29 02:19:49,760][ INFO][PID:1835408] Sending fedora-messaging bus message in build.start [2024-11-29 02:19:53,125][ INFO][PID:1835408] Sending fedora-messaging bus message in chroot.start [2024-11-29 02:19:53,253][ INFO][PID:1835408] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8324975-fedora-40-x86_64 --chroot fedora-40-x86_64 --detached [2024-11-29 02:19:53,743][ INFO][PID:1835408] Downloading the builder-live.log file, attempt 1 [2024-11-29 02:19:53,748][ INFO][PID:1835408] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.202.24.9 copr-rpmbuild-log [2024-11-29 02:19:58,798][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 02:29:58,944][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 02:39:59,016][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 02:49:59,087][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 02:59:59,158][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 03:09:59,241][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 03:19:59,324][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 03:29:59,498][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 03:39:59,588][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 03:49:59,672][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 03:59:59,753][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 04:09:59,959][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 04:20:00,303][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 04:30:00,417][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 04:40:00,503][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 04:50:00,591][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 05:00:00,677][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 05:10:00,811][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 05:20:00,902][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 05:30:01,003][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 05:40:01,093][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 05:50:01,180][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 06:00:01,267][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 06:10:01,375][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 06:20:01,488][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 06:30:01,572][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 06:40:01,657][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 06:50:01,743][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 07:00:01,860][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 07:10:02,002][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 07:20:02,131][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 07:30:02,231][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 07:40:02,321][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 07:50:02,410][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 08:00:02,497][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 08:10:02,620][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 08:20:02,814][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 08:30:02,903][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 08:40:02,990][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 08:50:03,124][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 09:00:03,210][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 09:10:03,325][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 09:20:03,484][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 09:30:03,576][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 09:40:03,667][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 09:50:03,774][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 10:00:04,020][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 10:10:04,166][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 10:20:04,374][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 10:30:04,523][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 10:40:04,662][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 10:50:04,792][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 11:00:04,892][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 11:10:04,988][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 11:20:05,119][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 11:30:05,222][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 11:40:05,299][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 11:50:05,399][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 12:00:05,489][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 12:10:05,606][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 12:20:05,752][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 12:30:05,844][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 12:34:28,439][ ERROR][PID:1835408] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 12:34:38,455][WARNING][PID:1835408] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-29 12:34:48,460][ INFO][PID:1835408] Downloading the builder-live.log file, attempt 2 [2024-11-29 12:34:48,534][ INFO][PID:1835408] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.202.24.9 copr-rpmbuild-log [2024-11-29 12:34:48,537][ ERROR][PID:1835408] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 12:34:58,537][WARNING][PID:1835408] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-29 12:35:08,538][ INFO][PID:1835408] Downloading the builder-live.log file, attempt 3 [2024-11-29 12:35:08,540][ INFO][PID:1835408] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.202.24.9 copr-rpmbuild-log [2024-11-29 12:36:08,608][ ERROR][PID:1835408] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 12:36:18,610][WARNING][PID:1835408] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-29 12:36:28,613][ INFO][PID:1835408] Downloading the builder-live.log file, attempt 4 [2024-11-29 12:36:28,617][ INFO][PID:1835408] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.202.24.9 copr-rpmbuild-log [2024-11-29 12:37:28,697][ ERROR][PID:1835408] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 12:37:38,749][ ERROR][PID:1835408] Giving up for unstable SSH, failures: 1732883678.4523973, 1732883698.5376394, 1732883778.6106036, 1732883858.70663 [2024-11-29 12:37:38,778][ ERROR][PID:1835408] Re-try request for task on 'ResallocHost, ticket_id=6123671, hostname=44.202.24.9': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-29 12:37:38,782][ INFO][PID:1835408] Releasing VM back to pool [2024-11-29 12:37:38,803][ INFO][PID:1835408] Retry #1 (on other host) [2024-11-29 12:37:38,814][ INFO][PID:1835408] Trying to allocate VM [2024-11-29 12:37:41,935][ INFO][PID:1835408] Allocated host ResallocHost, ticket_id=6129183, hostname=2620:52:3:1:dead:beef:cafe:c1d5 [2024-11-29 12:37:41,938][ INFO][PID:1835408] Allocating ssh connection to builder [2024-11-29 12:37:41,942][ INFO][PID:1835408] Checking that builder machine is OK [2024-11-29 12:37:42,500][ INFO][PID:1835408] Running remote command: copr-builder-ready fedora-40-x86_64 [2024-11-29 12:37:42,829][ INFO][PID:1835408] Red Hat subscription not needed for fedora-40-x86_64 Builder is ready to be used [2024-11-29 12:37:42,830][ INFO][PID:1835408] Filling build.info file with builder info [2024-11-29 12:37:42,842][ INFO][PID:1835408] Not re-notifying FE and msg buses for the new host. [2024-11-29 12:37:42,849][ INFO][PID:1835408] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8324975-fedora-40-x86_64 --chroot fedora-40-x86_64 --detached [2024-11-29 12:37:43,377][ INFO][PID:1835408] Downloading the builder-live.log file, attempt 1 [2024-11-29 12:37:43,378][ INFO][PID:1835408] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c1d5 copr-rpmbuild-log [2024-11-29 12:40:08,409][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 12:50:08,487][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 13:00:08,561][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 13:10:08,655][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 13:20:08,744][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 13:30:08,845][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 13:40:08,922][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 13:50:09,063][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 14:00:09,169][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 14:10:09,258][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 14:20:09,374][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 14:30:09,477][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 14:40:09,564][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 14:50:09,684][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 15:00:09,791][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 15:10:09,885][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 15:20:09,989][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 15:30:10,080][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 15:40:10,158][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 15:50:10,234][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 16:00:10,308][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 16:10:10,405][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 16:20:10,518][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 16:30:10,700][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 16:40:10,858][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 16:50:11,043][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 17:00:11,128][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 17:10:11,271][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 17:20:11,441][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 17:30:11,516][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 17:40:11,589][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 17:50:11,666][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 18:00:11,745][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 18:10:11,880][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 18:20:11,969][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 18:30:12,045][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 18:40:12,116][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 18:50:12,220][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 19:00:12,328][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 19:10:12,427][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 19:20:12,512][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 19:30:12,617][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 19:40:12,702][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 19:50:12,788][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 20:00:12,863][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 20:10:12,987][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 20:20:13,252][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 20:30:13,447][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 20:40:13,557][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 20:50:13,664][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 21:00:13,753][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 21:10:13,863][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 21:20:13,972][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 21:30:14,077][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 21:40:14,188][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 21:50:14,301][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 22:00:14,394][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 22:10:14,499][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 22:20:14,638][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 22:30:14,756][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 22:40:14,858][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 22:50:14,940][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 23:00:15,047][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 23:10:15,170][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 23:20:15,280][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 23:30:15,363][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 23:40:15,439][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-29 23:50:15,514][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 00:00:15,655][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 00:10:15,841][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 00:20:15,995][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 00:30:16,143][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 00:40:16,269][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 00:50:16,372][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 01:00:16,458][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 01:10:16,564][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 01:20:16,666][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 01:30:16,738][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 01:40:16,816][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 01:50:16,940][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 02:00:17,020][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 02:10:17,144][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 02:20:17,564][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 02:30:17,663][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 02:40:17,743][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 02:50:17,819][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 03:00:17,892][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 03:10:18,005][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 03:20:18,213][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 03:30:18,303][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 03:40:18,391][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 03:50:18,475][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 04:00:18,558][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 04:10:18,732][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 04:20:18,929][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 04:30:19,022][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 04:40:19,116][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 04:50:19,204][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 05:00:19,294][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 05:10:19,390][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 05:20:19,501][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 05:30:19,588][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 05:40:19,679][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 05:50:19,777][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 06:00:19,864][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 06:10:19,976][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 06:20:20,101][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 06:30:20,189][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 06:40:20,278][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 06:50:20,372][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 07:00:20,506][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 07:10:20,653][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 07:20:20,751][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 07:30:20,832][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 07:40:20,917][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 07:50:21,003][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 08:00:21,090][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 08:10:21,188][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 08:20:21,300][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 08:30:21,386][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 08:40:21,476][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 08:50:21,567][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 09:00:21,639][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 09:10:21,731][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 09:20:21,815][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 09:30:21,897][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 09:40:21,988][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 09:50:22,076][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 10:00:22,163][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 10:10:22,247][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 10:20:22,352][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 10:30:22,427][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 10:40:22,527][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 10:50:22,599][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 11:00:22,671][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 11:10:22,758][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 11:20:22,838][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 11:30:22,906][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 11:40:22,979][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 11:50:23,051][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 12:00:23,130][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 12:10:23,209][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 12:20:23,290][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 12:30:23,370][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 12:40:23,449][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 12:50:23,520][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 13:00:23,594][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 13:10:23,682][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 13:20:23,767][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 13:30:23,843][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 13:40:23,917][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 13:50:24,000][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 14:00:24,077][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 14:10:24,178][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 14:20:24,259][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 14:30:24,337][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 14:40:24,423][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 14:50:24,505][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 15:00:24,590][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 15:10:24,689][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 15:20:24,794][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 15:30:24,866][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 15:40:24,955][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 15:50:25,025][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 16:00:25,104][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 16:10:25,214][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 16:20:25,360][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 16:30:25,437][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 16:40:25,522][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 16:50:25,610][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 17:00:25,686][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 17:10:25,768][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 17:20:25,843][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 17:30:25,922][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 17:40:26,012][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 17:50:26,087][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 18:00:26,168][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 18:10:26,288][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 18:20:26,366][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 18:30:26,483][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 18:40:26,555][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 18:50:26,627][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 19:00:26,698][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 19:10:26,781][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 19:20:26,897][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 19:30:26,980][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 19:40:27,065][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 19:50:27,138][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 20:00:27,218][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 20:10:27,291][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 20:20:27,364][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 20:30:27,442][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 20:40:27,514][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 20:50:27,586][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 21:00:27,672][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 21:10:27,770][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 21:20:27,857][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 21:30:27,932][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 21:40:28,018][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 21:50:28,105][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 22:00:28,176][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 22:10:28,266][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 22:20:28,348][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 22:30:28,419][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 22:40:28,492][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 22:50:28,573][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 23:00:28,648][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 23:10:28,758][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 23:20:28,845][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 23:30:28,922][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 23:40:29,007][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-11-30 23:50:29,086][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 00:00:29,163][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 00:10:29,258][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 00:20:29,375][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 00:30:29,460][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 00:40:29,542][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 00:50:29,624][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 01:00:29,696][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 01:10:29,785][ INFO][PID:1835408] Periodic builder liveness probe: alive [2024-12-01 01:12:28,184][ INFO][PID:1835408] Downloading results from builder [2024-12-01 01:12:28,210][ INFO][PID:1835408] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c1d5]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass started [2024-12-01 01:12:28,231][ INFO][PID:1835408] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c1d5]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/ &> /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/build-08324975.rsync.log [2024-12-01 01:12:32,517][ INFO][PID:1835408] rsyncing finished. [2024-12-01 01:12:32,519][ INFO][PID:1835408] Releasing VM back to pool [2024-12-01 01:12:32,533][ INFO][PID:1835408] Searching for 'success' file in resultdir [2024-12-01 01:12:32,545][ INFO][PID:1835408] Going to sign pkgs from source: 8324975-fedora-40-x86_64 in chroot: /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64 [2024-12-01 01:12:32,557][ INFO][PID:1835408] Calling '/bin/sign -u rezso#ML@copr.fedorahosted.org -p' (attempt #1) [2024-12-01 01:12:32,889][ INFO][PID:1835408] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.src.rpm' (attempt #1) [2024-12-01 01:12:33,760][ INFO][PID:1835408] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.src.rpm [2024-12-01 01:12:33,763][ INFO][PID:1835408] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-devel-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.x86_64.rpm' (attempt #1) [2024-12-01 01:12:34,682][ INFO][PID:1835408] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-devel-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.x86_64.rpm [2024-12-01 01:12:34,684][ INFO][PID:1835408] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-static-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.x86_64.rpm' (attempt #1) [2024-12-01 01:12:38,767][ INFO][PID:1835408] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-static-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.x86_64.rpm [2024-12-01 01:12:38,767][ INFO][PID:1835408] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.x86_64.rpm' (attempt #1) [2024-12-01 01:12:42,726][ INFO][PID:1835408] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc40.x86_64.rpm [2024-12-01 01:12:42,727][ INFO][PID:1835408] Sign done [2024-12-01 01:12:42,732][ INFO][PID:1835408] Incremental createrepo run, adding 08324975-cutlass into https://download.copr.fedorainfracloud.org/results/rezso/ML/fedora-40-x86_64, (auto-create-repo=True) [2024-12-01 01:12:42,757][ INFO][PID:1835408] Running command 'copr-repo --batched /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64 --add 08324975-cutlass --no-appstream-metadata' as PID 2400034 [2024-12-01 01:12:44,300][ INFO][PID:1835408] Finished after 1 seconds with exit code 0 (copr-repo --batched /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64 --add 08324975-cutlass --no-appstream-metadata) [2024-12-01 01:12:44,319][ INFO][PID:1835408] Getting build details [2024-12-01 01:12:44,321][ INFO][PID:1835408] Listing built binary packages in /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass [2024-12-01 01:12:44,322][ INFO][PID:1835408] Built packages: ['cutlass-static 3.6.0', 'cutlass 3.6.0', 'cutlass-devel 3.6.0'] [2024-12-01 01:12:44,324][ INFO][PID:1835408] build details: {'built_packages': 'cutlass-static 3.6.0\ncutlass 3.6.0\ncutlass-devel 3.6.0'} [2024-12-01 01:12:44,326][ INFO][PID:1835408] Retrieving pubkey [2024-12-01 01:12:44,326][ INFO][PID:1835408] Calling '/bin/sign -u rezso#ML@copr.fedorahosted.org -p' (attempt #1) [2024-12-01 01:12:44,654][ INFO][PID:1835408] Added pubkey for user rezso project ML into: /var/lib/copr/public_html/results/rezso/ML/pubkey.gpg [2024-12-01 01:12:44,661][ INFO][PID:1835408] Finished build: id=8324975 failed=False timeout=172800 destdir=/var/lib/copr/public_html/results/rezso/ML chroot=fedora-40-x86_64 [2024-12-01 01:12:44,763][ INFO][PID:1835408] Worker succeeded build, took 131701.92218661308 [2024-12-01 01:12:44,780][ INFO][PID:1835408] Sending build state back to frontend: { "builds": [ { "timeout": 172800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "rezso", "project_name": "ML", "project_dirname": "ML", "submitter": "rezso", "ended_on": 1733015564.7618606, "started_on": 1732883862.839674, "submitted_on": null, "status": 1, "chroot": "fedora-40-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8324975-fedora-40-x86_64", "build_id": 8324975, "package_name": "cutlass", "package_version": "3.6.0-20241009.0.gitcc3c29a8.cu12_6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rezso/ML/cutlass", "git_hash": "5a03c12ed7a917e73aa37a7de99f8da105a49ac0", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rezso/ML--rezso", "results": { "packages": [ { "name": "cutlass", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc40", "arch": "src" }, { "name": "cutlass-static", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc40", "arch": "x86_64" }, { "name": "cutlass", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc40", "arch": "x86_64" }, { "name": "cutlass-devel", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc40", "arch": "x86_64" } ] }, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/ML/fedora-40-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/CUDA/fedora-40-x86_64/", "id": "copr_rezso_CUDA", "name": "Additional repo copr_rezso_CUDA" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rezso/ML", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rezso/ML", "result_dir": "08324975-cutlass", "built_packages": "cutlass-static 3.6.0\ncutlass 3.6.0\ncutlass-devel 3.6.0", "id": 8324975, "mockchain_macros": { "copr_username": "rezso", "copr_projectname": "ML", "vendor": "Fedora Project COPR (rezso/ML)" } } ] } [2024-12-01 01:12:45,849][ INFO][PID:1835408] Sending fedora-messaging bus message in build.end [2024-12-01 01:12:46,177][ INFO][PID:1835408] Compressing /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/builder-live.log by gzip [2024-12-01 01:12:46,180][ INFO][PID:1835408] Running command 'gzip /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/builder-live.log' as PID 2400375 [2024-12-01 01:12:46,715][ INFO][PID:1835408] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/builder-live.log) [2024-12-01 01:12:46,716][ INFO][PID:1835408] Compressing /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/backend.log by gzip [2024-12-01 01:12:46,717][ INFO][PID:1835408] Running command 'gzip /var/lib/copr/public_html/results/rezso/ML/fedora-40-x86_64/08324975-cutlass/backend.log' as PID 2400415