[2024-11-29 02:18:39,786][ INFO][PID:1835404] Marking build as starting [2024-11-29 02:18:39,863][ INFO][PID:1835404] Trying to allocate VM [2024-11-29 02:19:48,220][ INFO][PID:1835404] Allocated host ResallocHost, ticket_id=6123672, hostname=44.213.64.13 [2024-11-29 02:19:48,226][ INFO][PID:1835404] Allocating ssh connection to builder [2024-11-29 02:19:48,251][ INFO][PID:1835404] Checking that builder machine is OK [2024-11-29 02:19:48,897][ INFO][PID:1835404] Running remote command: copr-builder-ready fedora-39-x86_64 [2024-11-29 02:19:49,273][ INFO][PID:1835404] Red Hat subscription not needed for fedora-39-x86_64 Builder is ready to be used [2024-11-29 02:19:49,274][ INFO][PID:1835404] Filling build.info file with builder info [2024-11-29 02:19:49,281][ INFO][PID:1835404] Sending build state back to frontend: { "builds": [ { "timeout": 172800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "rezso", "project_name": "ML", "project_dirname": "ML", "submitter": "rezso", "ended_on": null, "started_on": 1732846789.2811835, "submitted_on": null, "status": 3, "chroot": "fedora-39-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8324975-fedora-39-x86_64", "build_id": 8324975, "package_name": "cutlass", "package_version": "3.6.0-20241009.0.gitcc3c29a8.cu12_6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rezso/ML/cutlass", "git_hash": "67f394ffca7a2f91abfbee4b994b11590d3f91d8", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rezso/ML--rezso", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/ML/fedora-39-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/CUDA/fedora-39-x86_64/", "id": "copr_rezso_CUDA", "name": "Additional repo copr_rezso_CUDA" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rezso/ML", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rezso/ML", "result_dir": "08324975-cutlass", "built_packages": "", "id": 8324975, "mockchain_macros": { "copr_username": "rezso", "copr_projectname": "ML", "vendor": "Fedora Project COPR (rezso/ML)" } } ] } [2024-11-29 02:19:49,744][ INFO][PID:1835404] Sending fedora-messaging bus message in build.start [2024-11-29 02:19:53,368][ INFO][PID:1835404] Sending fedora-messaging bus message in chroot.start [2024-11-29 02:19:53,498][ INFO][PID:1835404] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8324975-fedora-39-x86_64 --chroot fedora-39-x86_64 --detached [2024-11-29 02:19:53,994][ INFO][PID:1835404] Downloading the builder-live.log file, attempt 1 [2024-11-29 02:19:54,004][ INFO][PID:1835404] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.213.64.13 copr-rpmbuild-log [2024-11-29 02:19:59,042][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 02:29:59,184][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 02:39:59,256][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 02:49:59,328][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 02:59:59,403][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 03:09:59,485][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 03:19:59,564][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 03:29:59,743][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 03:39:59,831][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 03:49:59,916][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 03:59:59,997][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 04:10:00,178][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 04:20:00,475][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 04:30:00,574][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 04:40:00,657][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 04:50:00,748][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 05:00:00,835][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 05:10:00,958][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 05:20:01,095][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 05:30:01,185][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 05:40:01,275][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 05:50:01,368][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 06:00:01,451][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 06:10:01,561][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 06:20:01,675][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 06:30:01,758][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 06:40:01,843][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 06:50:01,934][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 07:00:02,050][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 07:10:02,209][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 07:20:02,326][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 07:30:02,423][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 07:40:02,515][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 07:50:02,605][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 08:00:02,694][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 08:10:02,799][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 08:20:02,983][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 08:30:03,073][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 08:40:03,156][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 08:50:03,260][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 09:00:03,349][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 09:10:03,452][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 09:20:03,597][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 09:30:03,694][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 09:40:03,780][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 09:50:03,882][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 10:00:04,137][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 10:10:04,289][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 10:20:04,482][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 10:30:04,639][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 10:40:04,761][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 10:50:04,879][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 11:00:04,979][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 11:10:05,079][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 11:20:05,209][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 11:30:05,327][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 11:40:05,409][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 11:50:05,522][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 12:00:05,616][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 12:10:05,725][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 12:20:05,850][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 12:30:05,947][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 12:40:06,041][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 12:50:06,124][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:00:06,198][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:10:06,314][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:20:06,414][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:30:06,509][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:40:06,595][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:50:06,724][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 13:55:03,467][ ERROR][PID:1835404] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 13:55:13,491][WARNING][PID:1835404] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-29 13:55:23,513][ INFO][PID:1835404] Downloading the builder-live.log file, attempt 2 [2024-11-29 13:55:23,656][ INFO][PID:1835404] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.213.64.13 copr-rpmbuild-log [2024-11-29 13:56:23,715][ ERROR][PID:1835404] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 13:56:33,716][WARNING][PID:1835404] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-29 13:56:43,717][ INFO][PID:1835404] Downloading the builder-live.log file, attempt 3 [2024-11-29 13:56:43,720][ INFO][PID:1835404] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.213.64.13 copr-rpmbuild-log [2024-11-29 13:57:43,787][ ERROR][PID:1835404] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 13:57:53,788][WARNING][PID:1835404] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-29 13:58:03,789][ INFO][PID:1835404] Downloading the builder-live.log file, attempt 4 [2024-11-29 13:58:03,791][ INFO][PID:1835404] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.213.64.13 copr-rpmbuild-log [2024-11-29 13:59:03,857][ ERROR][PID:1835404] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-29 13:59:13,888][ ERROR][PID:1835404] Giving up for unstable SSH, failures: 1732888513.488493, 1732888593.7169027, 1732888673.7888167, 1732888753.85836 [2024-11-29 13:59:13,909][ ERROR][PID:1835404] Re-try request for task on 'ResallocHost, ticket_id=6123672, hostname=44.213.64.13': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-29 13:59:13,914][ INFO][PID:1835404] Releasing VM back to pool [2024-11-29 13:59:13,952][ INFO][PID:1835404] Retry #1 (on other host) [2024-11-29 13:59:13,958][ INFO][PID:1835404] Trying to allocate VM [2024-11-29 13:59:17,056][ INFO][PID:1835404] Allocated host ResallocHost, ticket_id=6130039, hostname=2620:52:3:1:dead:beef:cafe:c1d3 [2024-11-29 13:59:17,059][ INFO][PID:1835404] Allocating ssh connection to builder [2024-11-29 13:59:17,062][ INFO][PID:1835404] Checking that builder machine is OK [2024-11-29 13:59:17,662][ INFO][PID:1835404] Running remote command: copr-builder-ready fedora-39-x86_64 [2024-11-29 13:59:17,985][ INFO][PID:1835404] Red Hat subscription not needed for fedora-39-x86_64 Builder is ready to be used [2024-11-29 13:59:17,987][ INFO][PID:1835404] Filling build.info file with builder info [2024-11-29 13:59:17,999][ INFO][PID:1835404] Not re-notifying FE and msg buses for the new host. [2024-11-29 13:59:18,010][ INFO][PID:1835404] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8324975-fedora-39-x86_64 --chroot fedora-39-x86_64 --detached [2024-11-29 13:59:18,546][ INFO][PID:1835404] Downloading the builder-live.log file, attempt 1 [2024-11-29 13:59:18,547][ INFO][PID:1835404] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c1d3 copr-rpmbuild-log [2024-11-29 14:00:08,558][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 14:10:08,667][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 14:20:08,775][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 14:30:08,889][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 14:40:08,974][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 14:50:09,078][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 15:00:09,167][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 15:10:09,261][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 15:20:09,360][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 15:30:09,450][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 15:40:09,529][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 15:50:09,602][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 16:00:09,679][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 16:10:09,778][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 16:20:09,887][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 16:30:10,088][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 16:40:10,234][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 16:50:10,448][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 17:00:10,534][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 17:10:10,701][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 17:20:10,849][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 17:30:10,924][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 17:40:10,996][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 17:50:11,072][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 18:00:11,153][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 18:10:11,274][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 18:20:11,402][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 18:30:11,476][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 18:40:11,546][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 18:50:11,619][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 19:00:11,724][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 19:10:11,860][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 19:20:11,951][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 19:30:12,105][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 19:40:12,190][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 19:50:12,275][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 20:00:12,350][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 20:10:12,450][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 20:20:12,669][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 20:30:12,910][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 20:40:13,011][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 20:50:13,130][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 21:00:13,220][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 21:10:13,336][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 21:20:13,453][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 21:30:13,554][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 21:40:13,676][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 21:50:13,805][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 22:00:13,933][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 22:10:14,043][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 22:20:14,197][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 22:30:14,316][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 22:40:14,427][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 22:50:14,514][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 23:00:14,591][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 23:10:14,703][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 23:20:14,828][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 23:30:14,914][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 23:40:14,992][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-29 23:50:15,065][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 00:00:15,210][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 00:10:15,386][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 00:20:15,537][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 00:30:15,677][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 00:40:15,774][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 00:50:15,862][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 01:00:15,936][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 01:10:16,037][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 01:20:16,152][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 01:30:16,228][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 01:40:16,308][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 01:50:16,474][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 02:00:16,553][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 02:10:16,663][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 02:20:17,132][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 02:30:17,232][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 02:40:17,308][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 02:50:17,385][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 03:00:17,458][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 03:10:17,536][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 03:20:17,785][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 03:30:17,879][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 03:40:17,966][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 03:50:18,050][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 04:00:18,141][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 04:10:18,309][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 04:20:18,469][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 04:30:18,576][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 04:40:18,669][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 04:50:18,757][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 05:00:18,848][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 05:10:18,965][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 05:20:19,070][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 05:30:19,157][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 05:40:19,250][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 05:50:19,346][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 06:00:19,439][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 06:10:19,545][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 06:20:19,669][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 06:30:19,753][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 06:40:19,840][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 06:50:19,927][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 07:00:20,060][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 07:10:20,198][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 07:20:20,308][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 07:30:20,396][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 07:40:20,518][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 07:50:20,604][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 08:00:20,699][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 08:10:20,797][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 08:20:20,914][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 08:30:21,000][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 08:40:21,092][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 08:50:21,184][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 09:00:21,257][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 09:10:21,349][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 09:20:21,433][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 09:30:21,515][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 09:40:21,603][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 09:50:21,688][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 10:00:21,772][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 10:10:21,849][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 10:20:21,944][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 10:30:22,022][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 10:40:22,095][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 10:50:22,171][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 11:00:22,242][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 11:10:22,334][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 11:20:22,415][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 11:30:22,489][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 11:40:22,562][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 11:50:22,636][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 12:00:22,715][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 12:10:22,793][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 12:20:22,870][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 12:30:22,949][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 12:40:23,023][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 12:50:23,091][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 13:00:23,163][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 13:10:23,254][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 13:20:23,336][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 13:30:23,409][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 13:40:23,484][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 13:50:23,569][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 14:00:23,642][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 14:10:23,732][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 14:20:23,815][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 14:30:23,894][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 14:40:24,002][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 14:50:24,101][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 15:00:24,184][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 15:10:24,326][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 15:20:24,417][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 15:30:24,489][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 15:40:24,567][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 15:50:24,643][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 16:00:24,722][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 16:10:24,873][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 16:20:25,027][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 16:30:25,107][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 16:40:25,190][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 16:50:25,278][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 17:00:25,353][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 17:10:25,438][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 17:20:25,513][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 17:30:25,590][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 17:40:25,668][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 17:50:25,742][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 18:00:25,823][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 18:10:25,937][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 18:20:26,017][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 18:30:26,094][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 18:40:26,166][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 18:50:26,240][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 19:00:26,311][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 19:10:26,390][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 19:20:26,481][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 19:30:26,562][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 19:40:26,648][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 19:50:26,720][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 20:00:26,799][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 20:10:26,871][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 20:20:26,944][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 20:30:27,017][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 20:40:27,087][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 20:50:27,161][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 21:00:27,251][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 21:10:27,337][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 21:20:27,427][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 21:30:27,504][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 21:40:27,592][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 21:50:27,681][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 22:00:27,755][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 22:10:27,853][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 22:20:27,938][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 22:30:28,010][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 22:40:28,087][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 22:50:28,177][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 23:00:28,252][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 23:10:28,331][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 23:20:28,411][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 23:30:28,485][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 23:40:28,569][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-11-30 23:50:28,652][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 00:00:28,730][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 00:10:28,823][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 00:20:28,911][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 00:30:28,993][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 00:40:29,078][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 00:50:29,154][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 01:00:29,226][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 01:10:29,311][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 01:20:29,414][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 01:30:29,560][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 01:40:29,644][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 01:50:29,774][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 02:00:29,897][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 02:10:30,116][ INFO][PID:1835404] Periodic builder liveness probe: alive [2024-12-01 02:12:27,220][ INFO][PID:1835404] Downloading results from builder [2024-12-01 02:12:27,251][ INFO][PID:1835404] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c1d3]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass started [2024-12-01 02:12:27,298][ INFO][PID:1835404] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c1d3]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/ &> /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/build-08324975.rsync.log [2024-12-01 02:12:32,940][ INFO][PID:1835404] rsyncing finished. [2024-12-01 02:12:32,945][ INFO][PID:1835404] Releasing VM back to pool [2024-12-01 02:12:32,964][ INFO][PID:1835404] Searching for 'success' file in resultdir [2024-12-01 02:12:32,977][ INFO][PID:1835404] Going to sign pkgs from source: 8324975-fedora-39-x86_64 in chroot: /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64 [2024-12-01 02:12:32,992][ INFO][PID:1835404] Calling '/bin/sign -u rezso#ML@copr.fedorahosted.org -p' (attempt #1) [2024-12-01 02:12:33,331][ INFO][PID:1835404] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-devel-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.x86_64.rpm' (attempt #1) [2024-12-01 02:12:34,218][ INFO][PID:1835404] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-devel-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.x86_64.rpm [2024-12-01 02:12:34,219][ INFO][PID:1835404] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.x86_64.rpm' (attempt #1) [2024-12-01 02:12:38,836][ INFO][PID:1835404] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.x86_64.rpm [2024-12-01 02:12:38,839][ INFO][PID:1835404] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-static-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.x86_64.rpm' (attempt #1) [2024-12-01 02:12:43,006][ INFO][PID:1835404] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-static-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.x86_64.rpm [2024-12-01 02:12:43,007][ INFO][PID:1835404] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.src.rpm' (attempt #1) [2024-12-01 02:12:43,755][ INFO][PID:1835404] signed rpm: /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/cutlass-3.6.0-20241009.0.gitcc3c29a8.cu12_6.fc39.src.rpm [2024-12-01 02:12:43,757][ INFO][PID:1835404] Sign done [2024-12-01 02:12:43,763][ INFO][PID:1835404] Incremental createrepo run, adding 08324975-cutlass into https://download.copr.fedorainfracloud.org/results/rezso/ML/fedora-39-x86_64, (auto-create-repo=True) [2024-12-01 02:12:43,797][ INFO][PID:1835404] Running command 'copr-repo --batched /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64 --add 08324975-cutlass --no-appstream-metadata' as PID 2743005 [2024-12-01 02:12:45,760][ INFO][PID:1835404] Finished after 1 seconds with exit code 0 (copr-repo --batched /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64 --add 08324975-cutlass --no-appstream-metadata) [2024-12-01 02:12:45,778][ INFO][PID:1835404] Getting build details [2024-12-01 02:12:45,780][ INFO][PID:1835404] Listing built binary packages in /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass [2024-12-01 02:12:45,781][ INFO][PID:1835404] Built packages: ['cutlass-devel 3.6.0', 'cutlass 3.6.0', 'cutlass-static 3.6.0'] [2024-12-01 02:12:45,784][ INFO][PID:1835404] build details: {'built_packages': 'cutlass-devel 3.6.0\ncutlass 3.6.0\ncutlass-static 3.6.0'} [2024-12-01 02:12:45,787][ INFO][PID:1835404] Retrieving pubkey [2024-12-01 02:12:45,787][ INFO][PID:1835404] Calling '/bin/sign -u rezso#ML@copr.fedorahosted.org -p' (attempt #1) [2024-12-01 02:12:45,985][ INFO][PID:1835404] Added pubkey for user rezso project ML into: /var/lib/copr/public_html/results/rezso/ML/pubkey.gpg [2024-12-01 02:12:45,995][ INFO][PID:1835404] Finished build: id=8324975 failed=False timeout=172800 destdir=/var/lib/copr/public_html/results/rezso/ML chroot=fedora-39-x86_64 [2024-12-01 02:12:46,090][ INFO][PID:1835404] Worker succeeded build, took 130408.0951731205 [2024-12-01 02:12:46,104][ INFO][PID:1835404] Sending build state back to frontend: { "builds": [ { "timeout": 172800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "rezso", "project_name": "ML", "project_dirname": "ML", "submitter": "rezso", "ended_on": 1733019166.0896704, "started_on": 1732888757.9944973, "submitted_on": null, "status": 1, "chroot": "fedora-39-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8324975-fedora-39-x86_64", "build_id": 8324975, "package_name": "cutlass", "package_version": "3.6.0-20241009.0.gitcc3c29a8.cu12_6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rezso/ML/cutlass", "git_hash": "67f394ffca7a2f91abfbee4b994b11590d3f91d8", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rezso/ML--rezso", "results": { "packages": [ { "name": "cutlass-devel", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc39", "arch": "x86_64" }, { "name": "cutlass", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc39", "arch": "x86_64" }, { "name": "cutlass-static", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc39", "arch": "x86_64" }, { "name": "cutlass", "epoch": null, "version": "3.6.0", "release": "20241009.0.gitcc3c29a8.cu12_6.fc39", "arch": "src" } ] }, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/ML/fedora-39-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/CUDA/fedora-39-x86_64/", "id": "copr_rezso_CUDA", "name": "Additional repo copr_rezso_CUDA" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rezso/ML", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rezso/ML", "result_dir": "08324975-cutlass", "built_packages": "cutlass-devel 3.6.0\ncutlass 3.6.0\ncutlass-static 3.6.0", "id": 8324975, "mockchain_macros": { "copr_username": "rezso", "copr_projectname": "ML", "vendor": "Fedora Project COPR (rezso/ML)" } } ] } [2024-12-01 02:12:47,343][ INFO][PID:1835404] Sending fedora-messaging bus message in build.end [2024-12-01 02:12:47,807][ INFO][PID:1835404] Compressing /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/builder-live.log by gzip [2024-12-01 02:12:47,816][ INFO][PID:1835404] Running command 'gzip /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/builder-live.log' as PID 2743400 [2024-12-01 02:12:48,523][ INFO][PID:1835404] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/builder-live.log) [2024-12-01 02:12:48,525][ INFO][PID:1835404] Compressing /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/backend.log by gzip [2024-12-01 02:12:48,531][ INFO][PID:1835404] Running command 'gzip /var/lib/copr/public_html/results/rezso/ML/fedora-39-x86_64/08324975-cutlass/backend.log' as PID 2743452