[2024-11-21 14:01:16,208][ INFO][PID:3501875] Marking build as starting [2024-11-21 14:01:16,556][ INFO][PID:3501875] Trying to allocate VM [2024-11-21 14:02:04,868][ INFO][PID:3501875] Allocated host ResallocHost, ticket_id=5999032, hostname=3.236.151.126 [2024-11-21 14:02:04,870][ INFO][PID:3501875] Allocating ssh connection to builder [2024-11-21 14:02:04,871][ INFO][PID:3501875] Checking that builder machine is OK [2024-11-21 14:02:05,268][ INFO][PID:3501875] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2024-11-21 14:02:05,509][ INFO][PID:3501875] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2024-11-21 14:02:05,511][ INFO][PID:3501875] Filling build.info file with builder info [2024-11-21 14:02:05,514][ INFO][PID:3501875] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "thrnciar", "project_name": "setuptools-impact-check-control", "project_dirname": "setuptools-impact-check-control", "submitter": "https://src.fedoraproject.org/user/tbaeder", "ended_on": null, "started_on": 1732197725.514521, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8295486-fedora-rawhide-x86_64", "build_id": 8295486, "package_name": "llvm", "package_version": "19.1.4-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/thrnciar/setuptools-impact-check-control/llvm", "git_hash": "52a623a8624c197af8db5b3b28bc1100becacf83", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "thrnciar/setuptools-impact-check-control--https://src.fedoraproject.org/user/tbaeder", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control", "result_dir": "08295486-llvm", "built_packages": "", "id": 8295486, "mockchain_macros": { "copr_username": "thrnciar", "copr_projectname": "setuptools-impact-check-control", "vendor": "Fedora Project COPR (thrnciar/setuptools-impact-check-control)" } } ] } [2024-11-21 14:02:05,707][ INFO][PID:3501875] Sending fedora-messaging bus message in build.start [2024-11-21 14:02:06,996][ INFO][PID:3501875] Sending fedora-messaging bus message in chroot.start [2024-11-21 14:02:07,039][ INFO][PID:3501875] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8295486-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2024-11-21 14:02:07,435][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 1 [2024-11-21 14:02:07,447][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.236.151.126 copr-rpmbuild-log [2024-11-21 14:02:12,458][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 14:12:12,616][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 14:22:12,758][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 14:32:13,013][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 14:42:13,214][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 14:52:13,398][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 15:02:13,593][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 15:12:14,023][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 15:22:14,523][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 15:32:14,910][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 15:42:15,240][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 15:52:15,381][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:02:15,503][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:03:42,350][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 16:03:52,351][WARNING][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 16:04:02,352][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 2 [2024-11-21 16:04:02,395][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.236.151.126 copr-rpmbuild-log [2024-11-21 16:05:02,444][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 16:05:12,463][WARNING][PID:3501875] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 16:05:22,463][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 3 [2024-11-21 16:05:22,465][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.236.151.126 copr-rpmbuild-log [2024-11-21 16:06:22,532][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 16:06:32,535][WARNING][PID:3501875] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 16:06:42,536][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 4 [2024-11-21 16:06:42,538][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.236.151.126 copr-rpmbuild-log [2024-11-21 16:07:42,608][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 16:07:52,626][ ERROR][PID:3501875] Giving up for unstable SSH, failures: 1732205032.3512754, 1732205112.4630253, 1732205192.5357313, 1732205272.626064 [2024-11-21 16:07:52,629][ ERROR][PID:3501875] Re-try request for task on 'ResallocHost, ticket_id=5999032, hostname=3.236.151.126': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-21 16:07:52,631][ INFO][PID:3501875] Releasing VM back to pool [2024-11-21 16:07:52,648][ INFO][PID:3501875] Retry #1 (on other host) [2024-11-21 16:07:52,677][ INFO][PID:3501875] Trying to allocate VM [2024-11-21 16:07:55,723][ INFO][PID:3501875] Allocated host ResallocHost, ticket_id=6003322, hostname=3.238.240.140 [2024-11-21 16:07:55,724][ INFO][PID:3501875] Allocating ssh connection to builder [2024-11-21 16:07:55,726][ INFO][PID:3501875] Checking that builder machine is OK [2024-11-21 16:07:56,127][ INFO][PID:3501875] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2024-11-21 16:07:56,390][ INFO][PID:3501875] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2024-11-21 16:07:56,391][ INFO][PID:3501875] Filling build.info file with builder info [2024-11-21 16:07:56,416][ INFO][PID:3501875] Not re-notifying FE and msg buses for the new host. [2024-11-21 16:07:56,417][ INFO][PID:3501875] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8295486-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2024-11-21 16:07:56,817][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 1 [2024-11-21 16:07:56,819][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.240.140 copr-rpmbuild-log [2024-11-21 16:12:16,927][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:22:17,077][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:32:17,240][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:42:17,391][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:52:17,499][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 16:58:16,363][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 16:58:26,398][WARNING][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 16:58:36,400][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 2 [2024-11-21 16:58:36,403][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.240.140 copr-rpmbuild-log [2024-11-21 16:59:36,477][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 16:59:46,479][WARNING][PID:3501875] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 16:59:56,480][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 3 [2024-11-21 16:59:56,481][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.240.140 copr-rpmbuild-log [2024-11-21 17:00:56,549][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 17:01:06,569][WARNING][PID:3501875] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 17:01:16,570][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 4 [2024-11-21 17:01:16,571][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.240.140 copr-rpmbuild-log [2024-11-21 17:02:16,640][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 17:02:17,614][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 17:02:26,656][ ERROR][PID:3501875] Giving up for unstable SSH, failures: 1732208306.398894, 1732208386.4797835, 1732208466.569552, 1732208546.656751 [2024-11-21 17:02:26,657][ ERROR][PID:3501875] Re-try request for task on 'ResallocHost, ticket_id=6003322, hostname=3.238.240.140': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-21 17:02:26,660][ INFO][PID:3501875] Releasing VM back to pool [2024-11-21 17:02:26,674][ INFO][PID:3501875] Retry #2 (on other host) [2024-11-21 17:02:26,676][ INFO][PID:3501875] Trying to allocate VM [2024-11-21 17:02:29,702][ INFO][PID:3501875] Allocated host ResallocHost, ticket_id=6004447, hostname=3.238.70.3 [2024-11-21 17:02:29,703][ INFO][PID:3501875] Allocating ssh connection to builder [2024-11-21 17:02:29,703][ INFO][PID:3501875] Checking that builder machine is OK [2024-11-21 17:02:30,051][ INFO][PID:3501875] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2024-11-21 17:02:30,255][ INFO][PID:3501875] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2024-11-21 17:02:30,259][ INFO][PID:3501875] Filling build.info file with builder info [2024-11-21 17:02:30,275][ INFO][PID:3501875] Not re-notifying FE and msg buses for the new host. [2024-11-21 17:02:30,275][ INFO][PID:3501875] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8295486-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2024-11-21 17:02:30,629][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 1 [2024-11-21 17:02:30,631][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.70.3 copr-rpmbuild-log [2024-11-21 17:12:20,742][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 17:22:20,849][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 17:32:20,932][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 17:42:21,038][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 17:49:21,969][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 17:49:31,975][WARNING][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 17:49:41,979][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 2 [2024-11-21 17:49:42,003][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.70.3 copr-rpmbuild-log [2024-11-21 17:50:42,103][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 17:50:52,117][WARNING][PID:3501875] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 17:51:02,123][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 3 [2024-11-21 17:51:02,137][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.70.3 copr-rpmbuild-log [2024-11-21 17:52:02,243][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 17:52:12,255][WARNING][PID:3501875] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-21 17:52:21,341][ INFO][PID:3501875] Periodic builder liveness probe: alive [2024-11-21 17:52:22,257][ INFO][PID:3501875] Downloading the builder-live.log file, attempt 4 [2024-11-21 17:52:22,260][ INFO][PID:3501875] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.238.70.3 copr-rpmbuild-log [2024-11-21 17:53:22,329][ ERROR][PID:3501875] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-21 17:53:32,333][ ERROR][PID:3501875] Giving up for unstable SSH, failures: 1732211371.9757783, 1732211452.11773, 1732211532.2559419, 1732211612.3335161 [2024-11-21 17:53:32,336][ ERROR][PID:3501875] Re-try request for task on 'ResallocHost, ticket_id=6004447, hostname=3.238.70.3': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-21 17:53:32,337][ INFO][PID:3501875] Releasing VM back to pool [2024-11-21 17:53:32,385][ ERROR][PID:3501875] Backend process error: Three host tried without success: {'3.236.151.126', '3.238.240.140', '3.238.70.3'} [2024-11-21 17:53:32,389][WARNING][PID:3501875] Switching not-finished job state to 'failed' [2024-11-21 17:53:32,392][ INFO][PID:3501875] Worker failed build, took 3062.1135959625244 [2024-11-21 17:53:32,419][ INFO][PID:3501875] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "thrnciar", "project_name": "setuptools-impact-check-control", "project_dirname": "setuptools-impact-check-control", "submitter": "https://src.fedoraproject.org/user/tbaeder", "ended_on": 1732211612.3887618, "started_on": 1732208550.2751658, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8295486-fedora-rawhide-x86_64", "build_id": 8295486, "package_name": "llvm", "package_version": "19.1.4-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/thrnciar/setuptools-impact-check-control/llvm", "git_hash": "52a623a8624c197af8db5b3b28bc1100becacf83", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "thrnciar/setuptools-impact-check-control--https://src.fedoraproject.org/user/tbaeder", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control", "result_dir": "08295486-llvm", "built_packages": "", "id": 8295486, "mockchain_macros": { "copr_username": "thrnciar", "copr_projectname": "setuptools-impact-check-control", "vendor": "Fedora Project COPR (thrnciar/setuptools-impact-check-control)" } } ] } [2024-11-21 17:53:33,653][ INFO][PID:3501875] Sending fedora-messaging bus message in build.end [2024-11-21 17:53:34,023][ INFO][PID:3501875] Compressing /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/08295486-llvm/builder-live.log by gzip [2024-11-21 17:53:34,036][ INFO][PID:3501875] Running command 'gzip /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/08295486-llvm/builder-live.log' as PID 946963 [2024-11-21 17:53:34,103][ INFO][PID:3501875] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/08295486-llvm/builder-live.log) [2024-11-21 17:53:34,111][ INFO][PID:3501875] Compressing /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/08295486-llvm/backend.log by gzip [2024-11-21 17:53:34,113][ INFO][PID:3501875] Running command 'gzip /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-rawhide-x86_64/08295486-llvm/backend.log' as PID 946972