[2024-11-12 14:41:13,345][ INFO][PID:1775065] Marking build as starting [2024-11-12 14:41:13,448][ INFO][PID:1775065] Trying to allocate VM [2024-11-12 14:41:16,481][ INFO][PID:1775065] Allocated host ResallocHost, ticket_id=5831095, hostname=44.221.82.15 [2024-11-12 14:41:16,482][ INFO][PID:1775065] Allocating ssh connection to builder [2024-11-12 14:41:16,482][ INFO][PID:1775065] Checking that builder machine is OK [2024-11-12 14:41:16,827][ INFO][PID:1775065] Running remote command: copr-builder-ready centos-stream-9-x86_64 [2024-11-12 14:41:17,051][ INFO][PID:1775065] Red Hat subscription not needed for centos-stream-9-x86_64 Builder is ready to be used [2024-11-12 14:41:17,052][ INFO][PID:1775065] Filling build.info file with builder info [2024-11-12 14:41:17,053][ INFO][PID:1775065] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "networkmanager", "project_name": "NetworkManager-main", "project_dirname": "NetworkManager-main", "submitter": null, "ended_on": null, "started_on": 1731422477.0534687, "submitted_on": null, "status": 3, "chroot": "centos-stream-9-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8246887-centos-stream-9-x86_64", "build_id": 8246887, "package_name": "NetworkManager", "package_version": "1:1.51.3-33601.copr.03ff197b96", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/networkmanager/NetworkManager-main/NetworkManager", "git_hash": "be395a258fa44c13045fee819e105d6c78bfa754", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "networkmanager/NetworkManager-main--092f44e6-021b-40ff-a0dc-c0c90a43f102", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://copr-be.cloud.fedoraproject.org/results/nmstate/nm-build-deps/centos-stream-9-$basearch/", "id": "https_copr_be_cloud_fedoraproject_org_results_nmstate_nm_build_deps_centos_stream_9_basearch", "name": "Additional repo https_copr_be_cloud_fedoraproject_org_results_nmstate_nm_build_deps_centos_stream_9_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/networkmanager/NetworkManager-main", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/networkmanager/NetworkManager-main", "result_dir": "08246887-NetworkManager", "built_packages": "", "id": 8246887, "mockchain_macros": { "copr_username": "networkmanager", "copr_projectname": "NetworkManager-main", "vendor": "Fedora Project COPR (networkmanager/NetworkManager-main)" } } ] } [2024-11-12 14:41:17,148][ INFO][PID:1775065] Sending fedora-messaging bus message in build.start [2024-11-12 14:41:17,902][ INFO][PID:1775065] Sending fedora-messaging bus message in chroot.start [2024-11-12 14:41:17,923][ INFO][PID:1775065] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8246887-centos-stream-9-x86_64 --chroot centos-stream-9-x86_64 --detached [2024-11-12 14:41:18,296][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 1 [2024-11-12 14:41:18,298][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.221.82.15 copr-rpmbuild-log [2024-11-12 14:41:23,308][ INFO][PID:1775065] Periodic builder liveness probe: alive [2024-11-12 14:51:23,448][ INFO][PID:1775065] Periodic builder liveness probe: alive [2024-11-12 14:53:31,121][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 14:53:41,122][WARNING][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 14:53:51,123][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 2 [2024-11-12 14:53:51,126][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.221.82.15 copr-rpmbuild-log [2024-11-12 14:54:51,194][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 14:55:01,199][WARNING][PID:1775065] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 14:55:11,200][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 3 [2024-11-12 14:55:11,201][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.221.82.15 copr-rpmbuild-log [2024-11-12 14:56:11,270][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 14:56:21,289][WARNING][PID:1775065] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 14:56:31,290][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 4 [2024-11-12 14:56:31,292][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.221.82.15 copr-rpmbuild-log [2024-11-12 14:57:31,362][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 14:57:41,375][ ERROR][PID:1775065] Giving up for unstable SSH, failures: 1731423221.1222613, 1731423301.1996026, 1731423381.289736, 1731423461.3748374 [2024-11-12 14:57:41,377][ ERROR][PID:1775065] Re-try request for task on 'ResallocHost, ticket_id=5831095, hostname=44.221.82.15': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-12 14:57:41,378][ INFO][PID:1775065] Releasing VM back to pool [2024-11-12 14:57:41,419][ INFO][PID:1775065] Retry #1 (on other host) [2024-11-12 14:57:41,435][ INFO][PID:1775065] Trying to allocate VM [2024-11-12 15:02:59,724][ INFO][PID:1775065] Allocated host ResallocHost, ticket_id=5831401, hostname=18.206.71.82 [2024-11-12 15:02:59,725][ INFO][PID:1775065] Allocating ssh connection to builder [2024-11-12 15:02:59,725][ INFO][PID:1775065] Checking that builder machine is OK [2024-11-12 15:03:00,063][ INFO][PID:1775065] Running remote command: copr-builder-ready centos-stream-9-x86_64 [2024-11-12 15:03:00,291][ INFO][PID:1775065] Red Hat subscription not needed for centos-stream-9-x86_64 Builder is ready to be used [2024-11-12 15:03:00,292][ INFO][PID:1775065] Filling build.info file with builder info [2024-11-12 15:03:00,337][ INFO][PID:1775065] Not re-notifying FE and msg buses for the new host. [2024-11-12 15:03:00,338][ INFO][PID:1775065] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8246887-centos-stream-9-x86_64 --chroot centos-stream-9-x86_64 --detached [2024-11-12 15:03:00,702][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 1 [2024-11-12 15:03:00,704][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.206.71.82 copr-rpmbuild-log [2024-11-12 15:03:05,709][ INFO][PID:1775065] Periodic builder liveness probe: alive [2024-11-12 15:06:36,912][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:06:46,914][WARNING][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 15:06:56,915][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 2 [2024-11-12 15:06:56,918][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.206.71.82 copr-rpmbuild-log [2024-11-12 15:07:56,987][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:08:06,992][WARNING][PID:1775065] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 15:08:16,997][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 3 [2024-11-12 15:08:17,000][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.206.71.82 copr-rpmbuild-log [2024-11-12 15:09:17,071][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:09:27,093][WARNING][PID:1775065] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 15:09:37,096][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 4 [2024-11-12 15:09:37,103][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@18.206.71.82 copr-rpmbuild-log [2024-11-12 15:10:37,196][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:10:47,207][ ERROR][PID:1775065] Giving up for unstable SSH, failures: 1731424006.9146724, 1731424086.9925623, 1731424167.093049, 1731424247.2076354 [2024-11-12 15:10:47,209][ ERROR][PID:1775065] Re-try request for task on 'ResallocHost, ticket_id=5831401, hostname=18.206.71.82': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-12 15:10:47,209][ INFO][PID:1775065] Releasing VM back to pool [2024-11-12 15:10:47,234][ INFO][PID:1775065] Retry #2 (on other host) [2024-11-12 15:10:47,236][ INFO][PID:1775065] Trying to allocate VM [2024-11-12 15:16:05,515][ INFO][PID:1775065] Allocated host ResallocHost, ticket_id=5831675, hostname=44.197.120.155 [2024-11-12 15:16:05,517][ INFO][PID:1775065] Allocating ssh connection to builder [2024-11-12 15:16:05,519][ INFO][PID:1775065] Checking that builder machine is OK [2024-11-12 15:16:05,990][ INFO][PID:1775065] Running remote command: copr-builder-ready centos-stream-9-x86_64 [2024-11-12 15:16:06,276][ INFO][PID:1775065] Red Hat subscription not needed for centos-stream-9-x86_64 Builder is ready to be used [2024-11-12 15:16:06,277][ INFO][PID:1775065] Filling build.info file with builder info [2024-11-12 15:16:06,284][ INFO][PID:1775065] Not re-notifying FE and msg buses for the new host. [2024-11-12 15:16:06,285][ INFO][PID:1775065] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8246887-centos-stream-9-x86_64 --chroot centos-stream-9-x86_64 --detached [2024-11-12 15:16:06,766][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 1 [2024-11-12 15:16:06,769][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.197.120.155 copr-rpmbuild-log [2024-11-12 15:16:11,808][ INFO][PID:1775065] Periodic builder liveness probe: alive [2024-11-12 15:22:52,259][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:23:02,291][WARNING][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 15:23:12,292][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 2 [2024-11-12 15:23:12,298][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.197.120.155 copr-rpmbuild-log [2024-11-12 15:24:12,377][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:24:22,401][WARNING][PID:1775065] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 15:24:32,402][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 3 [2024-11-12 15:24:32,411][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.197.120.155 copr-rpmbuild-log [2024-11-12 15:25:32,490][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:25:42,500][WARNING][PID:1775065] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 15:25:52,501][ INFO][PID:1775065] Downloading the builder-live.log file, attempt 4 [2024-11-12 15:25:52,511][ INFO][PID:1775065] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.197.120.155 copr-rpmbuild-log [2024-11-12 15:26:12,104][ INFO][PID:1775065] Periodic builder liveness probe: alive [2024-11-12 15:26:52,605][ ERROR][PID:1775065] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 15:27:02,619][ ERROR][PID:1775065] Giving up for unstable SSH, failures: 1731424982.2914083, 1731425062.401724, 1731425142.500486, 1731425222.6193974 [2024-11-12 15:27:02,621][ ERROR][PID:1775065] Re-try request for task on 'ResallocHost, ticket_id=5831675, hostname=44.197.120.155': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-12 15:27:02,622][ INFO][PID:1775065] Releasing VM back to pool [2024-11-12 15:27:02,661][ ERROR][PID:1775065] Backend process error: Three host tried without success: {'44.197.120.155', '44.221.82.15', '18.206.71.82'} [2024-11-12 15:27:02,663][WARNING][PID:1775065] Switching not-finished job state to 'failed' [2024-11-12 15:27:02,664][ INFO][PID:1775065] Worker failed build, took 656.3792757987976 [2024-11-12 15:27:02,667][ INFO][PID:1775065] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "networkmanager", "project_name": "NetworkManager-main", "project_dirname": "NetworkManager-main", "submitter": null, "ended_on": 1731425222.6635144, "started_on": 1731424566.2842386, "submitted_on": null, "status": 0, "chroot": "centos-stream-9-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8246887-centos-stream-9-x86_64", "build_id": 8246887, "package_name": "NetworkManager", "package_version": "1:1.51.3-33601.copr.03ff197b96", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/networkmanager/NetworkManager-main/NetworkManager", "git_hash": "be395a258fa44c13045fee819e105d6c78bfa754", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "networkmanager/NetworkManager-main--092f44e6-021b-40ff-a0dc-c0c90a43f102", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://copr-be.cloud.fedoraproject.org/results/nmstate/nm-build-deps/centos-stream-9-$basearch/", "id": "https_copr_be_cloud_fedoraproject_org_results_nmstate_nm_build_deps_centos_stream_9_basearch", "name": "Additional repo https_copr_be_cloud_fedoraproject_org_results_nmstate_nm_build_deps_centos_stream_9_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/networkmanager/NetworkManager-main", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/networkmanager/NetworkManager-main", "result_dir": "08246887-NetworkManager", "built_packages": "", "id": 8246887, "mockchain_macros": { "copr_username": "networkmanager", "copr_projectname": "NetworkManager-main", "vendor": "Fedora Project COPR (networkmanager/NetworkManager-main)" } } ] } [2024-11-12 15:27:02,741][ INFO][PID:1775065] Sending fedora-messaging bus message in build.end [2024-11-12 15:27:02,766][ INFO][PID:1775065] Compressing /var/lib/copr/public_html/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/08246887-NetworkManager/builder-live.log by gzip [2024-11-12 15:27:02,769][ INFO][PID:1775065] Running command 'gzip /var/lib/copr/public_html/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/08246887-NetworkManager/builder-live.log' as PID 2113374 [2024-11-12 15:27:02,770][ INFO][PID:1775065] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/08246887-NetworkManager/builder-live.log) [2024-11-12 15:27:02,771][ INFO][PID:1775065] Compressing /var/lib/copr/public_html/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/08246887-NetworkManager/backend.log by gzip [2024-11-12 15:27:02,773][ INFO][PID:1775065] Running command 'gzip /var/lib/copr/public_html/results/networkmanager/NetworkManager-main/centos-stream-9-x86_64/08246887-NetworkManager/backend.log' as PID 2113375