[2024-09-17 12:47:29,026][ INFO][PID:2516591] Marking build as starting [2024-09-17 12:47:29,438][ INFO][PID:2516591] Trying to allocate VM [2024-09-17 12:48:57,842][ INFO][PID:2516591] Allocated host ResallocHost, ticket_id=4916889, hostname=44.200.144.236 [2024-09-17 12:48:57,843][ INFO][PID:2516591] Allocating ssh connection to builder [2024-09-17 12:48:57,846][ INFO][PID:2516591] Checking that builder machine is OK [2024-09-17 12:48:58,311][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.200.144.236 '/usr/bin/test -f /etc/mock/fedora-41-x86_64.cfg' [2024-09-17 12:48:58,637][ INFO][PID:2516591] Filling build.info file with builder info [2024-09-17 12:48:58,641][ INFO][PID:2516591] Sending build state back to frontend: { "builds": [ { "timeout": 28800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "kwizart", "project_name": "kernel-longterm-6.6", "project_dirname": "kernel-longterm-6.6", "submitter": "kwizart", "ended_on": null, "started_on": 1726577338.6405933, "submitted_on": null, "status": 3, "chroot": "fedora-41-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8027346-fedora-41-x86_64", "build_id": 8027346, "package_name": "kernel-longterm", "package_version": "6.6.51-200", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/kwizart/kernel-longterm-6.6/kernel-longterm", "git_hash": "e790c8d6fc19f4dcba43a1910ec3fea31bbf67de", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "kwizart/kernel-longterm-6.6--kwizart", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/kwizart/kernel-longterm-6.6", "result_dir": "08027346-kernel-longterm", "built_packages": "", "id": 8027346, "mockchain_macros": { "copr_username": "kwizart", "copr_projectname": "kernel-longterm-6.6", "vendor": "Fedora Project COPR (kwizart/kernel-longterm-6.6)" } } ] } [2024-09-17 12:48:58,955][ INFO][PID:2516591] Sending fedora-messaging bus message in build.start [2024-09-17 12:49:00,785][ INFO][PID:2516591] Sending fedora-messaging bus message in chroot.start [2024-09-17 12:49:00,853][ INFO][PID:2516591] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8027346-fedora-41-x86_64 --chroot fedora-41-x86_64 --detached [2024-09-17 12:49:01,322][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 1 [2024-09-17 12:49:01,332][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.200.144.236 copr-rpmbuild-log [2024-09-17 12:49:06,361][ INFO][PID:2516591] Status: False [2024-09-17 12:59:06,582][ INFO][PID:2516591] Status: False [2024-09-17 13:09:06,692][ INFO][PID:2516591] Status: False [2024-09-17 13:19:06,930][ INFO][PID:2516591] Status: False [2024-09-17 13:29:07,064][ INFO][PID:2516591] Status: False [2024-09-17 13:39:07,154][ INFO][PID:2516591] Status: False [2024-09-17 13:49:07,251][ INFO][PID:2516591] Status: False [2024-09-17 13:59:07,402][ INFO][PID:2516591] Status: False [2024-09-17 14:09:07,522][ INFO][PID:2516591] Status: False [2024-09-17 14:19:07,681][ INFO][PID:2516591] Status: False [2024-09-17 14:29:07,828][ INFO][PID:2516591] Status: False [2024-09-17 14:39:07,961][ INFO][PID:2516591] Status: False [2024-09-17 14:49:08,239][ INFO][PID:2516591] Status: False [2024-09-17 14:59:08,409][ INFO][PID:2516591] Status: False [2024-09-17 15:09:08,631][ INFO][PID:2516591] Status: False [2024-09-17 15:19:08,981][ INFO][PID:2516591] Status: False [2024-09-17 15:29:09,180][ INFO][PID:2516591] Status: False [2024-09-17 15:39:09,296][ INFO][PID:2516591] Status: False [2024-09-17 15:43:52,252][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 15:44:02,253][WARNING][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-09-17 15:44:12,254][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 2 [2024-09-17 15:44:12,285][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.200.144.236 copr-rpmbuild-log [2024-09-17 15:44:12,293][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 15:44:22,327][WARNING][PID:2516591] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-09-17 15:44:32,327][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 3 [2024-09-17 15:44:32,329][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.200.144.236 copr-rpmbuild-log [2024-09-17 15:45:32,396][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 15:45:42,399][WARNING][PID:2516591] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-09-17 15:45:52,400][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 4 [2024-09-17 15:45:52,402][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.200.144.236 copr-rpmbuild-log [2024-09-17 15:46:52,471][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 15:47:02,479][ ERROR][PID:2516591] Giving up for unstable SSH, failures: 1726587842.2537374, 1726587862.3269832, 1726587942.3998854, 1726588022.4789631 [2024-09-17 15:47:02,480][ ERROR][PID:2516591] Re-try request for task on 'ResallocHost, ticket_id=4916889, hostname=44.200.144.236': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-09-17 15:47:02,481][ INFO][PID:2516591] Releasing VM back to pool [2024-09-17 15:47:02,494][ INFO][PID:2516591] Retry #1 (on other host) [2024-09-17 15:47:02,523][ INFO][PID:2516591] Trying to allocate VM [2024-09-17 15:50:20,694][ INFO][PID:2516591] Allocated host ResallocHost, ticket_id=4919843, hostname=98.82.134.209 [2024-09-17 15:50:20,697][ INFO][PID:2516591] Allocating ssh connection to builder [2024-09-17 15:50:20,702][ INFO][PID:2516591] Checking that builder machine is OK [2024-09-17 15:50:21,067][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.134.209 '/usr/bin/test -f /etc/mock/fedora-41-x86_64.cfg' [2024-09-17 15:50:21,286][ INFO][PID:2516591] Filling build.info file with builder info [2024-09-17 15:50:21,334][ INFO][PID:2516591] Not re-notifying FE and msg buses for the new host. [2024-09-17 15:50:21,342][ INFO][PID:2516591] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8027346-fedora-41-x86_64 --chroot fedora-41-x86_64 --detached [2024-09-17 15:50:21,720][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 1 [2024-09-17 15:50:21,723][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.134.209 copr-rpmbuild-log [2024-09-17 15:50:26,733][ INFO][PID:2516591] Status: False [2024-09-17 16:00:26,863][ INFO][PID:2516591] Status: False [2024-09-17 16:10:27,064][ INFO][PID:2516591] Status: False [2024-09-17 16:20:27,256][ INFO][PID:2516591] Status: False [2024-09-17 16:30:27,458][ INFO][PID:2516591] Status: False [2024-09-17 16:40:27,577][ INFO][PID:2516591] Status: False [2024-09-17 16:50:27,668][ INFO][PID:2516591] Status: False [2024-09-17 17:00:27,790][ INFO][PID:2516591] Status: False [2024-09-17 17:10:27,983][ INFO][PID:2516591] Status: False [2024-09-17 17:20:28,306][ INFO][PID:2516591] Status: False [2024-09-17 17:30:28,431][ INFO][PID:2516591] Status: False [2024-09-17 17:40:28,531][ INFO][PID:2516591] Status: False [2024-09-17 17:50:28,618][ INFO][PID:2516591] Status: False [2024-09-17 18:00:28,698][ INFO][PID:2516591] Status: False [2024-09-17 18:10:28,931][ INFO][PID:2516591] Status: False [2024-09-17 18:20:29,175][ INFO][PID:2516591] Status: False [2024-09-17 18:30:29,472][ INFO][PID:2516591] Status: False [2024-09-17 18:40:29,650][ INFO][PID:2516591] Status: False [2024-09-17 18:48:21,822][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 18:48:31,843][WARNING][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-09-17 18:48:41,851][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 2 [2024-09-17 18:48:41,965][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.134.209 copr-rpmbuild-log [2024-09-17 18:49:42,031][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 18:49:52,063][WARNING][PID:2516591] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-09-17 18:50:02,064][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 3 [2024-09-17 18:50:02,066][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.134.209 copr-rpmbuild-log [2024-09-17 18:50:30,326][ INFO][PID:2516591] Status: False [2024-09-17 18:51:02,135][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 18:51:12,151][WARNING][PID:2516591] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-09-17 18:51:22,153][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 4 [2024-09-17 18:51:22,160][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.82.134.209 copr-rpmbuild-log [2024-09-17 18:52:22,235][ ERROR][PID:2516591] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-09-17 18:52:32,315][ ERROR][PID:2516591] Giving up for unstable SSH, failures: 1726598911.8409765, 1726598992.0630648, 1726599072.1516302, 1726599152.2603443 [2024-09-17 18:52:32,351][ ERROR][PID:2516591] Re-try request for task on 'ResallocHost, ticket_id=4919843, hostname=98.82.134.209': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-09-17 18:52:32,356][ INFO][PID:2516591] Releasing VM back to pool [2024-09-17 18:52:32,395][ INFO][PID:2516591] Retry #2 (on other host) [2024-09-17 18:52:32,406][ INFO][PID:2516591] Trying to allocate VM [2024-09-17 18:54:00,760][ INFO][PID:2516591] Allocated host ResallocHost, ticket_id=4922948, hostname=54.198.171.109 [2024-09-17 18:54:00,767][ INFO][PID:2516591] Allocating ssh connection to builder [2024-09-17 18:54:00,779][ INFO][PID:2516591] Checking that builder machine is OK [2024-09-17 18:54:01,370][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.198.171.109 '/usr/bin/test -f /etc/mock/fedora-41-x86_64.cfg' [2024-09-17 18:54:01,642][ INFO][PID:2516591] Filling build.info file with builder info [2024-09-17 18:54:01,721][ INFO][PID:2516591] Not re-notifying FE and msg buses for the new host. [2024-09-17 18:54:01,731][ INFO][PID:2516591] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8027346-fedora-41-x86_64 --chroot fedora-41-x86_64 --detached [2024-09-17 18:54:02,231][ INFO][PID:2516591] Downloading the builder-live.log file, attempt 1 [2024-09-17 18:54:02,238][ INFO][PID:2516591] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.198.171.109 copr-rpmbuild-log [2024-09-17 19:00:32,348][ INFO][PID:2516591] Status: False [2024-09-17 19:10:32,530][ INFO][PID:2516591] Status: False [2024-09-17 19:20:32,885][ INFO][PID:2516591] Status: False [2024-09-17 19:30:33,234][ INFO][PID:2516591] Status: False [2024-09-17 19:40:33,479][ INFO][PID:2516591] Status: False [2024-09-17 19:50:33,563][ INFO][PID:2516591] Status: False [2024-09-17 20:00:33,663][ INFO][PID:2516591] Status: False [2024-09-17 20:10:33,780][ INFO][PID:2516591] Status: False [2024-09-17 20:20:33,988][ INFO][PID:2516591] Status: False [2024-09-17 20:30:34,070][ INFO][PID:2516591] Status: False [2024-09-17 20:40:34,173][ INFO][PID:2516591] Status: False [2024-09-17 20:50:34,249][ INFO][PID:2516591] Status: False [2024-09-17 21:00:34,338][ INFO][PID:2516591] Status: False [2024-09-17 21:10:34,440][ INFO][PID:2516591] Status: False [2024-09-17 21:20:34,525][ INFO][PID:2516591] Status: False [2024-09-17 21:30:34,618][ INFO][PID:2516591] Status: False [2024-09-17 21:40:34,735][ INFO][PID:2516591] Status: False [2024-09-17 21:50:34,818][ INFO][PID:2516591] Status: False [2024-09-17 22:00:34,894][ INFO][PID:2516591] Status: False [2024-09-17 22:10:35,035][ INFO][PID:2516591] Status: False [2024-09-17 22:20:35,137][ INFO][PID:2516591] Status: False [2024-09-17 22:30:35,210][ INFO][PID:2516591] Status: False [2024-09-17 22:40:35,293][ INFO][PID:2516591] Status: False [2024-09-17 22:48:19,622][ INFO][PID:2516591] Downloading results from builder [2024-09-17 22:48:19,643][ INFO][PID:2516591] rsyncing of mockbuilder@54.198.171.109:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm started [2024-09-17 22:48:19,660][ INFO][PID:2516591] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@54.198.171.109:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/ &> /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/build-08027346.rsync.log [2024-09-17 22:48:20,982][ INFO][PID:2516591] rsyncing finished. [2024-09-17 22:48:20,984][ INFO][PID:2516591] Releasing VM back to pool [2024-09-17 22:48:20,993][ INFO][PID:2516591] Searching for 'success' file in resultdir [2024-09-17 22:48:21,001][ ERROR][PID:2516591] Build failed: Backend process error: No success file => build failure [2024-09-17 22:48:21,080][ INFO][PID:2516591] Finished build: id=8027346 failed=True timeout=28800 destdir=/var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6 chroot=fedora-41-x86_64 [2024-09-17 22:48:21,146][ ERROR][PID:2516591] Backend process error: No success file => build failure [2024-09-17 22:48:21,147][ INFO][PID:2516591] Worker failed build, took 14059.43275308609 [2024-09-17 22:48:21,158][ INFO][PID:2516591] Sending build state back to frontend: { "builds": [ { "timeout": 28800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "kwizart", "project_name": "kernel-longterm-6.6", "project_dirname": "kernel-longterm-6.6", "submitter": "kwizart", "ended_on": 1726613301.147598, "started_on": 1726599241.714845, "submitted_on": null, "status": 0, "chroot": "fedora-41-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8027346-fedora-41-x86_64", "build_id": 8027346, "package_name": "kernel-longterm", "package_version": "6.6.51-200", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/kwizart/kernel-longterm-6.6/kernel-longterm", "git_hash": "e790c8d6fc19f4dcba43a1910ec3fea31bbf67de", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "kwizart/kernel-longterm-6.6--kwizart", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/kwizart/kernel-longterm-6.6", "result_dir": "08027346-kernel-longterm", "built_packages": "", "id": 8027346, "mockchain_macros": { "copr_username": "kwizart", "copr_projectname": "kernel-longterm-6.6", "vendor": "Fedora Project COPR (kwizart/kernel-longterm-6.6)" } } ] } [2024-09-17 22:48:21,877][ INFO][PID:2516591] Sending fedora-messaging bus message in build.end [2024-09-17 22:48:22,120][ INFO][PID:2516591] Compressing /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/builder-live.log by gzip [2024-09-17 22:48:22,128][ INFO][PID:2516591] Running command 'gzip /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/builder-live.log' as PID 1991315 [2024-09-17 22:48:22,305][ INFO][PID:2516591] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/builder-live.log) [2024-09-17 22:48:22,309][ INFO][PID:2516591] Compressing /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/backend.log by gzip [2024-09-17 22:48:22,310][ INFO][PID:2516591] Running command 'gzip /var/lib/copr/public_html/results/kwizart/kernel-longterm-6.6/fedora-41-x86_64/08027346-kernel-longterm/backend.log' as PID 1991319