[2025-01-08 11:46:43,828][ INFO][PID:3019876] Marking build as starting [2025-01-08 11:46:43,860][ INFO][PID:3019876] Trying to allocate VM [2025-01-08 11:46:46,890][ INFO][PID:3019876] Allocated host ResallocHost, ticket_id=564173, hostname=54.144.172.206 [2025-01-08 11:46:46,891][ INFO][PID:3019876] Allocating ssh connection to builder [2025-01-08 11:46:46,891][ INFO][PID:3019876] Checking that builder machine is OK [2025-01-08 11:46:47,081][ INFO][PID:3019876] Running remote command: copr-builder-ready fedora-40-x86_64 [2025-01-08 11:46:47,295][ INFO][PID:3019876] Red Hat subscription not needed for fedora-40-x86_64 Builder is ready to be used [2025-01-08 11:46:47,296][ INFO][PID:3019876] Filling build.info file with builder info [2025-01-08 11:46:47,297][ INFO][PID:3019876] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "churchyard", "project_name": "jinja2-3.1.5-f40", "project_dirname": "jinja2-3.1.5-f40:custom:isolated", "submitter": "churchyard", "ended_on": null, "started_on": 1736336807.2975001, "submitted_on": null, "status": 3, "chroot": "fedora-40-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8485868-fedora-40-x86_64", "build_id": 8485868, "package_name": "chromium", "package_version": "131.0.6778.204-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/churchyard/jinja2-3.1.5-f40:custom:isolated/chromium", "git_hash": "ab64298c6755c15d45935a5766c3a6413ee15e02", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "churchyard/jinja2-3.1.5-f40--churchyard", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/churchyard/jinja2-3.1.5-f40/fedora-40-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/f40-build/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_f40_build_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_f40_build_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/churchyard/jinja2-3.1.5-f40:custom:isolated", "result_dir": "08485868-chromium", "built_packages": "", "id": 8485868, "mockchain_macros": { "copr_username": "churchyard", "copr_projectname": "jinja2-3.1.5-f40", "vendor": "Fedora Project COPR (churchyard/jinja2-3.1.5-f40)" } } ] } [2025-01-08 11:46:47,330][ INFO][PID:3019876] Sending fedora-messaging bus message in build.start [2025-01-08 11:46:47,908][ INFO][PID:3019876] Sending fedora-messaging bus message in chroot.start [2025-01-08 11:46:47,926][ INFO][PID:3019876] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8485868-fedora-40-x86_64 --chroot fedora-40-x86_64 --detached [2025-01-08 11:46:48,286][ INFO][PID:3019876] Downloading the builder-live.log file, attempt 1 [2025-01-08 11:46:48,288][ INFO][PID:3019876] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.144.172.206 copr-rpmbuild-log [2025-01-08 11:46:53,292][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 11:56:53,371][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 12:06:53,523][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 12:16:53,778][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 12:26:53,934][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 12:36:54,044][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 12:46:54,151][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 12:56:54,244][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:06:54,347][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:16:54,559][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:26:54,664][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:36:54,787][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:46:54,890][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:53:28,601][ ERROR][PID:3019876] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-01-08 13:53:38,603][WARNING][PID:3019876] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-01-08 13:53:48,603][ INFO][PID:3019876] Downloading the builder-live.log file, attempt 2 [2025-01-08 13:53:48,616][ INFO][PID:3019876] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.144.172.206 copr-rpmbuild-log [2025-01-08 13:54:48,684][ ERROR][PID:3019876] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-01-08 13:54:58,696][WARNING][PID:3019876] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-01-08 13:55:08,704][ INFO][PID:3019876] Downloading the builder-live.log file, attempt 3 [2025-01-08 13:55:08,706][ INFO][PID:3019876] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.144.172.206 copr-rpmbuild-log [2025-01-08 13:56:08,777][ ERROR][PID:3019876] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-01-08 13:56:18,785][WARNING][PID:3019876] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-01-08 13:56:28,787][ INFO][PID:3019876] Downloading the builder-live.log file, attempt 4 [2025-01-08 13:56:28,807][ INFO][PID:3019876] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.144.172.206 copr-rpmbuild-log [2025-01-08 13:56:55,038][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 13:57:28,885][ ERROR][PID:3019876] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-01-08 13:57:38,886][ ERROR][PID:3019876] Giving up for unstable SSH, failures: 1736344418.6029503, 1736344498.6967587, 1736344578.7859297, 1736344658.8860853 [2025-01-08 13:57:38,887][ ERROR][PID:3019876] Re-try request for task on 'ResallocHost, ticket_id=564173, hostname=54.144.172.206': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-01-08 13:57:38,887][ INFO][PID:3019876] Releasing VM back to pool [2025-01-08 13:57:38,899][ INFO][PID:3019876] Retry #1 (on other host) [2025-01-08 13:57:38,900][ INFO][PID:3019876] Trying to allocate VM [2025-01-08 13:58:47,031][ INFO][PID:3019876] Allocated host ResallocHost, ticket_id=567159, hostname=2620:52:3:1:dead:beef:cafe:c14e [2025-01-08 13:58:47,032][ INFO][PID:3019876] Allocating ssh connection to builder [2025-01-08 13:58:47,032][ INFO][PID:3019876] Checking that builder machine is OK [2025-01-08 13:58:47,683][ INFO][PID:3019876] Running remote command: copr-builder-ready fedora-40-x86_64 [2025-01-08 13:58:48,033][ INFO][PID:3019876] Red Hat subscription not needed for fedora-40-x86_64 Builder is ready to be used [2025-01-08 13:58:48,033][ INFO][PID:3019876] Filling build.info file with builder info [2025-01-08 13:58:48,042][ INFO][PID:3019876] Not re-notifying FE and msg buses for the new host. [2025-01-08 13:58:48,043][ INFO][PID:3019876] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8485868-fedora-40-x86_64 --chroot fedora-40-x86_64 --detached [2025-01-08 13:58:48,685][ INFO][PID:3019876] Downloading the builder-live.log file, attempt 1 [2025-01-08 13:58:48,687][ INFO][PID:3019876] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c14e copr-rpmbuild-log [2025-01-08 14:06:58,765][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 14:16:58,924][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 14:26:59,084][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 14:36:59,302][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 14:46:59,425][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 14:56:59,521][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 15:06:59,631][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 15:16:59,755][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 15:26:59,854][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 15:36:59,929][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 15:47:00,008][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 15:57:00,084][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 16:07:00,173][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 16:17:00,310][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 16:27:00,393][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 16:37:00,482][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 16:47:00,590][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 16:57:00,708][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 17:07:00,804][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 17:17:00,921][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 17:27:01,013][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 17:37:01,083][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 17:47:01,153][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 17:57:01,221][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 18:07:01,304][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 18:17:01,421][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 18:27:01,506][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 18:37:01,593][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 18:47:01,674][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 18:57:01,758][ INFO][PID:3019876] Periodic builder liveness probe: alive [2025-01-08 19:00:22,368][ INFO][PID:3019876] Downloading results from builder [2025-01-08 19:00:22,383][ INFO][PID:3019876] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c14e]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium started [2025-01-08 19:00:22,403][ INFO][PID:3019876] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c14e]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/ &> /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/build-08485868.rsync.log [2025-01-08 19:01:15,498][ INFO][PID:3019876] rsyncing finished. [2025-01-08 19:01:15,501][ ERROR][PID:3019876] Failed to download data from builder due to rsync error, see the rsync log file for details. [2025-01-08 19:01:15,501][ ERROR][PID:3019876] SSH connection lost on #1 attempt, let's retry after 10s, Failed to download data from builder due to rsync error, see the rsync log file for details. [2025-01-08 19:01:25,503][ INFO][PID:3019876] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c14e]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium started [2025-01-08 19:01:25,504][ INFO][PID:3019876] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c14e]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/ &> /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/build-08485868.rsync.log [2025-01-08 19:01:26,429][ INFO][PID:3019876] rsyncing finished. [2025-01-08 19:01:26,430][ INFO][PID:3019876] Releasing VM back to pool [2025-01-08 19:01:26,447][ INFO][PID:3019876] Searching for 'success' file in resultdir [2025-01-08 19:01:26,450][ ERROR][PID:3019876] Build failed: Backend process error: No success file => build failure [2025-01-08 19:01:26,528][ INFO][PID:3019876] Finished build: id=8485868 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated chroot=fedora-40-x86_64 [2025-01-08 19:01:26,595][ ERROR][PID:3019876] Backend process error: No success file => build failure [2025-01-08 19:01:26,597][ INFO][PID:3019876] Worker failed build, took 18158.55441069603 [2025-01-08 19:01:26,607][ INFO][PID:3019876] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "churchyard", "project_name": "jinja2-3.1.5-f40", "project_dirname": "jinja2-3.1.5-f40:custom:isolated", "submitter": "churchyard", "ended_on": 1736362886.597034, "started_on": 1736344728.0426233, "submitted_on": null, "status": 0, "chroot": "fedora-40-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8485868-fedora-40-x86_64", "build_id": 8485868, "package_name": "chromium", "package_version": "131.0.6778.204-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/churchyard/jinja2-3.1.5-f40:custom:isolated/chromium", "git_hash": "ab64298c6755c15d45935a5766c3a6413ee15e02", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "churchyard/jinja2-3.1.5-f40--churchyard", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/churchyard/jinja2-3.1.5-f40/fedora-40-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/f40-build/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_f40_build_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_f40_build_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/churchyard/jinja2-3.1.5-f40:custom:isolated", "result_dir": "08485868-chromium", "built_packages": "", "id": 8485868, "mockchain_macros": { "copr_username": "churchyard", "copr_projectname": "jinja2-3.1.5-f40", "vendor": "Fedora Project COPR (churchyard/jinja2-3.1.5-f40)" } } ] } [2025-01-08 19:01:27,015][ INFO][PID:3019876] Sending fedora-messaging bus message in build.end [2025-01-08 19:01:27,259][ INFO][PID:3019876] Compressing /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/builder-live.log by gzip [2025-01-08 19:01:27,276][ INFO][PID:3019876] Running command 'gzip /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/builder-live.log' as PID 1140429 [2025-01-08 19:01:27,371][ INFO][PID:3019876] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/builder-live.log) [2025-01-08 19:01:27,377][ INFO][PID:3019876] Compressing /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/backend.log by gzip [2025-01-08 19:01:27,378][ INFO][PID:3019876] Running command 'gzip /var/lib/copr/public_html/results/churchyard/jinja2-3.1.5-f40:custom:isolated/fedora-40-x86_64/08485868-chromium/backend.log' as PID 1140433