[2024-11-19 15:54:08,366][ INFO][PID:1414368] Marking build as starting [2024-11-19 15:54:08,464][ INFO][PID:1414368] Trying to allocate VM [2024-11-19 15:54:36,544][ INFO][PID:1414368] Allocated host ResallocHost, ticket_id=5951395, hostname=44.210.121.140 [2024-11-19 15:54:36,545][ INFO][PID:1414368] Allocating ssh connection to builder [2024-11-19 15:54:36,545][ INFO][PID:1414368] Checking that builder machine is OK [2024-11-19 15:54:36,860][ INFO][PID:1414368] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2024-11-19 15:54:37,058][ INFO][PID:1414368] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2024-11-19 15:54:37,059][ INFO][PID:1414368] Filling build.info file with builder info [2024-11-19 15:54:37,061][ INFO][PID:1414368] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "thrnciar", "project_name": "python-sphinx", "project_dirname": "python-sphinx", "submitter": "https://src.fedoraproject.org/user/kkleine", "ended_on": null, "started_on": 1732031677.0606184, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8280603-fedora-rawhide-x86_64", "build_id": 8280603, "package_name": "llvm", "package_version": "19.1.3-4", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/thrnciar/python-sphinx/llvm", "git_hash": "24e7d7ad27876fcf64ff53f352f493739fe4be96", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "thrnciar/python-sphinx--https://src.fedoraproject.org/user/kkleine", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/thrnciar/python-sphinx", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/thrnciar/python-sphinx", "result_dir": "08280603-llvm", "built_packages": "", "id": 8280603, "mockchain_macros": { "copr_username": "thrnciar", "copr_projectname": "python-sphinx", "vendor": "Fedora Project COPR (thrnciar/python-sphinx)" } } ] } [2024-11-19 15:54:37,117][ INFO][PID:1414368] Sending fedora-messaging bus message in build.start [2024-11-19 15:54:37,761][ INFO][PID:1414368] Sending fedora-messaging bus message in chroot.start [2024-11-19 15:54:37,782][ INFO][PID:1414368] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8280603-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2024-11-19 15:54:38,129][ INFO][PID:1414368] Downloading the builder-live.log file, attempt 1 [2024-11-19 15:54:38,131][ INFO][PID:1414368] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.210.121.140 copr-rpmbuild-log [2024-11-19 15:54:43,147][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 15:58:43,695][ ERROR][PID:1414368] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-19 15:58:53,695][WARNING][PID:1414368] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-19 15:59:03,696][ INFO][PID:1414368] Downloading the builder-live.log file, attempt 2 [2024-11-19 15:59:03,699][ INFO][PID:1414368] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.210.121.140 copr-rpmbuild-log [2024-11-19 16:00:03,767][ ERROR][PID:1414368] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-19 16:00:13,785][WARNING][PID:1414368] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-19 16:00:23,786][ INFO][PID:1414368] Downloading the builder-live.log file, attempt 3 [2024-11-19 16:00:23,788][ INFO][PID:1414368] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.210.121.140 copr-rpmbuild-log [2024-11-19 16:01:23,857][ ERROR][PID:1414368] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-19 16:01:33,862][WARNING][PID:1414368] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-19 16:01:43,863][ INFO][PID:1414368] Downloading the builder-live.log file, attempt 4 [2024-11-19 16:01:43,865][ INFO][PID:1414368] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@44.210.121.140 copr-rpmbuild-log [2024-11-19 16:02:43,934][ ERROR][PID:1414368] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-19 16:02:53,951][ ERROR][PID:1414368] Giving up for unstable SSH, failures: 1732031933.695929, 1732032013.78577, 1732032093.8620615, 1732032173.9516175 [2024-11-19 16:02:53,952][ ERROR][PID:1414368] Re-try request for task on 'ResallocHost, ticket_id=5951395, hostname=44.210.121.140': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-19 16:02:53,953][ INFO][PID:1414368] Releasing VM back to pool [2024-11-19 16:02:53,965][ INFO][PID:1414368] Retry #1 (on other host) [2024-11-19 16:02:53,966][ INFO][PID:1414368] Trying to allocate VM [2024-11-19 16:02:56,994][ INFO][PID:1414368] Allocated host ResallocHost, ticket_id=5951478, hostname=2620:52:3:1:dead:beef:cafe:c194 [2024-11-19 16:02:56,995][ INFO][PID:1414368] Allocating ssh connection to builder [2024-11-19 16:02:56,995][ INFO][PID:1414368] Checking that builder machine is OK [2024-11-19 16:02:57,647][ INFO][PID:1414368] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2024-11-19 16:02:58,018][ INFO][PID:1414368] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2024-11-19 16:02:58,018][ INFO][PID:1414368] Filling build.info file with builder info [2024-11-19 16:02:58,022][ INFO][PID:1414368] Not re-notifying FE and msg buses for the new host. [2024-11-19 16:02:58,023][ INFO][PID:1414368] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8280603-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2024-11-19 16:02:58,634][ INFO][PID:1414368] Downloading the builder-live.log file, attempt 1 [2024-11-19 16:02:58,636][ INFO][PID:1414368] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c194 copr-rpmbuild-log [2024-11-19 16:04:43,654][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 16:14:43,780][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 16:24:43,929][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 16:34:44,094][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 16:44:44,362][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 16:54:44,498][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 17:04:44,591][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 17:14:44,698][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 17:24:44,799][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 17:34:44,905][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 17:44:45,016][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 17:54:45,128][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 18:04:45,226][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 18:14:45,342][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 18:24:45,438][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 18:34:45,548][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 18:44:45,686][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 18:54:45,785][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 19:04:45,895][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 19:14:46,029][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 19:24:46,170][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 19:34:46,347][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 19:44:46,557][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 19:54:46,693][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 20:04:46,863][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 20:14:47,033][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 20:24:47,165][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 20:34:47,333][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 20:44:47,540][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 20:54:47,707][ INFO][PID:1414368] Periodic builder liveness probe: alive [2024-11-19 21:03:07,897][ INFO][PID:1414368] Downloading results from builder [2024-11-19 21:03:07,928][ INFO][PID:1414368] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c194]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm started [2024-11-19 21:03:07,951][ INFO][PID:1414368] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c194]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/ &> /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/build-08280603.rsync.log [2024-11-19 21:03:09,754][ INFO][PID:1414368] rsyncing finished. [2024-11-19 21:03:09,755][ INFO][PID:1414368] Releasing VM back to pool [2024-11-19 21:03:09,770][ INFO][PID:1414368] Searching for 'success' file in resultdir [2024-11-19 21:03:09,775][ ERROR][PID:1414368] Build failed: Backend process error: No success file => build failure [2024-11-19 21:03:09,852][ INFO][PID:1414368] Finished build: id=8280603 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/thrnciar/python-sphinx chroot=fedora-rawhide-x86_64 [2024-11-19 21:03:09,906][ ERROR][PID:1414368] Backend process error: No success file => build failure [2024-11-19 21:03:09,907][ INFO][PID:1414368] Worker failed build, took 18011.885280370712 [2024-11-19 21:03:09,915][ INFO][PID:1414368] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "thrnciar", "project_name": "python-sphinx", "project_dirname": "python-sphinx", "submitter": "https://src.fedoraproject.org/user/kkleine", "ended_on": 1732050189.9078267, "started_on": 1732032178.0225463, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8280603-fedora-rawhide-x86_64", "build_id": 8280603, "package_name": "llvm", "package_version": "19.1.3-4", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/thrnciar/python-sphinx/llvm", "git_hash": "24e7d7ad27876fcf64ff53f352f493739fe4be96", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "thrnciar/python-sphinx--https://src.fedoraproject.org/user/kkleine", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/thrnciar/python-sphinx", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/thrnciar/python-sphinx", "result_dir": "08280603-llvm", "built_packages": "", "id": 8280603, "mockchain_macros": { "copr_username": "thrnciar", "copr_projectname": "python-sphinx", "vendor": "Fedora Project COPR (thrnciar/python-sphinx)" } } ] } [2024-11-19 21:03:10,683][ INFO][PID:1414368] Sending fedora-messaging bus message in build.end [2024-11-19 21:03:10,887][ INFO][PID:1414368] Compressing /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/builder-live.log by gzip [2024-11-19 21:03:10,898][ INFO][PID:1414368] Running command 'gzip /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/builder-live.log' as PID 3282939 [2024-11-19 21:03:11,247][ INFO][PID:1414368] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/builder-live.log) [2024-11-19 21:03:11,252][ INFO][PID:1414368] Compressing /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/backend.log by gzip [2024-11-19 21:03:11,254][ INFO][PID:1414368] Running command 'gzip /var/lib/copr/public_html/results/thrnciar/python-sphinx/fedora-rawhide-x86_64/08280603-llvm/backend.log' as PID 3282988