[2024-11-18 17:21:07,858][ INFO][PID:3388441] Marking build as starting [2024-11-18 17:21:08,041][ INFO][PID:3388441] Trying to allocate VM [2024-11-18 17:24:26,401][ INFO][PID:3388441] Allocated host ResallocHost, ticket_id=5935586, hostname=3.239.54.44 [2024-11-18 17:24:26,404][ INFO][PID:3388441] Allocating ssh connection to builder [2024-11-18 17:24:26,405][ INFO][PID:3388441] Checking that builder machine is OK [2024-11-18 17:24:26,758][ INFO][PID:3388441] Running remote command: copr-builder-ready fedora-39-x86_64 [2024-11-18 17:24:26,983][ INFO][PID:3388441] Red Hat subscription not needed for fedora-39-x86_64 Builder is ready to be used [2024-11-18 17:24:26,987][ INFO][PID:3388441] Filling build.info file with builder info [2024-11-18 17:24:26,990][ INFO][PID:3388441] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "@python", "project_name": "python3.12", "project_dirname": "python3.12", "submitter": "https://src.fedoraproject.org/user/kkeithle", "ended_on": null, "started_on": 1731950666.9904268, "submitted_on": null, "status": 3, "chroot": "fedora-39-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8276676-fedora-39-x86_64", "build_id": 8276676, "package_name": "ceph", "package_version": "2:19.2.0-4", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@python/python3.12/ceph", "git_hash": "4a0753b97b34f47f0c9cacc0d58092f89514bbac", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@python/python3.12--https://src.fedoraproject.org/user/kkeithle", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@python/python3.12/fedora-39-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@python/python3.12", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@python/python3.12", "result_dir": "08276676-ceph", "built_packages": "", "id": 8276676, "mockchain_macros": { "copr_username": "@python", "copr_projectname": "python3.12", "vendor": "Fedora Project COPR (@python/python3.12)" } } ] } [2024-11-18 17:24:27,070][ INFO][PID:3388441] Sending fedora-messaging bus message in build.start [2024-11-18 17:24:27,895][ INFO][PID:3388441] Sending fedora-messaging bus message in chroot.start [2024-11-18 17:24:27,921][ INFO][PID:3388441] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8276676-fedora-39-x86_64 --chroot fedora-39-x86_64 --detached [2024-11-18 17:24:28,314][ INFO][PID:3388441] Downloading the builder-live.log file, attempt 1 [2024-11-18 17:24:28,318][ INFO][PID:3388441] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.239.54.44 copr-rpmbuild-log [2024-11-18 17:24:33,338][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 17:34:33,527][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 17:44:33,663][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 17:54:33,783][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 18:04:33,949][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 18:14:35,914][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 18:24:36,090][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 18:34:36,283][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 18:35:49,892][ ERROR][PID:3388441] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-18 18:35:59,893][WARNING][PID:3388441] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-18 18:36:09,894][ INFO][PID:3388441] Downloading the builder-live.log file, attempt 2 [2024-11-18 18:36:09,919][ INFO][PID:3388441] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.239.54.44 copr-rpmbuild-log [2024-11-18 18:37:09,987][ ERROR][PID:3388441] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-18 18:37:19,996][WARNING][PID:3388441] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-18 18:37:29,997][ INFO][PID:3388441] Downloading the builder-live.log file, attempt 3 [2024-11-18 18:37:30,000][ INFO][PID:3388441] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.239.54.44 copr-rpmbuild-log [2024-11-18 18:38:30,069][ ERROR][PID:3388441] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-18 18:38:40,135][WARNING][PID:3388441] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-18 18:38:50,136][ INFO][PID:3388441] Downloading the builder-live.log file, attempt 4 [2024-11-18 18:38:50,149][ INFO][PID:3388441] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.239.54.44 copr-rpmbuild-log [2024-11-18 18:39:50,220][ ERROR][PID:3388441] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-18 18:40:00,225][ ERROR][PID:3388441] Giving up for unstable SSH, failures: 1731954959.893815, 1731955039.9968793, 1731955120.1356702, 1731955200.2248142 [2024-11-18 18:40:00,229][ ERROR][PID:3388441] Re-try request for task on 'ResallocHost, ticket_id=5935586, hostname=3.239.54.44': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-18 18:40:00,230][ INFO][PID:3388441] Releasing VM back to pool [2024-11-18 18:40:00,252][ INFO][PID:3388441] Retry #1 (on other host) [2024-11-18 18:40:00,257][ INFO][PID:3388441] Trying to allocate VM [2024-11-18 18:41:08,380][ INFO][PID:3388441] Allocated host ResallocHost, ticket_id=5937084, hostname=2620:52:3:1:dead:beef:cafe:c1cb [2024-11-18 18:41:08,380][ INFO][PID:3388441] Allocating ssh connection to builder [2024-11-18 18:41:08,381][ INFO][PID:3388441] Checking that builder machine is OK [2024-11-18 18:41:09,019][ INFO][PID:3388441] Running remote command: copr-builder-ready fedora-39-x86_64 [2024-11-18 18:41:09,385][ INFO][PID:3388441] Red Hat subscription not needed for fedora-39-x86_64 Builder is ready to be used [2024-11-18 18:41:09,386][ INFO][PID:3388441] Filling build.info file with builder info [2024-11-18 18:41:09,399][ INFO][PID:3388441] Not re-notifying FE and msg buses for the new host. [2024-11-18 18:41:09,400][ INFO][PID:3388441] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8276676-fedora-39-x86_64 --chroot fedora-39-x86_64 --detached [2024-11-18 18:41:10,257][ INFO][PID:3388441] Downloading the builder-live.log file, attempt 1 [2024-11-18 18:41:10,258][ INFO][PID:3388441] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c1cb copr-rpmbuild-log [2024-11-18 18:44:40,304][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 18:54:40,388][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 19:04:40,477][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 19:14:40,657][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 19:24:40,814][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 19:34:40,894][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 19:44:40,974][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 19:54:41,094][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 20:04:41,194][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 20:14:41,443][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 20:24:41,741][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 20:34:42,024][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 20:44:42,252][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 20:54:42,433][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 21:04:42,546][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 21:14:42,744][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 21:24:42,868][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 21:34:42,968][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 21:44:43,094][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 21:54:43,227][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 22:04:43,380][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 22:14:43,602][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 22:24:43,822][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 22:34:43,943][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 22:44:44,029][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 22:54:44,112][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 23:04:44,195][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 23:14:44,307][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 23:24:44,404][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 23:34:44,481][ INFO][PID:3388441] Periodic builder liveness probe: alive [2024-11-18 23:41:34,401][ INFO][PID:3388441] Downloading results from builder [2024-11-18 23:41:34,421][ INFO][PID:3388441] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c1cb]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph started [2024-11-18 23:41:34,440][ INFO][PID:3388441] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c1cb]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/ &> /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/build-08276676.rsync.log [2024-11-18 23:41:38,595][ INFO][PID:3388441] rsyncing finished. [2024-11-18 23:41:38,597][ INFO][PID:3388441] Releasing VM back to pool [2024-11-18 23:41:38,610][ INFO][PID:3388441] Searching for 'success' file in resultdir [2024-11-18 23:41:38,613][ ERROR][PID:3388441] Build failed: Backend process error: No success file => build failure [2024-11-18 23:41:38,683][ INFO][PID:3388441] Finished build: id=8276676 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/@python/python3.12 chroot=fedora-39-x86_64 [2024-11-18 23:41:38,736][ ERROR][PID:3388441] Backend process error: No success file => build failure [2024-11-18 23:41:38,738][ INFO][PID:3388441] Worker failed build, took 18029.339098215103 [2024-11-18 23:41:38,749][ INFO][PID:3388441] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "@python", "project_name": "python3.12", "project_dirname": "python3.12", "submitter": "https://src.fedoraproject.org/user/kkeithle", "ended_on": 1731973298.7384512, "started_on": 1731955269.399353, "submitted_on": null, "status": 0, "chroot": "fedora-39-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8276676-fedora-39-x86_64", "build_id": 8276676, "package_name": "ceph", "package_version": "2:19.2.0-4", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@python/python3.12/ceph", "git_hash": "4a0753b97b34f47f0c9cacc0d58092f89514bbac", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@python/python3.12--https://src.fedoraproject.org/user/kkeithle", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@python/python3.12/fedora-39-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@python/python3.12", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@python/python3.12", "result_dir": "08276676-ceph", "built_packages": "", "id": 8276676, "mockchain_macros": { "copr_username": "@python", "copr_projectname": "python3.12", "vendor": "Fedora Project COPR (@python/python3.12)" } } ] } [2024-11-18 23:41:39,356][ INFO][PID:3388441] Sending fedora-messaging bus message in build.end [2024-11-18 23:41:39,582][ INFO][PID:3388441] Compressing /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/builder-live.log by gzip [2024-11-18 23:41:39,594][ INFO][PID:3388441] Running command 'gzip /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/builder-live.log' as PID 1464243 [2024-11-18 23:41:39,914][ INFO][PID:3388441] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/builder-live.log) [2024-11-18 23:41:39,919][ INFO][PID:3388441] Compressing /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/backend.log by gzip [2024-11-18 23:41:39,921][ INFO][PID:3388441] Running command 'gzip /var/lib/copr/public_html/results/@python/python3.12/fedora-39-x86_64/08276676-ceph/backend.log' as PID 1464263