[2024-04-19 13:32:58,476][ INFO][PID:2905384] Marking build as starting [2024-04-19 13:32:58,563][ INFO][PID:2905384] Trying to allocate VM [2024-04-19 13:41:16,778][ INFO][PID:2905384] Allocated host ResallocHost, ticket_id=2361038, hostname=140.211.168.158 [2024-04-19 13:41:16,780][ INFO][PID:2905384] Allocating ssh connection to builder [2024-04-19 13:41:16,781][ INFO][PID:2905384] Checking that builder machine is OK [2024-04-19 13:41:18,280][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 '/usr/bin/test -f /etc/mock/fedora-rawhide-ppc64le.cfg' [2024-04-19 13:41:19,297][ INFO][PID:2905384] Filling build.info file with builder info [2024-04-19 13:41:19,299][ INFO][PID:2905384] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "psimovec", "project_name": "openblas-0.3.23", "project_dirname": "openblas-0.3.23:pr:9", "submitter": "https://src.fedoraproject.org/user/packit", "ended_on": null, "started_on": 1713534079.2994475, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "7328294-fedora-rawhide-ppc64le", "build_id": 7328294, "package_name": "python-neurom", "package_version": "3.2.10-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/psimovec/openblas-0.3.23:pr:9/python-neurom", "git_hash": "9bf531e345be2914e8344936ce6ab5a223ffa0cc", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "psimovec/openblas-0.3.23--https://src.fedoraproject.org/user/packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/psimovec/openblas-0.3.23/fedora-rawhide-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/psimovec/openblas-0.3.23:pr:9", "result_dir": "07328294-python-neurom", "built_packages": "", "id": 7328294, "mockchain_macros": { "copr_username": "psimovec", "copr_projectname": "openblas-0.3.23", "vendor": "Fedora Project COPR (psimovec/openblas-0.3.23)" } } ] } [2024-04-19 13:41:19,427][ INFO][PID:2905384] Sending fedora-messaging bus message in build.start [2024-04-19 13:41:20,341][ INFO][PID:2905384] Sending fedora-messaging bus message in chroot.start [2024-04-19 13:41:20,374][ INFO][PID:2905384] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/7328294-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2024-04-19 13:41:21,784][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 1 [2024-04-19 13:41:21,786][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2024-04-19 13:41:26,795][ INFO][PID:2905384] Status: False [2024-04-19 13:51:26,895][ INFO][PID:2905384] Status: False [2024-04-19 13:52:54,257][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 13:53:04,258][WARNING][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-04-19 13:53:14,259][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 2 [2024-04-19 13:53:14,261][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2024-04-19 13:54:14,329][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 13:54:24,330][WARNING][PID:2905384] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-04-19 13:54:34,331][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 3 [2024-04-19 13:54:34,333][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2024-04-19 13:55:34,401][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 13:55:44,406][WARNING][PID:2905384] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-04-19 13:55:54,407][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 4 [2024-04-19 13:55:54,408][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2024-04-19 13:56:54,476][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 13:57:04,479][ ERROR][PID:2905384] Giving up for unstable SSH, failures: 1713534784.258089, 1713534864.330444, 1713534944.4064069, 1713535024.4797156 [2024-04-19 13:57:04,481][ ERROR][PID:2905384] Re-try request for task on 'ResallocHost, ticket_id=2361038, hostname=140.211.168.158': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-04-19 13:57:04,481][ INFO][PID:2905384] Releasing VM back to pool [2024-04-19 13:57:04,491][ INFO][PID:2905384] Retry #1 (on other host) [2024-04-19 13:57:04,492][ INFO][PID:2905384] Trying to allocate VM [2024-04-19 14:02:22,610][ INFO][PID:2905384] Allocated host ResallocHost, ticket_id=2361338, hostname=140.211.168.171 [2024-04-19 14:02:22,611][ INFO][PID:2905384] Allocating ssh connection to builder [2024-04-19 14:02:22,611][ INFO][PID:2905384] Checking that builder machine is OK [2024-04-19 14:02:23,932][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 '/usr/bin/test -f /etc/mock/fedora-rawhide-ppc64le.cfg' [2024-04-19 14:02:24,948][ INFO][PID:2905384] Filling build.info file with builder info [2024-04-19 14:02:24,952][ INFO][PID:2905384] Not re-notifying FE and msg buses for the new host. [2024-04-19 14:02:24,953][ INFO][PID:2905384] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/7328294-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2024-04-19 14:02:26,297][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 1 [2024-04-19 14:02:26,299][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2024-04-19 14:02:31,303][ INFO][PID:2905384] Status: False [2024-04-19 14:12:31,395][ INFO][PID:2905384] Status: False [2024-04-19 14:17:10,258][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 14:17:20,259][WARNING][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-04-19 14:17:30,261][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 2 [2024-04-19 14:17:30,264][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2024-04-19 14:18:30,334][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 14:18:40,338][WARNING][PID:2905384] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-04-19 14:18:50,338][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 3 [2024-04-19 14:18:50,340][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2024-04-19 14:19:50,408][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 14:20:00,411][WARNING][PID:2905384] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-04-19 14:20:10,412][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 4 [2024-04-19 14:20:10,414][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2024-04-19 14:21:10,482][ ERROR][PID:2905384] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-04-19 14:21:20,484][ ERROR][PID:2905384] Giving up for unstable SSH, failures: 1713536240.2597547, 1713536320.3380117, 1713536400.4119525, 1713536480.4844637 [2024-04-19 14:21:20,485][ ERROR][PID:2905384] Re-try request for task on 'ResallocHost, ticket_id=2361338, hostname=140.211.168.171': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-04-19 14:21:20,486][ INFO][PID:2905384] Releasing VM back to pool [2024-04-19 14:21:20,498][ INFO][PID:2905384] Retry #2 (on other host) [2024-04-19 14:21:20,504][ INFO][PID:2905384] Trying to allocate VM [2024-04-19 14:21:23,546][ INFO][PID:2905384] Allocated host ResallocHost, ticket_id=2361727, hostname=2620:52:3:1:dead:beef:cafe:c289 [2024-04-19 14:21:23,547][ INFO][PID:2905384] Allocating ssh connection to builder [2024-04-19 14:21:23,548][ INFO][PID:2905384] Checking that builder machine is OK [2024-04-19 14:21:24,241][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c289 '/usr/bin/test -f /etc/mock/fedora-rawhide-ppc64le.cfg' [2024-04-19 14:21:24,606][ INFO][PID:2905384] Filling build.info file with builder info [2024-04-19 14:21:24,608][ INFO][PID:2905384] Not re-notifying FE and msg buses for the new host. [2024-04-19 14:21:24,609][ INFO][PID:2905384] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/7328294-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2024-04-19 14:21:25,311][ INFO][PID:2905384] Downloading the builder-live.log file, attempt 1 [2024-04-19 14:21:25,312][ INFO][PID:2905384] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c289 copr-rpmbuild-log [2024-04-19 14:22:35,328][ INFO][PID:2905384] Status: False [2024-04-19 14:23:45,142][ INFO][PID:2905384] Downloading results from builder [2024-04-19 14:23:45,143][ INFO][PID:2905384] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c289]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom started [2024-04-19 14:23:45,144][ INFO][PID:2905384] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c289]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/ &> /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/build-07328294.rsync.log [2024-04-19 14:23:45,671][ INFO][PID:2905384] rsyncing finished. [2024-04-19 14:23:45,672][ INFO][PID:2905384] Releasing VM back to pool [2024-04-19 14:23:45,691][ INFO][PID:2905384] Searching for 'success' file in resultdir [2024-04-19 14:23:45,692][ INFO][PID:2905384] Going to sign pkgs from source: 7328294-fedora-rawhide-ppc64le in chroot: /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le [2024-04-19 14:23:45,693][ INFO][PID:2905384] Calling '/bin/sign -u psimovec#openblas-0.3.23@copr.fedorahosted.org -p' (attempt #1) [2024-04-19 14:23:45,976][ INFO][PID:2905384] Calling '/bin/sign -4 -h sha256 -u psimovec#openblas-0.3.23@copr.fedorahosted.org -r /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python-neurom-doc-3.2.10-1.fc41.noarch.rpm' (attempt #1) [2024-04-19 14:23:46,805][ INFO][PID:2905384] signed rpm: /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python-neurom-doc-3.2.10-1.fc41.noarch.rpm [2024-04-19 14:23:46,806][ INFO][PID:2905384] Calling '/bin/sign -4 -h sha256 -u psimovec#openblas-0.3.23@copr.fedorahosted.org -r /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python3-neurom+plotly-3.2.10-1.fc41.noarch.rpm' (attempt #1) [2024-04-19 14:23:47,633][ INFO][PID:2905384] signed rpm: /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python3-neurom+plotly-3.2.10-1.fc41.noarch.rpm [2024-04-19 14:23:47,634][ INFO][PID:2905384] Calling '/bin/sign -4 -h sha256 -u psimovec#openblas-0.3.23@copr.fedorahosted.org -r /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python3-neurom-3.2.10-1.fc41.noarch.rpm' (attempt #1) [2024-04-19 14:23:48,459][ INFO][PID:2905384] signed rpm: /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python3-neurom-3.2.10-1.fc41.noarch.rpm [2024-04-19 14:23:48,460][ INFO][PID:2905384] Calling '/bin/sign -4 -h sha256 -u psimovec#openblas-0.3.23@copr.fedorahosted.org -r /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python-neurom-3.2.10-1.fc41.src.rpm' (attempt #1) [2024-04-19 14:23:49,351][ INFO][PID:2905384] signed rpm: /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/python-neurom-3.2.10-1.fc41.src.rpm [2024-04-19 14:23:49,352][ INFO][PID:2905384] Sign done [2024-04-19 14:23:49,353][ INFO][PID:2905384] Incremental createrepo run, adding 07328294-python-neurom into https://download.copr.fedorainfracloud.org/results/psimovec/openblas-0.3.23/fedora-rawhide-ppc64le, (auto-create-repo=True) [2024-04-19 14:23:49,354][ INFO][PID:2905384] Running command 'copr-repo --batched /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le --add 07328294-python-neurom --no-appstream-metadata' as PID 3218390 [2024-04-19 14:23:50,472][ INFO][PID:2905384] Finished after 1 seconds with exit code 0 (copr-repo --batched /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le --add 07328294-python-neurom --no-appstream-metadata) [2024-04-19 14:23:50,473][ INFO][PID:2905384] Getting build details [2024-04-19 14:23:50,473][ INFO][PID:2905384] Listing built binary packages in /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom [2024-04-19 14:23:50,474][ INFO][PID:2905384] Built packages: ['python3-neurom+plotly 3.2.10', 'python3-neurom 3.2.10', 'python-neurom-doc 3.2.10'] [2024-04-19 14:23:50,475][ INFO][PID:2905384] build details: {'built_packages': 'python3-neurom+plotly 3.2.10\npython3-neurom 3.2.10\npython-neurom-doc 3.2.10'} [2024-04-19 14:23:50,475][ INFO][PID:2905384] Retrieving pubkey [2024-04-19 14:23:50,475][ INFO][PID:2905384] Calling '/bin/sign -u psimovec#openblas-0.3.23@copr.fedorahosted.org -p' (attempt #1) [2024-04-19 14:23:50,759][ INFO][PID:2905384] Added pubkey for user psimovec project openblas-0.3.23 into: /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/pubkey.gpg [2024-04-19 14:23:50,760][ INFO][PID:2905384] Finished build: id=7328294 failed=False timeout=18000 destdir=/var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9 chroot=fedora-rawhide-ppc64le [2024-04-19 14:23:50,762][ INFO][PID:2905384] Worker succeeded build, took 146.15314054489136 [2024-04-19 14:23:50,762][ INFO][PID:2905384] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "psimovec", "project_name": "openblas-0.3.23", "project_dirname": "openblas-0.3.23:pr:9", "submitter": "https://src.fedoraproject.org/user/packit", "ended_on": 1713536630.7620337, "started_on": 1713536484.6088932, "submitted_on": null, "status": 1, "chroot": "fedora-rawhide-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "7328294-fedora-rawhide-ppc64le", "build_id": 7328294, "package_name": "python-neurom", "package_version": "3.2.10-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/psimovec/openblas-0.3.23:pr:9/python-neurom", "git_hash": "9bf531e345be2914e8344936ce6ab5a223ffa0cc", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "psimovec/openblas-0.3.23--https://src.fedoraproject.org/user/packit", "results": { "packages": [ { "name": "python3-neurom+plotly", "epoch": null, "version": "3.2.10", "release": "1.fc41", "arch": "noarch" }, { "name": "python3-neurom", "epoch": null, "version": "3.2.10", "release": "1.fc41", "arch": "noarch" }, { "name": "python-neurom-doc", "epoch": null, "version": "3.2.10", "release": "1.fc41", "arch": "noarch" }, { "name": "python-neurom", "epoch": null, "version": "3.2.10", "release": "1.fc41", "arch": "src" } ] }, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/psimovec/openblas-0.3.23/fedora-rawhide-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/psimovec/openblas-0.3.23:pr:9", "result_dir": "07328294-python-neurom", "built_packages": "python3-neurom+plotly 3.2.10\npython3-neurom 3.2.10\npython-neurom-doc 3.2.10", "id": 7328294, "mockchain_macros": { "copr_username": "psimovec", "copr_projectname": "openblas-0.3.23", "vendor": "Fedora Project COPR (psimovec/openblas-0.3.23)" } } ] } [2024-04-19 14:23:50,858][ INFO][PID:2905384] Sending fedora-messaging bus message in build.end [2024-04-19 14:23:50,891][ INFO][PID:2905384] Compressing /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/builder-live.log by gzip [2024-04-19 14:23:50,892][ INFO][PID:2905384] Running command 'gzip /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/builder-live.log' as PID 3218517 [2024-04-19 14:23:50,901][ INFO][PID:2905384] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/builder-live.log) [2024-04-19 14:23:50,902][ INFO][PID:2905384] Compressing /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/backend.log by gzip [2024-04-19 14:23:50,903][ INFO][PID:2905384] Running command 'gzip /var/lib/copr/public_html/results/psimovec/openblas-0.3.23:pr:9/fedora-rawhide-ppc64le/07328294-python-neurom/backend.log' as PID 3218519