[2024-11-12 01:10:23,461][ INFO][PID:2498704] Marking build as starting [2024-11-12 01:10:23,572][ INFO][PID:2498704] Trying to allocate VM [2024-11-12 01:18:41,825][ INFO][PID:2498704] Allocated host ResallocHost, ticket_id=5823519, hostname=98.80.253.92 [2024-11-12 01:18:41,826][ INFO][PID:2498704] Allocating ssh connection to builder [2024-11-12 01:18:41,827][ INFO][PID:2498704] Checking that builder machine is OK [2024-11-12 01:18:42,202][ INFO][PID:2498704] Running remote command: copr-builder-ready epel-9-x86_64 [2024-11-12 01:18:42,428][ INFO][PID:2498704] Checking Red Hat subscription (attempt #1) Red Hat subscription active Builder is ready to be used [2024-11-12 01:18:42,428][ INFO][PID:2498704] Filling build.info file with builder info [2024-11-12 01:18:42,430][ INFO][PID:2498704] Sending build state back to frontend: { "builds": [ { "timeout": 172800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "rezso", "project_name": "ML", "project_dirname": "ML", "submitter": "rezso", "ended_on": null, "started_on": 1731374322.4297435, "submitted_on": null, "status": 3, "chroot": "epel-9-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8245083-epel-9-x86_64", "build_id": 8245083, "package_name": "faiss", "package_version": "1.9.0-20241106.0.gitadb18841.cu12_6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rezso/ML/faiss", "git_hash": "9772dd205f6882fca6bb68e586340cabe5fb6d0c", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rezso/ML--rezso", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/ML/epel-9-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/CUDA/epel-9-x86_64/", "id": "copr_rezso_CUDA", "name": "Additional repo copr_rezso_CUDA" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rezso/ML", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rezso/ML", "result_dir": "08245083-faiss", "built_packages": "", "id": 8245083, "mockchain_macros": { "copr_username": "rezso", "copr_projectname": "ML", "vendor": "Fedora Project COPR (rezso/ML)" } } ] } [2024-11-12 01:18:42,519][ INFO][PID:2498704] Sending fedora-messaging bus message in build.start [2024-11-12 01:18:43,411][ INFO][PID:2498704] Sending fedora-messaging bus message in chroot.start [2024-11-12 01:18:43,592][ INFO][PID:2498704] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8245083-epel-9-x86_64 --chroot epel-9-x86_64 --detached [2024-11-12 01:18:44,013][ INFO][PID:2498704] Downloading the builder-live.log file, attempt 1 [2024-11-12 01:18:44,018][ INFO][PID:2498704] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.80.253.92 copr-rpmbuild-log [2024-11-12 01:18:49,040][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 01:21:50,062][ ERROR][PID:2498704] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 01:22:00,064][WARNING][PID:2498704] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 01:22:10,065][ INFO][PID:2498704] Downloading the builder-live.log file, attempt 2 [2024-11-12 01:22:10,067][ INFO][PID:2498704] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.80.253.92 copr-rpmbuild-log [2024-11-12 01:23:10,136][ ERROR][PID:2498704] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 01:23:20,158][WARNING][PID:2498704] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 01:23:30,162][ INFO][PID:2498704] Downloading the builder-live.log file, attempt 3 [2024-11-12 01:23:30,163][ INFO][PID:2498704] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.80.253.92 copr-rpmbuild-log [2024-11-12 01:24:30,233][ ERROR][PID:2498704] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 01:24:40,259][WARNING][PID:2498704] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-12 01:24:50,260][ INFO][PID:2498704] Downloading the builder-live.log file, attempt 4 [2024-11-12 01:24:50,268][ INFO][PID:2498704] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@98.80.253.92 copr-rpmbuild-log [2024-11-12 01:25:50,348][ ERROR][PID:2498704] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-12 01:26:00,364][ ERROR][PID:2498704] Giving up for unstable SSH, failures: 1731374520.0646772, 1731374600.158618, 1731374680.2596042, 1731374760.36436 [2024-11-12 01:26:00,365][ ERROR][PID:2498704] Re-try request for task on 'ResallocHost, ticket_id=5823519, hostname=98.80.253.92': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-12 01:26:00,366][ INFO][PID:2498704] Releasing VM back to pool [2024-11-12 01:26:00,379][ INFO][PID:2498704] Retry #1 (on other host) [2024-11-12 01:26:00,380][ INFO][PID:2498704] Trying to allocate VM [2024-11-12 01:26:06,421][ INFO][PID:2498704] Allocated host ResallocHost, ticket_id=5823898, hostname=2620:52:3:1:dead:beef:cafe:c117 [2024-11-12 01:26:06,422][ INFO][PID:2498704] Allocating ssh connection to builder [2024-11-12 01:26:06,423][ INFO][PID:2498704] Checking that builder machine is OK [2024-11-12 01:26:07,205][ INFO][PID:2498704] Running remote command: copr-builder-ready epel-9-x86_64 [2024-11-12 01:26:07,633][ INFO][PID:2498704] Checking Red Hat subscription (attempt #1) Red Hat subscription active Builder is ready to be used [2024-11-12 01:26:07,634][ INFO][PID:2498704] Filling build.info file with builder info [2024-11-12 01:26:07,646][ INFO][PID:2498704] Not re-notifying FE and msg buses for the new host. [2024-11-12 01:26:07,647][ INFO][PID:2498704] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8245083-epel-9-x86_64 --chroot epel-9-x86_64 --detached [2024-11-12 01:26:08,459][ INFO][PID:2498704] Downloading the builder-live.log file, attempt 1 [2024-11-12 01:26:08,461][ INFO][PID:2498704] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c117 copr-rpmbuild-log [2024-11-12 01:28:53,502][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 01:38:53,674][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 01:48:53,835][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 01:58:53,949][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 02:08:54,105][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 02:18:54,241][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 02:28:54,333][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 02:38:54,415][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 02:48:54,591][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 02:58:54,668][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 03:08:54,763][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 03:18:54,897][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 03:28:54,972][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 03:38:55,050][ INFO][PID:2498704] Periodic builder liveness probe: alive [2024-11-12 03:39:01,837][ INFO][PID:2498704] Downloading results from builder [2024-11-12 03:39:01,844][ INFO][PID:2498704] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c117]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss started [2024-11-12 03:39:01,846][ INFO][PID:2498704] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c117]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/ &> /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/build-08245083.rsync.log [2024-11-12 03:39:03,475][ INFO][PID:2498704] rsyncing finished. [2024-11-12 03:39:03,477][ INFO][PID:2498704] Releasing VM back to pool [2024-11-12 03:39:03,486][ INFO][PID:2498704] Searching for 'success' file in resultdir [2024-11-12 03:39:03,488][ INFO][PID:2498704] Going to sign pkgs from source: 8245083-epel-9-x86_64 in chroot: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64 [2024-11-12 03:39:03,496][ INFO][PID:2498704] Calling '/bin/sign -u rezso#ML@copr.fedorahosted.org -p' (attempt #1) [2024-11-12 03:39:03,837][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm' (attempt #1) [2024-11-12 03:39:06,855][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm [2024-11-12 03:39:06,856][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-debugsource-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm' (attempt #1) [2024-11-12 03:39:07,773][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-debugsource-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm [2024-11-12 03:39:07,774][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-debuginfo-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm' (attempt #1) [2024-11-12 03:39:08,823][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-debuginfo-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm [2024-11-12 03:39:08,826][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-1.9.0-20241106.0.gitadb18841.cu12_6.el9.src.rpm' (attempt #1) [2024-11-12 03:39:09,564][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-1.9.0-20241106.0.gitadb18841.cu12_6.el9.src.rpm [2024-11-12 03:39:09,565][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-devel-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm' (attempt #1) [2024-11-12 03:39:10,341][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-devel-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm [2024-11-12 03:39:10,341][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-python3-debuginfo-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm' (attempt #1) [2024-11-12 03:39:11,084][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-python3-debuginfo-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm [2024-11-12 03:39:11,085][ INFO][PID:2498704] Calling '/bin/sign -4 -h sha256 -u rezso#ML@copr.fedorahosted.org -r /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-python3-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm' (attempt #1) [2024-11-12 03:39:12,075][ INFO][PID:2498704] signed rpm: /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/faiss-python3-1.9.0-20241106.0.gitadb18841.cu12_6.el9.x86_64.rpm [2024-11-12 03:39:12,076][ INFO][PID:2498704] Sign done [2024-11-12 03:39:12,077][ INFO][PID:2498704] Incremental createrepo run, adding 08245083-faiss into https://download.copr.fedorainfracloud.org/results/rezso/ML/epel-9-x86_64, (auto-create-repo=True) [2024-11-12 03:39:12,084][ INFO][PID:2498704] Running command 'copr-repo --batched /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64 --add 08245083-faiss --no-appstream-metadata' as PID 3144811 [2024-11-12 03:39:13,606][ INFO][PID:2498704] Finished after 1 seconds with exit code 0 (copr-repo --batched /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64 --add 08245083-faiss --no-appstream-metadata) [2024-11-12 03:39:13,616][ INFO][PID:2498704] Getting build details [2024-11-12 03:39:13,617][ INFO][PID:2498704] Listing built binary packages in /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss [2024-11-12 03:39:13,618][ INFO][PID:2498704] Built packages: ['faiss-debuginfo 1.9.0', 'faiss-python3 1.9.0', 'faiss-python3-debuginfo 1.9.0', 'faiss-debugsource 1.9.0', 'faiss-devel 1.9.0', 'faiss 1.9.0'] [2024-11-12 03:39:13,618][ INFO][PID:2498704] build details: {'built_packages': 'faiss-debuginfo 1.9.0\nfaiss-python3 1.9.0\nfaiss-python3-debuginfo 1.9.0\nfaiss-debugsource 1.9.0\nfaiss-devel 1.9.0\nfaiss 1.9.0'} [2024-11-12 03:39:13,621][ INFO][PID:2498704] Retrieving pubkey [2024-11-12 03:39:13,621][ INFO][PID:2498704] Calling '/bin/sign -u rezso#ML@copr.fedorahosted.org -p' (attempt #1) [2024-11-12 03:39:13,829][ INFO][PID:2498704] Added pubkey for user rezso project ML into: /var/lib/copr/public_html/results/rezso/ML/pubkey.gpg [2024-11-12 03:39:13,830][ INFO][PID:2498704] Finished build: id=8245083 failed=False timeout=172800 destdir=/var/lib/copr/public_html/results/rezso/ML chroot=epel-9-x86_64 [2024-11-12 03:39:13,880][ INFO][PID:2498704] Worker succeeded build, took 7986.233958482742 [2024-11-12 03:39:13,886][ INFO][PID:2498704] Sending build state back to frontend: { "builds": [ { "timeout": 172800, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "rezso", "project_name": "ML", "project_dirname": "ML", "submitter": "rezso", "ended_on": 1731382753.8804781, "started_on": 1731374767.6465197, "submitted_on": null, "status": 1, "chroot": "epel-9-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8245083-epel-9-x86_64", "build_id": 8245083, "package_name": "faiss", "package_version": "1.9.0-20241106.0.gitadb18841.cu12_6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rezso/ML/faiss", "git_hash": "9772dd205f6882fca6bb68e586340cabe5fb6d0c", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rezso/ML--rezso", "results": { "packages": [ { "name": "faiss", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "src" }, { "name": "faiss-debuginfo", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "x86_64" }, { "name": "faiss-python3", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "x86_64" }, { "name": "faiss-python3-debuginfo", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "x86_64" }, { "name": "faiss-debugsource", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "x86_64" }, { "name": "faiss-devel", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "x86_64" }, { "name": "faiss", "epoch": null, "version": "1.9.0", "release": "20241106.0.gitadb18841.cu12_6.el9", "arch": "x86_64" } ] }, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/ML/epel-9-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "https://download.copr.fedorainfracloud.org/results/rezso/CUDA/epel-9-x86_64/", "id": "copr_rezso_CUDA", "name": "Additional repo copr_rezso_CUDA" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_x86_64" }, { "baseurl": "http://developer.download.nvidia.com/compute/cuda/repos/rhel9/sbsa", "id": "http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa", "name": "Additional repo http_developer_download_nvidia_com_compute_cuda_repos_rhel9_sbsa" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rezso/ML", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rezso/ML", "result_dir": "08245083-faiss", "built_packages": "faiss-debuginfo 1.9.0\nfaiss-python3 1.9.0\nfaiss-python3-debuginfo 1.9.0\nfaiss-debugsource 1.9.0\nfaiss-devel 1.9.0\nfaiss 1.9.0", "id": 8245083, "mockchain_macros": { "copr_username": "rezso", "copr_projectname": "ML", "vendor": "Fedora Project COPR (rezso/ML)" } } ] } [2024-11-12 03:39:14,476][ INFO][PID:2498704] Sending fedora-messaging bus message in build.end [2024-11-12 03:39:14,636][ INFO][PID:2498704] Compressing /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/builder-live.log by gzip [2024-11-12 03:39:14,638][ INFO][PID:2498704] Running command 'gzip /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/builder-live.log' as PID 3144895 [2024-11-12 03:39:14,693][ INFO][PID:2498704] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/builder-live.log) [2024-11-12 03:39:14,694][ INFO][PID:2498704] Compressing /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/backend.log by gzip [2024-11-12 03:39:14,695][ INFO][PID:2498704] Running command 'gzip /var/lib/copr/public_html/results/rezso/ML/epel-9-x86_64/08245083-faiss/backend.log' as PID 3144897