[2024-11-28 02:18:08,492][ INFO][PID:273761] Marking build as starting [2024-11-28 02:18:08,956][ INFO][PID:273761] Trying to allocate VM [2024-11-28 02:18:37,231][ INFO][PID:273761] Allocated host ResallocHost, ticket_id=6109267, hostname=2620:52:3:1:dead:beef:cafe:c290 [2024-11-28 02:18:37,233][ INFO][PID:273761] Allocating ssh connection to builder [2024-11-28 02:18:37,236][ INFO][PID:273761] Checking that builder machine is OK [2024-11-28 02:18:38,030][ INFO][PID:273761] Running remote command: copr-builder-ready srpm-builds [2024-11-28 02:18:38,779][ INFO][PID:273761] Red Hat subscription not needed for srpm-builds Builder is ready to be used [2024-11-28 02:18:38,780][ INFO][PID:273761] Filling build.info file with builder info [2024-11-28 02:18:38,782][ INFO][PID:273761] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": null, "enable_net": true, "project_owner": "@fedora-llvm-team", "project_name": "llvm-snapshots-pgo-20241128", "project_dirname": "llvm-snapshots-pgo-20241128", "submitter": "tstellar", "ended_on": null, "started_on": 1732760318.7824166, "submitted_on": null, "status": 3, "chroot": "srpm-builds", "arch": null, "buildroot_pkgs": null, "task_id": "8321173", "build_id": 8321173, "package_name": "llvm", "package_version": null, "git_repo": null, "git_hash": null, "git_branch": null, "source_type": 8, "source_json": "{\"type\": \"git\", \"clone_url\": \"https://src.fedoraproject.org/forks/kkleine/rpms/llvm.git\", \"subdirectory\": \"\", \"committish\": \"pgo\", \"spec\": \"llvm.spec\", \"srpm_build_method\": \"make_srpm\"}", "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": null, "sandbox": "@fedora-llvm-team/llvm-snapshots-pgo-20241128--tstellar", "results": null, "appstream": false, "allow_user_ssh": null, "ssh_public_keys": null, "storage": null, "background": false, "repos": [], "destdir": "/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128", "result_dir": "08321173", "built_packages": "", "tags": [ "arch_noarch" ], "id": 8321173, "mockchain_macros": { "copr_username": "@fedora-llvm-team", "copr_projectname": "llvm-snapshots-pgo-20241128", "vendor": "Fedora Project COPR (@fedora-llvm-team/llvm-snapshots-pgo-20241128)" } } ] } [2024-11-28 02:18:39,003][ INFO][PID:273761] Sending fedora-messaging bus message in build.start [2024-11-28 02:18:42,563][ INFO][PID:273761] Sending fedora-messaging bus message in chroot.start [2024-11-28 02:18:42,642][ INFO][PID:273761] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --srpm --task-url https://copr.fedorainfracloud.org/backend/get-srpm-build-task/8321173 --detached [2024-11-28 02:18:43,438][ INFO][PID:273761] Downloading the builder-live.log file, attempt 1 [2024-11-28 02:18:43,451][ INFO][PID:273761] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c290 copr-rpmbuild-log [2024-11-28 02:18:48,470][ INFO][PID:273761] Periodic builder liveness probe: alive [2024-11-28 02:22:46,948][ ERROR][PID:273761] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-28 02:22:56,956][WARNING][PID:273761] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-28 02:23:06,956][ INFO][PID:273761] Downloading the builder-live.log file, attempt 2 [2024-11-28 02:23:06,958][ INFO][PID:273761] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c290 copr-rpmbuild-log [2024-11-28 02:24:07,026][ ERROR][PID:273761] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-28 02:24:17,027][WARNING][PID:273761] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-28 02:24:27,028][ INFO][PID:273761] Downloading the builder-live.log file, attempt 3 [2024-11-28 02:24:27,030][ INFO][PID:273761] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c290 copr-rpmbuild-log [2024-11-28 02:25:27,098][ ERROR][PID:273761] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-28 02:25:37,100][WARNING][PID:273761] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2024-11-28 02:25:47,101][ INFO][PID:273761] Downloading the builder-live.log file, attempt 4 [2024-11-28 02:25:47,102][ INFO][PID:273761] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c290 copr-rpmbuild-log [2024-11-28 02:26:47,170][ ERROR][PID:273761] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2024-11-28 02:26:57,172][ ERROR][PID:273761] Giving up for unstable SSH, failures: 1732760576.9559598, 1732760657.0276475, 1732760737.100071, 1732760817.1719103 [2024-11-28 02:26:57,173][ ERROR][PID:273761] Re-try request for task on 'ResallocHost, ticket_id=6109267, hostname=2620:52:3:1:dead:beef:cafe:c290': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2024-11-28 02:26:57,173][ INFO][PID:273761] Releasing VM back to pool [2024-11-28 02:26:57,181][ INFO][PID:273761] Retry #1 (on other host) [2024-11-28 02:26:57,182][ INFO][PID:273761] Trying to allocate VM [2024-11-28 02:27:00,272][ INFO][PID:273761] Allocated host ResallocHost, ticket_id=6109378, hostname=54.81.67.160 [2024-11-28 02:27:00,272][ INFO][PID:273761] Allocating ssh connection to builder [2024-11-28 02:27:00,273][ INFO][PID:273761] Checking that builder machine is OK [2024-11-28 02:27:00,649][ INFO][PID:273761] Running remote command: copr-builder-ready srpm-builds [2024-11-28 02:27:00,878][ INFO][PID:273761] Red Hat subscription not needed for srpm-builds Builder is ready to be used [2024-11-28 02:27:00,879][ INFO][PID:273761] Filling build.info file with builder info [2024-11-28 02:27:00,887][ INFO][PID:273761] Not re-notifying FE and msg buses for the new host. [2024-11-28 02:27:00,888][ INFO][PID:273761] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --srpm --task-url https://copr.fedorainfracloud.org/backend/get-srpm-build-task/8321173 --detached [2024-11-28 02:27:01,322][ INFO][PID:273761] Downloading the builder-live.log file, attempt 1 [2024-11-28 02:27:01,324][ INFO][PID:273761] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.81.67.160 copr-rpmbuild-log [2024-11-28 02:27:44,265][ INFO][PID:273761] Downloading results from builder [2024-11-28 02:27:44,266][ INFO][PID:273761] rsyncing of mockbuilder@54.81.67.160:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173 started [2024-11-28 02:27:44,267][ INFO][PID:273761] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@54.81.67.160:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/ &> /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/build-08321173.rsync.log [2024-11-28 02:27:47,023][ INFO][PID:273761] rsyncing finished. [2024-11-28 02:27:47,024][ INFO][PID:273761] Releasing VM back to pool [2024-11-28 02:27:47,064][ INFO][PID:273761] Searching for 'success' file in resultdir [2024-11-28 02:27:47,065][ INFO][PID:273761] Getting build details [2024-11-28 02:27:47,065][ INFO][PID:273761] Retrieving SRPM info from /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173 [2024-11-28 02:27:47,066][ INFO][PID:273761] SRPM URL: https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/llvm-20.0.0~pre20241128.g9bdf683ba6cd9a-1.src.rpm [2024-11-28 02:27:47,067][ INFO][PID:273761] build details: {'srpm_url': 'https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/llvm-20.0.0~pre20241128.g9bdf683ba6cd9a-1.src.rpm', 'pkg_name': 'llvm', 'pkg_version': '19.1.3-1'} [2024-11-28 02:27:47,067][ INFO][PID:273761] Finished build: id=8321173 failed=False timeout=108000 destdir=/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128 chroot=srpm-builds [2024-11-28 02:27:47,069][ INFO][PID:273761] Worker succeeded build, took 46.18211603164673 [2024-11-28 02:27:47,070][ INFO][PID:273761] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": null, "enable_net": true, "project_owner": "@fedora-llvm-team", "project_name": "llvm-snapshots-pgo-20241128", "project_dirname": "llvm-snapshots-pgo-20241128", "submitter": "tstellar", "ended_on": 1732760867.0696816, "started_on": 1732760820.8875656, "submitted_on": null, "status": 1, "chroot": "srpm-builds", "arch": null, "buildroot_pkgs": null, "task_id": "8321173", "build_id": 8321173, "package_name": "llvm", "package_version": null, "git_repo": null, "git_hash": null, "git_branch": null, "source_type": 8, "source_json": "{\"type\": \"git\", \"clone_url\": \"https://src.fedoraproject.org/forks/kkleine/rpms/llvm.git\", \"subdirectory\": \"\", \"committish\": \"pgo\", \"spec\": \"llvm.spec\", \"srpm_build_method\": \"make_srpm\"}", "pkg_name": "llvm", "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/llvm-20.0.0~pre20241128.g9bdf683ba6cd9a-1.src.rpm", "uses_devel_repo": null, "sandbox": "@fedora-llvm-team/llvm-snapshots-pgo-20241128--tstellar", "results": { "name": "llvm", "epoch": null, "version": "19.1.3", "release": "1", "exclusivearch": [], "excludearch": [] }, "appstream": false, "allow_user_ssh": null, "ssh_public_keys": null, "storage": null, "background": false, "repos": [], "destdir": "/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128", "result_dir": "08321173", "built_packages": "", "tags": [ "arch_noarch" ], "pkg_version": "19.1.3-1", "id": 8321173, "mockchain_macros": { "copr_username": "@fedora-llvm-team", "copr_projectname": "llvm-snapshots-pgo-20241128", "vendor": "Fedora Project COPR (@fedora-llvm-team/llvm-snapshots-pgo-20241128)" } } ] } [2024-11-28 02:27:47,122][ INFO][PID:273761] Sending fedora-messaging bus message in build.end [2024-11-28 02:27:47,142][ INFO][PID:273761] Compressing /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/builder-live.log by gzip [2024-11-28 02:27:47,143][ INFO][PID:273761] Running command 'gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/builder-live.log' as PID 363726 [2024-11-28 02:27:47,147][ INFO][PID:273761] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/builder-live.log) [2024-11-28 02:27:47,148][ INFO][PID:273761] Compressing /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/backend.log by gzip [2024-11-28 02:27:47,149][ INFO][PID:273761] Running command 'gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-pgo-20241128/srpm-builds/08321173/backend.log' as PID 363727