[2024-04-26 11:28:37,258][  INFO][PID:471696] Marking build as starting
[2024-04-26 11:28:37,364][  INFO][PID:471696] Trying to allocate VM
[2024-04-26 11:29:25,489][  INFO][PID:471696] Allocated host ResallocHost, ticket_id=2483638, hostname=2620:52:3:1:dead:beef:cafe:c213
[2024-04-26 11:29:25,491][  INFO][PID:471696] Allocating ssh connection to builder
[2024-04-26 11:29:25,493][  INFO][PID:471696] Checking that builder machine is OK
[2024-04-26 11:29:27,322][  INFO][PID:471696] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c213 '/usr/bin/test -f /etc/mock/fedora-rawhide-ppc64le.cfg'
[2024-04-26 11:29:28,239][  INFO][PID:471696] Filling build.info file with builder info
[2024-04-26 11:29:28,241][  INFO][PID:471696] Sending build state back to frontend: {
    "builds": [
        {
            "timeout": 115200,
            "frontend_base_url": "https://copr.fedorainfracloud.org",
            "memory_reqs": 2048,
            "enable_net": false,
            "project_owner": "nickc",
            "project_name": "binutils-test",
            "project_dirname": "binutils-test",
            "submitter": "nickc",
            "ended_on": null,
            "started_on": 1714130968.240958,
            "submitted_on": null,
            "status": 3,
            "chroot": "fedora-rawhide-ppc64le",
            "arch": "ppc64le",
            "buildroot_pkgs": [],
            "task_id": "7372082-fedora-rawhide-ppc64le",
            "build_id": 7372082,
            "package_name": "texlive",
            "package_version": "11:2023-71",
            "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/nickc/binutils-test/texlive",
            "git_hash": "000ad146a4bad8daf2e6d258212521014ea82193",
            "git_branch": null,
            "source_type": null,
            "source_json": null,
            "pkg_name": null,
            "pkg_main_version": null,
            "pkg_epoch": null,
            "pkg_release": null,
            "srpm_url": null,
            "uses_devel_repo": false,
            "sandbox": "nickc/binutils-test--nickc",
            "results": null,
            "appstream": false,
            "allow_user_ssh": false,
            "ssh_public_keys": null,
            "background": true,
            "fedora_review": false,
            "isolation": "default",
            "repo_priority": null,
            "repos": [
                {
                    "baseurl": "https://download.copr.fedorainfracloud.org/results/nickc/binutils-test/fedora-rawhide-ppc64le/",
                    "id": "copr_base",
                    "name": "Copr repository",
                    "priority": null
                }
            ],
            "tags": [
                "arch_ppc64le"
            ],
            "with_opts": [],
            "without_opts": [],
            "destdir": "/var/lib/copr/public_html/results/nickc/binutils-test",
            "results_repo_url": "https://download.copr.fedorainfracloud.org/results/nickc/binutils-test",
            "result_dir": "07372082-texlive",
            "built_packages": "",
            "id": 7372082,
            "mockchain_macros": {
                "copr_username": "nickc",
                "copr_projectname": "binutils-test",
                "vendor": "Fedora Project COPR (nickc/binutils-test)"
            }
        }
    ]
}
[2024-04-26 11:29:28,373][  INFO][PID:471696] Sending fedora-messaging bus message in build.start
[2024-04-26 11:29:29,615][  INFO][PID:471696] Sending fedora-messaging bus message in chroot.start
[2024-04-26 11:29:29,658][  INFO][PID:471696] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/7372082-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached
[2024-04-26 11:29:30,984][  INFO][PID:471696] Downloading the builder-live.log file, attempt 1
[2024-04-26 11:29:30,987][  INFO][PID:471696] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c213 copr-rpmbuild-log
[2024-04-26 11:29:36,004][  INFO][PID:471696] Status: False
[2024-04-26 11:39:36,154][  INFO][PID:471696] Status: False
[2024-04-26 11:49:36,317][  INFO][PID:471696] Status: False
[2024-04-26 11:59:36,942][  INFO][PID:471696] Status: False
[2024-04-26 12:09:37,149][  INFO][PID:471696] Status: False
[2024-04-26 12:19:37,373][  INFO][PID:471696] Status: False
[2024-04-26 12:29:37,718][  INFO][PID:471696] Status: False
[2024-04-26 12:39:37,973][  INFO][PID:471696] Status: False
[2024-04-26 12:49:38,263][  INFO][PID:471696] Status: False
[2024-04-26 12:59:38,533][  INFO][PID:471696] Status: False
[2024-04-26 13:09:38,797][  INFO][PID:471696] Status: False
[2024-04-26 13:19:39,079][  INFO][PID:471696] Status: False
[2024-04-26 13:29:39,392][  INFO][PID:471696] Status: False
[2024-04-26 13:39:39,615][  INFO][PID:471696] Status: False
[2024-04-26 13:49:39,739][  INFO][PID:471696] Status: False
[2024-04-26 13:59:39,850][  INFO][PID:471696] Status: False
[2024-04-26 14:09:40,004][  INFO][PID:471696] Status: False
[2024-04-26 14:19:40,182][  INFO][PID:471696] Status: False
[2024-04-26 14:29:40,327][  INFO][PID:471696] Status: False
[2024-04-26 14:39:40,419][  INFO][PID:471696] Status: False
[2024-04-26 14:49:40,520][  INFO][PID:471696] Status: False
[2024-04-26 14:59:40,630][  INFO][PID:471696] Status: False
[2024-04-26 15:09:40,722][  INFO][PID:471696] Status: False
[2024-04-26 15:19:40,834][  INFO][PID:471696] Status: False
[2024-04-26 15:29:40,935][  INFO][PID:471696] Status: False
[2024-04-26 15:39:41,072][  INFO][PID:471696] Status: False
[2024-04-26 15:49:41,220][  INFO][PID:471696] Status: False
[2024-04-26 15:59:41,330][  INFO][PID:471696] Status: False
[2024-04-26 16:09:41,435][  INFO][PID:471696] Status: False
[2024-04-26 16:19:41,565][  INFO][PID:471696] Status: False
[2024-04-26 16:29:41,686][  INFO][PID:471696] Status: False
[2024-04-26 16:39:41,785][  INFO][PID:471696] Status: False
[2024-04-26 16:49:41,870][  INFO][PID:471696] Status: False
[2024-04-26 16:57:55,768][  INFO][PID:471696] Downloading results from builder
[2024-04-26 16:57:55,769][  INFO][PID:471696] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c213]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive started
[2024-04-26 16:57:55,770][  INFO][PID:471696] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c213]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/ &> /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/build-07372082.rsync.log
[2024-04-26 17:00:14,689][  INFO][PID:471696] rsyncing finished.
[2024-04-26 17:00:14,689][ ERROR][PID:471696] Failed to download data from builder due to rsync error, see the rsync log file for details.
[2024-04-26 17:00:14,690][ ERROR][PID:471696] SSH connection lost on #1 attempt, let's retry after 10s, Failed to download data from builder due to rsync error, see the rsync log file for details.
[2024-04-26 17:00:24,691][  INFO][PID:471696] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c213]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive started
[2024-04-26 17:00:24,692][  INFO][PID:471696] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c213]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/ &> /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/build-07372082.rsync.log
[2024-04-26 17:03:03,426][  INFO][PID:471696] rsyncing finished.
[2024-04-26 17:03:03,427][ ERROR][PID:471696] Failed to download data from builder due to rsync error, see the rsync log file for details.
[2024-04-26 17:03:03,428][ ERROR][PID:471696] SSH connection lost on #2 attempt, let's retry after 10s, Failed to download data from builder due to rsync error, see the rsync log file for details.
[2024-04-26 17:03:13,428][  INFO][PID:471696] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c213]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive started
[2024-04-26 17:03:13,430][  INFO][PID:471696] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c213]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/ &> /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/build-07372082.rsync.log
[2024-04-26 17:07:50,261][  INFO][PID:471696] rsyncing finished.
[2024-04-26 17:07:50,261][ ERROR][PID:471696] Failed to download data from builder due to rsync error, see the rsync log file for details.
[2024-04-26 17:07:50,262][ ERROR][PID:471696] SSH connection lost on #3 attempt, let's retry after 10s, Failed to download data from builder due to rsync error, see the rsync log file for details.
[2024-04-26 17:08:00,263][ ERROR][PID:471696] Re-try request for task on 'ResallocHost, ticket_id=2483638, hostname=2620:52:3:1:dead:beef:cafe:c213': Unable to finish after 3 SSH attempts
[2024-04-26 17:08:00,263][  INFO][PID:471696] Releasing VM back to pool
[2024-04-26 17:08:00,276][  INFO][PID:471696] Retry #1 (on other host)
[2024-04-26 17:08:00,279][  INFO][PID:471696] Trying to allocate VM
[2024-04-26 17:08:03,313][  INFO][PID:471696] Allocated host ResallocHost, ticket_id=2493556, hostname=2620:52:3:1:dead:beef:cafe:c29d
[2024-04-26 17:08:03,313][  INFO][PID:471696] Allocating ssh connection to builder
[2024-04-26 17:08:03,314][  INFO][PID:471696] Checking that builder machine is OK
[2024-04-26 17:08:04,279][  INFO][PID:471696] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c29d '/usr/bin/test -f /etc/mock/fedora-rawhide-ppc64le.cfg'
[2024-04-26 17:08:04,746][  INFO][PID:471696] Filling build.info file with builder info
[2024-04-26 17:08:04,757][  INFO][PID:471696] Not re-notifying FE and msg buses for the new host.
[2024-04-26 17:08:04,758][  INFO][PID:471696] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/7372082-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached
[2024-04-26 17:08:05,668][  INFO][PID:471696] Downloading the builder-live.log file, attempt 1
[2024-04-26 17:08:06,154][  INFO][PID:471696] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c29d copr-rpmbuild-log
[2024-04-26 17:08:10,678][  INFO][PID:471696] Status: False
[2024-04-26 17:18:10,780][  INFO][PID:471696] Status: False
[2024-04-26 17:28:10,883][  INFO][PID:471696] Status: False
[2024-04-26 17:38:11,002][  INFO][PID:471696] Status: False
[2024-04-26 17:48:11,094][  INFO][PID:471696] Status: False
[2024-04-26 17:58:11,218][  INFO][PID:471696] Status: False
[2024-04-26 18:08:11,316][  INFO][PID:471696] Status: False
[2024-04-26 18:18:11,422][  INFO][PID:471696] Status: False
[2024-04-26 18:28:11,520][  INFO][PID:471696] Status: False
[2024-04-26 18:38:11,694][  INFO][PID:471696] Status: False
[2024-04-26 18:48:11,809][  INFO][PID:471696] Status: False
[2024-04-26 18:58:11,898][  INFO][PID:471696] Status: False
[2024-04-26 19:08:12,008][  INFO][PID:471696] Status: False
[2024-04-26 19:14:13,713][  INFO][PID:471696] Downloading results from builder
[2024-04-26 19:14:13,714][  INFO][PID:471696] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c29d]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive started
[2024-04-26 19:14:13,715][  INFO][PID:471696] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c29d]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/ &> /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/build-07372082.rsync.log
[2024-04-26 19:16:10,418][  INFO][PID:471696] rsyncing finished.
[2024-04-26 19:16:10,419][  INFO][PID:471696] Releasing VM back to pool
[2024-04-26 19:16:10,474][  INFO][PID:471696] Searching for 'success' file in resultdir
[2024-04-26 19:16:10,475][ ERROR][PID:471696] Build failed: Backend process error: No success file => build failure
[2024-04-26 19:16:10,477][  INFO][PID:471696] Finished build: id=7372082 failed=True timeout=115200 destdir=/var/lib/copr/public_html/results/nickc/binutils-test chroot=fedora-rawhide-ppc64le 
[2024-04-26 19:16:10,480][ ERROR][PID:471696] Backend process error: No success file => build failure
[2024-04-26 19:16:10,481][  INFO][PID:471696] Worker failed build, took 7685.723817110062
[2024-04-26 19:16:10,482][  INFO][PID:471696] Sending build state back to frontend: {
    "builds": [
        {
            "timeout": 115200,
            "frontend_base_url": "https://copr.fedorainfracloud.org",
            "memory_reqs": 2048,
            "enable_net": false,
            "project_owner": "nickc",
            "project_name": "binutils-test",
            "project_dirname": "binutils-test",
            "submitter": "nickc",
            "ended_on": 1714158970.4812422,
            "started_on": 1714151284.757425,
            "submitted_on": null,
            "status": 0,
            "chroot": "fedora-rawhide-ppc64le",
            "arch": "ppc64le",
            "buildroot_pkgs": [],
            "task_id": "7372082-fedora-rawhide-ppc64le",
            "build_id": 7372082,
            "package_name": "texlive",
            "package_version": "11:2023-71",
            "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/nickc/binutils-test/texlive",
            "git_hash": "000ad146a4bad8daf2e6d258212521014ea82193",
            "git_branch": null,
            "source_type": null,
            "source_json": null,
            "pkg_name": null,
            "pkg_main_version": null,
            "pkg_epoch": null,
            "pkg_release": null,
            "srpm_url": null,
            "uses_devel_repo": false,
            "sandbox": "nickc/binutils-test--nickc",
            "results": null,
            "appstream": false,
            "allow_user_ssh": false,
            "ssh_public_keys": null,
            "background": true,
            "fedora_review": false,
            "isolation": "default",
            "repo_priority": null,
            "repos": [
                {
                    "baseurl": "https://download.copr.fedorainfracloud.org/results/nickc/binutils-test/fedora-rawhide-ppc64le/",
                    "id": "copr_base",
                    "name": "Copr repository",
                    "priority": null
                }
            ],
            "tags": [
                "arch_ppc64le"
            ],
            "with_opts": [],
            "without_opts": [],
            "destdir": "/var/lib/copr/public_html/results/nickc/binutils-test",
            "results_repo_url": "https://download.copr.fedorainfracloud.org/results/nickc/binutils-test",
            "result_dir": "07372082-texlive",
            "built_packages": "",
            "id": 7372082,
            "mockchain_macros": {
                "copr_username": "nickc",
                "copr_projectname": "binutils-test",
                "vendor": "Fedora Project COPR (nickc/binutils-test)"
            }
        }
    ]
}
[2024-04-26 19:16:10,577][  INFO][PID:471696] Sending fedora-messaging bus message in build.end
[2024-04-26 19:16:10,621][  INFO][PID:471696] Compressing /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/builder-live.log by gzip
[2024-04-26 19:16:10,624][  INFO][PID:471696] Running command 'gzip /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/builder-live.log' as PID 3545973
[2024-04-26 19:17:35,864][  INFO][PID:471696] Finished after 85 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/builder-live.log)
[2024-04-26 19:17:35,866][  INFO][PID:471696] Compressing /var/lib/copr/public_html/results/nickc/binutils-test/fedora-rawhide-ppc64le/07372082-texlive/backend.log by gzip