// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ambient

import (
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sv1 "sigs.k8s.io/gateway-api/apis/v1"

	"istio.io/api/label"
	apiv1alpha3 "istio.io/client-go/pkg/apis/networking/v1"
	clientsecurityv1beta1 "istio.io/client-go/pkg/apis/security/v1"
	"istio.io/istio/pilot/pkg/features"
	"istio.io/istio/pilot/pkg/model"
	"istio.io/istio/pkg/cluster"
	"istio.io/istio/pkg/config/constants"
	"istio.io/istio/pkg/kube/controllers"
	"istio.io/istio/pkg/kube/kclient/clienttest"
	"istio.io/istio/pkg/kube/krt"
	"istio.io/istio/pkg/kube/multicluster"
	"istio.io/istio/pkg/ptr"
	"istio.io/istio/pkg/test"
	"istio.io/istio/pkg/test/util/assert"
	"istio.io/istio/pkg/test/util/retry"
	"istio.io/istio/pkg/workloadapi"
	"istio.io/istio/tests/util/leak"
)

type ambientclients struct {
	pc    clienttest.TestClient[*corev1.Pod]
	sc    clienttest.TestClient[*corev1.Service]
	sec   clienttest.TestWriter[*corev1.Secret]
	ns    clienttest.TestWriter[*corev1.Namespace]
	grc   clienttest.TestWriter[*k8sv1.Gateway]
	gwcls clienttest.TestWriter[*k8sv1.GatewayClass]
	se    clienttest.TestWriter[*apiv1alpha3.ServiceEntry]
	we    clienttest.TestWriter[*apiv1alpha3.WorkloadEntry]
	pa    clienttest.TestWriter[*clientsecurityv1beta1.PeerAuthentication]
	authz clienttest.TestWriter[*clientsecurityv1beta1.AuthorizationPolicy]
}

type remoteAmbientClients struct {
	clusterID cluster.ID
	*ambientclients
}

func (r *remoteAmbientClients) ResourceName() string {
	return string(r.clusterID)
}

func TestAmbientMulticlusterIndex_WaypointForWorkloadTraffic(t *testing.T) {
	test.SetForTest(t, &features.EnableAmbientMultiNetwork, true)
	cases := []struct {
		name         string
		trafficType  string
		podAssertion func(s *ambientTestServer, c cluster.ID)
		svcAssertion func(s *ambientTestServer, svcKey string)
	}{
		{
			name:        "service traffic",
			trafficType: constants.ServiceTraffic,
			podAssertion: func(s *ambientTestServer, c cluster.ID) {
				s.t.Helper()
				// Test that there's no events for workloads in our cluster
				// as a result of our actions (we can't control pushes for
				// workloads from other clusters)
				s.assertNoMatchingEvent(s.t, c.String())
			},
			svcAssertion: func(s *ambientTestServer, _ string) {
				s.t.Helper()
				s.assertEvent(s.t, s.svcXdsName("svc2"))
			},
		},
		{
			name:        "all traffic",
			trafficType: constants.AllTraffic,
			podAssertion: func(s *ambientTestServer, _ cluster.ID) {
				s.t.Helper()
				s.assertEvent(s.t, s.podXdsName("pod1"))
			},
			svcAssertion: func(s *ambientTestServer, _ string) {
				s.t.Helper()
				s.assertEvent(s.t, s.svcXdsName("svc2"))
			},
		},
		{
			name:        "workload traffic",
			trafficType: constants.WorkloadTraffic,
			podAssertion: func(s *ambientTestServer, c cluster.ID) {
				s.t.Helper()
				s.assertEvent(s.t, s.podXdsName("pod1"))
			},
			svcAssertion: func(s *ambientTestServer, svcKey string) {
				s.t.Helper()
				s.assertNoMatchingEvent(s.t, svcKey)
			},
		},
		{
			name:        "no traffic",
			trafficType: constants.NoTraffic,
			podAssertion: func(s *ambientTestServer, c cluster.ID) {
				s.t.Helper()
				s.assertNoMatchingEvent(s.t, c.String())
			},
			svcAssertion: func(s *ambientTestServer, svcKey string) {
				s.t.Helper()
				s.assertNoMatchingEvent(s.t, svcKey)
			},
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			s := newAmbientTestServer(t, testC, testNW, "")
			s.AddSecret("s1", "c1") // overlapping ips
			s.AddSecret("s2", "c2") // Non-overlapping ips
			remoteClients := krt.NewCollection(s.mcController.Clusters(), func(_ krt.HandlerContext, c *multicluster.Cluster) **remoteAmbientClients {
				cl := c.Client
				return ptr.Of(&remoteAmbientClients{
					clusterID: c.ID,
					ambientclients: &ambientclients{
						pc:    clienttest.NewDirectClient[*corev1.Pod, corev1.Pod, *corev1.PodList](t, cl),
						sc:    clienttest.NewDirectClient[*corev1.Service, corev1.Service, *corev1.ServiceList](t, cl),
						ns:    clienttest.NewWriter[*corev1.Namespace](t, cl),
						grc:   clienttest.NewWriter[*k8sv1.Gateway](t, cl),
						gwcls: clienttest.NewWriter[*k8sv1.GatewayClass](t, cl),
						se:    clienttest.NewWriter[*apiv1alpha3.ServiceEntry](t, cl),
						we:    clienttest.NewWriter[*apiv1alpha3.WorkloadEntry](t, cl),
						pa:    clienttest.NewWriter[*clientsecurityv1beta1.PeerAuthentication](t, cl),
						authz: clienttest.NewWriter[*clientsecurityv1beta1.AuthorizationPolicy](t, cl),
						sec:   clienttest.NewWriter[*corev1.Secret](t, cl),
					},
				})
			})

			assert.EventuallyEqual(t, func() int {
				return len(remoteClients.List())
			}, 2)

			differentCIDRIPs := map[string]string{
				"waypoint": "10.1.0.10",
				"pod1":     "127.0.0.6",
				"svc2":     "10.1.0.1",
			}
			duplicateCIDRIPs := map[string]string{
				"waypoint": "10.0.0.10",
				"pod1":     "127.0.0.1",
				"svc2":     "10.0.0.1",
			}
			clusterToIPs := map[cluster.ID]map[string]string{
				"cluster0": duplicateCIDRIPs,
				"c1":       duplicateCIDRIPs,
				"c2":       differentCIDRIPs,
			}
			clusterToNetwork := map[cluster.ID]string{
				"cluster0": testNW,
				"c1":       "testnetwork-2", // overlapping ips
				"c2":       testNW,          // different ips
			}
			networkGatewayIps := map[cluster.ID]string{ // these have to be global
				"cluster0": "77.1.2.4",
				"c1":       "77.1.2.49",
				"c2":       "", // c2 and cluster0 share the same network, so no gw
			}
			rClients := remoteClients.List()
			clients := append([]*remoteAmbientClients{
				{
					clusterID: s.clusterID,
					ambientclients: &ambientclients{
						pc:    s.pc,
						sc:    s.sc,
						ns:    s.ns,
						grc:   s.grc,
						gwcls: s.gwcls,
						se:    s.se,
						we:    s.we,
						pa:    s.pa,
						authz: s.authz,
						sec:   s.sec,
					},
				},
			}, rClients...)
			for _, client := range clients {
				// Test ambient index already creates istio-system namespace
				if client.clusterID != s.clusterID {
					client.ns.Create(&corev1.Namespace{
						ObjectMeta: metav1.ObjectMeta{
							Name:   systemNS,
							Labels: map[string]string{label.TopologyNetwork.Name: clusterToNetwork[client.clusterID]},
						},
					})
					client.ns.Create(&corev1.Namespace{
						ObjectMeta: metav1.ObjectMeta{
							Name:   testNS,
							Labels: map[string]string{label.TopologyNetwork.Name: clusterToNetwork[client.clusterID]},
						},
					})
					client.gwcls.Create(&k8sv1.GatewayClass{
						ObjectMeta: metav1.ObjectMeta{
							Name: constants.EastWestGatewayClassName,
						},
						Spec: k8sv1.GatewayClassSpec{
							ControllerName: constants.ManagedGatewayMeshController,
						},
					})

					// A cluster's network is read from the topology label on its own system namespace, so wait
					// for that namespace to reach the cluster's informer before making other assertions.
					assert.EventuallyEqual(t, func() string {
						cl := s.mcController.ClusterStore().GetByID(client.clusterID)
						if cl == nil || !cl.HasSynced() {
							return ""
						}
						ns := cl.Namespaces().GetKey(systemNS)
						if ns == nil {
							return ""
						}
						return (*ns).Labels[label.TopologyNetwork.Name]
					}, clusterToNetwork[client.clusterID])
				}
				if networkGatewayIps[client.clusterID] != "" {
					s.addNetworkGatewayForClient(t, networkGatewayIps[client.clusterID], clusterToNetwork[client.clusterID], client.grc)
				}
				if networkGatewayIps[client.clusterID] != "" {
					s.addNetworkGatewayForClient(t, networkGatewayIps[client.clusterID], clusterToNetwork[client.clusterID], client.grc)
				}
			}

			// Need the namespaces to exist before creating services
			assert.EventuallyEqual(t, func() int {
				ns := s.namespaces.List()
				return len(ns)
			}, 2)

			for _, client := range clients {
				ips := clusterToIPs[client.clusterID]
				// These steps happen for every test regardless of traffic type.
				// It involves creating a waypoint for the specified traffic type
				// then creating a workload and a service with no annotations set
				// on these objects yet.
				s.addWaypointForClient(t, ips["waypoint"], "test-wp", c.trafficType, true, client.grc)

				s.addServiceForClient(t, "svc2",
					map[string]string{
						"istio.io/global": "true",
					},
					map[string]string{},
					[]int32{80}, map[string]string{"app": "a"}, ips["svc2"], client.sc)
				s.assertEvent(t, s.svcXdsName("svc2"))
			}

			// Service configuration needs to be uniform, so we add services to all clusters first,
			// then check for events
			events := make([]string, 0, len(clients))
			for _, client := range clients {
				ips := clusterToIPs[client.clusterID]
				s.addPodsForClient(t, ips["pod1"], "pod1", "sa1",
					map[string]string{"app": "a"}, nil, true, corev1.PodRunning, client.pc)
				if clusterToNetwork[client.clusterID] == clusterToNetwork[s.clusterID] {
					events = append(events, s.podXdsNameForCluster("pod1", client.clusterID))
				} else if networkGatewayIps[client.clusterID] != "" {
					events = append(events, fmt.Sprintf(
						"%s/SplitHorizonWorkload/ns1/east-west/%s/%s",
						clusterToNetwork[client.clusterID],
						networkGatewayIps[client.clusterID],
						s.svcXdsName("svc2"),
					))
				}
			}
			// the service sans get updated.
			events = append(events, s.svcXdsName("svc2"))
			s.assertEvent(t, events...)

			// Label the pod and check that the correct event is produced.
			s.labelPod(t, "pod1", testNS,
				map[string]string{"app": "a", label.IoIstioUseWaypoint.Name: "test-wp"})
			c.podAssertion(s, testC)

			// Label the service and check that the correct event is produced.
			s.labelService(t, "svc2", testNS,
				map[string]string{
					label.IoIstioUseWaypoint.Name: "test-wp",
					"istio.io/global":             "true",
				})
			c.svcAssertion(s, s.svcXdsName("svc2"))

			// clean up resources
			s.deleteService(t, "svc2")
			s.assertEvent(t, s.podXdsName("pod1"), s.svcXdsName("svc2"))
			s.deletePod(t, "pod1")
			s.assertEvent(t, s.podXdsName("pod1"))
			s.deleteWaypoint(t, "test-wp")

			for _, rc := range remoteClients.List() {
				// Removing the service changes the WDS workload in that cluster due to service attachments.
				// Note that we should NOT get an event changing the service attachment in our local cluster.
				// We also get a service event because we lost an IP
				if clusterToNetwork[rc.clusterID] == clusterToNetwork[s.clusterID] {
					s.deleteServiceForClient(t, "svc2", rc.sc)
					s.assertEvent(t, s.podXdsNameForCluster("pod1", rc.clusterID), s.svcXdsName("svc2"))
					s.deletePodForClient(t, "pod1", rc.pc)
					s.assertEvent(t, s.podXdsNameForCluster("pod1", rc.clusterID))
				} else {
					s.deleteServiceForClient(t, "svc2", rc.sc)
					s.assertEvent(t, fmt.Sprintf(
						"%s/SplitHorizonWorkload/ns1/east-west/%s/%s",
						clusterToNetwork[rc.clusterID],
						networkGatewayIps[rc.clusterID],
						s.svcXdsName("svc2"),
					), s.svcXdsName("svc2"))
					s.deletePodForClient(t, "pod1", rc.pc)
				}
				s.deleteWaypointForClient(t, "test-wp", rc.grc)
			}
			s.clearEvents()
		})
	}
}

func TestMulticlusterAmbientIndex_ServicesForWaypoint(t *testing.T) {
	test.SetForTest(t, &features.EnableAmbientMultiNetwork, true)
	wpKey := model.WaypointKey{
		Namespace: testNS,
		Hostnames: []string{fmt.Sprintf("%s.%s.svc.company.com", "wp", testNS)},
		Addresses: []string{"10.0.0.1"},
		Network:   testNW,
	}
	t.Run("waypoint eds", func(t *testing.T) {
		s := newAmbientTestServer(t, testC, testNW, "")
		s.addService(t, "svc1",
			map[string]string{
				label.IoIstioUseWaypoint.Name: "wp",
				"istio.io/global":             "true",
			},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "app1"}, "11.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		s.addWaypointSpecificAddress(t, "", s.hostnameForService("wp"), "wp", constants.AllTraffic, true)
		s.addService(t, "wp",
			map[string]string{},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "waypoint"}, "10.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		s.addPodsForClient(t, "10.0.1.5", "wp-pod", "sa1",
			map[string]string{"app": "waypoint"}, nil, true, corev1.PodRunning, s.pc)
		// Event IDs do not have the namespace prefix for EDS
		s.assertEvent(s.t, "svc1.ns1.svc.company.com")
	})

	t.Run("hostname (multicluster but unused)", func(t *testing.T) {
		s := newAmbientTestServer(t, testC, testNW, "")
		s.addService(t, "svc1",
			map[string]string{label.IoIstioUseWaypoint.Name: "wp"},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "app1"}, "11.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		s.addWaypointSpecificAddress(t, "", s.hostnameForService("wp"), "wp", constants.AllTraffic, true)
		s.addService(t, "wp",
			map[string]string{},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "waypoint"}, "10.0.0.2")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		svc1Host := ptr.ToList(s.services.GetKey(fmt.Sprintf("%s/%s", testNS, s.hostnameForService("svc1"))))
		assert.Equal(t, len(svc1Host), 1)
		assert.EventuallyEqual(t, func() []model.ServiceInfo {
			return s.ServicesForWaypoint(wpKey)
		}, svc1Host)
	})
	t.Run("ip (multicluster but unused)", func(t *testing.T) {
		s := newAmbientTestServer(t, testC, testNW, "")
		s.addService(t, "svc1",
			map[string]string{label.IoIstioUseWaypoint.Name: "wp"},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "app1"}, "11.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		s.addWaypointSpecificAddress(t, "10.0.0.1", "", "wp", constants.AllTraffic, true)
		s.addService(t, "wp",
			map[string]string{},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "waypoint"}, "10.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		svc1Host := ptr.ToList(s.services.GetKey(fmt.Sprintf("%s/%s", testNS, s.hostnameForService("svc1"))))
		assert.Equal(t, len(svc1Host), 1)
		assert.EventuallyEqual(t, func() []model.ServiceInfo {
			return s.ServicesForWaypoint(wpKey)
		}, svc1Host)
	})
	t.Run("mixed (multicluster but unused)", func(t *testing.T) {
		s := newAmbientTestServer(t, testC, testNW, "")
		s.addService(t, "svc1",
			map[string]string{label.IoIstioUseWaypoint.Name: "wp"},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "app1"}, "11.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		s.addWaypointSpecificAddress(t, "10.0.0.1", s.hostnameForService("wp"), "wp", constants.AllTraffic, true)
		s.addService(t, "wp",
			map[string]string{},
			map[string]string{},
			[]int32{80}, map[string]string{"app": "waypoint"}, "10.0.0.1")
		s.assertEvent(s.t, s.svcXdsName("svc1"))

		svc1Host := ptr.ToList(s.services.GetKey(fmt.Sprintf("%s/%s", testNS, s.hostnameForService("svc1"))))
		assert.Equal(t, len(svc1Host), 1)
		assert.EventuallyEqual(t, func() []model.ServiceInfo {
			return s.ServicesForWaypoint(wpKey)
		}, svc1Host)
	})
}

func TestMulticlusterAmbientIndex_TestServiceMerging(t *testing.T) {
	test.SetForTest(t, &features.EnableAmbientMultiNetwork, true)
	s := newAmbientTestServer(t, testC, testNW, "")
	s.AddSecret("s1", "remote-cluster") // overlapping ips
	remoteClients := krt.NewCollection(s.mcController.Clusters(), func(_ krt.HandlerContext, c *multicluster.Cluster) **remoteAmbientClients {
		cl := c.Client
		return ptr.Of(&remoteAmbientClients{
			clusterID: c.ID,
			ambientclients: &ambientclients{
				pc:    clienttest.NewDirectClient[*corev1.Pod, corev1.Pod, *corev1.PodList](t, cl),
				sc:    clienttest.NewDirectClient[*corev1.Service, corev1.Service, *corev1.ServiceList](t, cl),
				ns:    clienttest.NewWriter[*corev1.Namespace](t, cl),
				grc:   clienttest.NewWriter[*k8sv1.Gateway](t, cl),
				gwcls: clienttest.NewWriter[*k8sv1.GatewayClass](t, cl),
				se:    clienttest.NewWriter[*apiv1alpha3.ServiceEntry](t, cl),
				we:    clienttest.NewWriter[*apiv1alpha3.WorkloadEntry](t, cl),
				pa:    clienttest.NewWriter[*clientsecurityv1beta1.PeerAuthentication](t, cl),
				authz: clienttest.NewWriter[*clientsecurityv1beta1.AuthorizationPolicy](t, cl),
				sec:   clienttest.NewWriter[*corev1.Secret](t, cl),
			},
		})
	})

	const remoteNetwork = "remote-network"

	assert.EventuallyEqual(t, func() int {
		return len(remoteClients.List())
	}, 1)

	remoteClient := remoteClients.List()[0]
	localClient := remoteAmbientClients{
		clusterID: s.clusterID,
		ambientclients: &ambientclients{
			pc:    s.pc,
			sc:    s.sc,
			ns:    s.ns,
			grc:   s.grc,
			gwcls: s.gwcls,
			se:    s.se,
			we:    s.we,
			pa:    s.pa,
			authz: s.authz,
			sec:   s.sec,
		},
	}
	remoteClient.ns.Create(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name:   systemNS,
			Labels: map[string]string{label.TopologyNetwork.Name: remoteNetwork},
		},
	})
	remoteClient.ns.Create(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: testNS,
		},
	})
	s.addServiceForClient(
		t, "svc2",

		map[string]string{},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "10.0.0.1", localClient.sc,
	)
	s.addServiceForClient(
		t, "svc2",
		map[string]string{},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "127.1.0.1", remoteClient.sc,
	)
	retry.UntilSuccessOrFail(t, func() error {
		svc := s.lookupService("ns1/svc2.ns1.svc.company.com")
		if svc == nil {
			return fmt.Errorf("service not found")
		}
		if svc.Scope == model.Global {
			return fmt.Errorf("expected service scope to be Local, got %s", svc.Scope)
		}
		if len(svc.Service.Addresses) != 1 {
			return fmt.Errorf("expected service to have 1 address, got %d", len(svc.Service.Addresses))
		}
		return nil
	})
	s.deleteService(t, "svc2")
	retry.UntilSuccessOrFail(t, func() error {
		svc := s.lookupService("ns1/svc2.ns1.svc.company.com")
		if svc != nil {
			return fmt.Errorf("expected service to be deleted, but it still exists")
		}
		return nil
	})
	s.labelServiceForClient(
		t, "svc2", testNS,
		map[string]string{"istio.io/global": "true"},
		remoteClient.sc,
	)
	retry.UntilSuccessOrFail(t, func() error {
		svc := s.lookupService("ns1/svc2.ns1.svc.company.com")
		if svc == nil {
			return fmt.Errorf("service not found")
		}
		if svc.Scope != model.Global {
			return fmt.Errorf("expected service scope to be Global, got %s", svc.Scope)
		}
		if len(svc.Service.Addresses) != 1 {
			return fmt.Errorf("expected service to have 1 addresses, got %d", len(svc.Service.Addresses))
		}
		return nil
	})
	// Add a service to local cluster
	s.addService(
		t, "svc2",
		map[string]string{"istio.io/global": "true"},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "10.0.0.1",
	)
	retry.UntilSuccessOrFail(t, func() error {
		svc := s.lookupService("ns1/svc2.ns1.svc.company.com")
		if svc == nil {
			return fmt.Errorf("service not found")
		}
		if svc.Scope != model.Global {
			return fmt.Errorf("expected service scope to be Global, got %s", svc.Scope)
		}
		if len(svc.Service.Addresses) != 2 {
			return fmt.Errorf("expected service to have 2 addresses, got %d", len(svc.Service.Addresses))
		}
		return nil
	})
}

func TestMulticlusterAmbientIndex_SplitHorizon(t *testing.T) {
	test.SetForTest(t, &features.EnableAmbientMultiNetwork, true)
	s := newAmbientTestServer(t, testC, testNW, "")
	// Test that we're propagating the trust domain correctly
	s.meshConfig.Mesh().TrustDomain = s.DomainSuffix
	s.AddSecret("s1", "remote-cluster") // overlapping ips
	remoteClients := krt.NewCollection(s.mcController.Clusters(), func(_ krt.HandlerContext, c *multicluster.Cluster) **remoteAmbientClients {
		cl := c.Client
		return ptr.Of(&remoteAmbientClients{
			clusterID: c.ID,
			ambientclients: &ambientclients{
				pc:    clienttest.NewDirectClient[*corev1.Pod, corev1.Pod, *corev1.PodList](t, cl),
				sc:    clienttest.NewDirectClient[*corev1.Service, corev1.Service, *corev1.ServiceList](t, cl),
				ns:    clienttest.NewWriter[*corev1.Namespace](t, cl),
				grc:   clienttest.NewWriter[*k8sv1.Gateway](t, cl),
				gwcls: clienttest.NewWriter[*k8sv1.GatewayClass](t, cl),
				se:    clienttest.NewWriter[*apiv1alpha3.ServiceEntry](t, cl),
				we:    clienttest.NewWriter[*apiv1alpha3.WorkloadEntry](t, cl),
				pa:    clienttest.NewWriter[*clientsecurityv1beta1.PeerAuthentication](t, cl),
				authz: clienttest.NewWriter[*clientsecurityv1beta1.AuthorizationPolicy](t, cl),
				sec:   clienttest.NewWriter[*corev1.Secret](t, cl),
			},
		})
	})

	const remoteNetwork = "remote-network"

	assert.EventuallyEqual(t, func() int {
		return len(remoteClients.List())
	}, 1)

	remoteClient := remoteClients.List()[0]
	localClient := remoteAmbientClients{
		clusterID: s.clusterID,
		ambientclients: &ambientclients{
			pc:    s.pc,
			sc:    s.sc,
			ns:    s.ns,
			grc:   s.grc,
			gwcls: s.gwcls,
			se:    s.se,
			we:    s.we,
			pa:    s.pa,
			authz: s.authz,
			sec:   s.sec,
		},
	}
	remoteClient.ns.Create(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name:   systemNS,
			Labels: map[string]string{label.TopologyNetwork.Name: remoteNetwork},
		},
	})
	remoteClient.ns.Create(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: testNS,
		},
	})
	networkGatewayIP := "172.0.1.2"
	s.addNetworkGatewayForClient(t, networkGatewayIP, remoteNetwork, remoteClient.grc)
	s.addServiceForClient(
		t, "svc2",
		map[string]string{
			"istio.io/global": "true",
		},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "10.0.0.1", localClient.sc,
	)
	s.addServiceForClient(
		t, "svc2",
		map[string]string{
			"istio.io/global": "true",
		},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "127.1.0.1", remoteClient.sc,
	)
	retry.UntilSuccessOrFail(t, func() error {
		svc := s.lookupService("ns1/svc2.ns1.svc.company.com")
		if svc == nil {
			return fmt.Errorf("service not found")
		}
		if svc.Scope != model.Global {
			return fmt.Errorf("expected service scope to be Global, got %s", svc.Scope)
		}
		gwwl := s.workloads.GetKey("NetworkGateway/remote-network/172.0.1.2/0")
		if gwwl == nil {
			return fmt.Errorf("expected network gateway workload to exist, but it does not")
		}
		if len(gwwl.Workload.Addresses) != 1 {
			return fmt.Errorf("expected network gateway workload to have addresses, got %v", gwwl.Workload.Addresses)
		}
		if gwwl.Workload.TrustDomain != s.DomainSuffix {
			return fmt.Errorf(
				"expected network gateway workload to have trust domain %s, got %s",
				s.DomainSuffix,
				gwwl.Workload.TrustDomain,
			)
		}
		expectedAddress := []uint8{172, 0, 1, 2}
		if !reflect.DeepEqual(gwwl.Workload.Addresses[0], expectedAddress) {
			return fmt.Errorf(
				"expected network gateway workload to have address %s, got %s",
				networkGatewayIP,
				gwwl.Workload.Addresses[0],
			)
		}
		return nil
	})

	s.addPodsForClient(t, "10.0.1.1", "pod1", "sa1",
		map[string]string{"app": "a"}, nil, true, corev1.PodRunning, localClient.pc)
	s.addPodsForClient(t, "127.0.0.1", "pod1-abc", "sa1",
		map[string]string{"app": "a"}, nil, true, corev1.PodRunning, remoteClient.pc)

	splitHorizonName := fmt.Sprintf("%s/SplitHorizonWorkload/ns1/east-west/%s/%s",
		remoteNetwork, networkGatewayIP, s.svcXdsName("svc2"))
	retry.UntilSuccessOrFail(t, func() error {
		ais := s.Lookup("ns1/svc2.ns1.svc.company.com")
		if len(ais) != 3 {
			return fmt.Errorf("expected 3 pods, got %d", len(ais))
		}
		shwl := s.workloads.GetKey(splitHorizonName)
		if shwl == nil {
			return fmt.Errorf("expected split horizon workload to exist, but it does not")
		}
		if len(shwl.Workload.Addresses) != 0 {
			return fmt.Errorf("expected no addresses in split horizon workload, got %v", shwl.Workload.Addresses)
		}
		if !reflect.DeepEqual(
			shwl.Workload.NetworkGateway.Destination.(*workloadapi.GatewayAddress_Address).Address.Address,
			[]byte{172, 0, 1, 2},
		) {
			return fmt.Errorf(
				"expected split horizon workload to have network gateway address %s, got %s",
				networkGatewayIP,
				shwl.Workload.NetworkGateway.Destination.(*workloadapi.GatewayAddress_Address).Address.Address,
			)
		}
		if shwl.Workload.NetworkGateway.Destination.(*workloadapi.GatewayAddress_Address).Address.Network != remoteNetwork {
			return fmt.Errorf(
				"expected split horizon workload to have network %s, got %s",
				remoteNetwork,
				shwl.Workload.NetworkGateway.Destination.(*workloadapi.GatewayAddress_Address).Address.Network,
			)
		}
		if shwl.Workload.Capacity.GetValue() != 1 {
			return fmt.Errorf(
				"expected split horizon workload to have capacity 1, got %d",
				shwl.Workload.Capacity.GetValue(),
			)
		}
		if len(shwl.Workload.Addresses) != 0 {
			return fmt.Errorf(
				"expected no addresses in split horizon workload, got %v",
				shwl.Workload.Addresses,
			)
		}
		return nil
	})

	s.addPodsForClient(t, "127.0.0.2", "pod2", "sa1",
		map[string]string{"app": "a"}, nil, true, corev1.PodRunning, remoteClient.pc)
	s.assertEvent(t, s.podXdsName("pod1"), splitHorizonName, s.svcXdsName("svc2"))

	retry.UntilSuccessOrFail(t, func() error {
		ais := s.Lookup("ns1/svc2.ns1.svc.company.com")
		if len(ais) != 3 {
			return fmt.Errorf("expected 3 pods, got %d", len(ais))
		}
		shwl := s.workloads.GetKey(splitHorizonName)
		if shwl == nil {
			return fmt.Errorf("expected split horizon workload to exist, but it does not")
		}
		if shwl.Workload.Capacity.GetValue() != 2 {
			return fmt.Errorf("expected split horizon workload to have capacity 2, got %d", shwl.Workload.Capacity.GetValue())
		}
		return nil
	})

	// Remove network gateway: Split horizon workload should be removed
	s.deleteNetworkGatewayForClient(t, "east-west", remoteClient.grc)
	assert.EventuallyEqual(t, func() int {
		return len(s.Lookup("ns1/svc2.ns1.svc.company.com"))
	}, 2) // 2 pods, no split horizon workload
	retry.UntilSuccessOrFail(t, func() error {
		if s.workloads.GetKey(splitHorizonName) != nil {
			return fmt.Errorf("expected split horizon workload to be removed, but it still exists")
		}
		return nil
	})

	// Add the network gateway back: Split horizon workload should be recreated
	s.addNetworkGatewayForClient(t, networkGatewayIP, remoteNetwork, remoteClient.grc)
	retry.UntilSuccessOrFail(t, func() error {
		ais := s.Lookup("ns1/svc2.ns1.svc.company.com")
		if len(ais) != 3 {
			return fmt.Errorf("expected 3 pods, got %d", len(ais))
		}
		shwl := s.workloads.GetKey(splitHorizonName)
		if shwl == nil {
			return fmt.Errorf("expected split horizon workload to exist, but it does not")
		}
		if shwl.Workload.Capacity.GetValue() != 2 {
			return fmt.Errorf("expected split horizon workload to have capacity 2, got %d", shwl.Workload.Capacity.GetValue())
		}
		return nil
	})
	// make the service local local
	s.labelServiceForClient(t, "svc2", testNS, map[string]string{}, localClient.sc)
	s.labelServiceForClient(t, "svc2", testNS, map[string]string{}, remoteClient.sc)
	assert.EventuallyEqual(t, func() int {
		ais := s.Lookup("ns1/svc2.ns1.svc.company.com")
		return len(ais)
	}, 2)
	retry.UntilSuccessOrFail(t, func() error {
		if s.workloads.GetKey(splitHorizonName) != nil {
			return fmt.Errorf("expected split horizon workload to be removed, but it still exists")
		}
		return nil
	})

	// label remote cluster to have same network local and mark the service as global
	remoteClient.ns.Update(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name:   systemNS,
			Labels: map[string]string{label.TopologyNetwork.Name: testNW},
		},
	})

	s.labelServiceForClient(t, "svc2", testNS,
		map[string]string{"istio.io/global": "true"}, localClient.sc)
	s.labelServiceForClient(t, "svc2", testNS,
		map[string]string{"istio.io/global": "true"}, remoteClient.sc)
	s.assertEvent(
		t, s.podXdsNameForCluster("pod2", remoteClient.clusterID),
		s.podXdsNameForCluster("pod1-abc", remoteClient.clusterID),
	)
	assert.EventuallyEqual(t, func() int {
		ais := s.Lookup("ns1/svc2.ns1.svc.company.com")
		return len(ais)
	}, 4)

	// Finally, change the network back
	remoteClient.ns.Update(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name:   systemNS,
			Labels: map[string]string{label.TopologyNetwork.Name: remoteNetwork},
		},
	})

	assert.EventuallyEqual(t, func() int {
		ais := s.Lookup("ns1/svc2.ns1.svc.company.com")
		return len(ais)
	}, 3)
	s.assertEvent(
		t, s.podXdsNameForCluster("pod2", remoteClient.clusterID),
		s.podXdsNameForCluster("pod1-abc", remoteClient.clusterID),
		splitHorizonName,
	)
}

func TestMulticlusterAmbientIndex_SplitHorizonPreservesWaypoint(t *testing.T) {
	test.SetForTest(t, &features.EnableAmbientMultiNetwork, true)
	s := newAmbientTestServer(t, testC, testNW, "")
	s.meshConfig.Mesh().TrustDomain = s.DomainSuffix
	s.AddSecret("s1", "remote-cluster")
	remoteClients := krt.NewCollection(s.mcController.Clusters(), func(_ krt.HandlerContext, c *multicluster.Cluster) **remoteAmbientClients {
		cl := c.Client
		return ptr.Of(&remoteAmbientClients{
			clusterID: c.ID,
			ambientclients: &ambientclients{
				pc:    clienttest.NewDirectClient[*corev1.Pod, corev1.Pod, *corev1.PodList](t, cl),
				sc:    clienttest.NewDirectClient[*corev1.Service, corev1.Service, *corev1.ServiceList](t, cl),
				ns:    clienttest.NewWriter[*corev1.Namespace](t, cl),
				grc:   clienttest.NewWriter[*k8sv1.Gateway](t, cl),
				gwcls: clienttest.NewWriter[*k8sv1.GatewayClass](t, cl),
				se:    clienttest.NewWriter[*apiv1alpha3.ServiceEntry](t, cl),
				we:    clienttest.NewWriter[*apiv1alpha3.WorkloadEntry](t, cl),
				pa:    clienttest.NewWriter[*clientsecurityv1beta1.PeerAuthentication](t, cl),
				authz: clienttest.NewWriter[*clientsecurityv1beta1.AuthorizationPolicy](t, cl),
				sec:   clienttest.NewWriter[*corev1.Secret](t, cl),
			},
		})
	})

	const remoteNetwork = "remote-network"

	assert.EventuallyEqual(t, func() int {
		return len(remoteClients.List())
	}, 1)

	remoteClient := remoteClients.List()[0]
	localClient := remoteAmbientClients{
		clusterID: s.clusterID,
		ambientclients: &ambientclients{
			pc:    s.pc,
			sc:    s.sc,
			ns:    s.ns,
			grc:   s.grc,
			gwcls: s.gwcls,
			se:    s.se,
			we:    s.we,
			pa:    s.pa,
			authz: s.authz,
			sec:   s.sec,
		},
	}
	remoteClient.ns.Create(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name:   systemNS,
			Labels: map[string]string{label.TopologyNetwork.Name: remoteNetwork},
		},
	})
	remoteClient.ns.Create(&corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: testNS,
		},
	})
	networkGatewayIP := "172.0.1.2"
	s.addNetworkGatewayForClient(t, networkGatewayIP, remoteNetwork, remoteClient.grc)

	// Create a waypoint for the service
	s.addWaypointSpecificAddress(t, "10.0.0.10", "", "wp-svc", constants.AllTraffic, true)
	s.addService(t, "wp-svc",
		map[string]string{},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "waypoint"}, "10.0.0.10")

	// Create a global service with ingress-use-waypoint on both clusters
	s.addServiceForClient(
		t, "svc1",
		map[string]string{
			"istio.io/global":               "true",
			label.IoIstioUseWaypoint.Name:   "wp-svc",
			"istio.io/ingress-use-waypoint": "true",
		},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "10.0.0.1", localClient.sc,
	)
	s.addServiceForClient(
		t, "svc1",
		map[string]string{
			"istio.io/global":               "true",
			label.IoIstioUseWaypoint.Name:   "wp-svc",
			"istio.io/ingress-use-waypoint": "true",
		},
		map[string]string{},
		[]int32{80}, map[string]string{"app": "a"}, "127.1.0.1", remoteClient.sc,
	)
	s.clearEvents()

	// Add local and remote workloads to trigger the SAN merge in SplitHorizonServices.
	// The remote pod must be on a different network so its SAN gets merged, which is
	// the code path where fields were being dropped.
	s.addPodsForClient(t, "10.0.1.1", "pod1", "sa1",
		map[string]string{"app": "a"}, nil, true, corev1.PodRunning, localClient.pc)
	s.addPodsForClient(t, "127.0.0.1", "pod1-remote", "sa-remote",
		map[string]string{"app": "a"}, nil, true, corev1.PodRunning, remoteClient.pc)

	// Wait for the SAN merge to happen — the service should have SANs from the remote workload.
	// This proves the SplitHorizonServices code path that creates a new ServiceInfo was hit.
	svcKey := s.svcXdsName("svc1")
	retry.UntilSuccessOrFail(t, func() error {
		svc := s.lookupService(fmt.Sprintf("%s/%s", testNS, s.hostnameForService("svc1")))
		if svc == nil {
			return fmt.Errorf("service not found")
		}
		if svc.Scope != model.Global {
			return fmt.Errorf("expected service scope to be Global, got %s", svc.Scope)
		}
		hasRemoteSAN := false
		for _, san := range svc.Service.SubjectAltNames {
			if strings.Contains(san, "sa-remote") {
				hasRemoteSAN = true
				break
			}
		}
		if !hasRemoteSAN {
			return fmt.Errorf("expected SANs to include remote workload SAN (sa-remote), got %v",
				svc.Service.SubjectAltNames)
		}
		// Now verify the fields that were previously dropped during SAN merge
		if !svc.Waypoint.IngressLabelPresent {
			return fmt.Errorf("expected Waypoint.IngressLabelPresent to be true, got false")
		}
		if !svc.Waypoint.IngressUseWaypoint {
			return fmt.Errorf("expected Waypoint.IngressUseWaypoint to be true, got false")
		}
		if svc.Source.Kind == 0 {
			return fmt.Errorf("expected Source.Kind to be preserved, got zero")
		}
		if len(svc.PortNames) == 0 {
			return fmt.Errorf("expected PortNames to be preserved, got empty")
		}
		return nil
	})

	// Verify ServicesWithWaypoint also returns the correct IngressUseWaypoint
	retry.UntilSuccessOrFail(t, func() error {
		svcs := s.ServicesWithWaypoint(svcKey)
		if len(svcs) == 0 {
			return fmt.Errorf("ServicesWithWaypoint returned 0 results")
		}
		if !svcs[0].IngressUseWaypoint {
			return fmt.Errorf("expected ServicesWithWaypoint to return IngressUseWaypoint=true, got false")
		}
		if svcs[0].WaypointHostname == "" {
			return fmt.Errorf("expected WaypointHostname to be set, got empty")
		}
		return nil
	})
}

func (a *ambientTestServer) DeleteSecret(secretName string) {
	a.t.Helper()
	a.sec.Delete(secretName, secretNamespace)
}

func (a *ambientTestServer) AddSecret(secretName, clusterID string) {
	kubeconfig++
	a.sec.CreateOrUpdate(makeSecret(secretNamespace, secretName, clusterCredential{clusterID, fmt.Appendf(nil, "kubeconfig-%d", kubeconfig)}))
}

// TestMulticlusterAmbientIndex_ClusterLifecycleNoLeak verifies that adding and then removing a
// remote cluster does not leak any goroutines backing that cluster's derived krt collections
// (e.g. the per-cluster node locality collection) or the informer handlers they register.
//
// The ambient index (and its top-level stop channel) lives on the parent test, while the
// add/remove cycles run in a subtest wrapped in leak.Check.
func TestMulticlusterAmbientIndex_ClusterLifecycleNoLeak(t *testing.T) {
	test.SetForTest(t, &features.EnableAmbientMultiNetwork, true)
	// Build the index WITHOUT starting the config cluster client. This lets us create the warmup
	// remote-cluster secret up front and have it processed as part of the very first sync, so that
	// waiting on HasSynced deterministically means "all global collections are wired up".
	s := newAmbientTestServerWithFlags(t, testC, testNW, FeatureFlags{
		DefaultAllowFromWaypoint:              features.DefaultAllowFromWaypoint,
		EnableK8SServiceSelectWorkloadEntries: features.EnableK8SServiceSelectWorkloadEntries,
	}, "", false /* runClient */)
	clusters := s.mcController.Clusters()
	cl := s.mcController.ConfigCluster().Client

	waitRemoteClusters := func(t *testing.T, n int) {
		t.Helper()
		assert.EventuallyEqual(t, func() int { return len(clusters.List()) }, n)
	}

	// Warm-up: create a remote cluster BEFORE running the client. Global collections lazily wire up
	// per-cluster dependency handlers the first time they see a remote cluster; doing this as part of
	// the initial sync (gated by HasSynced) guarantees they are all created before leak.Check snapshots
	// its baseline. We intentionally do NOT remove this cluster: teardown is async, and leaving the
	// cluster in place keeps us in a stable, fully-initialized steady state. The subtests exercise
	// add/remove of a *second* cluster and must return to this baseline.
	s.AddSecret("s1", "remote-cluster-1")
	t.Cleanup(cl.Shutdown)
	cl.RunAndWait(test.NewStop(t))
	assert.EventuallyEqual(t, s.HasSynced, true)
	waitRemoteClusters(t, 1)

	t.Run("add-remove-cluster", func(t *testing.T) {
		leak.Check(t)

		s.AddSecret("s2", "remote-cluster-2")
		waitRemoteClusters(t, 2)

		s.DeleteSecret("s2")
		waitRemoteClusters(t, 1)
	})

	t.Run("update-then-remove-cluster", func(t *testing.T) {
		leak.Check(t)

		swapped := make(chan struct{}, 1)
		reg := clusters.RegisterBatch(func(events []krt.Event[*multicluster.Cluster]) {
			for _, e := range events {
				if e.Event == controllers.EventUpdate && e.Latest().ID == "remote-cluster-2" {
					select {
					case swapped <- struct{}{}:
					default:
					}
				}
			}
		}, false)
		defer reg.UnregisterHandler()
		reg.WaitUntilSynced(test.NewStop(t))

		s.AddSecret("s2", "remote-cluster-2")
		waitRemoteClusters(t, 2)

		// Update the cluster: re-adding the secret with a fresh kubeconfig swaps in a new Cluster (new
		// stop + collections), tearing down the old one. This is the teardown path we verify doesn't
		// leak. The cluster count stays at 2 across the update, so waitRemoteClusters would return
		// immediately without the swap having propagated into the collection. Instead, observe the swap
		// on the Clusters collection directly: capture the current cluster's stop channel and wait for
		// an event carrying a remote-cluster-2 with a *different* stop (the swap surfaces as a delete of
		// the not-yet-ready cluster followed by an add once it syncs, not necessarily a single update).
		var oldStop <-chan struct{}
		for _, c := range clusters.List() {
			if c.ID == "remote-cluster-2" {
				oldStop = c.GetStop()
			}
		}
		s.AddSecret("s2", "remote-cluster-2")
		select {
		case <-swapped:
		case <-time.After(30 * time.Second):
			t.Fatal("timed out waiting for remote-cluster-2 swap event")
		}
		<-oldStop

		// Remove so the subtest ends back at the single warm-up cluster, matching the leak.Check baseline.
		s.DeleteSecret("s2")
		waitRemoteClusters(t, 1)
	})
}
