class MU::Cloud::Azure::VPC

Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).

Attributes

cloud_desc_cache[R]
resource_group[R]

Public Class Methods

cleanup(**args) click to toggle source

Stub method. Azure resources are cleaned up by removing the parent resource group. @return [void]

# File modules/mu/providers/azure/vpc.rb, line 327
def self.cleanup(**args)
end
find(**args) click to toggle source

Locate and return cloud provider descriptors of this resource type which match the provided parameters, or all visible resources if no filters are specified. At minimum, implementations of find must honor credentials and cloud_id arguments. We may optionally support other search methods, such as tag_key and tag_value, or cloud-specific arguments like project. See also {MU::MommaCat.findStray}. @param args [Hash]: Hash of named arguments passed via Ruby's double-splat @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources

# File modules/mu/providers/azure/vpc.rb, line 141
def self.find(**args)
  found = {}

  # Azure resources are namedspaced by resource group. If we weren't
  # told one, we may have to search all the ones we can see.
  resource_groups = if args[:resource_group]
    [args[:resource_group]]
  elsif args[:cloud_id] and args[:cloud_id].is_a?(MU::Cloud::Azure::Id)
    [args[:cloud_id].resource_group]
  else
    MU::Cloud::Azure.resources(credentials: args[:credentials]).resource_groups.list.map { |rg| rg.name }
  end

  if args[:cloud_id]
    id_str = args[:cloud_id].is_a?(MU::Cloud::Azure::Id) ? args[:cloud_id].name : args[:cloud_id]
    resource_groups.each { |rg|
      resp = MU::Cloud::Azure.network(credentials: args[:credentials]).virtual_networks.get(rg, id_str)

      found[Id.new(resp.id)] = resp if resp
    }
  else
    if args[:resource_group]
      MU::Cloud::Azure.network(credentials: args[:credentials]).virtual_networks.list(args[:resource_group]).each { |net|
        found[Id.new(net.id)] = net
      }
    else
      MU::Cloud::Azure.network(credentials: args[:credentials]).virtual_networks.list_all.each { |net|
        found[Id.new(net.id)] = net
      }
    end
  end

  found
end
haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil) click to toggle source

Check whether we (the Mu Master) have a direct route to a particular instance. Useful for skipping hops through bastion hosts to get directly at child nodes in peered VPCs, the public internet, and the like. @param target_instance [OpenStruct]: The cloud descriptor of the instance to check. @param region [String]: The cloud provider region of the target subnet. @return [Boolean]

# File modules/mu/providers/azure/vpc.rb, line 293
        def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil)

#          target_instance.network_profile.network_interfaces.each { |iface|
#            iface_id = Id.new(iface.is_a?(Hash) ? iface['id'] : iface.id)
#            iface_desc = MU::Cloud::Azure.network(credentials: credentials).network_interfaces.get(iface_id.resource_group, iface_id.to_s)
#            iface_desc.ip_configurations.each { |ipcfg|
#              if ipcfg.respond_to?(:public_ipaddress) and ipcfg.public_ipaddress
#                return true # XXX invalid if Mu can't talk to the internet
#              end
#            }
#          }

          return false if MU.myCloud != "Azure"
# XXX if we're in Azure, see if this is in our VPC or if we're peered to its VPC
          false
        end
isGlobal?() click to toggle source

Does this resource type exist as a global (cloud-wide) artifact, or is it localized to a region/zone? @return [Boolean]

# File modules/mu/providers/azure/vpc.rb, line 314
def self.isGlobal?
  false
end
new(**args) click to toggle source

Initialize this cloud resource object. Calling super will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like @vpc, for us. @param args [Hash]: Hash of named arguments passed via Ruby's double-splat

Calls superclass method
# File modules/mu/providers/azure/vpc.rb, line 26
def initialize(**args)
  super
  @subnets = []
  @subnetcachesemaphore = Mutex.new

  if !mu_name.nil?
    @mu_name = mu_name
    if @cloud_id
      cloud_desc
      @cloud_id = Id.new(cloud_desc.id)
      @resource_group ||= @cloud_id.resource_group
      loadSubnets(use_cache: true)
    end
  elsif @config['scrub_mu_isms']
    @mu_name = @config['name']
  else
    @mu_name = @deploy.getResourceName(@config['name'])
  end
end
quality() click to toggle source

Denote whether this resource implementation is experiment, ready for testing, or ready for production use.

# File modules/mu/providers/azure/vpc.rb, line 320
def self.quality
  MU::Cloud::BETA
end
schema(_config = nil) click to toggle source

Cloud-specific configuration properties. @param _config [MU::Config]: The calling MU::Config object @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource

# File modules/mu/providers/azure/vpc.rb, line 350
def self.schema(_config = nil)
  toplevel_required = []
  schema = {
    "peers" => {
      "items" => {
        "properties" => {
          "allow_forwarded_traffic" => {
            "type" => "boolean",
            "default" => false,
            "description" => "Allow traffic originating from outside peered networks"
          },
          "allow_gateway_traffic" => {
            "type" => "boolean",
            "default" => false,
            "description" => "Permit peered networks to use each others' VPN gateways"
          }
        }
      }
    }
  }
  [toplevel_required, schema]
end
validateConfig(vpc, configurator) click to toggle source

Cloud-specific pre-processing of {MU::Config::BasketofKittens::vpcs}, bare and unvalidated. @param vpc [Hash]: The resource to process and validate @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member @return [Boolean]: True if validation succeeded, False otherwise

# File modules/mu/providers/azure/vpc.rb, line 378
def self.validateConfig(vpc, configurator)
  ok = true
  vpc['region'] ||= MU::Cloud::Azure.myRegion(vpc['credentials'])

  if vpc['subnets']
    vpc['subnets'].each { |subnet|
      subnet_routes[subnet['route_table']] = Array.new if subnet_routes[subnet['route_table']].nil?
      subnet_routes[subnet['route_table']] << subnet['name']
    }
  end

  if (!vpc['subnets'] or vpc['subnets'].empty?) and vpc['create_standard_subnets']
    subnets = configurator.divideNetwork(vpc['ip_block'], vpc['route_tables'].size, 28)
    vpc['subnets'] ||= []
    vpc['route_tables'].each { |rtb|
      is_public = false
      rtb['routes'].each { |route|
        if route['gateway'] == "#INTERNET"
          is_public = true
          break
        end
      }
      vpc['subnets'] << {
        "name" => "Subnet#{rtb['name'].capitalize}",
        "is_public" => is_public,
        "ip_block" => subnets.shift,
        "route_table" => rtb['name']
      }
    }
  end

  vpc['route_tables'].each { |rtb|
    rtb['routes'] ||= []
    rtb['routes'] << { "destination_network" => vpc['ip_block'] }
    rtb['routes'].uniq!
  }

  default_acl = {
    "name" => vpc['name']+"-defaultfw",
    "cloud" => "Azure",
    "region" => vpc['region'],
    "credentials" => vpc['credentials'],
    "rules" => [
      {
        "ingress" => true, "proto" => "all", "hosts" => [vpc['ip_block']]
      },
      {
        "egress" => true, "proto" => "all", "hosts" => [vpc['ip_block']]
      }
    ]
  }
  MU::Config.addDependency(vpc, vpc['name']+"-defaultfw", "firewall_rule")

  if !configurator.insertKitten(default_acl, "firewall_rules", true)
    ok = false
  end

  ok
end

Public Instance Methods

cloud_desc(use_cache: true) click to toggle source

Describe this VPC from the cloud platform's perspective @return [Hash]

# File modules/mu/providers/azure/vpc.rb, line 123
def cloud_desc(use_cache: true)
  if @cloud_desc_cache and use_cache
    return @cloud_desc_cache
  end
  @cloud_desc_cache = MU::Cloud::Azure::VPC.find(cloud_id: @cloud_id, resource_group: @resource_group).values.first

  @cloud_id ||= Id.new(@cloud_desc_cache.id)
  @cloud_desc_cache
end
create() click to toggle source

Called automatically by {MU::Deploy#createResources}

# File modules/mu/providers/azure/vpc.rb, line 47
def create
  create_update
end
createRouteForInstance(route, server) click to toggle source

@param route [Hash]: A route description, per the Basket of Kittens schema @param server [MU::Cloud::Azure::Server]: Instance to which this route will apply

# File modules/mu/providers/azure/vpc.rb, line 440
def createRouteForInstance(route, server)
  createRoute(route, network: @url, tags: [MU::Cloud::Azure.nameStr(server.mu_name)])
end
findBastion(nat_name: nil, nat_cloud_id: nil, nat_tag_key: nil, nat_tag_value: nil, nat_ip: nil) click to toggle source

Given some search criteria for a {MU::Cloud::Server}, see if we can locate a NAT host in this VPC. @param nat_name [String]: The name of the resource as defined in its 'name' Basket of Kittens field, typically used in conjunction with deploy_id. @param nat_cloud_id [String]: The cloud provider's identifier for this NAT. @param nat_tag_key [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_value. @param nat_tag_value [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_key. @param nat_ip [String]: An IP address associated with the NAT instance.

# File modules/mu/providers/azure/vpc.rb, line 233
def findBastion(nat_name: nil, nat_cloud_id: nil, nat_tag_key: nil, nat_tag_value: nil, nat_ip: nil)
  [:nat_name, :nat_cloud_id, :nat_tag_key, :nat_tag_value, :nat_ip].each { |var|
    if binding.local_variable_get(var) != nil
      binding.local_variable_set(var, var.to_s)
    end

    # If we're searching by name, assume it's part of this here deploy.
    if nat_cloud_id.nil? and !@deploy.nil?
      deploy_id = @deploy.deploy_id
    end
    found = MU::MommaCat.findStray(
      "Azure",
      "server",
      name: nat_name,
      cloud_id: nat_cloud_id,
      deploy_id: deploy_id,
      tag_key: nat_tag_key,
      tag_value: nat_tag_value,
      allow_multi: true,
      dummy_ok: true,
      calling_deploy: @deploy
    )

    return nil if found.nil? || found.empty?
    if found.size == 1
      return found.first
    end

  }
  nil
end
findNat(nat_cloud_id: nil, nat_filter_key: nil, nat_filter_value: nil, region: MU.curRegion) click to toggle source

Given some search criteria try locating a NAT Gaateway in this VPC. @param nat_cloud_id [String]: The cloud provider's identifier for this NAT. @param nat_filter_key [String]: A cloud provider filter to help identify the resource, used in conjunction with nat_filter_value. @param nat_filter_value [String]: A cloud provider filter to help identify the resource, used in conjunction with nat_filter_key. @param region [String]: The cloud provider region of the target instance.

# File modules/mu/providers/azure/vpc.rb, line 222
def findNat(nat_cloud_id: nil, nat_filter_key: nil, nat_filter_value: nil, region: MU.curRegion)
  nil
end
getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil) click to toggle source

Check for a subnet in this VPC matching one or more of the specified criteria, and return it if found.

# File modules/mu/providers/azure/vpc.rb, line 267
def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil)
  loadSubnets
  if !cloud_id.nil? and cloud_id.match(/^https:\/\//)
    cloud_id.gsub!(/.*?\//, "")
  end
  MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block})", MU::DEBUG, details: caller[0]

  @subnets.each { |subnet|
    if !cloud_id.nil? and !subnet.cloud_id.nil? and subnet.cloud_id.to_s == cloud_id.to_s
      return subnet
    elsif !name.nil? and !subnet.name.nil? and subnet.name.to_s == name.to_s
      return subnet
    end
  }
  return nil
end
groom() click to toggle source

Called automatically by {MU::Deploy#createResources}

# File modules/mu/providers/azure/vpc.rb, line 53
def groom

  if @config['peers']
    @config['peers'].each { |peer|
      if peer['vpc']['name']
        peer_obj = @deploy.findLitterMate(name: peer['vpc']['name'], type: "vpcs", habitat: peer['vpc']['project'])
        next if peer_obj.mu_name < @mu_name # both of us would try to create this peering, otherwise, so don't step on each other
      else
        tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
        if peer['vpc']['deploy_id'].nil? and peer['vpc']['id'].nil? and tag_key.nil?
          peer['vpc']['deploy_id'] = @deploy.deploy_id
        end

        peer_obj = MU::MommaCat.findStray(
          "Azure",
          "vpcs",
          deploy_id: peer['vpc']['deploy_id'],
          cloud_id: peer['vpc']['id'],
          name: peer['vpc']['name'],
          tag_key: tag_key,
          tag_value: tag_value,
          dummy_ok: true
        ).first
      end

      raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil?
  
      ext_peerings = MU::Cloud::Azure.network(credentials: @credentials).virtual_network_peerings.list(@resource_group, @cloud_id)
      peer_name = @mu_name+"-"+@config['name'].upcase+"-"+peer_obj.config['name'].upcase
      peer_params = MU::Cloud::Azure.network(:VirtualNetworkPeering).new
      peer_params.remote_virtual_network = peer_obj.cloud_desc
      peer['allow_forwarded_traffic'] ||= false
      peer_params.allow_forwarded_traffic = peer['allow_forwarded_traffic']
      peer['allow_gateway_traffic'] ||= false
      peer_params.allow_gateway_transit = peer['allow_gateway_traffic']

      need_update = true
      exists = false
      ext_peerings.each { |ext_peering|
        if ext_peering.remote_virtual_network.id == peer_obj.cloud_desc.id
          exists = true
          need_update = (ext_peering.allow_forwarded_traffic != peer_params.allow_forwarded_traffic or ext_peering.allow_gateway_transit != peer_params.allow_gateway_transit)
        end
      }

      if need_update
        if !exists
          MU.log "Creating peering connection from #{@mu_name} to #{peer_obj.mu_name}", details: peer_params
        else
          MU.log "Updating peering connection from #{@mu_name} to #{peer_obj.mu_name}", MU::NOTICE, details: peer_params
        end
        MU::Cloud::Azure.network(credentials: @credentials).virtual_network_peerings.create_or_update(@resource_group, @cloud_id, peer_name, peer_params)
      end
    }
  end

  create_update
end
loadSubnets(use_cache: false) click to toggle source

Describe subnets associated with this VPC. We'll compose identifying information similar to what MU::Cloud.describe builds for first-class resources. @param use_cache [Boolean]: If available, use saved deployment metadata to describe subnets, instead of querying the cloud API @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.

# File modules/mu/providers/azure/vpc.rb, line 192
def loadSubnets(use_cache: false)
  @subnets = []

  MU::Cloud::Azure.network(credentials: @credentials).subnets.list(@resource_group, cloud_desc(use_cache: use_cache).name).each { |subnet|
    subnet_cfg = {
      "cloud_id" => subnet.name,
      "mu_name" => subnet.name,
      "credentials" => @config['credentials'],
      "region" => @config['region'],
      "ip_block" => subnet.address_prefix
    }
    if @config['subnets']
      @config['subnets'].each { |s|
        if s['ip_block'] == subnet_cfg['ip_block']
          subnet_cfg['name'] = s['name']
          break
        end
      }
    end
    subnet_cfg['name'] ||= subnet.name
    @subnets << MU::Cloud::Azure::VPC::Subnet.new(self, subnet_cfg)
  }
  @subnets
end
notify() click to toggle source

Describe this VPC @return [Hash]

# File modules/mu/providers/azure/vpc.rb, line 114
def notify
  base = MU.structToHash(cloud_desc)
  base["cloud_id"] = @cloud_id.name
  base.merge!(@config.to_h)
  base
end
subnets() click to toggle source

Return an array of MU::Cloud::Azure::VPC::Subnet objects describe the member subnets of this VPC.

@return [Array<MU::Cloud::Azure::VPC::Subnet>]

# File modules/mu/providers/azure/vpc.rb, line 180
def subnets
  if @subnets.nil? or @subnets.size == 0
    return loadSubnets
  end
  return @subnets
end
toKitten(**_args) click to toggle source

Reverse-map our cloud description into a runnable config hash. We assume that any values we have in +@config+ are placeholders, and calculate our own accordingly based on what's live in the cloud. XXX add flag to return the diff between @config and live cloud

# File modules/mu/providers/azure/vpc.rb, line 334
def toKitten(**_args)
  return nil if cloud_desc.name == "default" # parent project builds these
  bok = {
    "cloud" => "Azure",
    "name" => cloud_desc.name,
    "project" => @config['project'],
    "credentials" => @config['credentials'],
    "cloud_id" => @cloud_id.to_s
  }

  bok
end

Private Instance Methods

create_update() click to toggle source
# File modules/mu/providers/azure/vpc.rb, line 446
        def create_update
          @config = MU::Config.manxify(@config)
          @config['region'] ||= MU::Cloud::Azure.myRegion(@config['credentials'])
          tags = {}
          if !@config['scrub_mu_isms']
            tags = MU::MommaCat.listStandardTags
          end
          if @config['tags']
            @config['tags'].each { |tag|
              tags[tag['key']] = tag['value']
            }
          end

          vpc_obj =  MU::Cloud::Azure.network(:VirtualNetwork).new
          addr_space_obj = MU::Cloud::Azure.network(:AddressSpace).new
          addr_space_obj.address_prefixes = [
            @config['ip_block']
          ]
          vpc_obj.address_space = addr_space_obj
          vpc_obj.location = @config['region']
          vpc_obj.tags = tags

          my_fw = deploy.findLitterMate(type: "firewall_rule", name: @config['name']+"-defaultfw")

          @resource_group = @deploy.deploy_id+"-"+@config['region'].upcase

          need_apply = false
          ext_vpc = nil
          begin
            ext_vpc = MU::Cloud::Azure.network(credentials: @config['credentials']).virtual_networks.get(
              @resource_group,
              @mu_name
            )
          rescue ::MU::Cloud::Azure::APIError => e
            if e.message.match(/: ResourceNotFound:/)
              need_apply = true
            else
              raise e
            end
          end
# XXX raw update seems to destroy child resources; if we just need to update
# tags, do that with .update_tags
          if !ext_vpc
            MU.log "Creating VPC #{@mu_name} (#{@config['ip_block']}) in #{@config['region']}", details: vpc_obj
            need_apply = true
          elsif ext_vpc.location != vpc_obj.location or
#                ext_vpc.tags != vpc_obj.tags or
#                XXX updating tags is a different API call
                ext_vpc.address_space.address_prefixes != vpc_obj.address_space.address_prefixes
            MU.log "Updating VPC #{@mu_name} (#{@config['ip_block']}) in #{@config['region']}", MU::NOTICE, details: vpc_obj
MU.structToHash(ext_vpc).diff(MU.structToHash(vpc_obj))
            need_apply = true
          end

          if need_apply
            begin
              resp = MU::Cloud::Azure.network(credentials: @config['credentials']).virtual_networks.create_or_update(
                @resource_group,
                @mu_name,
                vpc_obj
              )
              @cloud_id = Id.new(resp.id)
            rescue ::MU::Cloud::Azure::APIError => e
              if e.message.match(/InUseSubnetCannotBeDeleted: /)
                MU.log "Cannot delete an in-use Azure subnet", MU::WARN
              else
                raise e
              end
            end
          end

          # this is slow, so maybe thread it
          rtb_map = {}
          routethreads = []
          @config['route_tables'].each { |rtb_cfg|
            routethreads << Thread.new(rtb_cfg) { |rtb|
              rtb_name = @mu_name+"-"+rtb['name'].upcase
              rtb_obj = MU::Cloud::Azure.network(:RouteTable).new
              rtb_obj.location = @config['region']

              rtb_obj.tags = tags
              rtb_ref_obj = MU::Cloud::Azure.network(:RouteTable).new
              rtb_ref_obj.name = rtb_name
              rtb_map[rtb['name']] = rtb_ref_obj

              need_apply = false
              ext_rtb = nil
              begin
                ext_rtb = MU::Cloud::Azure.network(credentials: @config['credentials']).route_tables.get(
                  @resource_group,
                  rtb_name
                )
                rtb_map[rtb['name']] = ext_rtb
              rescue MU::Cloud::Azure::APIError => e
                if e.message.match(/: ResourceNotFound:/)
                  need_apply = true
                else
                  raise e
                end
              end

              if !ext_rtb
                MU.log "Creating route table #{rtb_name} in VPC #{@mu_name}", details: rtb_obj
                need_apply = true
              elsif ext_rtb.location != rtb_obj.location or
                    ext_rtb.tags != rtb_obj.tags
                need_apply = true
                MU.log "Updating route table #{rtb_name} in VPC #{@mu_name}", MU::NOTICE, details: rtb_obj
              end

              if need_apply
                rtb_map[rtb['name']] = MU::Cloud::Azure.network(credentials: @config['credentials']).route_tables.create_or_update(
                  @resource_group,
                  rtb_name,
                  rtb_obj
                )
              end

              rtb['routes'].each { |route|
                route_obj = MU::Cloud::Azure.network(:Route).new
                route_obj.address_prefix = route['destination_network']
                routename = rtb_name+"-"+route['destination_network'].gsub(/[^a-z0-9]/i, "_")
                route_obj.next_hop_type = if route['gateway'] == "#NAT" and @config['bastion']
                  routename = rtb_name+"-NAT"
                  if @config['bastion'].is_a?(Hash) and !@config['bastion']['id'] and !@config['bastion']['deploy_id']
                    @config['bastion']['deploy_id'] = @deploy.deploy_id
                  end
                  bastion_ref = MU::Config::Ref.get(@config['bastion'])
                  if bastion_ref.kitten and bastion_ref.kitten.cloud_desc
                    iface_id = Id.new(bastion_ref.kitten.cloud_desc.network_profile.network_interfaces.first.id)
                    iface_desc = MU::Cloud::Azure.network(credentials: @credentials).network_interfaces.get(@resource_group, iface_id.name)
                    if iface_desc and iface_desc.ip_configurations and iface_desc.ip_configurations.size > 0
                      route_obj.next_hop_ip_address = iface_desc.ip_configurations.first.private_ipaddress
                      "VirtualAppliance"
                    else
                      "VnetLocal"
                    end
                  else
                    "VnetLocal"
                  end
#                  create_nat_gateway = true
                elsif route['gateway'] == "#INTERNET"
                  routename = rtb_name+"-INTERNET"
                  "Internet"
                else
                  routename = rtb_name+"-LOCAL"
                  "VnetLocal"
                end
#next_hop_type 'VirtualNetworkGateway' is for VPNs I think

                need_apply = false
                ext_route = nil
                begin
                  ext_route = MU::Cloud::Azure.network(credentials: @config['credentials']).routes.get(
                    @resource_group,
                    rtb_name,
                    routename
                  )
                rescue MU::Cloud::Azure::APIError => e
                  if e.message.match(/\bNotFound\b/)
                    need_apply = true
                  else
                    raise e
                  end
                end

                if !ext_route
                  MU.log "Creating route #{routename} for #{route['destination_network']} in route table #{rtb_name}", details: rtb_obj
                  need_apply = true
                elsif ext_route.next_hop_type != route_obj.next_hop_type or
                      ext_route.address_prefix != route_obj.address_prefix
                  MU.log "Updating route #{routename} for #{route['destination_network']} in route table #{rtb_name}", MU::NOTICE, details: [route_obj, ext_route]
                  need_apply = true
                end

                if need_apply
                  MU::Cloud::Azure.network(credentials: @config['credentials']).routes.create_or_update(
                    @resource_group,
                    rtb_name,
                    routename,
                    route_obj
                  )
                end
              }
            }
          }

          routethreads.each { |t|
            t.join
          }

# TODO this is only available in westus as of 2019-09-29
#          if create_nat_gateway
#            nat_obj = MU::Cloud::Azure.network(:NatGateway).new
#            nat_obj.location = @config['region']
#            nat_obj.tags = tags
#            MU.log "Creating NAT Gateway #{@mu_name}-NAT", details: nat_obj
#            MU::Cloud::Azure.network(credentials: @config['credentials']).nat_gateways.create_or_update(
#              @resource_group,
#              @mu_name+"-NAT",
#              nat_obj
#            )
#          end

          if @config['subnets']
            subnetthreads = []
            @config['subnets'].each { |subnet_cfg|
              subnetthreads << Thread.new(subnet_cfg) { |subnet|
                subnet_obj = MU::Cloud::Azure.network(:Subnet).new
                subnet_name = @mu_name+"-"+subnet['name'].upcase
                subnet_obj.address_prefix = subnet['ip_block']
                subnet_obj.route_table = rtb_map[subnet['route_table']]
                if my_fw and my_fw.cloud_desc
                  subnet_obj.network_security_group = my_fw.cloud_desc
                end

                need_apply = false
                ext_subnet = nil
                begin
                  ext_subnet = MU::Cloud::Azure.network(credentials: @config['credentials']).subnets.get(
                    @resource_group,
                    @cloud_id.to_s,
                    subnet_name
                  )
                rescue APIError => e
                  if e.message.match(/\bNotFound\b/)
                    need_apply = true
                  else
#                raise e
                  end
                end

                if !ext_subnet
                  MU.log "Creating Subnet #{subnet_name} in VPC #{@mu_name}", details: subnet_obj
                  need_apply = true
                elsif (!ext_subnet.route_table.nil? and !subnet_obj.route_table.nil? and ext_subnet.route_table.id != subnet_obj.route_table.id) or
                      ext_subnet.address_prefix != subnet_obj.address_prefix or
                      ext_subnet.network_security_group.nil? and !subnet_obj.network_security_group.nil? or
                      (!ext_subnet.network_security_group.nil? and !subnet_obj.network_security_group.nil? and ext_subnet.network_security_group.id != subnet_obj.network_security_group.id)
                  MU.log "Updating Subnet #{subnet_name} in VPC #{@mu_name}", MU::NOTICE, details: subnet_obj
MU.structToHash(ext_subnet).diff(MU.structToHash(subnet_obj))
                  need_apply = true

                end

                if need_apply
                  begin
                    MU::Cloud::Azure.network(credentials: @config['credentials']).subnets.create_or_update(
                      @resource_group,
                      @cloud_id.to_s,
                      subnet_name,
                      subnet_obj
                    )
                  rescue ::MU::Cloud::Azure::APIError => e
                    if e.message.match(/InUseSubnetCannotBeUpdated: /)
                      MU.log "Cannot alter an in-use Azure subnet", MU::WARN
                    else
                      raise e
                    end
                  end
                end
              }
            }

            subnetthreads.each { |t|
              t.join
            }
          end

          loadSubnets
        end