I have a VM template I'm deploying an Azure Virtual Desktop environment with terraform (via octopus deploy) to Azure. On top of the Virtual Machines, I'm installing a number of extensions which culminates with a vm extension to register the VM with the Host Pool.

I'd like to rebuild the VM each time the custom script extension is applied (Extension #2, after domain join). But in rebuilding the VM, I'd like to build out a new VM, complete with the host pool registration before any part of the existing VM is destroyed.

Please accept the cut down version below to understand what I am trying to do. I expect the largest number of machine recreations to come from enhancements to the configuration scripts that configure the server on creation. Not all of the commands are expected to be idempotent and we want the AVD vms to be ephemeral. If an issue is encountered, the support team is expected to be able to drain a server and destroy it once empty to get a replacement by terraform apply. In a case where the script gets updated though, we want to be able to replace all VMs quickly in an emergency, or at the very least minimize the nightly maintenance window.

Script Process: parameterized script > gets filled out as a template file > gets stored as an az blob > called by custom script extension > executed on the machine. VM build process: VM is provisioned > currently 8 extensions get applied one at a time, starting with the domain join, then the custom script extension, followed by several Azure monitoring extensions, and finally the host pool registration extension.

I've been trying to use the create_before_destroy lifecycle feature, but I can't get it to spin up the VM, and apply all extensions before it begins removing the hostpool registration from the existing VMs. I assume there's a way to do it using the triggers, but I'm not sure how to do it in such a way that it always has at least the current number of VMs.

It would also need to be able to stop if it encounters an error on the new vm, before destroying the existing vm (or better yet, be authorized to rebuild VMs if an extension fails part way through).

resource "random_pet" "avd_vm" {
    
  prefix = var.client_name
  length = 1
  keepers = {
    # Generate a new pet name each time we update the setup_host script
    source_content = "${data.template_file.setup_host.rendered}"
  }
}

data "template_file" "setup_host" {
  template = file("${path.module}\\scripts\\setup-host.tpl")

  vars = {
    storageAccountName   = azurerm_storage_account.storage.name
    storageAccountKey    = azurerm_storage_account.storage.primary_access_key
    domain               = var.domain
    aad_group_name       = var.aad_group_name
  }
}

resource "azurerm_storage_blob" "setup_host" {
  name                   = "setup-host.ps1"
  storage_account_name   = azurerm_storage_account.scripts.name
  storage_container_name = time_sleep.container_rbac.triggers["name"]
  type                   = "Block"
  source_content         = data.template_file.setup_host.rendered #"${path.module}\\scripts\\setup-host.ps1"
  depends_on = [
    azurerm_role_assignment.account1_write,
    data.template_file.setup_host,
    time_sleep.container_rbac
  ]
}

data "template_file" "client_r_drive_mapping" {
  template = file("${path.module}\\scripts\\client_r_drive_mapping.tpl")

  vars = {
    storageAccountName    = azurerm_storage_account.storage.name
    storageAccountKey = azurerm_storage_account.storage.primary_access_key
  }
}

resource "azurerm_windows_virtual_machine" "example" {
    count = length(random_pet.avd_vm)
    name = "${random_pet.avd_vm[count.index].id}"
    ...
    lifecycle {
    ignore_changes = [
      boot_diagnostics,
      identity
    ]
  }
}

resource "azurerm_virtual_machine_extension" "first-domain_join_extension" {
  count                      = var.rdsh_count
  name                       = "${var.client_name}-avd-${random_pet.avd_vm[count.index].id}-domainJoin"
  virtual_machine_id         = azurerm_windows_virtual_machine.avd_vm.*.id[count.index]
  publisher                  = "Microsoft.Compute"
  type                       = "JsonADDomainExtension"
  type_handler_version       = "1.3"
  auto_upgrade_minor_version = true

  settings = <<SETTINGS
    {
      "Name": "${var.domain_name}",
      "OUPath": "${var.ou_path}",
      "User": "${var.domain_user_upn}@${var.domain_name}",
      "Restart": "true",
      "Options": "3"
    }
SETTINGS

  protected_settings = <<PROTECTED_SETTINGS
    {
      "Password": "${var.admin_password}"
    }
PROTECTED_SETTINGS

  lifecycle {
    ignore_changes = [settings, protected_settings]
  }

  depends_on = [
    azurerm_virtual_network_peering.out-primary,
    azurerm_virtual_network_peering.in-primary,
    azurerm_virtual_network_peering.in-secondary
  ]
}


# Multiple scripts called by ./<scriptname referencing them in follow-up scripts
# https://web.archive.org/web/20220127015539/https://learn.microsoft.com/en-us/azure/virtual-machines/extensions/custom-script-windows
# https://learn.microsoft.com/en-us/azure/virtual-machines/extensions/custom-script-windows#using-multiple-scripts
resource "azurerm_virtual_machine_extension" "second-custom_scripts" {
  count                      = var.rdsh_count
  name                       = "${random_pet.avd_vm[count.index].id}-setup-host"
  virtual_machine_id         = azurerm_windows_virtual_machine.avd_vm.*.id[count.index]
  publisher                  = "Microsoft.Compute"
  type                       = "CustomScriptExtension"
  type_handler_version       = "1.10"
  auto_upgrade_minor_version = "true"

  protected_settings = <<PROTECTED_SETTINGS
    {
      "storageAccountName": "${azurerm_storage_account.scripts.name}",
      "storageAccountKey": "${azurerm_storage_account.scripts.primary_access_key}"
    }
  PROTECTED_SETTINGS

  settings = <<SETTINGS
      {
          "fileUris": ["https://${azurerm_storage_account.scripts.name}.blob.core.windows.net/scripts/setup-host.ps1","https://${azurerm_storage_account.scripts.name}.blob.core.windows.net/scripts/client_r_drive_mapping.ps1"],
          "commandToExecute": "powershell -ExecutionPolicy Unrestricted -file setup-host.ps1"      
      } 
  SETTINGS

  depends_on = [
    azurerm_virtual_machine_extension.first-domain_join_extension,
    azurerm_storage_blob.setup_host
  ]
}

resource "azurerm_virtual_machine_extension" "last_host_extension_hp_registration" {
  count                      = var.rdsh_count
  name                       = "${var.client_name}-${random_pet.avd_vm[count.index].id}-avd_dsc"
  virtual_machine_id         = azurerm_windows_virtual_machine.avd_vm.*.id[count.index]
  publisher                  = "Microsoft.Powershell"
  type                       = "DSC"
  type_handler_version       = "2.73"
  auto_upgrade_minor_version = true

  settings = <<-SETTINGS
    {
      "modulesUrl": "https://wvdportalstorageblob.blob.core.windows.net/galleryartifacts/Configuration_3-10-2021.zip",
      "configurationFunction": "Configuration.ps1\\AddSessionHost",
      "properties": {
        "HostPoolName":"${azurerm_virtual_desktop_host_pool.pooleddepthfirst.name}"
      }
    }
SETTINGS

  protected_settings = <<PROTECTED_SETTINGS
  {
    "properties": {
      "registrationInfoToken": "${azurerm_virtual_desktop_host_pool_registration_info.pooleddepthfirst.token}"
    }
  }
PROTECTED_SETTINGS

  lifecycle {
    ignore_changes = [settings, protected_settings]
  }

  depends_on = [
    azurerm_virtual_machine_extension.second-custom_scripts
  ]
}