#requires -Version 7.5 [CmdletBinding()] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'The controller requires explicit -Execute, exact ownership and reviewed saved plans; helper functions are private to that gate.')] param( [Parameter(Mandatory)][ValidateSet('Preflight','Plan','Deploy','Bootstrap','Validate','Fault','Repair','Cleanup','Export')][string]$Stage, [Parameter(Mandatory)][string]$StatePath, [ValidatePattern('^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$')][string]$SubscriptionId, [ValidateSet('Probe','Tls','Waf')][string]$Scenario, [switch]$Execute, [switch]$AcknowledgeCost, [switch]$ReviewedPlan, [switch]$Resume, [string]$Location = 'westeurope', [string]$VnetCidr = '10.171.0.0/24', [string]$ImageUrn = 'MicrosoftWindowsServer:WindowsServer:2025-datacenter-azure-edition-smalldisk:26100.33674.261025', [string]$WafExclusionRulesJson, [string]$WafFixture = '', [switch]$CollectLogs, [int]$HealthWaitSeconds = 900, [int]$LogWaitSeconds = 2700 ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' Import-Module (Join-Path $PSScriptRoot 'scripts/Workshop.Common.psm1') -Force $privateDirectory = Initialize-WorkshopPrivateDirectory -StatePath $StatePath -PackageRoot $PSScriptRoot $StatePath = [IO.Path]::GetFullPath($StatePath) $runLock = Enter-WorkshopLock -PrivateDirectory $privateDirectory $script:state = $null $infraDirectory = Join-Path $PSScriptRoot 'infra' $tfvarsPath = Join-Path $privateDirectory 'terraform.tfvars.json' $tfstatePath = Join-Path $privateDirectory 'terraform.tfstate' $planPath = Join-Path $privateDirectory 'reviewed.tfplan' $destroyPlanPath = Join-Path $privateDirectory 'cleanup.tfplan' $previousTfData = $env:TF_DATA_DIR $previousArmSubscription = $env:ARM_SUBSCRIPTION_ID $previousArmTenant = $env:ARM_TENANT_ID $env:TF_DATA_DIR = Join-Path $privateDirectory 'tfdata' $script:healthWaitLimit=$HealthWaitSeconds $script:logWaitLimit=$LogWaitSeconds function Save-Run { Write-WorkshopJson -Path $StatePath -Value $script:state } function Invoke-Az([string[]]$A, [int]$Timeout = 300) { Invoke-WorkshopAzJson -Arguments $A -PrivateDirectory $privateDirectory -TimeoutSeconds $Timeout } function Invoke-Tf([string[]]$A, [int]$Timeout = 1800, [switch]$AllowFailure) { Invoke-WorkshopNative -FilePath terraform -Arguments (@("-chdir=$infraDirectory") + $A) -PrivateDirectory $privateDirectory -TimeoutSeconds $Timeout -AllowFailure:$AllowFailure -Label 'terraform' } function Assert-Mutation { if (-not $Execute) { throw 'This stage changes the disposable lab. Supply -Execute after reviewing its scope.' } Assert-WorkshopContext -State $script:state -PrivateDirectory $privateDirectory | Out-Null if ($Stage -notin @('Cleanup','Repair') -and [DateTimeOffset]::UtcNow -ge [DateTimeOffset]::Parse($script:state.exercise_stop_utc)) { throw 'The 4.5-hour exercise window has ended. Repair if necessary and clean up now.' } if ($script:state.pending_operation -and -not $Resume -and $Stage -ne 'Cleanup') { throw 'A previous operation needs reconciliation. Inspect private logs and use the same stage with -Resume, or clean up.' } if ($Resume -and $script:state.pending_operation -and $Stage -ne 'Cleanup') { $sameStage=$script:state.pending_operation.stage -eq $Stage if($Stage -in @('Fault','Repair')){$sameStage=$sameStage -and $script:state.pending_operation.scenario -eq $Scenario} $repairInterruptedFault=$Stage -eq 'Repair' -and $script:state.pending_operation.stage -eq 'Fault' -and $script:state.pending_operation.scenario -eq $Scenario if(-not $sameStage -and -not $repairInterruptedFault){throw 'Resume is restricted to the recorded stage or repair of its exact interrupted fault.'} } } function Start-Operation([string]$Kind) { $script:state.pending_operation = @{ stage = $Stage; kind = $Kind; scenario = $Scenario; started_utc = [DateTimeOffset]::UtcNow.ToString('o') } Save-Run } function Complete-Operation { $script:state.operations += $script:state.pending_operation $script:state.pending_operation = $null; Save-Run } function Add-Check([string]$Name, [bool]$Passed, [hashtable]$Observed = @{}, [string]$Phase = '') { try { Add-WorkshopCheck -State $script:state -Name $Name -Passed $Passed -Scenario $Scenario -Phase $Phase -Observed $Observed } finally { Save-Run } } function Get-Output([string]$Name) { if (-not $script:state.outputs -or -not $script:state.outputs.ContainsKey($Name)) { throw "Missing recorded Terraform output: $Name" } return [string]$script:state.outputs[$Name] } function Initialize-OutputDictionary($State) { if ($State.outputs -is [Collections.IDictionary]) { return $false } if ($State.outputs -is [array] -and $State.outputs.Count -eq 0) { # Recover only the empty output container from the authoring revision. # Actual resource identities are reconciled from the private Terraform state. $State.outputs = @{} return $true } throw 'The output ledger must be a dictionary; a nonempty incompatible value requires review.' } function Get-PreflightOwnership { $ownership=@{vnet_ids=@();existing_vm=$false;required_additional_cores=2} if(-not (Test-Path -LiteralPath $tfstatePath)){return $ownership} $snapshot=(Invoke-Tf @('show','-json',$tfstatePath)).StdOut|ConvertFrom-Json -AsHashtable -Depth 80 if(-not $snapshot.ContainsKey('values') -or -not $snapshot.values -or -not $snapshot.values.ContainsKey('root_module')){return $ownership} $scope="/subscriptions/$($script:state.subscription_id)/resourceGroups/$($script:state.prefix)-rg" foreach($resource in @($snapshot.values.root_module.resources|Where-Object{$_.mode -eq 'managed' -and $_.type -in @('azurerm_virtual_network','azurerm_windows_virtual_machine')})){ $id=[string]$resource.values.id if(-not $id.StartsWith($scope+'/',[StringComparison]::OrdinalIgnoreCase) -or $id -in $script:state.baseline_resource_ids){throw 'Preflight refresh encountered an unowned or protected Terraform resource.'} if($resource.type -eq 'azurerm_virtual_network'){$ownership.vnet_ids+=$id;continue} $vm=Invoke-Az @('vm','show','--subscription',$script:state.subscription_id,'--ids',$id) if($vm.id -ne $id -or $vm.hardwareProfile.vmSize -ne 'Standard_B2s' -or $resource.values.size -ne 'Standard_B2s'){throw 'Preflight refresh could not verify the exact owned B2s VM.'} $ownership.existing_vm=$true;$ownership.required_additional_cores=0 } return $ownership } function Assert-OwnedResource { $groupName = Get-Output 'resource_group_name' $members = @(Invoke-Az @('resource','list','--subscription',$script:state.subscription_id,'--resource-group',$groupName)) foreach ($member in $members) { if ($member.id -notin $script:state.owned_ids) { throw 'An unrecorded resource exists in the workshop group. Reconcile ownership; group-wide deletion is prohibited.' } } } function Sync-PartialDeployment { # Terraform state records exact provider IDs even if apply failed before output values were committed. if (-not (Test-Path -LiteralPath $tfstatePath)) { return } $snapshot = (Invoke-Tf @('show','-json',$tfstatePath)).StdOut | ConvertFrom-Json -AsHashtable -Depth 80 if (-not $snapshot.ContainsKey('values')) { return } if (-not $snapshot.values -or -not $snapshot.values.ContainsKey('root_module')) { return } $resources = @($snapshot.values.root_module.resources | Where-Object { $_.mode -eq 'managed' -and $_.values.ContainsKey('id') }) $verifiedIds=@() foreach($resource in $resources){ $id=[string]$resource.values.id if($id -in $script:state.baseline_resource_ids){throw 'Terraform state refers to a protected preflight resource.'} $runScope="/subscriptions/$($script:state.subscription_id)/resourceGroups/$($script:state.prefix)-rg" if($id -ne $runScope -and -not $id.StartsWith($runScope+'/',[StringComparison]::OrdinalIgnoreCase)){throw 'Terraform state includes a resource outside the recorded run scope.'} if($id -notin $verifiedIds){$verifiedIds += $id} switch($resource.type){ 'azurerm_resource_group' {$script:state.outputs.resource_group_name=$resource.values.name;$script:state.outputs.resource_group_id=$id} 'azurerm_application_gateway' {$script:state.outputs.gateway_name=$resource.values.name;$script:state.outputs.gateway_id=$id} 'azurerm_web_application_firewall_policy' {$script:state.outputs.waf_policy_name=$resource.values.name;$script:state.outputs.waf_policy_id=$id} 'azurerm_windows_virtual_machine' { $script:state.outputs.vm_name=$resource.values.name;$script:state.outputs.vm_id=$id $liveVm=Invoke-Az @('vm','show','--subscription',$script:state.subscription_id,'--ids',$id) if($liveVm.id -ne $id){throw 'Live VM identity differs from its exact Terraform state record.'} $diskId=[string]$liveVm.storageProfile.osDisk.managedDisk.id $expectedDiskName=[string]$resource.values.os_disk[0].name $expectedDiskId=$runScope+'/providers/Microsoft.Compute/disks/'+$expectedDiskName if(-not $diskId -or $diskId -ne $expectedDiskId -or $diskId -in $script:state.baseline_resource_ids){throw 'The live managed OS disk does not match the owned Terraform VM disk configuration.'} if($diskId -notin $verifiedIds){$verifiedIds += $diskId} } } } if($script:state.ContainsKey('verified_extension_ids')){ foreach($extensionId in $script:state.verified_extension_ids){ if(-not $script:state.outputs.ContainsKey('vm_id') -or $extensionId -ne ($script:state.outputs.vm_id+'/extensions/RunCommandWindows')){throw 'Recorded extension does not belong to the exact verified VM.'} if($extensionId -notin $verifiedIds){$verifiedIds+=$extensionId} } } $script:state.owned_ids=$verifiedIds Save-Run } function Get-GatewayHealth { return Invoke-Az @('network','application-gateway','show-backend-health','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--name',(Get-Output 'gateway_name')) 300 } function Get-BackendState($Health) { $rows = @() foreach ($pool in $Health.backendAddressPools) { foreach ($setting in $pool.backendHttpSettingsCollection) { foreach ($server in $setting.servers) { $rows += @{ settings = ($setting.backendHttpSettings.id -split '/')[-1]; health = $server.health; detail = $server.healthProbeLog } } } } return $rows } function Wait-Backend([string]$Settings, [string]$Expected) { $deadline = [DateTimeOffset]::UtcNow.AddSeconds($script:healthWaitLimit) do { $health = Get-GatewayHealth; $rows = @(Get-BackendState $health) Write-WorkshopJson -Path (Join-Path $privateDirectory 'evidence/backend-health-latest.json') -Value $health $selected = @($rows | Where-Object settings -eq $Settings) if ($selected.Count -and @($selected | Where-Object health -ne $Expected).Count -eq 0) { return $rows } if ([DateTimeOffset]::UtcNow -ge $deadline) { throw "Backend convergence failed for $Settings ($Expected)." } Start-Sleep -Seconds 10 } while ($true) } function Invoke-Probe([string]$Path, [string]$ArgumentName = '', [string]$ArgumentValue = '') { $bodyPath = Join-Path $privateDirectory "probes/$([guid]::NewGuid().ToString('N')).body" $headersPath = "$bodyPath.headers" $certificate = $script:state.certificates.root_pem_path $arguments = @('--silent','--show-error','--noproxy','*','--connect-timeout','20','--max-time','60','--cacert',$certificate, '--resolve',"appgw.lab.test:443:$(Get-Output 'gateway_public_ip')",'--dump-header',$headersPath,'--output',$bodyPath,'--write-out','%{http_code}', '--get',"https://appgw.lab.test$Path") # This disposable private CA has no CRL distribution point. Schannel may # tolerate that absence while still verifying the CA chain and DNS name. # Do not replace this with --insecure or --ssl-no-revoke. if ($IsWindows) { $arguments = @('--ssl-revoke-best-effort') + $arguments } $arguments += @('--data-urlencode',"runMarker=$($script:state.marker)") if ($ArgumentName) { $arguments += @('--data-urlencode',"$ArgumentName=$ArgumentValue") } $result = Invoke-WorkshopNative -FilePath curl.exe -Arguments $arguments -PrivateDirectory $privateDirectory -TimeoutSeconds 90 -AllowFailure -Label 'https-probe' $status = 0; [void][int]::TryParse($result.StdOut.Trim(),[ref]$status) $body = if(Test-Path -LiteralPath $bodyPath){[IO.File]::ReadAllText($bodyPath)}else{''} return @{ status = $status; curl_exit = $result.ExitCode; marker_match = $body.Contains($script:state.marker); body_sha256 = $(if(Test-Path -LiteralPath $bodyPath){Get-WorkshopFileHash $bodyPath}else{''}) } } function Assert-Http([string]$Name, [string]$Path, [int]$ExpectedStatus, [string]$ArgumentName = '', [string]$ArgumentValue = '') { $deadline=[DateTimeOffset]::UtcNow.AddSeconds($script:healthWaitLimit) do{ $observed = Invoke-Probe $Path $ArgumentName $ArgumentValue $passed = $observed.curl_exit -eq 0 -and $observed.status -eq $ExpectedStatus if ($ExpectedStatus -eq 200) { $passed = $passed -and $observed.marker_match } if($passed -or $observed.curl_exit -eq 60 -or [DateTimeOffset]::UtcNow -ge $deadline){break} Start-Sleep -Seconds 10 }while($true) Add-Check $Name $passed $observed $(if($script:state.active_fault){'fault'}else{'healthy'}) } function Set-WafExclusion([hashtable]$Rules) { $policy = Invoke-Az @('network','application-gateway','waf-policy','show','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--name',(Get-Output 'waf_policy_name')) if ($policy.id -ne (Get-Output 'waf_policy_id')) { throw 'WAF policy ownership mismatch.' } foreach($existing in $policy.managedRules.exclusions){ if($existing.matchVariable -ne 'RequestArgValues' -or $existing.selectorMatchOperator -ne 'Equals' -or $existing.selector -ne 'labSnippet'){throw 'Unowned WAF exclusion encountered; do not overwrite it.'} foreach($set in $existing.exclusionManagedRuleSets){ if($set.ruleSetType -ne 'Microsoft_DefaultRuleSet' -or $set.ruleSetVersion -ne '2.2'){throw 'Unowned WAF ruleset exclusion encountered.'} foreach($group in $set.ruleGroups){foreach($rule in $group.rules){if(-not $script:state.waf_exclusion_rules.ContainsKey($group.ruleGroupName) -or $rule.ruleId -notin $script:state.waf_exclusion_rules[$group.ruleGroupName]){throw 'Unrecorded WAF rule exclusion encountered.'}}} } } $exclusions = @() if($Rules.Count){ $groups=@(foreach($group in ($Rules.Keys|Sort-Object)){@{ruleGroupName=$group;rules=@($Rules[$group]|ForEach-Object{@{ruleId=[string]$_}})}}) $exclusions=@(@{matchVariable='RequestArgValues';selectorMatchOperator='Equals';selector='labSnippet';exclusionManagedRuleSets=@(@{ruleSetType='Microsoft_DefaultRuleSet';ruleSetVersion='2.2';ruleGroups=$groups})}) } $json=ConvertTo-Json -InputObject $exclusions -Depth 30 -Compress Invoke-Az @('network','application-gateway','waf-policy','update','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--name',(Get-Output 'waf_policy_name'),'--set',"managedRules.exclusions=$json") 1200 | Out-Null } function Invoke-LogQuery([string]$Query,[string]$Label){ $bodyPath=Join-Path $privateDirectory "$Label-query.json";Write-WorkshopJson $bodyPath @{query=$Query} $result=Invoke-WorkshopNative -FilePath az -Arguments @('rest','--method','POST','--url',"https://api.loganalytics.azure.com/v1/workspaces/$(Get-Output 'workspace_id')/query",'--resource','https://api.loganalytics.io','--body',"@$bodyPath",'--only-show-errors','--output','json') -PrivateDirectory $privateDirectory -TimeoutSeconds 180 -AllowFailure -Label $Label if($result.ExitCode -ne 0){ if($result.StdErr -match 'Failed to resolve table|FailedToResolveTable|does not refer to any known table'){return @()} throw 'Log query failed for a reason other than a not-yet-created table. Inspect its protected native log.' } $response=$result.StdOut|ConvertFrom-Json -AsHashtable -Depth 80 Write-WorkshopJson (Join-Path $privateDirectory "evidence/$Label.json") $response $records=@() foreach($table in $response.tables){foreach($row in $table.rows){$record=@{};for($i=0;$i -lt $table.columns.Count;$i++){$record[$table.columns[$i].name]=$row[$i]};$records+=$record}} return $records } function Get-LogEvidence { $deadline=[DateTimeOffset]::UtcNow.AddSeconds($script:logWaitLimit) $marker=$script:state.marker $firewallQuery="AGWFirewallLogs | where TimeGenerated > ago(6h) | where RequestUri contains '$marker' | project TimeGenerated,Action,RuleId,RuleSetType,RuleSetVersion,TransactionId,RequestUri | take 2000" $accessQuery="AGWAccessLogs | where TimeGenerated > ago(6h) | extend ObservedUri=tostring(column_ifexists('OriginalRequestUriWithArgs', column_ifexists('RequestUri',''))) | where ObservedUri contains '$marker' | project TimeGenerated,HttpStatus,TransactionId,ObservedUri | take 2000" do{ $firewall=@(Invoke-LogQuery $firewallQuery 'firewall-logs') $access=@(Invoke-LogQuery $accessQuery 'access-logs') $statuses=@($access|ForEach-Object{[int]$_.HttpStatus}|Sort-Object -Unique) $blockedTransactions=@($access|Where-Object{[int]$_.HttpStatus -eq 403}|ForEach-Object{[string]$_.TransactionId}) $selectedRules=@($script:state.waf_exclusion_rules.Values|ForEach-Object{$_}|ForEach-Object{[string]$_}) $managedMatches=@($firewall|Where-Object{[string]$_.RuleId -in $selectedRules -and $_.RuleSetType -eq 'Microsoft_DefaultRuleSet' -and [string]$_.RuleSetVersion -eq '2.2' -and $_.RequestUri -match 'labSnippet=' -and [string]$_.TransactionId -in $blockedTransactions}) $customBlocks=@($firewall|Where-Object{$_.RuleId -eq 'BlockWorkshopMarker' -and $_.Action -match 'Block' -and [string]$_.TransactionId -in $blockedTransactions}) $hasAllStatuses=200 -in $statuses -and 403 -in $statuses -and 502 -in $statuses $probeFailure=@($access|Where-Object{[int]$_.HttpStatus -eq 502 -and $_.ObservedUri -match '/probe-fault/'}).Count -gt 0 $tlsFailure=@($access|Where-Object{[int]$_.HttpStatus -eq 502 -and $_.ObservedUri -match '/tls-fault/'}).Count -gt 0 if($hasAllStatuses -and $probeFailure -and $tlsFailure -and $managedMatches.Count -and $customBlocks.Count){ $ruleIds=@($firewall|ForEach-Object{[string]$_.RuleId}|Sort-Object -Unique) Add-Check 'firewall-logs-observed' $true @{rows=$firewall.Count;rule_ids=$ruleIds;custom_block_correlated=$true} 'logs' Add-Check 'access-logs-observed' $true @{rows=$access.Count;http_statuses=$statuses;probe_fault_502=$true;tls_fault_502=$true} 'logs' Add-Check 'managed-rule-block-correlated' $true @{matched_rule_ids=@($managedMatches|ForEach-Object{[string]$_.RuleId}|Sort-Object -Unique);ruleset_version='2.2';correlated_transactions=$managedMatches.Count} 'logs' $script:state.log_evidence_utc=[DateTimeOffset]::UtcNow.ToString('o');Save-Run;return } if([DateTimeOffset]::UtcNow -ge $deadline){throw 'Required correlated access, custom-rule and managed-rule evidence did not arrive within the bounded collection window.'} Start-Sleep -Seconds 20 }while($true) } function Export-Evidence { param([string]$DestinationPath) $instant={param($Value) if($Value -is [DateTimeOffset]){return $Value} if($Value -is [DateTime]){return [DateTimeOffset]$Value} return [DateTimeOffset]::Parse([string]$Value,[Globalization.CultureInfo]::InvariantCulture) } $required=@('preflight-read-only','terraform-deploy','windows-iis-bootstrap','terraform-no-drift','firewall-logs-observed','access-logs-observed','managed-rule-block-correlated','cleanup-absence','protected-baseline-preserved') $missing=@($required|Where-Object{$name=$_;-not @($script:state.checks|Where-Object{$_.name -eq $name -and $_.passed}).Count}) $passed=@($script:state.checks|Where-Object{$_.passed}) $mutations=@($script:state.operations|Where-Object{$_.stage -in @('Fault','Repair')}|Sort-Object {& $instant $_.started_utc}) $lastMutation=if($mutations.Count){& $instant $mutations[-1].started_utc}else{[DateTimeOffset]::MinValue} $finalDrift=@($passed|Where-Object{$_.name -eq 'terraform-no-drift' -and $_.phase -eq 'final' -and (& $instant $_.utc) -gt $lastMutation}|Sort-Object {& $instant $_.utc}) if(-not $finalDrift.Count){$missing+='fresh-final-no-drift'} $finalHealthy=@($passed|Where-Object{ $_.name -eq 'scenario-observation' -and -not $_.scenario -and $_.phase -eq 'recovery' -and -not $_.observed.expected_failure_observed -and (& $instant $_.utc) -gt $lastMutation -and $finalDrift.Count -and (& $instant $_.utc) -ge (& $instant $finalDrift[-1].utc) }) if(-not $finalHealthy.Count){$missing+='fresh-final-healthy-validation'} foreach($name in @('Probe','Tls','Waf')){ $faults=@($mutations|Where-Object{$_.stage -eq 'Fault' -and $_.scenario -eq $name}) if(-not $faults.Count){$missing+="$name-fault-operation";continue} $faultTime=& $instant $faults[-1].started_utc $repairs=@($mutations|Where-Object{$_.stage -eq 'Repair' -and $_.scenario -eq $name -and (& $instant $_.started_utc) -gt $faultTime}) $firstRepair=if($repairs.Count){& $instant $repairs[0].started_utc}else{[DateTimeOffset]::MaxValue} $faultEvidence=@($passed|Where-Object{ $_.name -eq 'scenario-observation' -and $_.scenario -eq $name -and $_.phase -eq 'fault' -and $_.observed.expected_failure_observed -and (& $instant $_.utc) -gt $faultTime -and (& $instant $_.utc) -lt $firstRepair }) if(-not $faultEvidence.Count){$missing+="$name-fault"} if($repairs.Count -lt 2 -or -not $script:state.repaired.ContainsKey($name) -or $script:state.repaired[$name] -lt 2){$missing+="$name-repeated-repair"} for($repairIndex=0;$repairIndex -lt 2;$repairIndex++){ if($repairs.Count -le $repairIndex){$missing+="$name-recovery-$($repairIndex+1)";continue} $repairTime=& $instant $repairs[$repairIndex].started_utc $nextRepair=if($repairs.Count -gt ($repairIndex+1)){& $instant $repairs[$repairIndex+1].started_utc}else{[DateTimeOffset]::MaxValue} $recoveryEvidence=@($passed|Where-Object{ $_.name -eq 'scenario-observation' -and $_.scenario -eq $name -and $_.phase -eq 'recovery' -and -not $_.observed.expected_failure_observed -and (& $instant $_.utc) -gt $repairTime -and (& $instant $_.utc) -lt $nextRepair }) if(-not $recoveryEvidence.Count){$missing+="$name-recovery-$($repairIndex+1)"} } } $absence=@($passed|Where-Object{$_.name -eq 'cleanup-absence' -and $_.observed.exact_group_absent -and (& $instant $_.utc) -gt $lastMutation}|Sort-Object {& $instant $_.utc}) if($absence.Count -lt 2 -or (& $instant $absence[-1].utc) -le (& $instant $absence[0].utc)){$missing+='repeated-cleanup'} $baseline=@($passed|Where-Object{$_.name -eq 'protected-baseline-preserved' -and $absence.Count -and (& $instant $_.utc) -ge (& $instant $absence[0].utc)}) if(-not $baseline.Count){$missing+='cleanup-baseline-preservation'} if($script:state.pending_operation){$missing+='pending-operation'} $missing=@($missing|Sort-Object -Unique) $public = @{ schema_version = 1; generated_utc = [DateTimeOffset]::UtcNow.ToString('o'); region = $script:state.location; status = $script:state.status; active_fault = $script:state.active_fault; checks = $script:state.checks; complete=($missing.Count -eq 0 -and -not $script:state.active_fault -and $script:state.status -eq 'cleaned');missing_required_checks=$missing; cost = $script:state.cost; private_material_included = $false; observed_live = $true } $path = if($DestinationPath){$DestinationPath}else{Join-Path $PSScriptRoot 'docs/evidence/sanitized-evidence.json'} [IO.Directory]::CreateDirectory((Split-Path $path -Parent)) | Out-Null Write-WorkshopJson -Path $path -Value $public Write-Information 'Exported sanitized live checks. Private identifiers, certificate material, state, plans, and raw logs remain private.' -InformationAction Continue } try { if (Test-Path -LiteralPath $StatePath) { $script:state = Get-Content -LiteralPath $StatePath -Raw | ConvertFrom-Json -AsHashtable -Depth 80 -DateKind String } if (-not $script:state -and $Stage -ne 'Preflight') { throw 'Start this run with read-only Preflight and a new private state path.' } if ($script:state) { Assert-WorkshopContext -State $script:state -PrivateDirectory $privateDirectory | Out-Null;$env:ARM_SUBSCRIPTION_ID=$script:state.subscription_id;$env:ARM_TENANT_ID=$script:state.tenant_id;if(Initialize-OutputDictionary $script:state){Save-Run} } if ($Stage -in @('Fault','Repair') -and -not $Scenario) { throw 'Fault and Repair require -Scenario Probe, Tls, or Waf.' } switch ($Stage) { 'Preflight' { $account = Assert-WorkshopContext -State $script:state -PrivateDirectory $privateDirectory -ExpectedSubscriptionId $SubscriptionId if (-not $script:state) { $now = [DateTimeOffset]::UtcNow $script:state = @{ schema_version = 1; subscription_id = $account.id; tenant_id = $account.tenantId; prefix = 'awz-waf-' + [guid]::NewGuid().ToString('N').Substring(0,8); marker = 'AWZ-' + [guid]::NewGuid().ToString('N'); location = $Location; vnet_cidr = $VnetCidr; image_urn = $ImageUrn; created_utc = $now.ToString('o'); exercise_stop_utc = $now.AddHours(4.5).ToString('o'); delete_deadline_utc = $now.AddHours(6).ToString('o'); status = 'preflight'; pending_operation = $null; operations = @(); checks = @(); outputs = @{}; owned_ids = @(); active_fault = ''; fault_counts = @{}; repaired = @{}; waf_exclusion_rules = @{}; waf_fixture=$WafFixture;certificates = @{}; plan = $null; cleanup_plan = $null } Save-Run } $env:ARM_SUBSCRIPTION_ID=$script:state.subscription_id;$env:ARM_TENANT_ID=$script:state.tenant_id $azVersion = Invoke-Az @('version') $tfVersion = (Invoke-Tf @('version','-json')).StdOut | ConvertFrom-Json -AsHashtable if ([version]$azVersion.'azure-cli' -lt [version]'2.88.0' -or [version]$tfVersion.terraform_version -lt [version]'1.15.8') { throw 'Installed Azure CLI/Terraform versions are below the workshop minimum. No upgrade is automatic.' } $curlVersion=Invoke-WorkshopNative -FilePath curl.exe -Arguments @('--version') -PrivateDirectory $privateDirectory -TimeoutSeconds 30 -Label 'curl-version' $curlVersionMatch=[regex]::Match($curlVersion.StdOut,'(?m)^curl\s+(?\d+\.\d+\.\d+)\b') if(-not $curlVersionMatch.Success -or [version]$curlVersionMatch.Groups['version'].Value -lt [version]'7.70.0'){throw 'curl.exe 7.70.0 or later is required for the bounded Windows private-CA revocation behavior. No upgrade is automatic.'} $refreshOwnership=Get-PreflightOwnership foreach ($provider in @('Microsoft.Network','Microsoft.Compute','Microsoft.OperationalInsights','Microsoft.Insights')) { $registration = Invoke-Az @('provider','show','--namespace',$provider,'--subscription',$account.id) if ($registration.registrationState -ne 'Registered') { throw "Required provider $provider is not registered. Preflight does not register it." } } $skus = @(Invoke-Az @('vm','list-skus','--subscription',$account.id,'--location',$script:state.location,'--size','Standard_B2s','--all')) $available = @($skus | Where-Object { $_.name -eq 'Standard_B2s' -and @($_.restrictions).Count -eq 0 }) if (-not $available.Count) { throw 'The required B2s VM SKU is restricted or unavailable.' } $usage = @(Invoke-Az @('vm','list-usage','--subscription',$account.id,'--location',$script:state.location)) foreach ($family in @('cores','standardBSFamily')) { $quota = @($usage | Where-Object { $_.name.value -eq $family }) if (-not $quota.Count -or ($refreshOwnership.required_additional_cores -gt 0 -and ($quota[0].limit - $quota[0].currentValue) -lt $refreshOwnership.required_additional_cores)) { throw "Insufficient verified compute quota for $family." } } $image = Invoke-Az @('vm','image','show','--subscription',$account.id,'--location',$script:state.location,'--urn',$script:state.image_urn) if (-not $image.id) { throw 'Pinned marketplace image could not be verified.' } $networks = @(Invoke-Az @('network','vnet','list','--subscription',$account.id)) foreach ($network in $networks) { if($network.id -in $refreshOwnership.vnet_ids){continue} foreach ($cidr in $network.addressSpace.addressPrefixes) { if(Test-WorkshopCidrOverlap $script:state.vnet_cidr $cidr){throw 'Workshop address space overlaps an existing VNet.'} } } $scope = '/subscriptions/' + $account.id $policies = Invoke-Az @('policy','assignment','list','--subscription',$account.id,'--scope',$scope) $locks = Invoke-Az @('lock','list','--subscription',$account.id) if(@($locks|Where-Object{$_.id -match '^/subscriptions/[^/]+/providers/Microsoft.Authorization/locks/[^/]+$'}).Count){throw 'A subscription-scope management lock prevents a guaranteed clean lab lifecycle.'} $ruleSets=Invoke-Az @('network','application-gateway','waf-config','list-dynamic-rule-sets','--subscription',$account.id,'--location',$script:state.location) if(-not @($ruleSets.availableRuleSets|Where-Object{$_.ruleSetType -eq 'Microsoft_DefaultRuleSet' -and $_.ruleSetVersion -eq '2.2'}).Count){throw 'The required dynamic DRS 2.2 catalog was not verified.'} Write-WorkshopJson (Join-Path $privateDirectory 'drs-2.2-catalog.json') $ruleSets $permissions = Invoke-Az @('rest','--method','GET','--url',"https://management.azure.com$scope/providers/Microsoft.Authorization/permissions?api-version=2022-04-01") foreach ($action in @('Microsoft.Resources/subscriptions/resourceGroups/write','Microsoft.Resources/subscriptions/resourceGroups/delete','Microsoft.Network/applicationGateways/write','Microsoft.Compute/virtualMachines/write','Microsoft.Compute/virtualMachines/runCommand/action','Microsoft.OperationalInsights/workspaces/write')) { if (-not (Test-WorkshopActionAllowed $permissions.value $action)) { throw "Required effective permission not established: $action" } } Write-WorkshopJson -Path (Join-Path $privateDirectory 'preflight-policy-lock-review.json') -Value @{ policy_assignments = $policies; locks = $locks; image = $image; skus = $skus; quota = $usage } $script:state.cost = Get-WorkshopBudgetEstimate -Location $script:state.location if (-not $script:state.ContainsKey('baseline_resource_ids')) { $script:state.baseline_resource_ids = @((Invoke-Az @('resource','list','--subscription',$account.id)).id) } $script:state.preflight_utc = [DateTimeOffset]::UtcNow.ToString('o'); Save-Run Add-Check 'preflight-read-only' $true @{ active_estimate_eur = $script:state.cost.active_estimate; cleanup_reserve_eur = 5; tools_checked = $true; provider_registration_changed = $false;required_additional_cores=$refreshOwnership.required_additional_cores } 'preflight' Write-Information 'Preflight passed. Review the protected policy/lock report and the saved plan before deployment.' -InformationAction Continue } 'Plan' { if ($script:state.pending_operation -and -not $Resume) { throw 'Resolve the pending operation before planning; -Resume is only for an inspected interrupted deployment.' } if ([DateTimeOffset]::UtcNow -ge [DateTimeOffset]::Parse($script:state.exercise_stop_utc)) { throw 'Exercise window ended. Prepare cleanup.' } $publishedMap=Join-Path $PSScriptRoot 'fixtures/waf-exclusions.json' $publishedFixture=Join-Path $PSScriptRoot 'fixtures/waf-fixture.json' if(-not $WafExclusionRulesJson -and -not $script:state.waf_exclusion_rules.Count -and (Test-Path -LiteralPath $publishedMap)){ $WafExclusionRulesJson=Get-Content -LiteralPath $publishedMap -Raw if(Test-Path -LiteralPath $publishedFixture){$fixtureMetadata=Get-Content -LiteralPath $publishedFixture -Raw|ConvertFrom-Json -AsHashtable;$script:state.waf_fixture=[string]$fixtureMetadata.payload} } if ($WafExclusionRulesJson) { $rules = $WafExclusionRulesJson | ConvertFrom-Json -AsHashtable if ($rules -isnot [hashtable]) { throw 'WafExclusionRulesJson must be a group-to-rule-ID JSON object.' } foreach ($group in $rules.Keys) { if($group -notmatch '^[A-Za-z0-9_-]+$'){throw 'Invalid WAF group name.'}; foreach($id in $rules[$group]){if([string]$id -notmatch '^\d{6}$'){throw 'Invalid WAF rule ID.'}} } $script:state.waf_exclusion_rules = $rules if($PSBoundParameters.ContainsKey('WafFixture')){$script:state.waf_fixture=$WafFixture} } if (-not $script:state.certificates.Count) { $certificateJson = & (Join-Path $PSScriptRoot 'scripts/New-WorkshopCertificates.ps1') -PrivateDirectory (Join-Path $privateDirectory 'certificates') $script:state.certificates = ($certificateJson | ConvertFrom-Json -AsHashtable); Save-Run } if (Test-Path -LiteralPath $tfvarsPath) { $variables = Get-Content -LiteralPath $tfvarsPath -Raw | ConvertFrom-Json -AsHashtable } else { $variables = @{ prefix = $script:state.prefix; location = $script:state.location; vnet_cidr = $script:state.vnet_cidr; image_urn = $script:state.image_urn; admin_password = 'Az!' + [Convert]::ToHexString([Security.Cryptography.RandomNumberGenerator]::GetBytes(20)) + 'z9'; frontend_pfx_base64 = $script:state.certificates.frontend_pfx_base64; pfx_password = $script:state.certificates.pfx_password; root_cer_base64 = $script:state.certificates.root_cer_base64 } } $variables.waf_exclusion_rules = $script:state.waf_exclusion_rules Write-WorkshopJson -Path $tfvarsPath -Value $variables Invoke-Tf @('init','-input=false','-lockfile=readonly') | Out-Null Invoke-Tf @('validate','-no-color') | Out-Null Invoke-Tf @('plan','-input=false','-no-color',"-state=$tfstatePath","-var-file=$tfvarsPath","-out=$planPath") | Out-Null $plan = (Invoke-Tf @('show','-json',$planPath)).StdOut | ConvertFrom-Json -AsHashtable -Depth 80 $changes = @($plan.resource_changes | Where-Object { @($_.change.actions | Where-Object { $_ -ne 'no-op' -and $_ -ne 'read' }).Count }) foreach ($change in $changes) { if ('delete' -in $change.change.actions) { throw 'Normal Plan cannot delete or replace resources.' } if ('update' -in $change.change.actions) { if ($change.type -ne 'azurerm_web_application_firewall_policy' -or $change.change.before.id -ne $script:state.outputs.waf_policy_id) { throw 'Calibration plan may update only the exact owned WAF policy.' } $before = $change.change.before.Clone(); $after = $change.change.after.Clone() $beforeManaged=$before.managed_rules[0].Clone();$afterManaged=$after.managed_rules[0].Clone();$beforeManaged.Remove('exclusion');$afterManaged.Remove('exclusion') if(($beforeManaged|ConvertTo-Json -Depth 80 -Compress) -ne ($afterManaged|ConvertTo-Json -Depth 80 -Compress)){throw 'Calibration must preserve the managed ruleset and its rule configuration.'} $before.Remove('managed_rules'); $after.Remove('managed_rules') if (($before|ConvertTo-Json -Depth 80 -Compress) -ne ($after|ConvertTo-Json -Depth 80 -Compress)) { throw 'WAF update contains changes outside managed-rule calibration.' } } if ($change.change.after.ContainsKey('name') -and -not ([string]$change.change.after.name).StartsWith($script:state.prefix)) { if ($change.type -notin @('azurerm_subnet','azurerm_subnet_nat_gateway_association','azurerm_subnet_network_security_group_association','azurerm_network_interface_security_group_association','azurerm_monitor_diagnostic_setting')) { throw 'Planned resource name lies outside this run prefix.' } } } $script:state.plan = @{ sha256 = Get-WorkshopFileHash $planPath; tfvars_sha256 = Get-WorkshopFileHash $tfvarsPath; creates = @($changes|Where-Object {'create' -in $_.change.actions}).Count; updates = @($changes|Where-Object {'update' -in $_.change.actions}).Count; utc = [DateTimeOffset]::UtcNow.ToString('o') } Save-Run Write-Information "PLAN saved: creates=$($script:state.plan.creates) updates=$($script:state.plan.updates) deletes=0. Inspect the protected plan before -ReviewedPlan." -InformationAction Continue } 'Deploy' { Assert-Mutation if (-not $AcknowledgeCost -or -not $ReviewedPlan -or -not $script:state.plan) { throw 'Deploy requires -AcknowledgeCost -ReviewedPlan and a saved reviewed plan.' } if ((Get-WorkshopFileHash $planPath) -ne $script:state.plan.sha256 -or (Get-WorkshopFileHash $tfvarsPath) -ne $script:state.plan.tfvars_sha256) { throw 'Plan or variables changed after review.' } if ([DateTimeOffset]::UtcNow -gt [DateTimeOffset]::Parse($script:state.cost.utc).AddMinutes(60)) { throw 'Retail estimate expired. Rerun Preflight and review a new Plan.' } if ($script:state.outputs.Count) { Assert-OwnedResource } Start-Operation 'terraform-apply' Invoke-Tf @('apply','-input=false','-no-color',"-state=$tfstatePath",$planPath) 3600 | Out-Null $outputs = (Invoke-Tf @('output',"-state=$tfstatePath",'-json')).StdOut | ConvertFrom-Json -AsHashtable -Depth 80 foreach ($name in $outputs.Keys) { $script:state.outputs[$name] = $outputs[$name].value } Sync-PartialDeployment Assert-OwnedResource $members = @(Invoke-Az @('resource','list','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'))) $script:state.status = 'deployed'; Complete-Operation Add-Check 'terraform-deploy' $true @{ owned_resource_count = $members.Count } 'deploy' } 'Bootstrap' { Assert-Mutation; Assert-OwnedResource; Start-Operation 'iis-bootstrap' $guestScript = & (Join-Path $PSScriptRoot 'scripts/New-IisBootstrapScript.ps1') -BackendPfxBase64 $script:state.certificates.backend_pfx_base64 -RootCerBase64 $script:state.certificates.root_cer_base64 -PfxPassword $script:state.certificates.pfx_password -Marker $script:state.marker $guestPath = Join-Path $privateDirectory 'bootstrap-guest.ps1'; [IO.File]::WriteAllText($guestPath,[string]$guestScript,[Text.UTF8Encoding]::new($false)) $result = Invoke-Az @('vm','run-command','invoke','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--name',(Get-Output 'vm_name'),'--command-id','RunPowerShellScript','--scripts',"@$guestPath") 1800 $readyMessage=@($result.value|Where-Object{$_.message -match '"status"\s*:\s*"ready"' -and $_.message.Contains($script:state.marker)}) if (-not $readyMessage.Count -or @($result.value | Where-Object { $_.code -match 'failed' }).Count) { throw 'Guest bootstrap did not report ready status and the exact marker.' } $extensions = @(Invoke-Az @('vm','extension','list','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--vm-name',(Get-Output 'vm_name'))) if(-not $script:state.ContainsKey('verified_extension_ids')){$script:state.verified_extension_ids=@()} foreach($extension in $extensions){ if($extension.publisher -ne 'Microsoft.CPlat.Core' -or $extension.virtualMachineExtensionType -ne 'RunCommandWindows' -or $extension.id -ne ((Get-Output 'vm_id')+'/extensions/RunCommandWindows')){throw 'Unexpected VM extension needs ownership review.'} if($extension.id -notin $script:state.owned_ids){$script:state.owned_ids += $extension.id} if($extension.id -notin $script:state.verified_extension_ids){$script:state.verified_extension_ids += $extension.id} } $script:state.status = 'bootstrapped'; Complete-Operation Add-Check 'windows-iis-bootstrap' $true @{ success = $true } 'bootstrap' } 'Fault' { Assert-Mutation; Assert-OwnedResource $resumingFault=$Resume -and $script:state.active_fault -eq $Scenario -and $script:state.pending_operation -and $script:state.pending_operation.stage -eq 'Fault' -and $script:state.pending_operation.scenario -eq $Scenario if($script:state.active_fault -and -not $resumingFault){throw 'Repair the active fault before introducing another.'} # Fresh controls prevent unrelated existing failures from being accepted as injected faults. Assert-Http 'pre-fault-control' '/control/index.html' 200 if(-not $resumingFault){switch($Scenario){ 'Probe' { Wait-Backend 'probe-settings' 'Healthy' | Out-Null;Assert-Http 'pre-fault-target' '/probe-fault/index.html' 200 } 'Tls' { Wait-Backend 'tls-settings' 'Healthy' | Out-Null;Assert-Http 'pre-fault-target' '/tls-fault/index.html' 200 } 'Waf' { if(-not $script:state.waf_exclusion_rules.Count){throw 'Calibrate and deploy exact WAF exclusions first.'}; Assert-Http 'pre-fault-legitimate' '/docs/index.html' 200 'labSnippet' $script:state.waf_fixture } }} $script:state.active_fault = $Scenario; Save-Run; Start-Operation "fault-$Scenario" switch($Scenario){ 'Probe' { Invoke-Az @('network','application-gateway','probe','update','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--gateway-name',(Get-Output 'gateway_name'),'--name','probe-probe','--path','/missing-awz-health.txt') 1200 | Out-Null } 'Tls' { Invoke-Az @('network','application-gateway','http-settings','update','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--gateway-name',(Get-Output 'gateway_name'),'--name','tls-settings','--host-name','wrong.lab.test') 1200 | Out-Null } 'Waf' { Set-WafExclusion @{} } } if(-not $script:state.fault_counts.ContainsKey($Scenario)){$script:state.fault_counts[$Scenario]=0};$script:state.fault_counts[$Scenario]++ Complete-Operation Write-Information "FAULT $Scenario applied. Run Validate with the same scenario to observe the expected failure." -InformationAction Continue } 'Repair' { Assert-Mutation; Assert-OwnedResource if($script:state.active_fault -and $script:state.active_fault -ne $Scenario){throw 'The active fault belongs to another scenario.'} $alreadyRepaired = -not $script:state.active_fault if($alreadyRepaired -and -not $script:state.repaired.ContainsKey($Scenario)){throw 'No owned scenario exists to repair.'} Start-Operation "repair-$Scenario" switch($Scenario){ 'Probe' { Invoke-Az @('network','application-gateway','probe','update','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--gateway-name',(Get-Output 'gateway_name'),'--name','probe-probe','--path','/health.txt') 1200 | Out-Null } 'Tls' { Invoke-Az @('network','application-gateway','http-settings','update','--subscription',$script:state.subscription_id,'--resource-group',(Get-Output 'resource_group_name'),'--gateway-name',(Get-Output 'gateway_name'),'--name','tls-settings','--host-name','backend.lab.test') 1200 | Out-Null } 'Waf' { Set-WafExclusion $script:state.waf_exclusion_rules } } $script:state.active_fault = ''; if(-not $script:state.repaired.ContainsKey($Scenario)){$script:state.repaired[$Scenario]=0};$script:state.repaired[$Scenario]++ Complete-Operation; Add-Check "repair-$Scenario" $true @{ repeated = [bool]$alreadyRepaired } 'repair' } 'Validate' { Assert-Mutation; Assert-OwnedResource $faultActive = $script:state.active_fault -eq $Scenario -and $Scenario Wait-Backend 'control-settings' 'Healthy' | Out-Null Assert-Http 'control-https-marker' '/control/index.html' 200 switch($Scenario){ 'Probe' { $rows=Wait-Backend 'probe-settings' $(if($faultActive){'Unhealthy'}else{'Healthy'});if($faultActive){Add-Check 'probe-fault-reason' (@($rows|Where-Object{$_.settings -eq 'probe-settings' -and $_.detail -match '404'}).Count -gt 0) @{} 'fault'}; Assert-Http 'probe-scenario-http' '/probe-fault/index.html' $(if($faultActive){502}else{200});Assert-Http 'tls-path-unaffected' '/tls-fault/index.html' 200 } 'Tls' { $rows=Wait-Backend 'tls-settings' $(if($faultActive){'Unhealthy'}else{'Healthy'});if($faultActive){Add-Check 'tls-fault-reason' (@($rows|Where-Object{$_.settings -eq 'tls-settings' -and $_.detail -match 'certificate|common.name|SSL|TLS|CN'}).Count -gt 0) @{} 'fault'}; Assert-Http 'tls-scenario-http' '/tls-fault/index.html' $(if($faultActive){502}else{200});Assert-Http 'probe-path-unaffected' '/probe-fault/index.html' 200 } 'Waf' { if(-not $script:state.waf_exclusion_rules.Count){throw 'WAF fixture is not calibrated.'}; Assert-Http 'waf-legitimate-fixture' '/docs/index.html' $(if($faultActive){403}else{200}) 'labSnippet' $script:state.waf_fixture; Assert-Http 'waf-nonexcluded-control' '/docs/index.html' 403 'untrustedSnippet' $script:state.waf_fixture } default { foreach($setting in @('control-settings','probe-settings','tls-settings')){Wait-Backend $setting 'Healthy'|Out-Null} foreach($path in @('/probe-fault/index.html','/tls-fault/index.html','/docs/index.html')){Assert-Http 'baseline-https-marker' $path 200} Assert-Http 'custom-marker-blocked' '/control/index.html' 403 'labMarker' 'awz-block-me' if($script:state.waf_exclusion_rules.Count){Assert-Http 'legitimate-fixture' '/docs/index.html' 200 'labSnippet' $script:state.waf_fixture;Assert-Http 'nonexcluded-control' '/docs/index.html' 403 'untrustedSnippet' $script:state.waf_fixture} $drift = Invoke-Tf @('plan','-input=false','-no-color','-detailed-exitcode',"-state=$tfstatePath","-var-file=$tfvarsPath") 1800 -AllowFailure Add-Check 'terraform-no-drift' ($drift.ExitCode -eq 0) @{ detailed_exit_code = $drift.ExitCode } 'final' } } Add-Check 'scenario-observation' $true @{ expected_failure_observed = [bool]$faultActive; intended_http_failure_is_not_wrapper_failure = [bool]$faultActive } $(if($faultActive){'fault'}else{'recovery'}) if($CollectLogs){Get-LogEvidence} } 'Cleanup' { Sync-PartialDeployment if(-not $script:state.outputs.ContainsKey('resource_group_name')){throw 'No owned group exists in private Terraform state. Inspect interrupted operations before claiming cleanup.'} $exists = Invoke-Az @('group','exists','--subscription',$script:state.subscription_id,'--name',(Get-Output 'resource_group_name')) if(-not $exists){ $current=@((Invoke-Az @('resource','list','--subscription',$script:state.subscription_id)).id) $missing=@($script:state.baseline_resource_ids|Where-Object{$_ -notin $current}) Add-Check 'protected-baseline-preserved' ($missing.Count -eq 0) @{missing_baseline_resources=$missing.Count} 'cleanup' $script:state.status='cleaned';$script:state.pending_operation=$null Add-Check 'cleanup-absence' $true @{ exact_group_absent = $true; repeated = $true } 'cleanup';Save-Run;Export-Evidence;break } Assert-OwnedResource if(-not $Execute){ Invoke-Tf @('plan','-destroy','-input=false','-no-color',"-state=$tfstatePath","-var-file=$tfvarsPath","-out=$destroyPlanPath") 1800 | Out-Null $script:state.cleanup_plan=@{sha256=Get-WorkshopFileHash $destroyPlanPath;utc=[DateTimeOffset]::UtcNow.ToString('o')};Save-Run Write-Information 'Cleanup plan saved. Inspect exact recorded targets, then run Cleanup -Execute -ReviewedPlan.' -InformationAction Continue;break } Assert-Mutation if(-not $ReviewedPlan -or -not $script:state.cleanup_plan -or (Get-WorkshopFileHash $destroyPlanPath) -ne $script:state.cleanup_plan.sha256){throw 'Cleanup requires its unchanged reviewed destroy plan.'} Start-Operation 'terraform-destroy';Invoke-Tf @('apply','-input=false','-no-color',"-state=$tfstatePath",$destroyPlanPath) 3600 | Out-Null $remaining=Invoke-Az @('group','exists','--subscription',$script:state.subscription_id,'--name',(Get-Output 'resource_group_name')) Add-Check 'cleanup-absence' (-not $remaining) @{ exact_group_absent = (-not $remaining); repeated = $false } 'cleanup' $current=@((Invoke-Az @('resource','list','--subscription',$script:state.subscription_id)).id) $missing=@($script:state.baseline_resource_ids | Where-Object { $_ -notin $current }) Add-Check 'protected-baseline-preserved' ($missing.Count -eq 0) @{ missing_baseline_resources = $missing.Count } 'cleanup' $script:state.status='cleaned';Complete-Operation;Export-Evidence } 'Export' { Export-Evidence } } } catch { if($script:state){$script:state.last_failure=@{stage=$Stage;scenario=$Scenario;utc=[DateTimeOffset]::UtcNow.ToString('o');message=$_.Exception.Message};Save-Run} throw } finally { $env:TF_DATA_DIR=$previousTfData $env:ARM_SUBSCRIPTION_ID=$previousArmSubscription;$env:ARM_TENANT_ID=$previousArmTenant if($runLock){$runLock.Dispose()} }