#requires -Version 7.6 [CmdletBinding()] param( [Parameter(Mandatory)][ValidateSet('Preflight','Plan','Deploy','Bootstrap','Pipeline','Validate','Fault','Repair','Exercise','Cleanup','Export')][string]$Stage, [Parameter(Mandatory)][string]$StatePath, [ValidateSet('Foundation','Federation')][string]$Phase='Foundation', [ValidateSet('Plan','Apply','Validate','Destroy')][string]$Action='Validate', [ValidateSet('Oidc','Drift','StateLock')][string]$Scenario, [string]$SubscriptionId,[string]$ExpectedSubscriptionSuffix,[string]$GitHubOwner,[string]$PlanRunId,[string]$PlanSha256,[string]$ArtifactId,[int]$PlanRunAttempt=1, [switch]$Execute,[switch]$AcknowledgeCost,[switch]$ReviewedPlan,[switch]$Resume,[switch]$CaptureAccessVerified,[switch]$ExpectedFailure,[switch]$CollectLogs,[switch]$DestroyPlan, [ValidateSet('westeurope')][string]$Location='westeurope' ) Set-StrictMode -Version Latest $ErrorActionPreference='Stop' Import-Module (Join-Path $PSScriptRoot 'scripts/Workshop.Common.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'scripts/TerraformDelivery.Common.psm1') -Force $private=Initialize-WorkshopPrivateDirectory -StatePath $StatePath -PackageRoot $PSScriptRoot $StatePath=[IO.Path]::GetFullPath($StatePath) $runLock=Enter-WorkshopLock -PrivateDirectory $private $script:s=$null $infra=Join-Path $PSScriptRoot 'infra/foundation' $tfstate=Join-Path $private 'foundation.tfstate' $tfvars=Join-Path $private 'foundation.tfvars.json' $savedEnv=@{} foreach($key in @([Environment]::GetEnvironmentVariables().Keys|Where-Object {$_ -match '^(ARM_|TF_CLI_ARGS|TF_VAR_)' -or $_ -in @('TF_DATA_DIR','TF_WORKSPACE')})){$savedEnv[$key]=[Environment]::GetEnvironmentVariable($key);Remove-Item -LiteralPath ('Env:'+$key)} foreach($key in @('TF_DATA_DIR','TF_WORKSPACE','ARM_SUBSCRIPTION_ID','ARM_TENANT_ID','ARM_USE_CLI','ARM_USE_OIDC','ARM_USE_MSI')){if(-not $savedEnv.ContainsKey($key)){$savedEnv[$key]=$null}} $env:TF_DATA_DIR=Join-Path $private 'tfdata' $env:TF_WORKSPACE='default' $env:ARM_USE_CLI='true';$env:ARM_USE_OIDC='false';$env:ARM_USE_MSI='false' function Save-Run {Write-WorkshopJson -Path $StatePath -Value $script:s} function Native([string]$Exe,[string[]]$A,[int]$Timeout=300,[switch]$AllowFailure){Invoke-WorkshopNative -FilePath $Exe -Arguments $A -PrivateDirectory $private -TimeoutSeconds $Timeout -AllowFailure:$AllowFailure -Label $Exe} function Az([string[]]$A,[int]$Timeout=300){if($script:s -and '--subscription' -notin $A){$A+=@('--subscription',$script:s.subscription_id)};Invoke-WorkshopAzJson -Arguments $A -PrivateDirectory $private -TimeoutSeconds $Timeout} function Gh([string[]]$A,[int]$Timeout=120){$r=Native gh $A $Timeout;if($r.StdOut.Trim()){return ($r.StdOut|ConvertFrom-Json -AsHashtable -Depth 100 -DateKind String)}} function Tf([string[]]$A,[int]$Timeout=1800,[switch]$AllowFailure){Native terraform (@("-chdir=$infra")+$A) $Timeout -AllowFailure:$AllowFailure} function Check([string]$Name,[bool]$Ok,[hashtable]$Observed=@{}){try{Add-WorkshopCheck -State $script:s -Name $Name -Passed $Ok -Scenario $Scenario -Observed $Observed}finally{Save-Run}} function Begin-Operation([string]$Kind){$script:s.pending=@{stage=$Stage;phase=$Phase;scenario=$Scenario;kind=$Kind;utc=[DateTimeOffset]::UtcNow.ToString('o');pid=$PID;process_start=(Get-Process -Id $PID).StartTime.ToUniversalTime().ToString('o')};Save-Run} function Complete-Operation {$script:s.operations+=@($script:s.pending);$script:s.pending=$null;Save-Run} function Scopes { $root='/subscriptions/'+$script:s.subscription_id+'/resourceGroups/'+$script:s.prefix; @("$root-foundation-rg", "$root-workload-rg") } function Assert-Mutation { if(-not $Execute){throw 'This stage requires -Execute after its scope has been reviewed.'} $null=Assert-WorkshopContext -State $script:s -PrivateDirectory $private -ExpectedSubscriptionId $SubscriptionId if($Stage -ne 'Cleanup' -and $script:s.cleanup_count -gt 0){throw 'The Azure run is closed after cleanup; use new private state for another deployment.'} if($Stage -notin @('Cleanup','Repair') -and $script:s.paid_started_utc -and [DateTimeOffset]::UtcNow -ge [DateTimeOffset]::Parse($script:s.exercise_stop_utc)){throw 'Cleanup deadline reached. Start Cleanup now.'} if($script:s.pending){ if(-not $Resume){throw 'An interrupted operation is recorded. Reconcile with -Resume; never start a competing run.'} $p=$script:s.pending $process=Get-Process -Id $p.pid -ErrorAction SilentlyContinue if($process -and $p.pid -ne $PID -and $process.StartTime.ToUniversalTime().ToString('o') -eq $p.process_start){throw 'The original controller is still running.'} if($p.ContainsKey('github_run_id')){$run=Gh @('api',"repos/$($script:s.repository)/actions/runs/$($p.github_run_id)");if($run.status -ne 'completed'){throw 'Original GitHub operation is not terminal; reconcile that exact run first.'}} if($p.ContainsKey('guest_id')){$guest=Az @('rest','--method','get','--url',($p.guest_id+'?api-version=2023-03-01&$expand=instanceView'));if($guest.properties.instanceView.executionState -notin @('Succeeded','Failed','Canceled','TimedOut')){throw 'Original guest command is not terminal.'}} foreach($rg in $script:s.group_names){if(Az @('group','exists','-n',$rg)){foreach($r in @(Az @('resource','list','-g',$rg))){if($r.type -notin @('Microsoft.Compute/virtualMachines','Microsoft.Network/privateEndpoints','Microsoft.Network/natGateways','Microsoft.Storage/storageAccounts')){continue};$live=Az @('resource','show','--ids',$r.id);if(-not $live.properties.ContainsKey('provisioningState') -or $live.properties.provisioningState -notin @('Succeeded','Failed','Canceled')){throw 'An Azure resource is still provisioning or lacks positive terminal evidence.'}}}} $same=$p.stage -eq $Stage -and $p.phase -eq $Phase -and $p.scenario -eq $Scenario $repair=$Stage -eq 'Repair' -and $p.stage -in @('Fault','Validate','Exercise') -and $p.phase -eq $Phase -and $p.scenario -eq $Scenario -and $script:s.active_fault -eq $Scenario if($Stage -ne 'Cleanup' -and -not ($same -or $repair)){throw 'Resume must match the interrupted phase/scenario, repair its exact fault, or clean up.'} $p.reconciled_utc=[DateTimeOffset]::UtcNow.ToString('o');$script:s.operations+=@($p);$script:s.pending=$null;Save-Run } } function Sync-Ownership { if(-not (Test-Path -LiteralPath $tfstate)){return} $snapshot=(Tf @('show','-json',$tfstate)).StdOut|ConvertFrom-Json -AsHashtable -Depth 100 if(-not $snapshot.ContainsKey('values') -or -not $snapshot.values -or -not $snapshot.values.ContainsKey('root_module')){return} foreach($r in $snapshot.values.root_module.resources){ if($r.mode -ne 'managed' -or -not $r.values.ContainsKey('id')){continue} $id=([string]$r.values.id).Split('|')[0] if($r.type -eq 'azurerm_monitor_diagnostic_setting'){$id=$id+'/providers/Microsoft.Insights/diagnosticSettings/'+$r.values.name} if(-not (Test-DeliveryOwnedId -Id $id -Scopes (Scopes) -Baseline $script:s.baseline_ids)){throw 'Terraform state contains an out-of-scope or pre-existing resource.'} if($id -notin $script:s.owned_ids){$script:s.owned_ids+=@($id)} if($r.type -eq 'azurerm_linux_virtual_machine'){ $vm=Az @('vm','show','--ids',$id) $expected=(Scopes)[0]+'/providers/Microsoft.Compute/disks/'+$r.values.os_disk[0].name if($vm.id -ine $id -or $vm.storageProfile.osDisk.managedDisk.id -ine $expected -or -not (Test-DeliveryOwnedId $expected (Scopes) $script:s.baseline_ids)){throw 'Owned VM actual OS disk differs from its exact declaration.'} if($expected -notin $script:s.owned_ids){$script:s.owned_ids+=@($expected)} foreach($extension in @(Az @('vm','extension','list','--ids',$id))){ if($extension.publisher -ceq 'Microsoft.CPlat.Core' -and $extension.name -in @('RunCommandLinux','RunCommandHandlerLinux') -and $extension.typePropertiesType -in @('RunCommandLinux','RunCommandHandlerLinux') -and $extension.id -ieq ($id+'/extensions/'+$extension.name) -and (Test-DeliveryOwnedId $extension.id (Scopes) $script:s.baseline_ids)){if($extension.id -notin $script:s.owned_ids){$script:s.owned_ids+=@($extension.id)}} } } if($r.type -eq 'azurerm_private_endpoint'){ $endpoint=Az @('network','private-endpoint','show','--ids',$id) $expected=(Scopes)[0]+'/providers/Microsoft.Network/networkInterfaces/'+$r.values.custom_network_interface_name if($endpoint.id -ine $id -or @($endpoint.networkInterfaces).Count -ne 1 -or $endpoint.networkInterfaces[0].id -ine $expected -or -not (Test-DeliveryOwnedId $expected (Scopes) $script:s.baseline_ids)){throw 'Owned Private Endpoint actual NIC differs from its exact declaration.'} if($expected -notin $script:s.owned_ids){$script:s.owned_ids+=@($expected)} } } foreach($k in $snapshot.values.outputs.Keys){$script:s.outputs[$k]=$snapshot.values.outputs[$k].value} if($script:s.outputs.ContainsKey('resource_ids')){foreach($id in $script:s.outputs.resource_ids){if(-not (Test-DeliveryOwnedId $id (Scopes) $script:s.baseline_ids)){throw 'An output ownership ID is invalid.'};if($id -match '/providers/Microsoft.Compute/disks/|/providers/Microsoft.Network/networkInterfaces/' -and $id -notin $script:s.owned_ids){throw 'An output implicit disk/NIC lacks exact live-parent verification.'};if($id -notin $script:s.owned_ids){$script:s.owned_ids+=@($id)}}} Save-Run } function Assert-GroupOwnership([string]$Name){ $g=Az @('group','show','-n',$Name) if($g.id -notin (Scopes) -or $g.tags.workshop_run -ne $script:s.run_id){throw 'Exact group ownership tag/ID mismatch.'} foreach($r in @(Az @('resource','list','-g',$Name))){ if($r.id -notin $script:s.owned_ids){throw 'Unexpected resource in disposable group; broad deletion refused.'} } } function Assert-CleanupReview { if(-not $ReviewedPlan){throw 'Cleanup -Execute requires -ReviewedPlan after reviewing the exact cleanup preview.'} } function Set-DeliveryOidcInjection { param([Parameter(Mandatory)][string]$OperationUtc,[string[]]$PriorLogNames=@()) if($script:s.ContainsKey('fault_injections') -and $script:s.fault_injections.ContainsKey('Oidc')){throw 'Original OIDC injection evidence is immutable and cannot be replaced.'} $operations=@($script:s.operations)+@($script:s.pending) $operation=@($operations|Where-Object {$_.stage -ceq 'Fault' -and $_.scenario -ceq 'Oidc'}|Sort-Object {[DateTimeOffset]::Parse($_.utc)}) if(-not $operation.Count -or $operation[0].utc -cne $OperationUtc -or $script:s.pending.utc -cne $OperationUtc -or $script:s.active_fault -cne 'Oidc'){throw 'OIDC evidence must bind the original recorded fault operation.'} $id=$script:s.outputs.deployment_fic_id if($id -notin $script:s.owned_ids -or $script:s.original_fic.subject -cne $script:s.claims.sub -or $script:s.original_fic.issuer -cne $script:s.claims.iss -or ($script:s.original_fic.audiences -join ',') -cne $script:s.claims.aud -or $script:s.fault_subject -cne ($script:s.claims.sub+':awz-owned-fault')){throw 'Original owned OIDC tuple is not valid for injection evidence.'} $proofMatches=@(foreach($file in Get-ChildItem -LiteralPath (Join-Path $private 'logs') -File -Filter '*-azure.json'){ if($file.Name -cin $PriorLogNames -or ($file.Attributes -band [IO.FileAttributes]::ReparsePoint)){continue} $bytes=[IO.File]::ReadAllBytes($file.FullName) $record=[Text.Encoding]::UTF8.GetString($bytes)|ConvertFrom-Json -AsHashtable -Depth 100 -DateKind String if($record.exit_code -isnot [long] -or $record.exit_code -ne 0 -or -not $record.ContainsKey('rest_method') -or $record.rest_method -ine 'put' -or $record.rest_url -ine ($id+'?api-version=2023-01-31')){continue} $response=$record.stdout|ConvertFrom-Json -AsHashtable -Depth 100 -DateKind String if($response.id -ine $id -or $response.properties.subject -cne $script:s.fault_subject -or $response.properties.issuer -cne $script:s.original_fic.issuer -or ($response.properties.audiences -join ',') -cne ($script:s.original_fic.audiences -join ',')){continue} $at=[DateTimeOffset]::Parse($record.utc) if($at -lt [DateTimeOffset]::Parse($OperationUtc) -or $at -gt [DateTimeOffset]::UtcNow){continue} $hash=[Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() if((Get-WorkshopFileHash $file.FullName) -cne $hash){throw 'Original native PUT evidence changed while being recorded.'} @{operation_utc=$OperationUtc;injected_utc=$record.utc;source_path=('logs/'+$file.Name);source_sha256=$hash} }) if($proofMatches.Count -ne 1){throw 'Exactly one new successful native PUT of the original owned OIDC fault tuple is required.'} if(-not $script:s.ContainsKey('fault_injections')){$script:s.fault_injections=@{}} $script:s.fault_injections.Oidc=$proofMatches[0];Save-Run } function Wait-DeliveryGuestTerminal { param( [Parameter(Mandatory)][string]$Id, [Parameter(Mandatory)][DateTimeOffset]$Deadline, [scriptblock]$ReadStatus={param($CommandId,$RequestTimeout) Az @('rest','--method','get','--url',($CommandId+'?api-version=2023-03-01&$expand=instanceView')) $RequestTimeout}, [scriptblock]$Clock={[DateTimeOffset]::UtcNow}, [scriptblock]$Delay={param($Seconds) Start-Sleep -Seconds $Seconds} ) if(-not $script:s.pending -or $script:s.pending.guest_id -cne $Id -or $Id -notin $script:s.owned_ids){throw 'Guest polling requires the exact recorded pending owned command.'} while($true){ $remaining=($Deadline-(& $Clock)).TotalSeconds if($remaining -le 0){throw 'Owned Run Command polling deadline reached; original pending command remains recorded for reconciliation.'} $snapshot=& $ReadStatus $Id ([int][math]::Min(60,[math]::Ceiling($remaining))) if($snapshot.id -ine $Id){throw 'Guest status does not match the exact recorded command.'} $executionState=if($snapshot.properties.ContainsKey('instanceView') -and $snapshot.properties.instanceView){$snapshot.properties.instanceView.executionState}else{''} if($executionState -in @('Succeeded','Failed','Canceled','TimedOut')){return $snapshot} # Pending/Running, including the initial absent instanceView, are not failure. $remaining=($Deadline-(& $Clock)).TotalSeconds if($remaining -le 0){throw 'Owned Run Command polling deadline reached; original pending command remains recorded for reconciliation.'} & $Delay ([int][math]::Min(10,[math]::Ceiling($remaining))) } } function Guest([hashtable]$configuration,[int]$Timeout=1900){ $nonce=[guid]::NewGuid().ToString('N');$name='awztf-'+$nonce.Substring(0,12) $id=$script:s.outputs.runner_vm_id+'/runCommands/'+$name if($script:s.outputs.runner_vm_id -notin $script:s.owned_ids){throw 'Run Command VM is not recorded as owned.'} $commands=Az @('rest','--method','get','--url',($script:s.outputs.runner_vm_id+'/runCommands?api-version=2023-03-01')) if(-not $commands.ContainsKey('value') -or @($commands.value).Count -ge 24){throw 'Managed Run Command headroom is exhausted; reconcile exact retained terminal commands before dispatch. No PUT was attempted.'} $body=@{location=$script:s.location;tags=@{workshop_run=$script:s.run_id};properties=@{source=@{script=[IO.File]::ReadAllText((Join-Path $PSScriptRoot 'scripts/runner.sh'))};protectedParameters=@(@{name='AWZ_INPUT';value=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes(($configuration|ConvertTo-Json -Depth 25 -Compress)))});timeoutInSeconds=$Timeout;treatFailureAsDeploymentFailure=$true;asyncExecution=$false}} $bodyPath=Join-Path $private ('guest-'+$nonce+'.json');Write-WorkshopJson $bodyPath $body $script:s.pending.guest_id=$id;$script:s.owned_ids+=@($id);Save-Run $guestDeadline=[DateTimeOffset]::UtcNow.AddSeconds($Timeout+120) $null=Az @('rest','--method','put','--url',($id+'?api-version=2023-03-01'),'--body',('@'+$bodyPath)) ($Timeout+180) $g=Wait-DeliveryGuestTerminal -Id $id -Deadline $guestDeadline if($g.properties.instanceView.executionState -ne 'Succeeded' -or $g.properties.instanceView.exitCode -ne 0){throw 'Owned Run Command failed; inspect private guest evidence.'} $script:s.guest_reports+=@(@{id=$id;utc=[DateTimeOffset]::UtcNow.ToString('o');result=$g.properties.instanceView.output});Sync-Ownership;Save-Run return $g.properties.instanceView.output } function Register-OneJob { $token=Gh @('api','--method','POST',"repos/$($script:s.repository)/actions/runners/registration-token") $nonce=[guid]::NewGuid().ToString('N') $configuration=@{mode='run';job_nonce=$nonce;repository_url=('https://github.com/'+$script:s.repository);registration_token=$token.token;runner_name=($script:s.prefix+'-'+$nonce.Substring(0,8));runner_label=$script:s.runner_label} $null=Guest $configuration } function Assert-RepositoryIdentity { $repo=Gh @('api',('repos/'+$script:s.repository)) if([string]$repo.id -cne [string]$script:s.repository_id -or [string]$repo.owner.id -cne [string]$script:s.github_owner_id -or $repo.full_name -ine $script:s.repository -or $repo.private -isnot [bool] -or -not $repo.private){throw 'Repository immutable identity/privacy no longer matches this owned run.'} return $repo } function Disable-DeliveryWorkflows { $disabled=@('disabled_manually','disabled_inactivity','disabled_fork') $listed=Gh @('api',"repos/$($script:s.repository)/actions/workflows?per_page=100") $workflows=@($listed.workflows) if($listed.total_count -ne $workflows.Count){throw 'Complete owned workflow inventory was not returned.'} $expected=@{} foreach($workflow in $workflows){ $id=[string]$workflow.id if($id -cnotmatch '^[1-9][0-9]*$' -or $expected.ContainsKey($id) -or $workflow.state -cnotin (@('active')+$disabled)){throw 'Unknown or duplicate owned workflow identity/state; cleanup refused.'} $expected[$id]=[string]$workflow.path } foreach($workflow in $workflows){if($workflow.state -ceq 'active'){$null=Gh @('api','--method','PUT',"repos/$($script:s.repository)/actions/workflows/$($workflow.id)/disable")}} # A successful PUT and a previously disabled state both need fresh readback. $readback=Gh @('api',"repos/$($script:s.repository)/actions/workflows?per_page=100") $fresh=@($readback.workflows) if($readback.total_count -ne $fresh.Count -or $fresh.Count -ne $expected.Count){throw 'Owned workflow inventory changed during disable verification.'} $verified=@{} foreach($workflow in $fresh){ $id=[string]$workflow.id if(-not $expected.ContainsKey($id) -or $verified.ContainsKey($id) -or $workflow.path -cne $expected[$id] -or $workflow.state -cnotin $disabled){throw 'Fresh workflow readback did not confirm the exact disabled inventory.'} $verified[$id]=$true } } function Dispatch([string]$Workflow,[hashtable]$Inputs=@{},[switch]$ExpectFailure){ $null=Assert-RepositoryIdentity $nonce=[guid]::NewGuid().ToString('N');$Inputs.correlation_id=$nonce $bodyPath=Join-Path $private ('dispatch-'+$nonce+'.json');Write-WorkshopJson $bodyPath @{ref='main';inputs=$Inputs} $null=Gh @('api','--method','POST',"repos/$($script:s.repository)/actions/workflows/$Workflow/dispatches",'--input',$bodyPath) $limit=[DateTimeOffset]::UtcNow.AddSeconds(90);$found=@() do{$runs=Gh @('api',"repos/$($script:s.repository)/actions/workflows/$Workflow/runs?event=workflow_dispatch&per_page=30");$found=@($runs.workflow_runs|Where-Object {$_.display_title -like "*$nonce*"});if($found.Count -eq 1){break};if([DateTimeOffset]::UtcNow -ge $limit){throw 'Could not reconcile the exact dispatched workflow.'};Start-Sleep -Seconds 3}while($true) $run=$found[0];$script:s.pending.github_run_id=$run.id;Save-Run Write-Host "GitHub run $($run.id): $Workflow (ephemeral runner starting)" Register-OneJob $limit=[DateTimeOffset]::UtcNow.AddMinutes(5) do{$run=Gh @('api',"repos/$($script:s.repository)/actions/runs/$($run.id)");if($run.status -eq 'completed'){break};if([DateTimeOffset]::UtcNow -ge $limit){throw 'GitHub result is not terminal; reconcile before resuming.'};Start-Sleep -Seconds 4}while($true) $dest=Join-Path $private ('evidence/run-'+$run.id+'-'+$run.run_attempt);New-Item -ItemType Directory -Path $dest -Force|Out-Null $logs=Native gh @('run','view',[string]$run.id,'--repo',$script:s.repository,'--log') 120 -AllowFailure [IO.File]::WriteAllText((Join-Path $dest 'run.log'),$logs.StdOut,[Text.UTF8Encoding]::new($false)) $artifacts=Gh @('api',"repos/$($script:s.repository)/actions/runs/$($run.id)/artifacts") if($artifacts.total_count){$null=Native gh @('run','download',[string]$run.id,'--repo',$script:s.repository,'--dir',$dest) 180} $safeInputs=@{};foreach($key in @('plan_run_id','plan_run_attempt','artifact_id','reviewed_plan_sha256')){if($Inputs.ContainsKey($key)){$safeInputs[$key]=[string]$Inputs[$key]}} $record=@{id=$run.id;attempt=$run.run_attempt;workflow=$Workflow;commit=$run.head_sha;status=$run.status;conclusion=$run.conclusion;utc=$run.updated_at;created_at=$run.created_at;run_started_at=$run.run_started_at;inputs=$safeInputs;artifacts=$artifacts.artifacts;directory=[IO.Path]::GetFileName($dest);expected_failure=[bool]$ExpectFailure;correlation_id=$nonce} $resultFiles=@(Get-ChildItem -LiteralPath $dest -Filter 'delivery-result.json' -File -Recurse) if($resultFiles.Count -ne 1){throw 'Expected exactly one genuine delivery result after artifact download.'} $record.result_path=[IO.Path]::GetRelativePath((Join-Path $private 'evidence'),$resultFiles[0].FullName).Replace('\','/') $record.result_sha256=Get-WorkshopFileHash $resultFiles[0].FullName $null=Result $record $script:s.github_runs+=@($record);Save-Run if($ExpectFailure){if($run.conclusion -ne 'failure'){throw 'An intentional guard rejection did not fail as expected.'}}elseif($run.conclusion -ne 'success'){throw 'GitHub workflow failed; inspect retained private logs.'} return $record } function Result([hashtable]$Run){ if(-not $Run.ContainsKey('result_path') -or -not $Run.ContainsKey('result_sha256') -or $Run.result_sha256 -cnotmatch '^[a-f0-9]{64}$'){throw 'The original result path/hash provenance is missing.'} $root=[IO.Path]::GetFullPath((Join-Path $private 'evidence')) $path=[IO.Path]::GetFullPath((Join-Path $root $Run.result_path)) if(-not $path.StartsWith(($root+[IO.Path]::DirectorySeparatorChar),[StringComparison]::OrdinalIgnoreCase) -or [IO.Path]::GetFileName($path) -cne 'delivery-result.json'){throw 'The retained result path is outside owned evidence.'} $item=Get-Item -LiteralPath $path while($item.FullName -ine $root){if(($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0){throw 'Linked evidence is prohibited.'};$item=Get-Item -LiteralPath ([IO.Path]::GetDirectoryName($item.FullName))} if((Get-WorkshopFileHash $path) -cne $Run.result_sha256){throw 'Retained result bytes changed after download.'} $r=Get-Content -LiteralPath $path -Raw|ConvertFrom-Json -AsHashtable -Depth 100 -DateKind String if([string]$r.context.run_id -cne [string]$Run.id -or [string]$r.context.run_attempt -cne [string]$Run.attempt -or $r.context.correlation_id -cne $Run.correlation_id -or $r.context.repository -ine $script:s.repository -or [string]$r.context.repository_id -cne [string]$script:s.repository_id -or [string]$r.context.repository_owner_id -cne [string]$script:s.github_owner_id -or $r.context.head_sha -cne $Run.commit -or $r.context.workflow_sha -cne $Run.commit){throw 'Result is not bound to the exact dispatched run/attempt/commit/nonce and immutable repository owner.'} return $r } function Adopt-Workload { $group=$script:s.prefix+'-workload-rg' $scope=(Scopes)[1] $expected=@{ ($scope+'/providers/Microsoft.Network/virtualNetworks/'+$script:s.prefix+'-workload-vnet')='Microsoft.Network/virtualNetworks' ($scope+'/providers/Microsoft.Network/networkSecurityGroups/'+$script:s.prefix+'-exercise-nsg')='Microsoft.Network/networkSecurityGroups' ($scope+'/providers/Microsoft.Network/networkSecurityGroups/'+$script:s.prefix+'-canary-nsg')='Microsoft.Network/networkSecurityGroups' } foreach($r in @(Az @('resource','list','-g',$group))){ if(-not $expected.ContainsKey($r.id) -or $expected[$r.id] -ine $r.type -or $r.tags.workshop_run -cne $script:s.run_id -or $r.id -in $script:s.baseline_ids){throw 'Pipeline workload inventory differs from exact owned declarations.'} if($r.id -notin $script:s.owned_ids){$script:s.owned_ids+=@($r.id)} } Save-Run } function Get-Control { $canary=Az @('network','nsg','show','-g',$script:s.outputs.workload_rg_name,'-n',($script:s.prefix+'-canary-nsg')) return @{id=$canary.id;tags=$canary.tags;rules=$canary.securityRules} } function Assert-OwnedFaultIntent([string]$Name,[switch]$RequireFault) { if($script:s.active_fault -cne $Name){throw 'The exact owned fault intent is not active.'} if($Name -ceq 'Oidc'){ $id=$script:s.outputs.deployment_fic_id if($id -notin $script:s.owned_ids -or -not $script:s.ContainsKey('original_fic') -or -not $script:s.ContainsKey('fault_subject')){throw 'Original owned federation intent is missing.'} $original=$script:s.original_fic if($original.subject -cne $script:s.claims.sub -or $original.issuer -cne $script:s.claims.iss -or ($original.audiences -join ',') -cne $script:s.claims.aud -or $script:s.fault_subject -cne ($script:s.claims.sub+':awz-owned-fault')){throw 'Original federation intent differs from the observed claims.'} $live=Az @('rest','--method','get','--url',($id+'?api-version=2023-01-31')) if($live.properties.issuer -cne $original.issuer -or ($live.properties.audiences -join ',') -cne ($original.audiences -join ',') -or $live.properties.subject -cnotin @($original.subject,$script:s.fault_subject) -or ($RequireFault -and $live.properties.subject -cne $script:s.fault_subject)){throw 'Current federation is not the exact recorded baseline/fault tuple.'} }elseif($Name -ceq 'Drift'){ $id=$script:s.outputs.workload_rg_id+'/providers/Microsoft.Network/networkSecurityGroups/'+$script:s.prefix+'-exercise-nsg' if($id -notin $script:s.owned_ids -or -not $script:s.ContainsKey('original_marker') -or -not $script:s.ContainsKey('fault_marker') -or $script:s.original_marker -cne ('baseline-'+$script:s.run_id) -or $script:s.fault_marker -cne ('fault-'+$script:s.run_id)){throw 'Original owned marker intent is missing or changed.'} $live=Az @('network','nsg','show','--ids',$id) if($live.id -ine $id -or $live.tags.workshop_run -cne $script:s.run_id -or $live.tags.marker -cnotin @($script:s.original_marker,$script:s.fault_marker) -or ($RequireFault -and $live.tags.marker -cne $script:s.fault_marker)){throw 'Current marker is not the exact recorded baseline/fault tuple.'} }else{throw 'Unknown owned fault intent.'} } function Preview-Cleanup { $groups=@() foreach($name in $script:s.group_names){ $exists=Az @('group','exists','-n',$name) if($exists){Assert-GroupOwnership $name;$group=Az @('group','show','-n',$name);$members=@(Az @('resource','list','-g',$name));$groups+=@{name=$name;id=$group.id;workshop_run=$group.tags.workshop_run;exists=$true;owned_members=@($members|ForEach-Object {$_.id})}}else{$groups+=@{name=$name;exists=$false;owned_members=@()}} } $github=@{repository=$script:s.repository;created=[bool]$script:s.repo_created;runners=0;artifacts=0;nonterminal_runs=0} if($script:s.repo_created){$null=Assert-RepositoryIdentity;$runners=Gh @('api',"repos/$($script:s.repository)/actions/runners?per_page=100");$artifacts=Gh @('api',"repos/$($script:s.repository)/actions/artifacts?per_page=100");$runs=Gh @('api',"repos/$($script:s.repository)/actions/runs?per_page=100");$github.runners=$runners.total_count;$github.artifacts=$artifacts.total_count;$github.nonterminal_runs=@($runs.workflow_runs|Where-Object {$_.status -ne 'completed'}).Count} return @{mode='read-only exact cleanup preview';groups=$groups;github=$github;repository_source_retained=$true;requires_execute=$true;pending_operation=[bool]$script:s.pending} } function Healthy { $r=Dispatch 'validate.yml';$result=Result $r Check 'healthy-control' ([bool]$result.passed -and $result.plan_exit_code -eq 0) @{run_id=$r.id;zero_drift=$true;canary=$result.canary} Adopt-Workload $script:s.control=Get-Control;Save-Run return $r } function Approved-Inputs { if(-not $ReviewedPlan -or -not $PlanRunId -or $PlanRunId -notmatch '^[1-9][0-9]*$' -or $PlanSha256 -notmatch '^[a-f0-9]{64}$' -or $ArtifactId -notmatch '^[1-9][0-9]*$'){throw 'Apply/Destroy requires reviewed plan run ID, artifact ID, lowercase SHA256 and -ReviewedPlan.'} return @{plan_run_id=$PlanRunId;plan_run_attempt=[string]$PlanRunAttempt;artifact_id=$ArtifactId;reviewed_plan_sha256=$PlanSha256} } function Plan-Pipeline([bool]$Destroy=$false){ $r=Dispatch 'plan.yml' @{destroy=$Destroy};$result=Result $r $artifact=@($r.artifacts|Where-Object {$_.name -eq ('plans-'+$r.id+'-'+$r.attempt)}) if($artifact.Count -ne 1 -or -not $result.passed){throw 'The trusted saved plan artifact is missing.'} $script:s.latest_pipeline_plan=@{run_id=$r.id;attempt=$r.attempt;artifact_id=$artifact[0].id;sha256=$result.plan_sha256;operation=$(if($Destroy){'destroy'}else{'apply'});utc=[DateTimeOffset]::UtcNow.ToString('o');consumed=$false};Save-Run Write-Host ('Review private plan-review.json, then use -PlanRunId '+$r.id+' -PlanRunAttempt '+$r.attempt+' -ArtifactId '+$artifact[0].id+' -PlanSha256 '+$result.plan_sha256+' -ReviewedPlan') return $r } try { if(Test-Path -LiteralPath $StatePath){$script:s=Get-Content -LiteralPath $StatePath -Raw|ConvertFrom-Json -AsHashtable -Depth 100 -DateKind String} if($Stage -eq 'Preflight'){ if($script:s){$null=Assert-WorkshopContext -State $script:s -PrivateDirectory $private;Write-Host 'Preflight already recorded. Use a new private state path for a new run.';exit} $account=Assert-WorkshopContext -State $null -PrivateDirectory $private -ExpectedSubscriptionId $SubscriptionId if($ExpectedSubscriptionSuffix -and -not $account.id.EndsWith($ExpectedSubscriptionSuffix)){throw 'Selected subscription does not match the explicitly expected suffix.'} if(-not $CaptureAccessVerified){throw 'Verify usable authenticated Portal/GitHub and retained CLI screenshot capture before recording preflight.'} $tools=@{powershell=$PSVersionTable.PSVersion.ToString();terraform=((Native terraform @('version','-json')).StdOut|ConvertFrom-Json).terraform_version;azure_cli=(Az @('version'))['azure-cli']} if($tools.powershell -ne '7.6.9' -or $tools.terraform -ne '1.15.8' -or $tools.azure_cli -ne '2.88.0'){throw 'Installed tool versions differ from the pinned workshop.'} $user=Gh @('api','user');if($GitHubOwner -and $user.login -cne $GitHubOwner){throw 'Authenticated GitHub owner differs from the explicitly requested owner.'};$GitHubOwner=$user.login $baseline=@(Az @('resource','list'));$vnets=@(Az @('network','vnet','list')) foreach($v in $vnets){foreach($cidr in $v.addressSpace.addressPrefixes){foreach($wanted in @('10.176.0.0/22','10.177.0.0/24')){if(Test-WorkshopCidrOverlap $cidr $wanted){throw 'Workshop CIDR overlaps an existing VNet.'}}}} $providers=@(Az @('provider','list'));foreach($ns in @('Microsoft.Compute','Microsoft.Network','Microsoft.Storage','Microsoft.ManagedIdentity','Microsoft.OperationalInsights','Microsoft.Insights')){if(@($providers|Where-Object {$_.namespace -eq $ns -and $_.registrationState -eq 'Registered'}).Count -ne 1){throw "Provider $ns is not Registered."}} $permissions=Az @('rest','--method','get','--url',('https://management.azure.com/subscriptions/'+$account.id+'/providers/Microsoft.Authorization/permissions?api-version=2022-04-01')) foreach($permission in @('Microsoft.Resources/subscriptions/resourceGroups/write','Microsoft.Authorization/roleAssignments/write','Microsoft.Compute/virtualMachines/write','Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/write')){if(-not (Test-WorkshopActionAllowed $permissions.value $permission)){throw "Missing operator permission $permission"}} $policy=@(Az @('policy','assignment','list','--disable-scope-strict-match')) $quota=@(Az @('vm','list-usage','-l',$Location));foreach($family in @('cores','standardBSFamily')){$q=@($quota|Where-Object {$_.name.value -eq $family});if($q.Count -ne 1 -or [int]$q[0].limit-[int]$q[0].currentValue -lt 2){throw 'Insufficient regional B2s quota.'}} $sku=@(Az @('vm','list-skus','-l',$Location,'--resource-type','virtualMachines','--size','Standard_B2s','--all'));if(@($sku|Where-Object {$_.name -eq 'Standard_B2s' -and $_.restrictions.Count -eq 0}).Count -ne 1){throw 'B2s has regional restrictions.'} $image=Az @('vm','image','show','-l',$Location,'--urn','Canonical:ubuntu-24_04-lts:server:latest') $cost=Get-DeliveryBudget -Location $Location $runner=Gh @('api','repos/actions/runner/releases/latest');$asset=@($runner.assets|Where-Object {$_.name -match '^actions-runner-linux-x64-[0-9.]+\.tar\.gz$'}) if($asset.Count -ne 1 -or $asset[0].digest -notmatch '^sha256:[a-f0-9]{64}$'){throw 'Runner release has no verifiable SHA256.'} $sums=(Invoke-WebRequest -Uri https://releases.hashicorp.com/terraform/1.15.8/terraform_1.15.8_SHA256SUMS -TimeoutSec 60).Content if($sums -is [byte[]]){$sums=[Text.Encoding]::UTF8.GetString($sums)} $tfHash=[regex]::Match([string]$sums,'(?m)^([a-f0-9]{64})\s+terraform_1\.15\.8_linux_amd64\.zip\s*$').Groups[1].Value if($tfHash -notmatch '^[a-f0-9]{64}$'){throw 'Linux Terraform release checksum unavailable.'} $runId=[DateTime]::UtcNow.ToString('yyMMdd')+'-'+[guid]::NewGuid().ToString('N').Substring(0,6);$prefix='awztf'+($runId -replace '-','').Substring(4) $groupRoot=$prefix $script:s=@{schema=1;run_id=$runId;prefix=$prefix;subscription_id=$account.id;tenant_id=$account.tenantId;location=$Location;created_utc=[DateTimeOffset]::UtcNow.ToString('o');paid_started_utc=$null;exercise_stop_utc=$null;cleanup_deadline_utc=$null;repository=($GitHubOwner+'/awz-terraform-delivery-'+$runId);github_owner=$GitHubOwner;github_owner_id=$user.id;repository_id=$null;runner_label=('awz-'+$runId);group_names=@("$groupRoot-foundation-rg","$groupRoot-workload-rg");baseline_ids=@($baseline|ForEach-Object {$_.id});owned_ids=@();outputs=@{};checks=@();operations=@();pending=$null;active_fault='';faults=@{};repairs=@{};github_runs=@();guest_reports=@();plan=$null;cleanup_count=0;tools=$tools;cost=$cost;image_urn=('Canonical:ubuntu-24_04-lts:server:'+$image.name);runner=@{version=$runner.tag_name.TrimStart('v');url=$asset[0].browser_download_url;sha256=$asset[0].digest.Substring(7)};terraform_linux_sha256=$tfHash;claims=$null;repo_created=$false} foreach($rg in $script:s.group_names){if(Az @('group','exists','-n',$rg)){throw 'Generated disposable resource group already exists.'}} Write-WorkshopJson (Join-Path $private 'preflight-private.json') @{account=$account;baseline=$baseline;vnets=$vnets;quota=$quota;sku=$sku;policies=$policy;permissions=$permissions;providers=$providers;image=$image;runner=$runner} Save-Run $keyPath=Join-Path $private 'runner-admin';$null=Native ssh-keygen @('-t','rsa','-b','3072','-f',$keyPath,'-N','','-C','disposable-workshop-admin') Write-WorkshopJson $tfvars @{prefix=$prefix;run_id=$runId;location=$Location;admin_username='awzadmin';ssh_public_key=[IO.File]::ReadAllText($keyPath+'.pub').Trim();image_urn=$script:s.image_urn;federation_enabled=$false} Check 'preflight' $true @{subscription_suffix=$account.id.Substring($account.id.Length-6);location=$Location;active_estimate_eur=$cost.active_estimate;cleanup_reserve_eur=5;image=$script:s.image_urn;tools=$tools;policies_observed=$policy.Count} Write-Host "Run $runId; six-hour estimate EUR$($cost.active_estimate) + EUR5 cleanup reserve. No paid resources created." exit } if(-not $script:s){throw 'Run Preflight first with an explicit subscription and private state path.'} $null=Assert-WorkshopContext -State $script:s -PrivateDirectory $private -ExpectedSubscriptionId $SubscriptionId $env:ARM_SUBSCRIPTION_ID=$script:s.subscription_id;$env:ARM_TENANT_ID=$script:s.tenant_id switch($Stage){ 'Plan' { if($script:s.pending){throw 'A pending operation must be positively reconciled before producing another foundation plan.'} if($Phase -eq 'Foundation' -and $script:s.outputs.ContainsKey('deployment_fic_id') -and $script:s.outputs.deployment_fic_id){throw 'Federation is already deployed; use Phase Federation instead of downgrading the foundation.'} if($Phase -eq 'Federation'){ if(-not $script:s.claims){throw 'Bootstrap must discover actual claims before federation planning.'} $vars=Get-Content $tfvars -Raw|ConvertFrom-Json -AsHashtable;$vars.federation_enabled=$true;$vars.oidc_issuer=$script:s.claims.iss;$vars.oidc_subject=$script:s.claims.sub;$vars.oidc_audience=$script:s.claims.aud;Write-WorkshopJson $tfvars $vars } $null=Tf @('init','-input=false',('-backend-config=path='+$tfstate)) $planPath=Join-Path $private ($Phase.ToLowerInvariant()+'.tfplan') $null=Tf @('plan','-input=false',('-var-file='+$tfvars),('-out='+$planPath)) $show=Tf @('show','-no-color',$planPath);[IO.File]::WriteAllText((Join-Path $private ($Phase+'.plan.txt')),$show.StdOut) $script:s.plan=@{phase=$Phase;sha256=(Get-WorkshopFileHash $planPath);vars_sha256=(Get-WorkshopFileHash $tfvars);source_sha256=(Get-DeliverySourceHash $infra);created_utc=[DateTimeOffset]::UtcNow.ToString('o');expires_utc=[DateTimeOffset]::UtcNow.AddMinutes(15).ToString('o');consumed=$false};Save-Run Write-Host $show.StdOut;Write-Host ('Saved plan SHA256: '+$script:s.plan.sha256) } 'Deploy' { Assert-Mutation if(-not $ReviewedPlan -or ($Phase -eq 'Foundation' -and -not $AcknowledgeCost)){throw 'Deploy requires -ReviewedPlan and initial -AcknowledgeCost.'} $planPath=Join-Path $private ($Phase.ToLowerInvariant()+'.tfplan');Assert-DeliveryPlan $script:s.plan $planPath (Get-DeliverySourceHash $infra) $Phase -VariablesPath $tfvars if(-not $script:s.paid_started_utc -and [DateTimeOffset]::UtcNow-[DateTimeOffset]::Parse($script:s.cost.utc) -gt [TimeSpan]::FromMinutes(60)){$script:s.cost=Get-DeliveryBudget -Location $script:s.location;Save-Run} if(-not $script:s.paid_started_utc){$start=[DateTimeOffset]::UtcNow;$script:s.paid_started_utc=$start.ToString('o');$script:s.exercise_stop_utc=$start.AddHours(4.5).ToString('o');$script:s.cleanup_deadline_utc=$start.AddHours(6).ToString('o');Save-Run} Begin-Operation 'terraform-apply';$script:s.plan.consumed=$true;Save-Run try{$null=Tf @('apply','-input=false',$planPath) 2400}finally{Sync-Ownership} Check ('deploy-'+$Phase) $true;Complete-Operation } 'Bootstrap' { Assert-Mutation;Begin-Operation 'bootstrap-private-repository-and-runner' if(-not $script:s.repo_created){ $exists=Native gh @('repo','view',$script:s.repository,'--json','id') 60 -AllowFailure if($exists.ExitCode -eq 0){throw 'Expected new repository already exists; refusing to adopt it.'} $null=Native gh @('repo','create',$script:s.repository,'--private','--disable-issues','--disable-wiki','--description','Disposable Azure Workshopz Terraform delivery execution repository') $repo=Gh @('api',('repos/'+$script:s.repository));if(-not $repo.private -or $repo.owner.login -cne $script:s.github_owner){throw 'Created repository is not the expected private owner.'} $script:s.repository_id=$repo.id;$script:s.repo_created=$true;Save-Run } $repoDir=Join-Path $private 'repository';if(-not (Test-Path -LiteralPath $repoDir)){New-Item -ItemType Directory $repoDir|Out-Null} foreach($part in @('.github','pipeline','infra')){Copy-Item -LiteralPath (Join-Path $PSScriptRoot $part) -Destination $repoDir -Recurse -Force} [IO.File]::WriteAllText((Join-Path $repoDir '.gitignore'),".terraform/`n*.tfstate*`n*.tfplan`n*.tfvars*`n*.log`n.env`n") if(-not (Test-Path (Join-Path $repoDir '.git'))){$null=Native git @('-C',$repoDir,'init','-b','main');$null=Native git @('-C',$repoDir,'remote','add','origin',('https://github.com/'+$script:s.repository+'.git'))} $null=Native git @('-C',$repoDir,'config','user.name',$script:s.github_owner);$null=Native git @('-C',$repoDir,'config','user.email',([string]$script:s.github_owner_id+'+'+$script:s.github_owner+'@users.noreply.github.com')) $null=Native git @('-C',$repoDir,'add','.');$null=Native git @('-C',$repoDir,'commit','-m','Add reviewed private Terraform delivery workshop') 60 -AllowFailure $null=Native git @('-C',$repoDir,'push','-u','origin','main') $repoVars=$script:s.outputs.repo_variables;$repoVars.RUNNER_LABEL=$script:s.runner_label;$repoVars.RUN_ID=$script:s.run_id foreach($k in $repoVars.Keys){$null=Native gh @('variable','set',$k,'--repo',$script:s.repository,'--body',[string]$repoVars[$k])} $configuration=@{mode='install';azure_cli_version='2.88.0';azure_cli_package='2.88.0-1~noble';terraform_version='1.15.8';terraform_url='https://releases.hashicorp.com/terraform/1.15.8/terraform_1.15.8_linux_amd64.zip';terraform_sha256=$script:s.terraform_linux_sha256;runner_url=$script:s.runner.url;runner_sha256=$script:s.runner.sha256;runner_version=$script:s.runner.version} $null=Guest $configuration 1800 $run=Dispatch 'claims.yml' $claims=(Result $run).claims if($claims.iss -ne 'https://token.actions.githubusercontent.com' -or $claims.aud -ne 'api://AzureADTokenExchange' -or [string]$claims.repository_id -ne [string]$script:s.repository_id -or $claims.ref -ne 'refs/heads/main'){throw 'Observed claims do not match the exact private main workflow.'} $script:s.claims=$claims;Save-Run;Check 'bootstrap-claims' $true @{issuer=$claims.iss;subject=$claims.sub;audience=$claims.aud};Complete-Operation } 'Pipeline' { Assert-Mutation;Begin-Operation ('pipeline-'+$Action) switch($Action){ 'Plan' {$null=Plan-Pipeline -Destroy ([bool]$DestroyPlan)} 'Apply' {$inputs=Approved-Inputs;$r=Dispatch 'apply.yml' $inputs -ExpectFailure:$ExpectedFailure;$result=Result $r;if(-not $ExpectedFailure){Check 'workload-applied' ([bool]$result.passed -and [bool]$result.applied_original_plan) @{run_id=$r.id;plan_sha256=$result.plan_sha256};Adopt-Workload}else{$guards=@{REVIEWED_HASH_MISMATCH='wrong-hash';PLAN_EXPIRED_OR_FUTURE='expired';PLAN_ALREADY_CONSUMED='consumed';ARTIFACT_PROVENANCE_MISMATCH='artifact-mismatch'};$guard=if($guards.ContainsKey([string]$result.error_code)){$guards[[string]$result.error_code]}else{''};Check 'plan-guard-rejected' ($result.passed -is [bool] -and -not $result.passed -and [bool]$guard -and $result.terraform_started -is [bool] -and -not $result.terraform_started -and $result.terraform_command_count -eq 0) @{run_id=$r.id;error_code=$result.error_code;guard=$guard}}} 'Validate' {$null=Healthy} 'Destroy' {$inputs=Approved-Inputs;$r=Dispatch 'destroy.yml' $inputs;$result=Result $r;Check 'workload-destroyed' ([bool]$result.passed -and $result.operation -eq 'destroy') @{run_id=$r.id}} } Complete-Operation } 'Validate' { Assert-Mutation;Begin-Operation 'validate' if($script:s.active_fault){throw 'Use the specific fault/repair stage to verify an active fault.'} $null=Healthy $v=Tf @('plan','-input=false','-detailed-exitcode',('-var-file='+$tfvars)) 900 -AllowFailure Check 'foundation-no-drift' ($v.ExitCode -eq 0) if($CollectLogs){ $query='StorageBlobLogs | where TimeGenerated > ago(6h) | project TimeGenerated, OperationName, StatusCode, StatusText, AuthenticationType, Uri, RequesterObjectId, CorrelationId | order by TimeGenerated asc' $bodyPath=Join-Path $private 'log-query.json';Write-WorkshopJson $bodyPath @{query=$query;timespan='PT6H'} $logs=Az @('rest','--method','post','--url',('https://api.loganalytics.azure.com/v1/workspaces/'+$script:s.outputs.workspace_id+'/query'),'--resource','https://api.loganalytics.io','--body',('@'+$bodyPath)) Write-WorkshopJson (Join-Path $private 'evidence/blob-logs.json') $logs $activity=@(Az @('monitor','activity-log','list','--resource-group',$script:s.outputs.workload_rg_name,'--start-time',$script:s.paid_started_utc,'--max-events','1000')) Write-WorkshopJson (Join-Path $private 'evidence/workload-activity.json') $activity $script:s.log_evidence=@{blob=@{path='blob-logs.json';sha256=(Get-WorkshopFileHash (Join-Path $private 'evidence/blob-logs.json'))};activity=@{path='workload-activity.json';sha256=(Get-WorkshopFileHash (Join-Path $private 'evidence/workload-activity.json'))};collected_utc=[DateTimeOffset]::UtcNow.ToString('o')};Save-Run Check 'activity-logs-observed' ($activity.Count -gt 0) @{events=$activity.Count} Check 'blob-logs-observed' ($logs.tables.Count -eq 1 -and $logs.tables[0].rows.Count -gt 0) @{rows=$logs.tables[0].rows.Count;columns=$logs.tables[0].columns.name} } Check 'final-healthy' $true @{run_id=$script:s.github_runs[-1].id};Complete-Operation } 'Fault' { $retryIntent=$Resume -and $script:s.active_fault -ceq $Scenario Assert-Mutation if($Scenario -notin @('Oidc','Drift')){throw 'Fault requires Oidc or Drift; use Exercise for StateLock.'} if($retryIntent){Assert-OwnedFaultIntent $Scenario -RequireFault}else{Assert-DeliveryFreshControl $script:s} Begin-Operation ('fault-'+$Scenario) $script:s.active_fault=$Scenario;Save-Run if($Scenario -eq 'Oidc'){ if(-not $retryIntent){ if($script:s.ContainsKey('fault_injections') -and $script:s.fault_injections.ContainsKey('Oidc')){throw 'Original OIDC injection is already recorded; resume its exact intent or repair it.'} $id=$script:s.outputs.deployment_fic_id if($id -notin $script:s.owned_ids){throw 'The deployment federation is not exactly recorded as owned.'} $fic=Az @('rest','--method','get','--url',($id+'?api-version=2023-01-31')) if($fic.properties.subject -cne $script:s.claims.sub){throw 'FIC already differs from the recorded subject.'} $script:s.original_fic=$fic.properties;$script:s.fault_subject=$script:s.claims.sub+':awz-owned-fault';Save-Run $props=@{issuer=$fic.properties.issuer;subject=$script:s.fault_subject;audiences=$fic.properties.audiences} $bodyPath=Join-Path $private 'fic-fault.json';Write-WorkshopJson $bodyPath @{properties=$props} $priorLogs=@(Get-ChildItem -LiteralPath (Join-Path $private 'logs') -File -Filter '*-azure.json'|ForEach-Object Name) $null=Az @('rest','--method','put','--url',($id+'?api-version=2023-01-31'),'--body',('@'+$bodyPath)) Set-DeliveryOidcInjection -OperationUtc $script:s.pending.utc -PriorLogNames $priorLogs } $r=Dispatch 'drill-oidc.yml' @{expect_failure=$true};$result=Result $r Check 'fault-Oidc' ([bool]$result.passed -and [bool]$result.deployment_exchange_rejected) @{run_id=$r.id;diagnostic=$result.observed_error_code;control=$result.control} }else{ if(-not $retryIntent){ $id=$script:s.outputs.workload_rg_id+'/providers/Microsoft.Network/networkSecurityGroups/'+$script:s.prefix+'-exercise-nsg' if($id -notin $script:s.owned_ids){throw 'The exercise NSG is not exactly recorded as owned.'} $nsg=Az @('network','nsg','show','--ids',$id) if($nsg.tags.marker -cne ('baseline-'+$script:s.run_id)){throw 'Exercise marker already differs from intended baseline.'} $script:s.original_marker=$nsg.tags.marker;$script:s.fault_marker='fault-'+$script:s.run_id;Save-Run $null=Az @('resource','update','--ids',$id,'--set',('tags.marker='+$script:s.fault_marker)) } $r=Plan-Pipeline $reviewFile=@(Get-ChildItem -LiteralPath (Join-Path $private ('evidence/'+$r.directory)) -Filter 'plan-review.json' -Recurse) if($reviewFile.Count -ne 1){throw 'Corrective plan review is missing.'} $review=Get-Content $reviewFile[0].FullName -Raw|ConvertFrom-Json -AsHashtable -Depth 100 $changes=@($review.resource_changes|Where-Object {($_.actions -join ',') -ne 'no-op'}) $canaryUnchanged=(Get-DeliveryCanonicalHash (Get-Control)) -ceq (Get-DeliveryCanonicalHash $script:s.control) Check 'fault-Drift' ($changes.Count -eq 1 -and $changes[0].type -eq 'azurerm_network_security_group' -and ($changes[0].actions -join ',') -eq 'update' -and $changes[0].before.tags.marker -eq $script:s.fault_marker -and $changes[0].after.tags.marker -eq $script:s.original_marker -and $canaryUnchanged) @{run_id=$r.id;changed_resources=$changes.Count;canary_unchanged=$canaryUnchanged} } $script:s.faults[$Scenario]=1;Save-Run;Complete-Operation } 'Repair' { Assert-Mutation if($Scenario -notin @('Oidc','Drift')){throw 'Repair requires a recorded owned Oidc/Drift exercise.'} if(-not $script:s.faults.ContainsKey($Scenario)){Assert-OwnedFaultIntent $Scenario} if($script:s.active_fault -and $script:s.active_fault -ne $Scenario){throw 'A different fault is active.'} Begin-Operation ('repair-'+$Scenario) $count=if($script:s.repairs.ContainsKey($Scenario)){[int]$script:s.repairs[$Scenario]}else{0} if($Scenario -eq 'Oidc'){ $id=$script:s.outputs.deployment_fic_id;$current=Az @('rest','--method','get','--url',($id+'?api-version=2023-01-31')) if($current.properties.subject -cnotin @($script:s.fault_subject,$script:s.claims.sub)){throw 'Unknown federation change; refusing to overwrite it.'} if($current.properties.issuer -cne $script:s.claims.iss -or ($current.properties.audiences -join ',') -cne $script:s.claims.aud){throw 'Unrelated federation properties changed.'} if($current.properties.subject -cne $script:s.claims.sub){$bodyPath=Join-Path $private 'fic-repair.json';Write-WorkshopJson $bodyPath @{properties=$script:s.original_fic};$null=Az @('rest','--method','put','--url',($id+'?api-version=2023-01-31'),'--body',('@'+$bodyPath))} $r=Dispatch 'drill-oidc.yml' @{expect_failure=$false};$result=Result $r Check $(if($count){'repeated-recovery-Oidc'}else{'recovery-Oidc'}) ([bool]$result.passed -and [bool]$result.deployment_exchange_succeeded) @{run_id=$r.id;control=$result.control} }else{ if(-not $count){$inputs=Approved-Inputs;$r=Dispatch 'apply.yml' $inputs;$result=Result $r;if(-not $result.passed){throw 'Reviewed drift repair did not apply.'}} $script:s.active_fault='';$r=Healthy Check $(if($count){'repeated-recovery-Drift'}else{'recovery-Drift'}) $true @{run_id=$r.id} } $script:s.repairs[$Scenario]=$count+1;$script:s.active_fault='';Save-Run;Complete-Operation } 'Exercise' { Assert-Mutation;if($Scenario -ne 'StateLock'){throw 'Exercise requires -Scenario StateLock.'} Assert-DeliveryFreshControl $script:s;Begin-Operation 'owned-blob-lease-drill' $r=Dispatch 'drill-lease.yml';$result=Result $r Check 'fault-StateLock' ([bool]$result.passed -and [bool]$result.expected_lock_failure) @{run_id=$r.id;lease_seconds=$result.lease_seconds} Check 'recovery-StateLock' ([bool]$result.released_owned_lease -and [bool]$result.state_unchanged) @{run_id=$r.id;serial_before=$result.serial_before;serial_after=$result.serial_after;state_sha256=$result.state_sha256} Check 'repeated-recovery-StateLock' ([bool]$result.repeated_repair_available -and [bool]$result.control_unchanged) @{run_id=$r.id} $script:s.faults.StateLock=1;$script:s.repairs.StateLock=2;Save-Run;Complete-Operation } 'Cleanup' { if(-not $Execute){Preview-Cleanup;break} Assert-CleanupReview Assert-Mutation;Begin-Operation 'owned-resource-cleanup' if($script:s.repo_created){ $null=Assert-RepositoryIdentity # Stop new CI first. Refuse to delete while any job can still mutate Azure. Disable-DeliveryWorkflows $runs=Gh @('api',"repos/$($script:s.repository)/actions/runs?per_page=100") if(@($runs.workflow_runs|Where-Object {$_.status -ne 'completed'}).Count){throw 'A queued/running workflow remains; reconcile or cancel it before cleanup.'} $runners=Gh @('api',"repos/$($script:s.repository)/actions/runners") foreach($runner in $runners.runners){if($runner.busy -or -not $runner.name.StartsWith($script:s.prefix+'-')){throw 'Unowned/busy runner registration blocks cleanup.'};$null=Gh @('api','--method','DELETE',"repos/$($script:s.repository)/actions/runners/$($runner.id)")} $artifacts=Gh @('api',"repos/$($script:s.repository)/actions/artifacts?per_page=100") foreach($artifact in $artifacts.artifacts){if($artifact.name -notmatch '^(plans|evidence)-\d+-\d+$'){throw 'Unrecognized repository artifact blocks cleanup.'};$null=Gh @('api','--method','DELETE',"repos/$($script:s.repository)/actions/artifacts/$($artifact.id)")} } # Normal teaching path: Pipeline Plan -DestroyPlan, reviewed Destroy, then # Cleanup. Exact owned group deletion also bounds costs if CI is unavailable. foreach($name in @($script:s.group_names|Sort-Object { $_.EndsWith('-foundation-rg') })){ if(Az @('group','exists','-n',$name)){ Sync-Ownership if($name -eq ($script:s.prefix+'-workload-rg')){Adopt-Workload} Assert-GroupOwnership $name $null=Az @('group','delete','-n',$name,'--yes') 1500 } } $remaining=@(Az @('resource','list'));$remainingIds=@($remaining|ForEach-Object {$_.id});$ownedLeft=@($remaining|Where-Object {Test-DeliveryOwnedId $_.id (Scopes) @()}) $groupsGone=$true;foreach($name in $script:s.group_names){if(Az @('group','exists','-n',$name)){$groupsGone=$false}} $preserved=@($script:s.baseline_ids|Where-Object {$_ -notin $remainingIds}) $script:s.cleanup_count++;$script:s.active_fault='';Save-Run Check $(if($script:s.cleanup_count -gt 1){'cleanup-repeat'}else{'cleanup-absence'}) ($groupsGone -and $ownedLeft.Count -eq 0) @{remaining_owned=$ownedLeft.Count;check=$script:s.cleanup_count} Check 'baseline-preserved' ($preserved.Count -eq 0) @{baseline_count=$script:s.baseline_ids.Count;missing_count=$preserved.Count} if($script:s.repo_created){$runners=Gh @('api',"repos/$($script:s.repository)/actions/runners");$artifacts=Gh @('api',"repos/$($script:s.repository)/actions/artifacts");Check 'github-runtime-removed' ($runners.total_count -eq 0 -and $artifacts.total_count -eq 0)} Complete-Operation } 'Export' { Import-Module (Join-Path $PSScriptRoot 'scripts/TerraformDelivery.Acceptance.psm1') $completion=Get-DeliveryAcceptanceAudit -State $script:s -PrivateDirectory $private $missing=@($completion.missing_checks) $out=Join-Path $PSScriptRoot 'docs';New-Item -ItemType Directory -Force $out|Out-Null # Allowlisted checks exclude account identifiers, token/state/plan bytes. $publicChecks=@($script:s.checks|ForEach-Object {@{name=$_.name;passed=$_.passed;scenario=$_.scenario;execution_utc=$_.utc}}) Write-WorkshopJson (Join-Path $out 'live-status.json') @{status=$(if($missing.Count){'INCOMPLETE'}else{'PASSED'});export_utc=[DateTimeOffset]::UtcNow.ToString('o');missing_checks=$missing;checks=$publicChecks;observations=$completion.observations;paid_started_utc=$script:s.paid_started_utc;cleanup_deadline_utc=$script:s.cleanup_deadline_utc;cleanup_checks=$script:s.cleanup_count;baseline_resource_count=$script:s.baseline_ids.Count;local_tests='Separate report required';evidence_mode='Observed Azure/GitHub operations, with retained output clearly identified'} if($missing.Count){throw ('Acceptance incomplete: '+($missing -join ', '))} Write-Host 'Sanitized live status exported. Audit screenshots, guard evidence and ZIP separately before publication.' } } } finally { foreach($key in $savedEnv.Keys){if($null -eq $savedEnv[$key]){Remove-Item -LiteralPath ('Env:'+$key) -ErrorAction SilentlyContinue}else{[Environment]::SetEnvironmentVariable($key,$savedEnv[$key])}} $runLock.Dispose() }