From 31bda4aed88b51d572d7d31ece9fd90a1d872f49 Mon Sep 17 00:00:00 2001 From: Hariharen Veerakumar Date: Tue, 23 Apr 2024 01:51:28 +0000 Subject: [PATCH 1/4] Have diffs done in order for deletes --- .github/actions/validate-deploy/action.yml | 45 ++++++++++---- .pipelines/.templates/validate-deploy.yml | 70 ++++++++++++++-------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/.github/actions/validate-deploy/action.yml b/.github/actions/validate-deploy/action.yml index 74985d6..994cc2b 100644 --- a/.github/actions/validate-deploy/action.yml +++ b/.github/actions/validate-deploy/action.yml @@ -12,26 +12,19 @@ runs: using: 'composite' steps: # - # Diff + # Diff List # List index changes # - - name: "Diff" + - name: "Diff List" shell: pwsh run: | - $gitDiff = git diff --name-status HEAD^ HEAD + Import-PSFConfig -Path settings.json -Schema MetaJson -EnableException + $stateDir = Get-PSFConfigValue -FullName 'AzOps.Core.State' + $gitDiff = git diff --name-status HEAD^ HEAD -- "$stateDir" if ($null -ne $gitDiff) { $gitDiff | Write-Host $gitDiff | Out-File -FilePath '/tmp/diff.txt' - - $deletedContent = git diff --diff-filter=D HEAD^ HEAD --no-prefix --no-renames - if($null -ne $deletedContent) { - $deletedContent = $deletedContent -match '^-' -replace '^([^-+ ]*)[-+ ]', '$1' - Write-Host '##[group]Deleted files content' - $deletedContent | Write-Host - Write-Host '##[endgroup]' - $deletedContent | Out-File -FilePath '/tmp/diffdeletedfiles.txt' - } } else { Write-Error -Message 'The validation pipeline failed because there is currently no change to be processed' @@ -50,7 +43,33 @@ runs: ./.scripts/customSorting.ps1 # - # Deploy + # Diff + # Get content from list + # + + - name: "Diff" + shell: pwsh + run: | + $diff = Get-Content -Path '/tmp/diff.txt' + $deletedContent = $(foreach ($change in $diff) { + $path = ($change -split "`t")[-1] + $fileDelContent = git diff --diff-filter=D HEAD^ HEAD --no-prefix --no-renames -- "$path" + if($null -ne $fileDelContent) { + $fileDelContent = $fileDelContent -match '^-' -replace '^([^-+ ]*)[-+ ]', '$1' + Write-Output -InputObject $fileDelContent + } + }) | Where-Object { $_ } + + if ($deletedContent.Count -gt 0) { + Write-Host '##[group]Deleted files content' + $deletedContent | Write-Host + Write-Host '##[endgroup]' + $deletedContent | Out-File -FilePath '/tmp/diffdeletedfiles.txt' + } + + # + # Validate or Deploy + # If parameter "deploy" is set to true, then deploy the changes, # Initial deployment of any index changes # diff --git a/.pipelines/.templates/validate-deploy.yml b/.pipelines/.templates/validate-deploy.yml index bc2391d..5c283e2 100644 --- a/.pipelines/.templates/validate-deploy.yml +++ b/.pipelines/.templates/validate-deploy.yml @@ -5,39 +5,32 @@ parameters: steps: - # - # Diff - # List index changes - # - + # + # Diff List + # List index changes + # + - task: PowerShell@2 - displayName: "Diff" + displayName: "Diff List" inputs: targetType: "inline" script: | - $gitDiff = git diff --name-status HEAD^ HEAD - if($null -ne $gitDiff) { - $gitDiff | Write-Host - $gitDiff | Out-File -FilePath '/tmp/diff.txt' - - $deletedContent = git diff --diff-filter=D HEAD^ HEAD --no-prefix --no-renames - if($null -ne $deletedContent) { - $deletedContent = $deletedContent -match '^-' -replace '^([^-+ ]*)[-+ ]', '$1' - Write-Host '##[group]Deleted files content' - $deletedContent | Write-Host - Write-Host '##[endgroup]' - $deletedContent | Out-File -FilePath '/tmp/diffdeletedfiles.txt' - } + Import-PSFConfig -Path settings.json -Schema MetaJson -EnableException + $stateDir = Get-PSFConfigValue -FullName 'AzOps.Core.State' + $gitDiff = git diff --name-status HEAD^ HEAD -- "$stateDir" + if ($null -ne $gitDiff) { + $gitDiff | Write-Host + $gitDiff | Out-File -FilePath '/tmp/diff.txt' } else { - Write-Host '##[error]The validation pipeline failed because there is currently no change to be processed' - exit 1 + Write-Host '##[error]The validation pipeline failed because there is currently no change to be processed' + exit 1 } - # - # CustomSorting - # If CustomSorting is enabled, sort files in diff by the .order file in each directory - # + # + # CustomSorting + # If CustomSorting is enabled, sort files in diff by the .order file in each directory + # - task: PowerShell@2 displayName: "CustomSorting" @@ -46,6 +39,33 @@ steps: targetType: "filePath" filePath: ".scripts/customSorting.ps1" + # + # Diff + # Get content from list + # + + - task: PowerShell@2 + displayName: "Diff" + inputs: + targetType: "inline" + script: | + $diff = Get-Content -Path '/tmp/diff.txt' + $deletedContent = $(foreach ($change in $diff) { + $path = ($change -split "`t")[-1] + $fileDelContent = git diff --diff-filter=D HEAD^ HEAD --no-prefix --no-renames -- "$path" + if($null -ne $fileDelContent) { + $fileDelContent = $fileDelContent -match '^-' -replace '^([^-+ ]*)[-+ ]', '$1' + Write-Output -InputObject $fileDelContent + } + }) | Where-Object { $_ } + + if ($deletedContent.Count -gt 0) { + Write-Host '##[group]Deleted files content' + $deletedContent | Write-Host + Write-Host '##[endgroup]' + $deletedContent | Out-File -FilePath '/tmp/diffdeletedfiles.txt' + } + # # Validate or Deploy # If parameter "deploy" is set to true, then deploy the changes, From c04be815bbeaa41d7a396eac226fab7324baa101 Mon Sep 17 00:00:00 2001 From: Hariharen Veerakumar Date: Tue, 23 Apr 2024 01:55:31 +0000 Subject: [PATCH 2/4] Log group fixed to smaller scope --- .scripts/customSorting.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.scripts/customSorting.ps1 b/.scripts/customSorting.ps1 index e48b38f..0b15ef4 100644 --- a/.scripts/customSorting.ps1 +++ b/.scripts/customSorting.ps1 @@ -9,8 +9,9 @@ $EndGroup = $ENV:CI ? '::endgroup::' : '##[endgroup]' $diff = Get-Content -Path $DiffFilePath Write-Host "${StartGroup}Files found in diff:" - $diff | Write-Host +Write-Host "$EndGroup" + $diffTable = @{} $diff | ForEach-Object -Process { $change = $_ @@ -40,7 +41,7 @@ $sortedDiff = foreach ($directoryPath in ($diffTable.Keys | Sort-Object)) { } Write-Output ($diffTable.$directoryPath.Values.diffString | Sort-Object) } -Write-Host "$EndGroup" + Write-Host "${StartGroup}Sorted files:" $sortedDiff | Write-Host From e8e31b510f9c9225041cb5b512fb6a3ea5c00f5b Mon Sep 17 00:00:00 2001 From: Hariharen Veerakumar Date: Tue, 23 Apr 2024 01:56:24 +0000 Subject: [PATCH 3/4] Deletes should occur in reverse of specified order --- .scripts/customSorting.ps1 | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.scripts/customSorting.ps1 b/.scripts/customSorting.ps1 index 0b15ef4..b01046d 100644 --- a/.scripts/customSorting.ps1 +++ b/.scripts/customSorting.ps1 @@ -15,10 +15,11 @@ Write-Host "$EndGroup" $diffTable = @{} $diff | ForEach-Object -Process { $change = $_ - $path = ($change -split "`t")[-1] + $operation, $path = ($change -split "`t") $entry = [pscustomobject]@{ fileName = Split-Path -Path $path -Leaf directory = Split-Path -Path $path -Parent + operation = $operation diffString = $change } if ($null -eq $diffTable[$entry.directory]) { @@ -28,19 +29,37 @@ $diff | ForEach-Object -Process { $diffTable[$entry.directory][$entry.fileName] = $entry } } -$sortedDiff = foreach ($directoryPath in ($diffTable.Keys | Sort-Object)) { +$sortedDiff = $(foreach ($directoryPath in ($diffTable.Keys | Sort-Object)) { $orderPath = [System.IO.Path]::Combine($directoryPath,'.order') if (Test-Path -Path $orderPath) { $order = Get-Content -Path $orderPath | ForEach-Object { $_.Trim() } - foreach ($orderName in $order) { + $deleteSortedDiffs = @() + $addSortedDiffs = foreach ($orderName in $order) { if ($null -ne $diffTable.$directoryPath.$orderName) { - Write-Output -InputObject $diffTable.$directoryPath.$orderName.diffString + $diffString = $diffTable.$directoryPath.$orderName.diffString + $operation = $diffTable.$directoryPath.$orderName.operation $diffTable.$directoryPath.Remove($orderName) + if ($operation -eq 'D') { + $deleteSortedDiffs += $diffString + continue + } + elseif ($operation -in 'A', 'M', 'R' -or $operation -match '^R0[0-9][0-9]$') { + Write-Output -InputObject $diffString + } + else { + Write-Error -Message "Invalid changeset type '$operation' for $diffString" + } } } + # Deletes should happen in reverse order to add/modifys + [array]::Reverse($deleteSortedDiffs) + + Write-Output -InputObject $addSortedDiffs + Write-Output -InputObject $deleteSortedDiffs } - Write-Output ($diffTable.$directoryPath.Values.diffString | Sort-Object) -} + # make sure to return unaddressed diffs too + Write-Output -InputObject ($diffTable.$directoryPath.Values.diffString | Sort-Object) +}) | Where-Object { $_ } Write-Host "${StartGroup}Sorted files:" From 951dfbdb5349dddebcaf10ebaa57532ff2285bd7 Mon Sep 17 00:00:00 2001 From: Hariharen Veerakumar Date: Fri, 3 May 2024 00:46:42 +0000 Subject: [PATCH 4/4] Need to accomodate for renames --- .scripts/customSorting.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.scripts/customSorting.ps1 b/.scripts/customSorting.ps1 index b01046d..40ae5cf 100644 --- a/.scripts/customSorting.ps1 +++ b/.scripts/customSorting.ps1 @@ -15,7 +15,12 @@ Write-Host "$EndGroup" $diffTable = @{} $diff | ForEach-Object -Process { $change = $_ - $operation, $path = ($change -split "`t") + # there can be 2 elements for Add, Delete, Modify operations + # there can be 3 elements if it's a rename + $changeParts = ($change -split "`t") + $operation = $changeParts[0] + $path = $changeParts[-1] + $entry = [pscustomobject]@{ fileName = Split-Path -Path $path -Leaf directory = Split-Path -Path $path -Parent