Improve preprocessor (#2579)

* Update documentation for 'Invoke-Preprocessing' Script Tool

* Improve Compile Script a bit Deduplicating a lot of un-needed pre-fixes - Improve implementation for 'Invoke-Preprocessing' Script Tool

* Fix RegEx in 'Invoke-Preprocessing' Script Tool

* Result of Preprocessing

* Update Replace Regex for Code Formatting in 'Invoke-Preprocessing' Script Tool

* Result of Preprocessing

* Update Exclude Files List for Preprocessing in 'Compile.ps1' Script

* Remove Extra Whitespace in some place for 'Invoke-Preprocessing.ps1' Script Tool

* Simplified and Improved the Exclude List Validation Step in 'Invoke-Preprocessing.ps1' Script Tool

* Restore 'workingdir' variable when using '-Run' Parameter with 'Compile.ps1' Script

* Revert "Update Exclude Files List for Preprocessing in 'Compile.ps1' Script"

This reverts commit 674ab0308b.

* Result of Preprocessing

Compile.ps1

@@ -8,6 +8,9 @@ $OFS = "`r`n"
 $scriptname = "winutil.ps1"
 $workingdir = $PSScriptRoot
 
+Push-Location
+Set-Location $workingdir
+
 # Variable to sync between runspaces
 $sync = [Hashtable]::Synchronized(@{})
 $sync.PSScriptRoot = $workingdir
@@ -42,11 +45,11 @@ if (-NOT $SkipPreprocessing) {
 
     # Dot source the 'Invoke-Preprocessing' Function from 'tools/Invoke-Preprocessing.ps1' Script
     $preprocessingFilePath = ".\tools\Invoke-Preprocessing.ps1"
-    . "$(($workingdir -replace ('\\$', '')) + '\' + ($preprocessingFilePath -replace ('\.\\', '')))"
+    . $preprocessingFilePath
 
     $excludedFiles = @('.\.git\', '.\.gitignore', '.\.gitattributes', '.\.github\CODEOWNERS', '.\LICENSE', "$preprocessingFilePath", '*.png', '*.exe')
     $msg = "Pre-req: Code Formatting"
-    Invoke-Preprocessing -WorkingDir "$workingdir" -ExcludedFiles $excludedFiles -ProgressStatusMessage $msg
+    Invoke-Preprocessing -WorkingDir "$workingdir" -ExcludedFiles $excludedFiles -ProgressStatusMessage $msg -ThrowExceptionOnEmptyFilesList
 }
 
 # Create the script in memory.
@@ -57,14 +60,14 @@ Update-Progress "Adding: Header" 5
 $script_content.Add($header)
 
 Update-Progress "Adding: Version" 10
-$script_content.Add($(Get-Content "$workingdir\scripts\start.ps1").replace('#{replaceme}',"$(Get-Date -Format yy.MM.dd)"))
+$script_content.Add($(Get-Content "scripts\start.ps1").replace('#{replaceme}',"$(Get-Date -Format yy.MM.dd)"))
 
 Update-Progress "Adding: Functions" 20
-Get-ChildItem "$workingdir\functions" -Recurse -File | ForEach-Object {
+Get-ChildItem "functions" -Recurse -File | ForEach-Object {
     $script_content.Add($(Get-Content $psitem.FullName))
     }
 Update-Progress "Adding: Config *.json" 40
-Get-ChildItem "$workingdir\config" | Where-Object {$psitem.extension -eq ".json"} | ForEach-Object {
+Get-ChildItem "config" | Where-Object {$psitem.extension -eq ".json"} | ForEach-Object {
     $json = (Get-Content $psitem.FullName).replace("'","''")
     $jsonAsObject = $json | convertfrom-json
 
@@ -85,34 +88,33 @@ Get-ChildItem "$workingdir\config" | Where-Object {$psitem.extension -eq ".json"
     $script_content.Add($(Write-output "`$sync.configs.$($psitem.BaseName) = '$json' `| convertfrom-json" ))
 }
 
-$xaml = (Get-Content "$workingdir\xaml\inputXML.xaml").replace("'","''")
+$xaml = (Get-Content "xaml\inputXML.xaml").replace("'","''")
 
 Update-Progress "Adding: Xaml " 90
 
 $script_content.Add($(Write-output "`$inputXML =  '$xaml'"))
 
-$script_content.Add($(Get-Content "$workingdir\scripts\main.ps1"))
+$script_content.Add($(Get-Content "scripts\main.ps1"))
 
 if ($Debug) {
     Update-Progress "Writing debug files" 95
-    $appXamlContent | Out-File -FilePath "$workingdir\xaml\inputApp.xaml" -Encoding ascii
+    $appXamlContent | Out-File -FilePath "xaml\inputApp.xaml" -Encoding ascii
-    $tweaksXamlContent | Out-File -FilePath "$workingdir\xaml\inputTweaks.xaml" -Encoding ascii
+    $tweaksXamlContent | Out-File -FilePath "xaml\inputTweaks.xaml" -Encoding ascii
-    $featuresXamlContent | Out-File -FilePath "$workingdir\xaml\inputFeatures.xaml" -Encoding ascii
+    $featuresXamlContent | Out-File -FilePath "xaml\inputFeatures.xaml" -Encoding ascii
 } else {
     Update-Progress "Removing temporary files" 99
-    Remove-Item "$workingdir\xaml\inputApp.xaml" -ErrorAction SilentlyContinue
+    Remove-Item "xaml\inputApp.xaml" -ErrorAction SilentlyContinue
-    Remove-Item "$workingdir\xaml\inputTweaks.xaml" -ErrorAction SilentlyContinue
+    Remove-Item "xaml\inputTweaks.xaml" -ErrorAction SilentlyContinue
-    Remove-Item "$workingdir\xaml\inputFeatures.xaml" -ErrorAction SilentlyContinue
+    Remove-Item "xaml\inputFeatures.xaml" -ErrorAction SilentlyContinue
 }
 
-Set-Content -Path "$workingdir\$scriptname" -Value ($script_content -join "`r`n") -Encoding ascii
+Set-Content -Path "$scriptname" -Value ($script_content -join "`r`n") -Encoding ascii
 Write-Progress -Activity "Compiling" -Completed
 
 Update-Progress -Activity "Validating" -StatusMessage "Checking winutil.ps1 Syntax" -Percent 0
 try {
     $null = Get-Command -Syntax .\winutil.ps1
-}
+} catch {
-catch {
     Write-Warning "Syntax Validation for 'winutil.ps1' has failed"
     Write-Host "$($Error[0])" -ForegroundColor Red
 }
@@ -128,3 +130,4 @@ if ($run) {
 
     break
 }
+Pop-Location

config/autounattend.xml

@@ -293,7 +293,7 @@
     <Extensions xmlns="https://schneegans.de/windows/unattend-generator/">
         <ExtractScript>
 param(
-    [xml] $Document
+    [xml]$Document
 );
 
 $scriptsDir = 'C:\Windows\Setup\Scripts\';

functions/private/Invoke-WinUtilMicroWin-Helper.ps1

@@ -259,7 +259,7 @@ function New-Unattend {
 
     param (
         [Parameter(Mandatory, Position = 0)] [string]$userName,
-        [Parameter(Position = 1)] [string] $userPassword
+        [Parameter(Position = 1)] [string]$userPassword
     )
 
     $unattend = @'

scripts/main.ps1

@@ -32,19 +32,19 @@ $sync.runspace.Open()
 # Create classes for different exceptions
 
     class WingetFailedInstall : Exception {
-        [string] $additionalData
+        [string]$additionalData
 
         WingetFailedInstall($Message) : base($Message) {}
     }
 
     class ChocoFailedInstall : Exception {
-        [string] $additionalData
+        [string]$additionalData
 
         ChocoFailedInstall($Message) : base($Message) {}
     }
 
     class GenericException : Exception {
-        [string] $additionalData
+        [string]$additionalData
 
         GenericException($Message) : base($Message) {}
     }

tools/Invoke-Preprocessing.ps1

@@ -39,7 +39,7 @@
         .EXAMPLE
         Invoke-Preprocessing -ThrowExceptionOnEmptyFilesList -WorkingDir "DRIVE:\Path\To\Folder\" -ExcludedFiles @('file.txt', '.\.git\', '*.png') -ProgressStatusMessage "Doing Preprocessing"
 
-        Same as Example No. 1, but will throw an exception when 'Invoke-Preprocessing' function doesn't find any files in 'WorkingDir' (not including 'ExcludedFiles' list).
+        Same as Example No. 1, but uses '-ThrowExceptionOnEmptyFilesList', which's an optional parameter that'll make 'Invoke-Preprocessing' throw an exception when no files are found in 'WorkingDir' (not including the ExcludedFiles, of course), useful when you want to double check your parameters & you're sure there's files to process in the 'WorkingDir'.
 
         .EXAMPLE
         Invoke-Preprocessing -Skip -WorkingDir "DRIVE:\Path\To\Folder\" -ExcludedFiles @('file.txt', '.\.git\', '*.png') -ProgressStatusMessage "Doing Preprocessing"
@@ -73,21 +73,79 @@
     }
 
     $count = $ExcludedFiles.Count
-    if ((-NOT ($count -eq 0)) -AND (-NOT $SkipExcludedFilesValidation)) {
+
+    # Make sure there's a * at the end of folders in ExcludedFiles list
+    for ($i = 0; $i -lt $count; $i++) {
+        $excludedFile = $ExcludedFiles[$i]
+        $isFolder = ($excludedFile) -match '\\$'
+        if ($isFolder) { $ExcludedFiles[$i] = $excludedFile + '*' }
+    }
+
+    # Validate the ExcludedFiles List before continuing on,
+    # that's if there's a list in the first place, and '-SkipExcludedFilesValidation' was not provided.
+    if (-not $SkipExcludedFilesValidation) {
         for ($i = 0; $i -lt $count; $i++) {
             $excludedFile = $ExcludedFiles[$i]
             $filePath = "$(($WorkingDir -replace ('\\$', '')) + '\' + ($excludedFile -replace ('\.\\', '')))"
+
+            # Handle paths with wildcards in a different implementation
+            $matches = ($filePath) -match '^.*?\*'
+
+            if ($matches) {
                 if (-NOT (Get-ChildItem -Recurse -Path "$filePath" -File)) {
                     $failedFilesList += "'$filePath', "
                 }
+            } else {
+                if (-NOT (Test-Path -Path "$filePath")) {
+                    $failedFilesList += "'$filePath', "
+                }
+            }
         }
         $failedFilesList = $failedFilesList -replace (',\s*$', '')
         if (-NOT $failedFilesList -eq "") {
-            throw "[Invoke-Preprocessing] One or more File Paths & File Patterns were not found, you can use '-SkipExcludedFilesValidation' switch to skip this check, and the failed files are: $failedFilesList"
+            throw "[Invoke-Preprocessing] One or more File Paths and/or File Patterns were not found, you can use '-SkipExcludedFilesValidation' switch to skip this check, the failed to validate are: $failedFilesList"
         }
     }
 
-    $files = Get-ChildItem $WorkingDir -Recurse -Exclude $ExcludedFiles -File
+    # Get Files List
+    [System.Collections.ArrayList]$files = Get-ChildItem $WorkingDir -Recurse -Exclude $ExcludedFiles -File
+    $numOfFiles = $files.Count
+
+    # Only keep the 'FullName' Property for every entry in the list
+    for ($i = 0; $i -lt $numOfFiles; $i++) {
+        $file = $files[$i]
+        $files[$i] = $file.FullName
+    }
+
+    # If a file(s) are found in Exclude List,
+    # Remove the file from files list.
+    for ($j = 0; $j -lt $excludedFiles.Count; $j++) {
+        # Prepare some variables
+        $excluded = $excludedFiles[$j]
+        $pathToFind = ($excluded) -replace ('^\.\\', '')
+        $pathToFind = $WorkingDir + '\' + $pathToFind
+        $index = -1 # reset index on every iteration
+
+        # Handle paths with wildcards in a different implementation
+        $matches = ($pathToFind) -match '^.*?\*'
+
+        if ($matches) {
+             $filesToCheck = Get-ChildItem -Recurse -Path "$pathToFind" -File
+             if ($filesToCheck) {
+                for ($k = 0; $k -lt $filesToCheck.Count; $k++) {
+                    $fileToCheck = $filesToCheck[$k]
+                    $index = $files.IndexOf("$fileToCheck")
+                    if ($index -ge 0) { $files.RemoveAt($index) }
+                }
+             }
+        } else {
+            $index = $files.IndexOf("$pathToFind")
+            if ($index -ge 0) { $files.RemoveAt($index) }
+        }
+    }
+
+    # Make sure 'numOfFiles' is synced with the actual Number of Files found in '$files'
+    # This's done because previous may or may not edit the files list, so we should update it
     $numOfFiles = $files.Count
 
     if ($numOfFiles -eq 0) {
@@ -99,26 +157,11 @@
     }
 
     for ($i = 0; $i -lt $numOfFiles; $i++) {
-        $file = $files[$i]
+        $fullFileName = $files[$i]
-
-        # If the file is in Exclude List, don't proceed to check/modify said file.
-        $fileIsExcluded = $False
-        for ($j = 0; $j -lt $excludedFiles.Count; $j++) {
-            $excluded = $excludedFiles[$j]
-            $strToCompare = ($excluded) -replace ('^\.\\', '')
-            if ($file.FullName.Contains("$strToCompare")) {
-                $fileIsExcluded = $True
-                break
-            }
-        }
-
-        if ($fileIsExcluded) {
-            continue
-        }
 
         # TODO:
         #   make more formatting rules, and document them in WinUtil Official Documentation
-        (Get-Content "$file").TrimEnd() `
+        (Get-Content "$fullFileName").TrimEnd() `
             -replace ('\t', '    ') `
             -replace ('\)\s*\{', ') {') `
             -replace ('(?<keyword>if|for|foreach)\s*(?<condition>\([.*?]\))\s*\{', '${keyword} ${condition} {') `
@@ -129,8 +172,8 @@
             -replace ('\}\s*Catch', '} catch') `
             -replace ('\}\s*Catch\s*(?<exceptions>(\[.*?\]\s*(\,)?\s*)+)\s*\{', '} catch ${exceptions} {') `
             -replace ('\}\s*Catch\s*(?<exceptions>\[.*?\])\s*\{', '} catch ${exceptions} {') `
-            -replace ('(?<parameter_type>\[.*?\])\s*(?<str_after_type>\$.*?(,|\s*\)))', '${parameter_type}${str_after_type}') `
+            -replace ('(?<parameter_type>\[[^$0-9]+\])\s*(?<str_after_type>\$.*?)', '${parameter_type}${str_after_type}') `
-        | Set-Content "$file"
+        | Set-Content "$fullFileName"
 
         Write-Progress -Activity $ProgressActivity -Status "$ProgressStatusMessage - Finished $i out of $numOfFiles" -PercentComplete (($i/$numOfFiles)*100)
     }