Improve preprocessor (#2579)

* Update documentation for 'Invoke-Preprocessing' Script Tool

* Improve Compile Script a bit Deduplicating a lot of un-needed pre-fixes - Improve implementation for 'Invoke-Preprocessing' Script Tool

* Fix RegEx in 'Invoke-Preprocessing' Script Tool

* Result of Preprocessing

* Update Replace Regex for Code Formatting in 'Invoke-Preprocessing' Script Tool

* Result of Preprocessing

* Update Exclude Files List for Preprocessing in 'Compile.ps1' Script

* Remove Extra Whitespace in some place for 'Invoke-Preprocessing.ps1' Script Tool

* Simplified and Improved the Exclude List Validation Step in 'Invoke-Preprocessing.ps1' Script Tool

* Restore 'workingdir' variable when using '-Run' Parameter with 'Compile.ps1' Script

* Revert "Update Exclude Files List for Preprocessing in 'Compile.ps1' Script"

This reverts commit 674ab0308b.

* Result of Preprocessing

Compile.ps1

@@ -8,6 +8,9 @@ $OFS = "`r`n"
 $scriptname = "winutil.ps1"
 $workingdir = $PSScriptRoot
 
+Push-Location
+Set-Location $workingdir
+
 # Variable to sync between runspaces
 $sync = [Hashtable]::Synchronized(@{})
 $sync.PSScriptRoot = $workingdir
@@ -42,11 +45,11 @@ if (-NOT $SkipPreprocessing) {
 
     # Dot source the 'Invoke-Preprocessing' Function from 'tools/Invoke-Preprocessing.ps1' Script
     $preprocessingFilePath = ".\tools\Invoke-Preprocessing.ps1"
-    . "$(($workingdir -replace ('\\$', '')) + '\' + ($preprocessingFilePath -replace ('\.\\', '')))"
+    . $preprocessingFilePath
 
     $excludedFiles = @('.\.git\', '.\.gitignore', '.\.gitattributes', '.\.github\CODEOWNERS', '.\LICENSE', "$preprocessingFilePath", '*.png', '*.exe')
     $msg = "Pre-req: Code Formatting"
-    Invoke-Preprocessing -WorkingDir "$workingdir" -ExcludedFiles $excludedFiles -ProgressStatusMessage $msg
+    Invoke-Preprocessing -WorkingDir "$workingdir" -ExcludedFiles $excludedFiles -ProgressStatusMessage $msg -ThrowExceptionOnEmptyFilesList
 }
 
 # Create the script in memory.
@@ -57,14 +60,14 @@ Update-Progress "Adding: Header" 5
 $script_content.Add($header)
 
 Update-Progress "Adding: Version" 10
-$script_content.Add($(Get-Content "$workingdir\scripts\start.ps1").replace('#{replaceme}',"$(Get-Date -Format yy.MM.dd)"))
+$script_content.Add($(Get-Content "scripts\start.ps1").replace('#{replaceme}',"$(Get-Date -Format yy.MM.dd)"))
 
 Update-Progress "Adding: Functions" 20
-Get-ChildItem "$workingdir\functions" -Recurse -File | ForEach-Object {
+Get-ChildItem "functions" -Recurse -File | ForEach-Object {
     $script_content.Add($(Get-Content $psitem.FullName))
     }
 Update-Progress "Adding: Config *.json" 40
-Get-ChildItem "$workingdir\config" | Where-Object {$psitem.extension -eq ".json"} | ForEach-Object {
+Get-ChildItem "config" | Where-Object {$psitem.extension -eq ".json"} | ForEach-Object {
     $json = (Get-Content $psitem.FullName).replace("'","''")
     $jsonAsObject = $json | convertfrom-json
 
@@ -85,34 +88,33 @@ Get-ChildItem "$workingdir\config" | Where-Object {$psitem.extension -eq ".json"
     $script_content.Add($(Write-output "`$sync.configs.$($psitem.BaseName) = '$json' `| convertfrom-json" ))
 }
 
-$xaml = (Get-Content "$workingdir\xaml\inputXML.xaml").replace("'","''")
+$xaml = (Get-Content "xaml\inputXML.xaml").replace("'","''")
 
 Update-Progress "Adding: Xaml " 90
 
 $script_content.Add($(Write-output "`$inputXML =  '$xaml'"))
 
-$script_content.Add($(Get-Content "$workingdir\scripts\main.ps1"))
+$script_content.Add($(Get-Content "scripts\main.ps1"))
 
 if ($Debug) {
     Update-Progress "Writing debug files" 95
-    $appXamlContent | Out-File -FilePath "$workingdir\xaml\inputApp.xaml" -Encoding ascii
+    $appXamlContent | Out-File -FilePath "xaml\inputApp.xaml" -Encoding ascii
-    $tweaksXamlContent | Out-File -FilePath "$workingdir\xaml\inputTweaks.xaml" -Encoding ascii
+    $tweaksXamlContent | Out-File -FilePath "xaml\inputTweaks.xaml" -Encoding ascii
-    $featuresXamlContent | Out-File -FilePath "$workingdir\xaml\inputFeatures.xaml" -Encoding ascii
+    $featuresXamlContent | Out-File -FilePath "xaml\inputFeatures.xaml" -Encoding ascii
 } else {
     Update-Progress "Removing temporary files" 99
-    Remove-Item "$workingdir\xaml\inputApp.xaml" -ErrorAction SilentlyContinue
+    Remove-Item "xaml\inputApp.xaml" -ErrorAction SilentlyContinue
-    Remove-Item "$workingdir\xaml\inputTweaks.xaml" -ErrorAction SilentlyContinue
+    Remove-Item "xaml\inputTweaks.xaml" -ErrorAction SilentlyContinue
-    Remove-Item "$workingdir\xaml\inputFeatures.xaml" -ErrorAction SilentlyContinue
+    Remove-Item "xaml\inputFeatures.xaml" -ErrorAction SilentlyContinue
 }
 
-Set-Content -Path "$workingdir\$scriptname" -Value ($script_content -join "`r`n") -Encoding ascii
+Set-Content -Path "$scriptname" -Value ($script_content -join "`r`n") -Encoding ascii
 Write-Progress -Activity "Compiling" -Completed
 
 Update-Progress -Activity "Validating" -StatusMessage "Checking winutil.ps1 Syntax" -Percent 0
 try {
     $null = Get-Command -Syntax .\winutil.ps1
-}
+} catch {
-catch {
     Write-Warning "Syntax Validation for 'winutil.ps1' has failed"
     Write-Host "$($Error[0])" -ForegroundColor Red
 }
@@ -128,3 +130,4 @@ if ($run) {
 
     break
 }
+Pop-Location

config/autounattend.xml

@@ -293,7 +293,7 @@
     <Extensions xmlns="https://schneegans.de/windows/unattend-generator/">
         <ExtractScript>
 param(
-    [xml] $Document
+    [xml]$Document
 );
 
 $scriptsDir = 'C:\Windows\Setup\Scripts\';

functions/private/Invoke-WinUtilMicroWin-Helper.ps1

@@ -259,7 +259,7 @@ function New-Unattend {
 
     param (
         [Parameter(Mandatory, Position = 0)] [string]$userName,
-        [Parameter(Position = 1)] [string] $userPassword
+        [Parameter(Position = 1)] [string]$userPassword
     )
 
     $unattend = @'

scripts/main.ps1

@@ -32,19 +32,19 @@ $sync.runspace.Open()
 # Create classes for different exceptions
 
     class WingetFailedInstall : Exception {
-        [string] $additionalData
+        [string]$additionalData
 
         WingetFailedInstall($Message) : base($Message) {}
     }
 
     class ChocoFailedInstall : Exception {
-        [string] $additionalData
+        [string]$additionalData
 
         ChocoFailedInstall($Message) : base($Message) {}
     }
 
     class GenericException : Exception {
-        [string] $additionalData
+        [string]$additionalData
 
         GenericException($Message) : base($Message) {}
     }

tools/Invoke-Preprocessing.ps1

@@ -39,7 +39,7 @@
         .EXAMPLE
         Invoke-Preprocessing -ThrowExceptionOnEmptyFilesList -WorkingDir "DRIVE:\Path\To\Folder\" -ExcludedFiles @('file.txt', '.\.git\', '*.png') -ProgressStatusMessage "Doing Preprocessing"
 
-        Same as Example No. 1, but will throw an exception when 'Invoke-Preprocessing' function doesn't find any files in 'WorkingDir' (not including 'ExcludedFiles' list).
+        Same as Example No. 1, but uses '-ThrowExceptionOnEmptyFilesList', which's an optional parameter that'll make 'Invoke-Preprocessing' throw an exception when no files are found in 'WorkingDir' (not including the ExcludedFiles, of course), useful when you want to double check your parameters & you're sure there's files to process in the 'WorkingDir'.
 
         .EXAMPLE
         Invoke-Preprocessing -Skip -WorkingDir "DRIVE:\Path\To\Folder\" -ExcludedFiles @('file.txt', '.\.git\', '*.png') -ProgressStatusMessage "Doing Preprocessing"
@@ -47,7 +47,7 @@
         Same as Example No. 1, but uses '-SkipExcludedFilesValidation', which'll skip the validation step for 'ExcludedFiles' list. This can be useful when 'ExcludedFiles' list is generated from another function, or from unreliable source (you can't guarantee every item in list is a valid path), but you want to silently continue through the function.
     #>
 
-     param (
+    param (
         [Parameter(position=0)]
         [switch]$SkipExcludedFilesValidation,
 
@@ -66,28 +66,86 @@
 
         [Parameter(position=5)]
         [string]$ProgressActivity = "Preprocessing"
-     )
+    )
 
     if (-NOT (Test-Path -PathType Container -Path "$WorkingDir")) {
         throw "[Invoke-Preprocessing] Invalid Paramter Value for 'WorkingDir', passed value: '$WorkingDir'. Either the path is a File or Non-Existing/Invlid, please double check your code."
     }
 
     $count = $ExcludedFiles.Count
-    if ((-NOT ($count -eq 0)) -AND (-NOT $SkipExcludedFilesValidation)) {
+
+    # Make sure there's a * at the end of folders in ExcludedFiles list
+    for ($i = 0; $i -lt $count; $i++) {
+        $excludedFile = $ExcludedFiles[$i]
+        $isFolder = ($excludedFile) -match '\\$'
+        if ($isFolder) { $ExcludedFiles[$i] = $excludedFile + '*' }
+    }
+
+    # Validate the ExcludedFiles List before continuing on,
+    # that's if there's a list in the first place, and '-SkipExcludedFilesValidation' was not provided.
+    if (-not $SkipExcludedFilesValidation) {
         for ($i = 0; $i -lt $count; $i++) {
             $excludedFile = $ExcludedFiles[$i]
             $filePath = "$(($WorkingDir -replace ('\\$', '')) + '\' + ($excludedFile -replace ('\.\\', '')))"
-            if (-NOT (Get-ChildItem -Recurse -Path "$filePath" -File)) {
+
-                $failedFilesList += "'$filePath', "
+            # Handle paths with wildcards in a different implementation
+            $matches = ($filePath) -match '^.*?\*'
+
+            if ($matches) {
+                if (-NOT (Get-ChildItem -Recurse -Path "$filePath" -File)) {
+                    $failedFilesList += "'$filePath', "
+                }
+            } else {
+                if (-NOT (Test-Path -Path "$filePath")) {
+                    $failedFilesList += "'$filePath', "
+                }
             }
         }
         $failedFilesList = $failedFilesList -replace (',\s*$', '')
         if (-NOT $failedFilesList -eq "") {
-            throw "[Invoke-Preprocessing] One or more File Paths & File Patterns were not found, you can use '-SkipExcludedFilesValidation' switch to skip this check, and the failed files are: $failedFilesList"
+            throw "[Invoke-Preprocessing] One or more File Paths and/or File Patterns were not found, you can use '-SkipExcludedFilesValidation' switch to skip this check, the failed to validate are: $failedFilesList"
         }
     }
 
-    $files = Get-ChildItem $WorkingDir -Recurse -Exclude $ExcludedFiles -File
+    # Get Files List
+    [System.Collections.ArrayList]$files = Get-ChildItem $WorkingDir -Recurse -Exclude $ExcludedFiles -File
+    $numOfFiles = $files.Count
+
+    # Only keep the 'FullName' Property for every entry in the list
+    for ($i = 0; $i -lt $numOfFiles; $i++) {
+        $file = $files[$i]
+        $files[$i] = $file.FullName
+    }
+
+    # If a file(s) are found in Exclude List,
+    # Remove the file from files list.
+    for ($j = 0; $j -lt $excludedFiles.Count; $j++) {
+        # Prepare some variables
+        $excluded = $excludedFiles[$j]
+        $pathToFind = ($excluded) -replace ('^\.\\', '')
+        $pathToFind = $WorkingDir + '\' + $pathToFind
+        $index = -1 # reset index on every iteration
+
+        # Handle paths with wildcards in a different implementation
+        $matches = ($pathToFind) -match '^.*?\*'
+
+        if ($matches) {
+             $filesToCheck = Get-ChildItem -Recurse -Path "$pathToFind" -File
+             if ($filesToCheck) {
+                for ($k = 0; $k -lt $filesToCheck.Count; $k++) {
+                    $fileToCheck = $filesToCheck[$k]
+                    $index = $files.IndexOf("$fileToCheck")
+                    if ($index -ge 0) { $files.RemoveAt($index) }
+                }
+             }
+        } else {
+            $index = $files.IndexOf("$pathToFind")
+            if ($index -ge 0) { $files.RemoveAt($index) }
+        }
+    }
+
+    # Make sure 'numOfFiles' is synced with the actual Number of Files found in '$files'
+    # This's done because previous may or may not edit the files list, so we should update it
     $numOfFiles = $files.Count
 
     if ($numOfFiles -eq 0) {
@@ -99,26 +157,11 @@
     }
 
     for ($i = 0; $i -lt $numOfFiles; $i++) {
-        $file = $files[$i]
+        $fullFileName = $files[$i]
-
-        # If the file is in Exclude List, don't proceed to check/modify said file.
-        $fileIsExcluded = $False
-        for ($j = 0; $j -lt $excludedFiles.Count; $j++) {
-            $excluded = $excludedFiles[$j]
-            $strToCompare = ($excluded) -replace ('^\.\\', '')
-            if ($file.FullName.Contains("$strToCompare")) {
-                $fileIsExcluded = $True
-                break
-            }
-        }
-
-        if ($fileIsExcluded) {
-            continue
-        }
 
         # TODO:
         #   make more formatting rules, and document them in WinUtil Official Documentation
-        (Get-Content "$file").TrimEnd() `
+        (Get-Content "$fullFileName").TrimEnd() `
             -replace ('\t', '    ') `
             -replace ('\)\s*\{', ') {') `
             -replace ('(?<keyword>if|for|foreach)\s*(?<condition>\([.*?]\))\s*\{', '${keyword} ${condition} {') `
@@ -129,8 +172,8 @@
             -replace ('\}\s*Catch', '} catch') `
             -replace ('\}\s*Catch\s*(?<exceptions>(\[.*?\]\s*(\,)?\s*)+)\s*\{', '} catch ${exceptions} {') `
             -replace ('\}\s*Catch\s*(?<exceptions>\[.*?\])\s*\{', '} catch ${exceptions} {') `
-            -replace ('(?<parameter_type>\[.*?\])\s*(?<str_after_type>\$.*?(,|\s*\)))', '${parameter_type}${str_after_type}') `
+            -replace ('(?<parameter_type>\[[^$0-9]+\])\s*(?<str_after_type>\$.*?)', '${parameter_type}${str_after_type}') `
-        | Set-Content "$file"
+        | Set-Content "$fullFileName"
 
         Write-Progress -Activity $ProgressActivity -Status "$ProgressStatusMessage - Finished $i out of $numOfFiles" -PercentComplete (($i/$numOfFiles)*100)
     }