fix(core): add Get-Encoding function to fix missing webClient encoding ([ScoopInstaller#4956](ScoopInstaller#4956))

Add Get-Encoding function in core.ps1

close ScoopInstaller#4911

Author: yi-Xu-0100

CHANGELOG.md

@@ -6,6 +6,7 @@
 - **core:** Using `Invoke-Command` instead of `Invoke-Expression` ([#4941](https://github.com/ScoopInstaller/Scoop/issues/4941))
 - **core:** Load config file before initialization ([#4932](https://github.com/ScoopInstaller/Scoop/issues/4932))
 - **core:** Allow to use '_' and '.' in bucket name ([#4952](https://github.com/ScoopInstaller/Scoop/pull/4952))
+- **core:** Add `Get-Encoding` function to fix missing webclient encoding ([#4956](https://github.com/ScoopInstaller/Scoop/pull/4956))
 - **depends:** Avoid digits in archive file extension (except for .7z and .001) ([#4915](https://github.com/ScoopInstaller/Scoop/issues/4915))
 - **bucket:** Don't check remote URL of non-git buckets ([#4923](https://github.com/ScoopInstaller/Scoop/issues/4923))
 - **bucket:** Don't write message OK before bucket is cloned ([#4925](https://github.com/ScoopInstaller/Scoop/issues/4925))

bin/checkver.ps1

@@ -190,6 +190,7 @@ $Queue | ForEach-Object {
     }
 
     $wc.Headers.Add('Referer', (strip_filename $url))
+    $wc.Encoding = Get-Encoding($url)
     $wc.DownloadStringAsync($url, $state)
 }
 

bin/describe.ps1

@@ -44,6 +44,7 @@ $Queue | ForEach-Object {
     try {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         $home_html = $wc.DownloadString($manifest.homepage)
     } catch {
         Write-Host "`n$($_.Exception.Message)" -ForegroundColor Red

lib/autoupdate.ps1

@@ -1,11 +1,15 @@
 # Must included with 'json.ps1'
+
+. "$PSScriptRoot\..\lib\core.ps1"
+
 function find_hash_in_rdf([String] $url, [String] $basename) {
     $data = $null
     try {
         # Download and parse RDF XML file
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         [xml]$data = $wc.downloadstring($url)
     } catch [system.net.webexception] {
         write-host -f darkred $_
@@ -35,6 +39,7 @@ function find_hash_in_textfile([String] $url, [Hashtable] $substitutions, [Strin
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         $hashfile = $wc.downloadstring($url)
     } catch [system.net.webexception] {
         write-host -f darkred $_
@@ -88,6 +93,7 @@ function find_hash_in_json([String] $url, [Hashtable] $substitutions, [String] $
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         $json = $wc.downloadstring($url)
     } catch [system.net.webexception] {
         write-host -f darkred $_
@@ -108,6 +114,7 @@ function find_hash_in_xml([String] $url, [Hashtable] $substitutions, [String] $x
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('Referer', (strip_filename $url))
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         $xml = [xml]$wc.downloadstring($url)
     } catch [system.net.webexception] {
         write-host -f darkred $_

lib/core.ps1

@@ -15,6 +15,15 @@ function Optimize-SecurityProtocol {
     }
 }
 
+function Get-Encoding($url) {
+    $rawData = $wc.downloadData($url)
+    if($wc.ResponseHeaders["Content-Type"] -match 'charset=([^;]*)') {
+        return [System.Text.Encoding]::GetEncoding($Matches[1])
+    } else {
+        return [System.Text.Encoding]::GetEncoding('utf-8')
+    }
+}
+
 function Get-UserAgent() {
     return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -eq 'AMD64'){'WOW64; '})$PSEdition)"
 }

lib/description.ps1

@@ -18,6 +18,7 @@ function find_description($url, $html, $redir = $false) {
     if($refresh -and !$redir) {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($refresh)
         $html = $wc.downloadstring($refresh)
         return find_description $refresh $html $true
     }

lib/manifest.ps1

@@ -1,3 +1,4 @@
+. "$PSScriptRoot\..\lib\core.ps1"
 function manifest_path($app, $bucket) {
     fullpath "$(Find-BucketDirectory $bucket)\$(sanitary_path $app).json"
 }
@@ -12,6 +13,7 @@ function url_manifest($url) {
     try {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         $str = $wc.downloadstring($url)
     } catch [system.management.automation.methodinvocationexception] {
         warn "error: $($_.exception.innerexception.message)"
@@ -31,6 +33,7 @@ function save_installed_manifest($app, $bucket, $dir, $url) {
     if($url) {
         $wc = New-Object Net.Webclient
         $wc.Headers.Add('User-Agent', (Get-UserAgent))
+        $wc.Encoding = Get-Encoding($url)
         $wc.downloadstring($url) > "$dir\manifest.json"
     } else {
         Copy-Item (manifest_path $app $bucket) "$dir\manifest.json"

libexec/scoop-virustotal.ps1

@@ -34,6 +34,7 @@
 #   -n, --no-depends          By default, all dependencies are checked too. This flag avoids it.
 #   -u, --no-update-scoop     Don't update Scoop before checking if it's outdated
 
+. "$PSScriptRoot\..\lib\core.ps1"
 . "$PSScriptRoot\..\lib\getopt.ps1"
 . "$PSScriptRoot\..\lib\manifest.ps1" # 'Find-Manifest' (indirectly)
 . "$PSScriptRoot\..\lib\json.ps1" # 'json_path'
@@ -85,6 +86,7 @@ Function Get-VirusTotalResult($hash, $app) {
     $url = "https://www.virustotal.com/ui/files/$hash"
     $wc = New-Object Net.Webclient
     $wc.Headers.Add('User-Agent', (Get-UserAgent))
+    $wc.Encoding = Get-Encoding($url)
     $result = $wc.downloadstring($url)
     $stats = json_path $result '$.data.attributes.last_analysis_stats'
     $malicious = json_path $stats '$.malicious'