feat: update dmgbuild and migrate to portable dmgbuild python bundle (#9516)

Author: mmaietta

.changeset/fuzzy-flowers-fix.md

@@ -0,0 +1,6 @@
+---
+"app-builder-lib": patch
+"dmg-builder": patch
+---
+
+feat: update `dmgbuild` and migrate to portable `dmgbuild` python bundle. Fixes `badge-icon` property

packages/app-builder-lib/package.json

@@ -50,6 +50,7 @@
     "@develar/schema-utils": "~2.6.5",
     "@electron/asar": "3.4.1",
     "@electron/fuses": "^1.8.0",
+    "@electron/get": "^3.0.0",
     "@electron/notarize": "2.5.0",
     "@electron/osx-sign": "1.3.3",
     "@electron/rebuild": "4.0.1",
@@ -75,6 +76,7 @@
     "lazy-val": "^1.0.5",
     "minimatch": "^10.0.3",
     "plist": "3.1.0",
+    "proper-lockfile": "^4.1.2",
     "resedit": "^1.7.0",
     "semver": "~7.7.3",
     "tar": "7.5.3",
@@ -108,6 +110,7 @@
     "@types/hosted-git-info": "3.0.2",
     "@types/js-yaml": "4.0.3",
     "@types/plist": "3.0.5",
+    "@types/proper-lockfile": "^4.1.4",
     "@types/semver": "7.7.1",
     "@types/tiny-async-pool": "^1",
     "@types/which": "^3.0.4",

packages/app-builder-lib/scheme.json

@@ -871,7 +871,7 @@
           ]
         },
         "badgeIcon": {
-          "description": "The path to DMG icon (badge icon), which will be shown when mounted, relative to the [build resources](./contents.md#extraresources) or to the project directory.\nDefaults to the application icon (`build/icon.icns`).",
+          "description": "The path to DMG icon (badge icon), which will be shown when mounted, relative to the [build resources](./contents.md#extraresources) or to the project directory.",
           "type": [
             "null",
             "string"

packages/app-builder-lib/src/binDownload.ts

@@ -1,7 +1,179 @@
-import { executeAppBuilder } from "builder-util"
+import * as get from "@electron/get"
+import { ElectronDownloadCacheMode, ElectronDownloadRequest, ElectronDownloadRequestOptions, GotDownloaderOptions } from "@electron/get"
+import { executeAppBuilder, exists, log, PADDING } from "builder-util"
 import { Nullish } from "builder-util-runtime"
 import { sanitizeFileName } from "builder-util/out/filename"
+import { MultiProgress } from "electron-publish/out/multiProgress"
+import * as fs from "fs/promises"
+import * as os from "os"
 import * as path from "path"
+import * as lockfile from "proper-lockfile"
+import * as tar from "tar"
+
+/**
+ * Deterministic <length>-character URL-safe hash (a–z0–9)
+ */
+export function hashUrlSafe(input: string, length = 6): string {
+  let hash = 5381
+  for (let i = 0; i < input.length; i++) {
+    hash = ((hash << 5) + hash) ^ input.charCodeAt(i) // hash * 33 ^ c
+  }
+  // Force unsigned 32-bit
+  hash >>>= 0
+  // Base-36 (0–9a–z)
+  const out = hash.toString(36)
+  // Ensure exactly `length` chars
+  if (out.length >= length) {
+    return out.slice(0, length)
+  }
+  return out.padStart(length, "0")
+}
+
+/**
+ * Get cache directory for electron-builder
+ */
+export function getCacheDirectory(isAvoidSystemOnWindows = false): string {
+  const env = process.env.ELECTRON_BUILDER_CACHE?.trim()
+  if (env) {
+    return env
+  }
+
+  const appName = "electron-builder"
+
+  const platform = os.platform()
+  const homeDir = os.homedir()
+
+  if (platform === "darwin") {
+    return path.join(homeDir, "Library", "Caches", appName)
+  }
+
+  if (platform === "win32") {
+    const localAppData = process.env.LOCALAPPDATA?.trim()
+    const username = process.env.USERNAME?.trim()?.toLowerCase()
+    const isSystemUser = isAvoidSystemOnWindows && (localAppData?.toLowerCase()?.includes("\\windows\\system32\\") || username === "system")
+    if (!localAppData || isSystemUser) {
+      return path.join(os.tmpdir(), `${appName}-cache`)
+    }
+    return path.join(localAppData, appName, "Cache")
+  }
+
+  // linux
+  const xdgCache = process.env.XDG_CACHE_HOME
+  if (xdgCache) {
+    return path.join(xdgCache, appName)
+  }
+
+  return path.join(homeDir, ".cache", appName)
+}
+
+/**
+ * Downloads an artifact from GitHub releases (convenience wrapper)
+ */
+export async function downloadArtifact(options: { releaseName: string; filenameWithExt: string; checksums: Record<string, string>; githubOrgRepo?: string }): Promise<string> {
+  const { releaseName, filenameWithExt, checksums, githubOrgRepo = "electron-userland/electron-builder-binaries" } = options
+
+  const file = await _downloadArtifact(`https://github.com/${githubOrgRepo}/releases/download/`, releaseName, filenameWithExt, checksums)
+
+  return file
+}
+
+/**
+ * Downloads, validates, and extracts a .tar.gz from a release URL
+ */
+async function _downloadArtifact(baseUrl: string, releaseName: string, filenameWithExt: string, checksums: Record<string, string>): Promise<string> {
+  const suffix = hashUrlSafe(`${baseUrl}-${releaseName}-${filenameWithExt}`, 5)
+  const folderName = `${filenameWithExt.replace(/\.(tar\.gz|tgz)$/, "")}-${suffix}`
+  const extractDir = path.join(getCacheDirectory(), releaseName, folderName)
+  const extractionCompleteMarker = `${extractDir}.complete`
+
+  // Ensure download directory exists before trying to lock
+  await fs.mkdir(extractDir, { recursive: true })
+
+  // Acquire the lock
+  let release: (() => Promise<void>) | undefined
+  try {
+    release = await lockfile.lock(extractDir, {
+      retries: {
+        retries: 5,
+        minTimeout: 1000,
+        maxTimeout: 5000,
+      },
+      stale: 60000,
+    })
+
+    const varName = "ELECTRON_DOWNLOAD_CACHE_MODE"
+    const cacheOverride = process.env[varName]?.trim()
+
+    let cacheMode: ElectronDownloadCacheMode = ElectronDownloadCacheMode.ReadWrite
+    if (cacheOverride && Number(cacheOverride) in ElectronDownloadCacheMode) {
+      cacheMode = Number(cacheOverride)
+      log.debug({ mode: cacheMode }, `cache mode overridden via env var ${varName}`)
+    }
+
+    if (await exists(extractionCompleteMarker)) {
+      log.debug({ file: filenameWithExt, path: extractDir }, "using cached artifact - skipping download/extract")
+      return extractDir
+    }
+
+    // These are just stubs. Actual url construction/file naming are in `mirrorOptions` below.
+    const details: ElectronDownloadRequest = {
+      // Needs to be higher than 1.3.2 to avoid @electron/get validation shortcut
+      // https://github.com/electron/get/blob/05c466d4fc60fa0c83064df28dce245eb83d63c9/src/index.ts#L60
+      version: "9.9.9",
+      artifactName: filenameWithExt, // also is the output filename
+    }
+
+    const progress = process.stdout.isTTY ? new MultiProgress() : null
+    const progressBar = progress?.createBar(`${" ".repeat(PADDING + 2)}[:bar] :percent | ${filenameWithExt}`, { total: 100 })
+
+    const downloadOptions: GotDownloaderOptions = {
+      getProgressCallback: info => {
+        progressBar?.update(info.percent != null ? Math.floor(info.percent * 100) : 0)
+        return Promise.resolve()
+      },
+    }
+    const options: ElectronDownloadRequestOptions = {
+      cacheRoot: path.resolve(getCacheDirectory(), "downloads"),
+      cacheMode,
+      downloadOptions,
+      checksums,
+      mirrorOptions: {
+        // `${opts.mirror}${opts.customDir}/${opts.customFilename}`
+        mirror: baseUrl,
+        customDir: releaseName,
+        customFilename: filenameWithExt,
+      },
+    }
+
+    log.info({ release: releaseName, file: filenameWithExt }, "downloading")
+    progressBar?.render()
+    const downloadedFile = await get.downloadArtifact({
+      ...details,
+      ...options,
+      isGeneric: true,
+    })
+
+    await tar.extract({
+      file: downloadedFile,
+      cwd: extractDir,
+      strip: 1, // Strip the top-level directory from the archive
+    })
+
+    // Write the extraction complete marker file to indicate successful extraction and prevent future re-extraction
+    await fs.writeFile(extractionCompleteMarker, "")
+
+    log.debug({ file: filenameWithExt, path: extractDir }, "downloaded")
+    progressBar?.update(100)
+    progressBar?.terminate()
+
+    return extractDir
+  } finally {
+    // Release the lock
+    if (release) {
+      await release()
+    }
+  }
+}
 
 const versionToPromise = new Map<string, Promise<string>>()
 

packages/app-builder-lib/src/options/macOptions.ts

@@ -253,7 +253,6 @@ export interface DmgOptions extends TargetSpecificOptions {
 
   /**
    * The path to DMG icon (badge icon), which will be shown when mounted, relative to the [build resources](./contents.md#extraresources) or to the project directory.
-   * Defaults to the application icon (`build/icon.icns`).
    */
   badgeIcon?: string | null
 

packages/app-builder-lib/src/platformPackager.ts

@@ -796,7 +796,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
   }
 
   private cachedIcnsFromIconFile = new Map<string, Promise<string>>()
-  private async generateIcnsFromIcon(iconPath: string): Promise<string> {
+  async generateIcnsFromIcon(iconPath: string): Promise<string> {
     const cachedPromise = this.cachedIcnsFromIconFile.get(iconPath)
     if (cachedPromise) {
       return cachedPromise

packages/dmg-builder/package.json

@@ -13,8 +13,7 @@
 	"homepage": "https://github.com/electron-userland/electron-builder",
 	"files": [
 		"out",
-		"templates",
-		"vendor"
+		"templates"
 	],
 	"dependencies": {
 		"app-builder-lib": "workspace:*",

packages/dmg-builder/src/dmgUtil.ts

@@ -1,9 +1,10 @@
 import { DmgOptions, MacPackager, PlatformPackager } from "app-builder-lib"
+import { downloadArtifact } from "app-builder-lib/out/binDownload"
 import { exec, executeFinally, exists, isEmptyOrSpaces, TmpDir } from "builder-util"
-import * as path from "path"
-import { hdiUtil, hdiutilTransientExitCodes } from "./hdiuil"
 import { writeFile } from "fs-extra"
+import * as path from "path"
 import { DmgBuildConfig } from "./dmg"
+import { hdiUtil, hdiutilTransientExitCodes } from "./hdiuil"
 
 export { DmgTarget } from "./dmg"
 
@@ -13,11 +14,30 @@ export function getDmgTemplatePath() {
   return path.join(root, "templates")
 }
 
-export function getDmgVendorPath() {
-  return path.join(root, "vendor")
+async function getDmgVendorPath(): Promise<string> {
+  const customDmgbuildPath = process.env.CUSTOM_DMGBUILD_PATH?.trim()
+  if (customDmgbuildPath) {
+    return path.resolve(customDmgbuildPath)
+  }
+
+  // https://github.com/electron-userland/electron-builder-binaries/releases/tag/dmg-builder%401.1.0
+  const releaseVersion = "9614277"
+  const arch = process.arch === "arm64" ? "arm64" : "x86_64"
+  const config = {
+    "dmgbuild-bundle-arm64-9614277.tar.gz": "28e11550cf990f78180a2d82090f35a24588beda3d9165098837714f90ee47ce",
+    "dmgbuild-bundle-x86_64-9614277.tar.gz": "4dbf1cc186af62921f8b6f4a5956b28d8622d211797a8b05eb75a260ee9c3fdb",
+  }
+  const filename: keyof typeof config = `dmgbuild-bundle-${arch}-${releaseVersion}.tar.gz`
+  const file = await downloadArtifact({
+    releaseName: "dmg-builder@1.1.0",
+    filenameWithExt: filename,
+    checksums: config,
+    githubOrgRepo: "electron-userland/electron-builder-binaries",
+  })
+  return path.resolve(file, "dmgbuild")
 }
 
-export async function attachAndExecute(dmgPath: string, readWrite: boolean, task: (devicePath: string) => Promise<any>) {
+export async function attachAndExecute(dmgPath: string, readWrite: boolean, forceDetach: boolean, task: (devicePath: string) => Promise<any>) {
   //noinspection SpellCheckingInspection
   const args = ["attach", "-noverify", "-noautoopen"]
   if (readWrite) {
@@ -36,7 +56,7 @@ export async function attachAndExecute(dmgPath: string, readWrite: boolean, task
     throw new Error(`Cannot find volume mount path for device: ${device}`)
   }
 
-  return await executeFinally(task(volumePath), () => detach(device))
+  return await executeFinally(task(volumePath), () => detach(device, forceDetach))
 }
 
 /**
@@ -58,9 +78,9 @@ async function findMountPath(devName: string, index: number = 1): Promise<string
   return matches.length >= index ? matches[index - 1] : null
 }
 
-export async function detach(name: string) {
+export async function detach(name: string, alwaysForce: boolean) {
   return hdiUtil(["detach", "-quiet", name]).catch(async e => {
-    if (hdiutilTransientExitCodes.has(e.code)) {
+    if (hdiutilTransientExitCodes.has(e.code) || alwaysForce) {
       // Delay then force unmount with verbose output
       await new Promise(resolve => setTimeout(resolve, 3000))
       return hdiUtil(["detach", "-force", name])
@@ -105,12 +125,9 @@ export async function customizeDmg({ appPath, artifactPath, volumeName, specific
   const iconTextSize = isValidIconTextSize ? specification.iconTextSize : 12
   const volumePath = path.join("/Volumes", volumeName)
   // https://github.com/electron-userland/electron-builder/issues/2115
-  const backgroundFile = specification.background == null ? null : await transformBackgroundFileIfNeed(specification.background, packager.info.tempDirManager)
 
   const settings: DmgBuildConfig = {
     title: path.basename(volumePath),
-    icon: await packager.getResource(specification.icon),
-    "badge-icon": await packager.getResource(specification.badgeIcon),
     "icon-size": specification.iconSize,
     "text-size": iconTextSize,
 
@@ -128,6 +145,16 @@ export async function customizeDmg({ appPath, artifactPath, volumeName, specific
       })) || [],
   }
 
+  if (specification.badgeIcon) {
+    let badgeIcon = await packager.getResource(specification.badgeIcon)
+    if (badgeIcon && badgeIcon.toLowerCase().endsWith(".icon")) {
+      badgeIcon = await packager.generateIcnsFromIcon(badgeIcon)
+    }
+    settings["badge-icon"] = badgeIcon
+  } else {
+    settings.icon = await packager.getResource(specification.icon)
+  }
+
   if (specification.backgroundColor != null || specification.background == null) {
     settings["background-color"] = specification.backgroundColor || "#ffffff"
 
@@ -145,8 +172,7 @@ export async function customizeDmg({ appPath, artifactPath, volumeName, specific
       }
     }
   } else {
-    settings.background = backgroundFile
-    delete settings["background-color"]
+    settings.background = specification.background == null ? null : await transformBackgroundFileIfNeed(specification.background, packager.info.tempDirManager)
   }
 
   if (!isEmptyOrSpaces(settings.background)) {
@@ -157,15 +183,8 @@ export async function customizeDmg({ appPath, artifactPath, volumeName, specific
   const settingsFile = await packager.getTempFile(".json")
   await writeFile(settingsFile, JSON.stringify(settings, null, 2))
 
-  const python3Check = () => exec("command", ["-v", "python3"])
-  const pythonCheck = () => exec("command", ["-v", "python"])
-  const pythonPath = process.env.PYTHON_PATH || (await python3Check().catch(pythonCheck)) || (await pythonCheck())
-  if (pythonPath == null || isEmptyOrSpaces(pythonPath.trim())) {
-    throw new Error("Cannot find 'python' or 'python3' executable, please ensure Python is installed and available in PATH or set PYTHON_PATH environment variable")
-  }
-  const vendorDir = getDmgVendorPath()
-  await exec(pythonPath.trim(), [path.join(vendorDir, "run_dmgbuild.py"), "-s", settingsFile, path.basename(volumePath), artifactPath], {
-    cwd: vendorDir,
+  const dmgbuild = await getDmgVendorPath()
+  await exec(dmgbuild, ["-s", settingsFile, path.basename(volumePath), artifactPath], {
     env: {
       ...process.env,
       PYTHONIOENCODING: "utf8",
@@ -175,7 +194,7 @@ export async function customizeDmg({ appPath, artifactPath, volumeName, specific
   // effectiveOptionComputed, when present, is purely for verifying result during test execution
   return (
     packager.packagerOptions.effectiveOptionComputed == null ||
-    (await attachAndExecute(artifactPath, false, async volumePath => {
+    (await attachAndExecute(artifactPath, false, true, async volumePath => {
       return !(await packager.packagerOptions.effectiveOptionComputed!({
         volumePath,
         specification: {