fix: clear runtime folders once before workers start, not per spec

.changeset/fix-clear-runtime-folder.md

@@ -0,0 +1,10 @@
+---
+"@wdio/visual-service": patch
+"@wdio/image-comparison-core": patch
+---
+
+Fix `clearRuntimeFolder` clearing the actual and diff folders after each spec/feature execution instead of once before all workers start. This caused only the last spec's visual data to be present in the output when running multiple specs.
+
+# Committers: 1
+
+- Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

packages/image-comparison-core/src/base.test.ts

@@ -45,12 +45,9 @@ describe('BaseClass', () => {
         expect(instance.folders.actualFolder).toContain('functional/screenshots')
     })
 
-    it('clears runtime folders if clearRuntimeFolder is true', () => {
-        const options = {
-            clearRuntimeFolder: true,
-        }
-        new BaseClass(options)
-
-        expect(rmSync).toHaveBeenCalledTimes(2)
+    it('should not clear runtime folders in the constructor - clearing should only happen once in the launcher (issue #683)', () => {
+        vi.mocked(rmSync).mockClear()
+        new BaseClass({ clearRuntimeFolder: true })
+        expect(rmSync).not.toHaveBeenCalled()
     })
 })

packages/image-comparison-core/src/base.ts

@@ -18,11 +18,6 @@ export default class BaseClass {
 
         // Setup folder structure
         this.folders = this._setupFolders(options)
-
-        // Clear runtime folders if requested
-        if (options.clearRuntimeFolder) {
-            this._clearRuntimeFolders()
-        }
     }
 
     /**
@@ -47,9 +42,9 @@ export default class BaseClass {
 
     /**
      * Clear the runtime folders (actual and diff)
-     * @private
+     * @protected
      */
-    private _clearRuntimeFolders(): void {
+    protected _clearRuntimeFolders(): void {
         log.info('\x1b[33m\n##############################\n!!CLEARING RUNTIME FOLDERS!!\n##############################\x1b[0m')
 
         try {

packages/visual-service/src/index.ts

@@ -1,6 +1,6 @@
 import type { WicElement } from '@wdio/image-comparison-core'
 import WdioImageComparisonService from './service.js'
-import VisualLauncher from './storybook/launcher.js'
+import VisualLauncher from './launcher.js'
 import type {
     Output,
     Result,

packages/visual-service/src/launcher.ts

@@ -0,0 +1,32 @@
+import type { Capabilities } from '@wdio/types'
+import type { ClassOptions } from '@wdio/image-comparison-core'
+import { BaseClass } from '@wdio/image-comparison-core'
+import { prepareStorybook, cleanupStorybook } from './storybook/hooks.js'
+import generateVisualReport from './reporter.js'
+
+export default class VisualLauncher extends BaseClass {
+    #options: ClassOptions
+
+    constructor(options: ClassOptions) {
+        super(options)
+        this.#options = options
+    }
+
+    async onPrepare(config: WebdriverIO.Config, capabilities: Capabilities.TestrunnerCapabilities) {
+        if (this.#options.clearRuntimeFolder) {
+            this._clearRuntimeFolders()
+        }
+
+        await prepareStorybook(config, capabilities, this.#options, this.folders)
+    }
+
+    async onComplete() {
+        cleanupStorybook()
+
+        if (this.#options.createJsonReportFiles) {
+            new generateVisualReport(
+                { directoryPath: this.folders.actualFolder }
+            ).generate()
+        }
+    }
+}

packages/visual-service/src/storybook/hooks.ts

@@ -0,0 +1,129 @@
+import { rmdirSync } from 'node:fs'
+import logger from '@wdio/logger'
+import { SevereServiceError } from 'webdriverio'
+import type { Capabilities } from '@wdio/types'
+import type { ClassOptions, CheckElementMethodOptions, Folders } from '@wdio/image-comparison-core'
+import {
+    createStorybookCapabilities,
+    createTestFiles,
+    getArgvValue,
+    isCucumberFramework,
+    isStorybookMode,
+    parseSkipStories,
+    scanStorybook,
+} from './utils.js'
+import { CLIP_SELECTOR, NUM_SHARDS, V6_CLIP_SELECTOR } from '../constants.js'
+
+const log = logger('@wdio/visual-service')
+
+export async function prepareStorybook(
+    config: WebdriverIO.Config,
+    capabilities: Capabilities.TestrunnerCapabilities,
+    options: ClassOptions,
+    folders: Folders,
+): Promise<void> {
+    const isStorybook = isStorybookMode()
+    const framework = config.framework as string
+    const isCucumber = isCucumberFramework(framework)
+
+    if (isCucumber && isStorybook) {
+        throw new SevereServiceError('\n\nRunning Storybook in combination with the cucumber framework adapter is not supported.\nOnly Jasmine and Mocha are supported.\n\n')
+    }
+
+    if (!isStorybook) {
+        return
+    }
+
+    log.info('Running `@wdio/visual-service` in Storybook mode.')
+
+    const { storiesJson, storybookUrl, tempDir } = await scanStorybook(config, options)
+    process.env.VISUAL_STORYBOOK_TEMP_SPEC_FOLDER = tempDir
+    process.env.VISUAL_STORYBOOK_URL = storybookUrl
+
+    if (typeof capabilities === 'object' && !Array.isArray(capabilities)) {
+        throw new SevereServiceError('\n\nRunning Storybook in combination with Multiremote is not supported.\nRemove your `capabilities` property from your config or assign an empty array to it like `capabilities: [],`.\n\n')
+    }
+
+    capabilities.length = 0
+    log.info('Clearing the current capabilities.')
+
+    const compareOptions: CheckElementMethodOptions = {
+        blockOutSideBar: options.blockOutSideBar,
+        blockOutStatusBar: options.blockOutStatusBar,
+        blockOutToolBar: options.blockOutToolBar,
+        ignoreAlpha: options.ignoreAlpha,
+        ignoreAntialiasing: options.ignoreAntialiasing,
+        ignoreColors: options.ignoreColors,
+        ignoreLess: options.ignoreLess,
+        ignoreNothing: options.ignoreNothing,
+        rawMisMatchPercentage: options.rawMisMatchPercentage,
+        returnAllCompareData: options.returnAllCompareData,
+        saveAboveTolerance: options.saveAboveTolerance,
+        scaleImagesToSameSize: options.scaleImagesToSameSize,
+    }
+
+    // --version
+    const versionOption = options?.storybook?.version
+    const versionArgv = getArgvValue('--version', value => Math.floor(parseFloat(value)))
+    const version = versionOption ?? versionArgv ?? 7
+    // --numShards
+    const maxInstances = config?.maxInstances ?? 1
+    const numShardsOption = options?.storybook?.numShards
+    const numShardsArgv = getArgvValue('--numShards', value => parseInt(value, 10))
+    const numShards = Math.min(numShardsOption || numShardsArgv || NUM_SHARDS, maxInstances)
+    // --clip
+    const clipOption = options?.storybook?.clip
+    const clipArgv = getArgvValue('--clip', value => value !== 'false')
+    const clip = clipOption ?? clipArgv ?? true
+    // --clipSelector
+    const clipSelectorOption = options?.storybook?.clipSelector
+    const clipSelectorArgv = getArgvValue('--clipSelector', value => value)
+    const clipSelector = (clipSelectorOption ?? clipSelectorArgv) ?? (version === 6 ? V6_CLIP_SELECTOR : CLIP_SELECTOR)
+    process.env.VISUAL_STORYBOOK_CLIP_SELECTOR = clipSelector
+    // --skipStories
+    const skipStoriesOption = options?.storybook?.skipStories
+    const skipStoriesArgv = getArgvValue('--skipStories', value => value)
+    const skipStories = skipStoriesOption ?? skipStoriesArgv ?? []
+    const parsedSkipStories = parseSkipStories(skipStories)
+    // --additionalSearchParams
+    const additionalSearchParamsOption = options?.storybook?.additionalSearchParams
+    const additionalSearchParamsArgv = getArgvValue('--additionalSearchParams', value => new URLSearchParams(value))
+    const additionalSearchParams = additionalSearchParamsOption ?? additionalSearchParamsArgv ?? new URLSearchParams()
+    const getStoriesBaselinePath = options?.storybook?.getStoriesBaselinePath
+
+    createTestFiles({
+        additionalSearchParams,
+        clip,
+        clipSelector,
+        compareOptions,
+        directoryPath: tempDir,
+        folders,
+        framework,
+        getStoriesBaselinePath,
+        numShards,
+        skipStories: parsedSkipStories,
+        storiesJson,
+        storybookUrl,
+    })
+
+    createStorybookCapabilities(capabilities as WebdriverIO.Capabilities[])
+}
+
+export function cleanupStorybook(): void {
+    const tempDir = process.env.VISUAL_STORYBOOK_TEMP_SPEC_FOLDER
+    if (!tempDir) {
+        return
+    }
+
+    log.info(`Cleaning up temporary folder for storybook specs: ${tempDir}`)
+    try {
+        rmdirSync(tempDir, { recursive: true })
+        log.info(`Temporary folder for storybook specs has been removed: ${tempDir}`)
+    } catch (err) {
+        log.error(`Failed to remove temporary folder for storybook specs: ${tempDir} due to: ${(err as Error).message}`)
+    }
+
+    delete process.env.VISUAL_STORYBOOK_TEMP_SPEC_FOLDER
+    delete process.env.VISUAL_STORYBOOK_URL
+    delete process.env.VISUAL_STORYBOOK_CLIP_SELECTOR
+}