feat(content): add includeShadow option to page.content()

docs/src/api/class-frame.md

@@ -299,6 +299,9 @@ When all steps combined have not finished during the specified [`option: timeout
 
 Gets the full HTML contents of the frame, including the doctype.
 
+### option: Frame.content.includeShadow = %%-content-option-include-shadow-%%
+* since: v1.64
+
 ## async method: Frame.dblclick
 * since: v1.8
 * discouraged: Use locator-based [`method: Locator.dblclick`] instead. Read more about [locators](../locators.md).

docs/src/api/class-page.md

@@ -866,6 +866,9 @@ Defaults to `false`. Whether to run the
 
 Gets the full HTML contents of the page, including the doctype.
 
+### option: Page.content.includeShadow = %%-content-option-include-shadow-%%
+* since: v1.64
+
 ## method: Page.context
 * since: v1.8
 - returns: <[BrowserContext]>

docs/src/api/params.md

@@ -7,6 +7,12 @@ When to consider operation succeeded, defaults to `load`. Events can be either:
 * `'networkidle'` - **DISCOURAGED** consider operation to be finished when there are no network connections for at least `500` ms. Don't use this method for testing, rely on web assertions to assess readiness instead.
 * `'commit'` - consider operation to be finished when network response is received and the document started loading.
 
+## content-option-include-shadow
+- `includeShadow` <[boolean]>
+
+When true, contents of open shadow roots are included as [declarative shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM#declaratively_with_html),
+i.e. `<template shadowrootmode="open">` elements nested inside their host elements. Closed shadow roots are never included. Defaults to `false`.
+
 ## navigation-timeout
 * langs: python, java, csharp
 - `timeout` <[float]>

packages/injected/src/injectedScript.ts

@@ -344,6 +344,31 @@ export class InjectedScript {
     return renderAriaSnapshotAsYaml(json);
   }
 
+  documentContent(includeShadow: boolean): string {
+    let content = '';
+    if (this.document.doctype)
+      content = new XMLSerializer().serializeToString(this.document.doctype);
+    const root = this.document.documentElement;
+    if (!root)
+      return content;
+    if (!includeShadow)
+      return content + root.outerHTML;
+    const shadowRoots: ShadowRoot[] = [];
+    const collectShadowRoots = (node: Document | ShadowRoot) => {
+      for (const element of node.querySelectorAll('*')) {
+        if (element.shadowRoot) {
+          shadowRoots.push(element.shadowRoot);
+          collectShadowRoots(element.shadowRoot);
+        }
+      }
+    };
+    collectShadowRoots(this.document);
+    // getHTML() serializes children only, wrap them with the root element tags.
+    const emptyRoot = (root.cloneNode(false) as Element).outerHTML;
+    const endTagIndex = emptyRoot.lastIndexOf('</');
+    return content + emptyRoot.slice(0, endTagIndex) + root.getHTML({ shadowRoots }) + emptyRoot.slice(endTagIndex);
+  }
+
   getAllElementsMatchingExpectAriaTemplate(document: Document, template: AriaTemplateNode): Element[] {
     return getAllElementsMatchingExpectAriaTemplate(document.documentElement, template);
   }

packages/playwright-client/types/types.d.ts

@@ -2458,8 +2458,17 @@ export interface Page {
 
   /**
    * Gets the full HTML contents of the page, including the doctype.
+   * @param options
    */
-  content(): Promise<string>;
+  content(options?: {
+    /**
+     * When true, contents of open shadow roots are included as
+     * [declarative shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM#declaratively_with_html),
+     * i.e. `<template shadowrootmode="open">` elements nested inside their host elements. Closed shadow roots are never
+     * included. Defaults to `false`.
+     */
+    includeShadow?: boolean;
+  }): Promise<string>;
 
   /**
    * Get the browser context that the page belongs to.
@@ -6945,8 +6954,17 @@ export interface Frame {
 
   /**
    * Gets the full HTML contents of the frame, including the doctype.
+   * @param options
    */
-  content(): Promise<string>;
+  content(options?: {
+    /**
+     * When true, contents of open shadow roots are included as
+     * [declarative shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM#declaratively_with_html),
+     * i.e. `<template shadowrootmode="open">` elements nested inside their host elements. Closed shadow roots are never
+     * included. Defaults to `false`.
+     */
+    includeShadow?: boolean;
+  }): Promise<string>;
 
   /**
    * **NOTE** Use locator-based [locator.dblclick([options])](https://playwright.dev/docs/api/class-locator#locator-dblclick)

packages/playwright-core/src/client/channels.d.ts

@@ -2449,8 +2449,12 @@ export type FrameClickOptions = {
   steps?: number,
 };
 export type FrameClickResult = void;
-export type FrameContentParams = {};
-export type FrameContentOptions = {};
+export type FrameContentParams = {
+  includeShadow?: boolean,
+};
+export type FrameContentOptions = {
+  includeShadow?: boolean,
+};
 export type FrameContentResult = {
   value: string,
 };

packages/playwright-core/src/client/frame.ts

@@ -272,8 +272,8 @@ export class Frame extends ChannelOwner<channels.FrameChannel> implements api.Fr
     return (await this._channel.queryCount({ selector }, kNoTimeout)).value;
   }
 
-  async content(): Promise<string> {
-    return (await this._channel.content({}, kNoTimeout)).value;
+  async content(options: channels.FrameContentOptions = {}): Promise<string> {
+    return (await this._channel.content({ ...options }, kNoTimeout)).value;
   }
 
   async setContent(html: string, options: channels.FrameSetContentOptions & TimeoutOptions = {}): Promise<void> {

packages/playwright-core/src/client/page.ts

@@ -407,8 +407,8 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
     return this._mainFrame.url();
   }
 
-  async content(): Promise<string> {
-    return await this._mainFrame.content();
+  async content(options?: channels.FrameContentOptions): Promise<string> {
+    return await this._mainFrame.content(options);
   }
 
   async setContent(html: string, options?: channels.FrameSetContentOptions & TimeoutOptions): Promise<void> {

packages/playwright-core/src/server/channels.d.ts

@@ -2450,8 +2450,12 @@ export type FrameClickOptions = {
   steps?: number,
 };
 export type FrameClickResult = void;
-export type FrameContentParams = {};
-export type FrameContentOptions = {};
+export type FrameContentParams = {
+  includeShadow?: boolean,
+};
+export type FrameContentOptions = {
+  includeShadow?: boolean,
+};
 export type FrameContentResult = {
   value: string,
 };

packages/playwright-core/src/server/dispatchers/frameDispatcher.ts

@@ -119,7 +119,7 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Br
   }
 
   async content(params: channels.FrameContentParams, progress: Progress): Promise<channels.FrameContentResult> {
-    return { value: await this._frame.content(progress) };
+    return { value: await this._frame.content(progress, params) };
   }
 
   async setContent(params: channels.FrameSetContentParams, progress: Progress): Promise<void> {

packages/playwright-core/src/server/frames.ts

@@ -939,21 +939,15 @@ export class Frame extends SdkObject<FrameEventMap> {
     }
   }
 
-  async content(progress: Progress): Promise<string> {
-    return progress.race(this._content());
+  async content(progress: Progress, options: channels.FrameContentParams): Promise<string> {
+    return progress.race(this._content(options));
   }
 
-  private async _content(): Promise<string> {
+  private async _content(options: channels.FrameContentParams): Promise<string> {
     try {
       const context = await this.utilityContext();
-      return await context.evaluate(() => {
-        let retVal = '';
-        if (document.doctype)
-          retVal = new XMLSerializer().serializeToString(document.doctype);
-        if (document.documentElement)
-          retVal += document.documentElement.outerHTML;
-        return retVal;
-      });
+      const injected = await context.injectedScript();
+      return await injected.evaluate((injected, includeShadow) => injected.documentContent(includeShadow), !!options.includeShadow);
     } catch (e) {
       if (this.isNonRetriableError(e))
         throw e;

packages/playwright-core/types/types.d.ts

@@ -2458,8 +2458,17 @@ export interface Page {
 
   /**
    * Gets the full HTML contents of the page, including the doctype.
+   * @param options
    */
-  content(): Promise<string>;
+  content(options?: {
+    /**
+     * When true, contents of open shadow roots are included as
+     * [declarative shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM#declaratively_with_html),
+     * i.e. `<template shadowrootmode="open">` elements nested inside their host elements. Closed shadow roots are never
+     * included. Defaults to `false`.
+     */
+    includeShadow?: boolean;
+  }): Promise<string>;
 
   /**
    * Get the browser context that the page belongs to.
@@ -6945,8 +6954,17 @@ export interface Frame {
 
   /**
    * Gets the full HTML contents of the frame, including the doctype.
+   * @param options
    */
-  content(): Promise<string>;
+  content(options?: {
+    /**
+     * When true, contents of open shadow roots are included as
+     * [declarative shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM#declaratively_with_html),
+     * i.e. `<template shadowrootmode="open">` elements nested inside their host elements. Closed shadow roots are never
+     * included. Defaults to `false`.
+     */
+    includeShadow?: boolean;
+  }): Promise<string>;
 
   /**
    * **NOTE** Use locator-based [locator.dblclick([options])](https://playwright.dev/docs/api/class-locator#locator-dblclick)

packages/protocol/spec/frame.yml

@@ -205,6 +205,8 @@ Frame:
 
     content:
       title: Get content
+      parameters:
+        includeShadow: boolean?
       returns:
         value: string
       flags:

packages/protocol/src/validator.ts

@@ -1328,7 +1328,9 @@ scheme.FrameClickParams = tObject({
   steps: tOptional(tInt),
 });
 scheme.FrameClickResult = tOptional(tObject({}));
-scheme.FrameContentParams = tOptional(tObject({}));
+scheme.FrameContentParams = tObject({
+  includeShadow: tOptional(tBoolean),
+});
 scheme.FrameContentResult = tObject({
   value: tString,
 });

tests/page/page-set-content.spec.ts

@@ -54,6 +54,14 @@ it('should work with HTML 4 doctype', async ({ page, server }) => {
   expect(result).toBe(`${doctype}${expectedOutput}`);
 });
 
+it('should include shadow roots', async ({ page }) => {
+  const html = '<!DOCTYPE html><html lang="en"><head></head><body><div id="host"><template shadowrootmode="open"><div id="nested"><template shadowrootmode="open"><span>nested</span></template><slot></slot></div></template><span>light</span></div><div id="closed"></div></body></html>';
+  // Closed shadow roots are not accessible from script and are never serialized.
+  await page.setContent(html.replace('<div id="closed">', '<div id="closed"><template shadowrootmode="closed"><span>closed</span></template>'));
+  expect(await page.content()).toBe('<!DOCTYPE html><html lang="en"><head></head><body><div id="host"><span>light</span></div><div id="closed"></div></body></html>');
+  expect(await page.content({ includeShadow: true })).toBe(html);
+});
+
 it('should respect timeout', async ({ page, server, playwright }) => {
   const imgPath = '/img.png';
   // stall for image