@@ -40,6 +40,51 @@ export const extractTitle = (html: string): string | undefined => {
4040 return match ?. [ 1 ] ?. trim ( ) || undefined
4141}
4242
43+ /** Download a single asset ref, returning true if successful */
44+ const downloadAsset = async (
45+ ref : string ,
46+ baseURL : URL ,
47+ srcDir : string ,
48+ downloaded : Set < string > ,
49+ content : Map < string , string > ,
50+ optional : boolean ,
51+ ) : Promise < boolean > => {
52+ if ( downloaded . has ( ref ) ) return false
53+ if ( ref . startsWith ( "data:" ) || ref . startsWith ( "#" ) || ref . startsWith ( "javascript:" ) ) return false
54+
55+ try {
56+ const assetURL = new URL ( ref , baseURL )
57+ // Only download same-origin assets
58+ if ( assetURL . origin !== baseURL . origin ) return false
59+
60+ const assetPath = assetURL . pathname . replace ( / ^ \/ / , "" )
61+ if ( ! assetPath ) return false
62+ const localPath = path . join ( srcDir , assetPath )
63+
64+ fs . mkdirSync ( path . dirname ( localPath ) , { recursive : true } )
65+
66+ const assetResponse = await fetchPage ( assetURL . href )
67+ if ( ! assetResponse . ok ) {
68+ if ( optional ) console . warn ( ` Warning: could not download optional asset ${ ref } (${ assetResponse . status } )` )
69+ return false
70+ }
71+
72+ const ab = await assetResponse . arrayBuffer ( )
73+ fs . writeFileSync ( localPath , new Uint8Array ( ab ) )
74+
75+ // Cache text content for JS/CSS so we can scan them for more refs
76+ if ( / \. ( j s | c s s | m j s ) ( \? .* ) ? $ / i. test ( assetPath ) ) {
77+ content . set ( ref , Buffer . from ( ab ) . toString ( "utf-8" ) )
78+ }
79+
80+ downloaded . add ( ref )
81+ return true
82+ } catch {
83+ if ( optional ) console . warn ( ` Warning: failed to download optional asset ${ ref } ` )
84+ return false
85+ }
86+ }
87+
4388/** Downloads an HTML page and its referenced assets into a src/ directory */
4489export const downloadPage = async ( url : string , outputDir : string ) : Promise < { title ?: string } > => {
4590 const srcDir = path . join ( outputDir , "src" )
@@ -52,36 +97,30 @@ export const downloadPage = async (url: string, outputDir: string): Promise<{ ti
5297
5398 const baseURL = new URL ( url )
5499 const downloaded = new Set < string > ( )
100+ const assetContent = new Map < string , string > ( )
55101
56102 // Find referenced assets (src, href, url())
57103 const assetRefs = extractAssetRefs ( html )
58104
59105 for ( const ref of assetRefs ) {
60- if ( downloaded . has ( ref ) ) continue
61- if ( ref . startsWith ( "data:" ) || ref . startsWith ( "#" ) || ref . startsWith ( "javascript:" ) ) continue
62-
63- try {
64- const assetURL = new URL ( ref , baseURL )
65- // Only download same-origin assets
66- if ( assetURL . origin !== baseURL . origin ) continue
67-
68- const assetPath = assetURL . pathname . replace ( / ^ \/ / , "" )
69- if ( ! assetPath ) continue
70- const localPath = path . join ( srcDir , assetPath )
71-
72- fs . mkdirSync ( path . dirname ( localPath ) , { recursive : true } )
73-
74- const assetResponse = await fetchPage ( assetURL . href )
75- if ( ! assetResponse . ok ) continue
106+ const ok = await downloadAsset ( ref , baseURL , srcDir , downloaded , assetContent , false )
107+ if ( ok ) html = html . replaceAll ( ref , new URL ( ref , baseURL ) . pathname . replace ( / ^ \/ / , "" ) )
108+ }
76109
77- const ab = await assetResponse . arrayBuffer ( )
78- fs . writeFileSync ( localPath , new Uint8Array ( ab ) )
110+ // Second pass: scan downloaded JS/CSS files for string literals referencing
111+ // paths with known asset extensions (images, fonts, audio, video).
112+ // These are optional — the game may still work without them.
113+ const jsAssetRefs = new Set < string > ( )
114+ for ( const [ , text ] of assetContent ) {
115+ for ( const ref of extractAssetFileRefs ( text ) ) {
116+ if ( ! downloaded . has ( ref ) ) jsAssetRefs . add ( ref )
117+ }
118+ }
79119
80- // Rewrite reference in HTML to local path
81- html = html . replaceAll ( ref , assetPath )
82- downloaded . add ( ref )
83- } catch {
84- // Skip assets that fail to download
120+ if ( jsAssetRefs . size > 0 ) {
121+ console . log ( ` Found ${ jsAssetRefs . size } additional asset reference${ jsAssetRefs . size === 1 ? "" : "s" } in JS/CSS files` )
122+ for ( const ref of jsAssetRefs ) {
123+ await downloadAsset ( ref , baseURL , srcDir , downloaded , assetContent , true )
85124 }
86125 }
87126
@@ -112,3 +151,18 @@ const extractAssetRefs = (html: string): string[] => {
112151
113152 return [ ...new Set ( refs ) ]
114153}
154+
155+ /** Known asset file extensions to look for in JS/CSS string literals */
156+ const ASSET_EXTENSIONS = "webp|png|jpe?g|gif|svg|ico|mp3|ogg|wav|mp4|webm|woff2?|ttf|eot"
157+
158+ /** Extracts paths with known asset extensions from JS/CSS string literals */
159+ const extractAssetFileRefs = ( source : string ) : string [ ] => {
160+ const refs : string [ ] = [ ]
161+
162+ const regex = new RegExp ( `["'\`](/[^"'\`\\s]*\\.(?:${ ASSET_EXTENSIONS } ))(?:\\?[^"'\`\\s]*)?["'\`]` , "gi" )
163+ let match
164+ while ( ( match = regex . exec ( source ) ) !== null ) {
165+ if ( match [ 1 ] ) refs . push ( match [ 1 ] )
166+ }
167+ return [ ...new Set ( refs ) ]
168+ }
0 commit comments