diff --git a/docker-compose.yml b/docker-compose.yml index a5c69d7..34713cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ services: app: image: 254201323850.dkr.ecr.ap-northeast-2.amazonaws.com/teaching-app:latest container_name: teaching-app + init: true ports: - "8080:8080" environment: diff --git a/src/main/java/com/teaching/backend/domain/material/exception/MaterialErrorCode.java b/src/main/java/com/teaching/backend/domain/material/exception/MaterialErrorCode.java index d5d712e..fb4c174 100644 --- a/src/main/java/com/teaching/backend/domain/material/exception/MaterialErrorCode.java +++ b/src/main/java/com/teaching/backend/domain/material/exception/MaterialErrorCode.java @@ -22,6 +22,7 @@ public enum MaterialErrorCode implements BaseErrorCode { CONTENT_REQUIRED(HttpStatus.BAD_REQUEST, "MATERIAL4007", "분석할 본문 내용을 입력해주세요."), UNSUPPORTED_MATERIAL_PLATFORM(HttpStatus.BAD_REQUEST, "MATERIAL4008", "지원하지 않는 자료 플랫폼입니다."), MATERIAL_ACCESS_DENIED(HttpStatus.FORBIDDEN, "MATERIAL4031", "해당 자료에 접근할 권한이 없습니다."), + MATERIAL_SOURCE_AUTH_REQUIRED(HttpStatus.FORBIDDEN, "MATERIAL4032", "로그인 또는 회원가입이 필요한 페이지는 분석할 수 없습니다."), MATERIAL_NOT_FOUND(HttpStatus.NOT_FOUND, "MATERIAL4041", "자료를 찾을 수 없습니다."), MATERIAL_ANALYSIS_NOT_FOUND(HttpStatus.NOT_FOUND, "MATERIAL4042", "자료의 AI 분석 결과를 찾을 수 없습니다."), MATERIAL_NOT_IN_TRASH(HttpStatus.CONFLICT, "MATERIAL4093", "휴지통에 있는 자료가 아닙니다."), diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/AbstractHtmlMaterialContentExtractor.java b/src/main/java/com/teaching/backend/domain/material/service/extract/AbstractHtmlMaterialContentExtractor.java index e54758a..a77e4ca 100644 --- a/src/main/java/com/teaching/backend/domain/material/service/extract/AbstractHtmlMaterialContentExtractor.java +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/AbstractHtmlMaterialContentExtractor.java @@ -117,6 +117,12 @@ protected ParsedHtmlContent parseDocument(String originalUrl, HtmlDocument docum } protected void validateDocument(HtmlDocument document) { + if (document == null) { + return; + } + if (ProtectedSourceDetector.isProtectedHtml(document.body(), document.originalUrl())) { + throw new MaterialException(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } } protected List contentClassSignals() { diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/CafeMaterialContentExtractor.java b/src/main/java/com/teaching/backend/domain/material/service/extract/CafeMaterialContentExtractor.java index 8cbaee4..8fbd945 100644 --- a/src/main/java/com/teaching/backend/domain/material/service/extract/CafeMaterialContentExtractor.java +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/CafeMaterialContentExtractor.java @@ -33,6 +33,10 @@ public CafeMaterialContentExtractor(ExternalHtmlDocumentClient htmlDocumentClien @Override protected void validateDocument(HtmlDocument document) { + super.validateDocument(document); + if (document == null) { + return; + } String body = document.body(); if (body == null || body.isBlank()) { return; @@ -44,7 +48,7 @@ protected void validateDocument(HtmlDocument document) { safeUrl(document.originalUrl()), body.length() ); - throw new MaterialException(MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED); + throw new MaterialException(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); } } diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClient.java b/src/main/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClient.java index a2e27fc..c21168a 100644 --- a/src/main/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClient.java +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClient.java @@ -105,6 +105,13 @@ public HtmlDocument fetch(String originalUrl) { .exchangeToMono(response -> { HttpStatusCode statusCode = response.statusCode(); if (!statusCode.is2xxSuccessful()) { + if (isAuthenticationRequiredStatus(statusCode)) { + logExtractionFailure("fetch-auth-required", host, statusCode, null, null); + return response.releaseBody() + .then(Mono.error(new MaterialException( + MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED + ))); + } return response.bodyToMono(String.class) .defaultIfEmpty("") .flatMap(body -> extractionFailed( @@ -158,6 +165,11 @@ public HtmlDocument fetch(String originalUrl) { false ); } + if (ProtectedSourceDetector.isProtectedHtml(body, originalUrl)) { + throw new MaterialException( + MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED + ); + } return new HtmlDocument(originalUrl, body, contentType.toString()); }); }) @@ -311,6 +323,13 @@ private boolean isRecoverableStatus(HttpStatusCode statusCode) { return statusCode != null && statusCode.is5xxServerError(); } + private boolean isAuthenticationRequiredStatus(HttpStatusCode statusCode) { + if (statusCode == null) { + return false; + } + return statusCode.value() == 401; + } + private String normalizeHost(String host) { String normalizedHost = host.toLowerCase(Locale.ROOT); if (normalizedHost.startsWith("[") && normalizedHost.endsWith("]")) { diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/GenericWebMaterialContentExtractor.java b/src/main/java/com/teaching/backend/domain/material/service/extract/GenericWebMaterialContentExtractor.java index ff02eb5..1fb5042 100644 --- a/src/main/java/com/teaching/backend/domain/material/service/extract/GenericWebMaterialContentExtractor.java +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/GenericWebMaterialContentExtractor.java @@ -78,6 +78,7 @@ private List contentClassSignals() { } private ParsedHtmlContent parse(String originalUrl, HtmlDocument document) { + validateDocument(document); return htmlContentParser.parse( originalUrl, document.body(), @@ -85,6 +86,12 @@ private ParsedHtmlContent parse(String originalUrl, HtmlDocument document) { ); } + private void validateDocument(HtmlDocument document) { + if (document != null && ProtectedSourceDetector.isProtectedHtml(document.body(), document.originalUrl())) { + throw new MaterialException(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } + } + private boolean hasSufficientContent(ParsedHtmlContent parsed) { return parsed != null && parsed.content() != null diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/NotionMaterialContentExtractor.java b/src/main/java/com/teaching/backend/domain/material/service/extract/NotionMaterialContentExtractor.java index 0eeed7b..6f1ad50 100644 --- a/src/main/java/com/teaching/backend/domain/material/service/extract/NotionMaterialContentExtractor.java +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/NotionMaterialContentExtractor.java @@ -25,9 +25,16 @@ public NotionMaterialContentExtractor(ExternalHtmlDocumentClient htmlDocumentCli @Override protected void validateDocument(HtmlDocument document) { + super.validateDocument(document); + if (document == null) { + return; + } String body = document.body(); + if (body == null || body.isBlank()) { + return; + } if (body.contains("This page is private") || body.contains("You do not have access")) { - throw new MaterialException(MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED); + throw new MaterialException(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); } } diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/ProtectedSourceDetector.java b/src/main/java/com/teaching/backend/domain/material/service/extract/ProtectedSourceDetector.java new file mode 100644 index 0000000..c0e149e --- /dev/null +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/ProtectedSourceDetector.java @@ -0,0 +1,247 @@ +package com.teaching.backend.domain.material.service.extract; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import java.net.URI; +import java.util.List; +import java.util.Locale; + +final class ProtectedSourceDetector { + + private static final List AUTH_HOSTS = List.of( + "accounts.google.com", + "login.microsoftonline.com", + "login.live.com", + "appleid.apple.com" + ); + private static final List AUTH_TEXT_SIGNALS = List.of( + "login", + "log in", + "sign in", + "signin", + "sign up", + "signup", + "register", + "로그인", + "회원가입" + ); + private static final List ACCESS_DENIED_SIGNALS = List.of( + "access denied", + "unauthorized", + "you do not have access", + "this page is private", + "권한이 없습니다" + ); + + private ProtectedSourceDetector() { + } + + static boolean isAuthenticationUrl(String url) { + try { + URI uri = URI.create(url); + String host = normalize(uri.getHost()); + String path = normalize(uri.getPath()); + if (host.isBlank()) { + return false; + } + if (isKnownAuthHost(host)) { + return true; + } + return hasAuthPathSegment(path); + } catch (RuntimeException e) { + return false; + } + } + + static boolean isProtectedHtml(String html, String currentUrl) { + return isProtectedHtml(html, currentUrl, currentUrl); + } + + static boolean isProtectedHtml(String html, String currentUrl, String originalUrl) { + if (html == null || html.isBlank()) { + return false; + } + if (isAuthenticationUrl(currentUrl)) { + return true; + } + + Document document = Jsoup.parse(html, currentUrl == null ? "" : currentUrl); + if (hasNotionPrivateSignal(document)) { + return true; + } + if (isNotionAppPrivateShell(document, originalUrl, currentUrl)) { + return true; + } + if (hasAccessDeniedHeading(document)) { + return true; + } + + boolean hasPasswordInput = !document.select("input[type=password]").isEmpty(); + if (!hasPasswordInput) { + return false; + } + + return hasAuthTitleOrHeading(document) || hasAuthFormSignal(document); + } + + static boolean hasNotionPrivateSignal(String html) { + if (html == null || html.isBlank()) { + return false; + } + return hasNotionPrivateSignal(Jsoup.parse(html)); + } + + private static boolean isKnownAuthHost(String host) { + return AUTH_HOSTS.stream().anyMatch(authHost -> host.equals(authHost) || host.endsWith("." + authHost)); + } + + private static boolean hasAuthPathSegment(String path) { + if (path.isBlank()) { + return false; + } + String[] segments = path.split("/+"); + for (int i = 0; i < segments.length; i++) { + String segment = segments[i]; + if (segment.isBlank()) { + continue; + } + if (segment.equals("login") || segment.equals("signin") || segment.equals("sign-in")) { + return true; + } + if ((segment.equals("auth") || segment.equals("oauth") || segment.equals("sso")) && isFirstPathSegment(segments, i)) { + return true; + } + } + return false; + } + + private static boolean isFirstPathSegment(String[] segments, int index) { + for (int i = 0; i < index; i++) { + if (!segments[i].isBlank()) { + return false; + } + } + return true; + } + + private static boolean hasAuthTitleOrHeading(Document document) { + String title = normalizeText(document.title()); + if (hasAuthTextSignal(title)) { + return true; + } + for (Element heading : document.select("h1, h2, h3, [role=heading]")) { + if (hasAuthTextSignal(normalizeText(heading.text()))) { + return true; + } + } + return false; + } + + private static boolean hasAuthFormSignal(Document document) { + for (Element form : document.select("form")) { + String signalSource = normalizeText(String.join(" ", + form.attr("action"), + form.id(), + form.className(), + form.attr("aria-label"), + form.attr("name") + )); + if (hasAuthTextSignal(signalSource) || hasAuthPathSegment(signalSource.replace(' ', '/'))) { + return true; + } + } + return false; + } + + private static boolean hasNotionPrivateSignal(Document document) { + String text = normalizeText(document.text()); + return text.contains("this page is private") || text.contains("you do not have access"); + } + + private static boolean isNotionAppPrivateShell(Document document, String originalUrl, String currentUrl) { + if (!isNotionAppDocumentUrl(originalUrl) && !isNotionAppDocumentUrl(currentUrl)) { + return false; + } + if (hasNotionDocumentContentSignal(document)) { + return false; + } + + String title = normalizeText(document.title()); + String text = normalizeText(document.text()); + boolean marketingTitle = title.contains("notion") + && title.contains("where teams and agents work together"); + boolean shellAuthSignal = hasNotionShellAuthSignal(document, text); + + return marketingTitle && shellAuthSignal; + } + + private static boolean isNotionAppDocumentUrl(String url) { + try { + URI uri = URI.create(url); + String host = normalize(uri.getHost()); + if (!host.equals("app.notion.com")) { + return false; + } + String[] segments = normalize(uri.getPath()).split("/+"); + for (String segment : segments) { + if (!segment.isBlank()) { + return segment.equals("p"); + } + } + return false; + } catch (RuntimeException e) { + return false; + } + } + + private static boolean hasNotionDocumentContentSignal(Document document) { + return document.selectFirst(".notion-page-content, .notion-page-block, [data-block-id], [data-content-editable-leaf]") != null; + } + + private static boolean hasNotionShellAuthSignal(Document document, String text) { + if (text.contains("log in") || text.contains("sign up") || text.contains("get notion free")) { + return true; + } + for (Element element : document.select("a[href], button, [role=button]")) { + String signalSource = normalizeText(String.join(" ", + element.attr("href"), + element.attr("aria-label"), + element.text() + )); + if (signalSource.contains("log in") + || signalSource.contains("login") + || signalSource.contains("sign in") + || signalSource.contains("sign up")) { + return true; + } + } + return false; + } + + private static boolean hasAccessDeniedHeading(Document document) { + for (Element heading : document.select("title, h1, h2, h3, [role=heading]")) { + String text = normalizeText(heading.text()); + if (ACCESS_DENIED_SIGNALS.stream().anyMatch(text::equals)) { + return true; + } + } + return false; + } + + private static boolean hasAuthTextSignal(String value) { + if (value == null || value.isBlank()) { + return false; + } + return AUTH_TEXT_SIGNALS.stream().anyMatch(value::contains); + } + + private static String normalize(String value) { + return value == null ? "" : value.toLowerCase(Locale.ROOT); + } + + private static String normalizeText(String value) { + return normalize(value).replaceAll("[\\r\\n\\t]+", " ").replaceAll("\\s+", " ").trim(); + } +} diff --git a/src/main/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClient.java b/src/main/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClient.java index 5fc3b71..6d8d528 100644 --- a/src/main/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClient.java +++ b/src/main/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClient.java @@ -1,8 +1,11 @@ package com.teaching.backend.domain.material.service.extract; +import com.teaching.backend.domain.material.exception.MaterialErrorCode; +import com.teaching.backend.domain.material.exception.MaterialException; import lombok.extern.slf4j.Slf4j; import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.PageLoadStrategy; +import org.openqa.selenium.ScriptTimeoutException; import org.openqa.selenium.TimeoutException; import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; @@ -28,6 +31,8 @@ public class RenderedHtmlDocumentClient { private final boolean enabled; private final Duration pageLoadTimeout; private final Duration scriptTimeout; + private final Duration notionPageLoadTimeout; + private final Duration notionScriptTimeout; private final Duration waitTimeout; private final int minVisibleTextLength; private final Duration acquireTimeout; @@ -40,6 +45,8 @@ public RenderedHtmlDocumentClient( @Value("${material.extract.rendered.enabled:true}") boolean enabled, @Value("${material.extract.rendered.page-load-timeout-ms:10000}") long pageLoadTimeoutMs, @Value("${material.extract.rendered.script-timeout-ms:5000}") long scriptTimeoutMs, + @Value("${material.extract.rendered.notion-page-load-timeout-ms:30000}") long notionPageLoadTimeoutMs, + @Value("${material.extract.rendered.notion-script-timeout-ms:10000}") long notionScriptTimeoutMs, @Value("${material.extract.rendered.wait-timeout-ms:7000}") long waitTimeoutMs, @Value("${material.extract.rendered.min-visible-text-length:20}") int minVisibleTextLength, @Value("${material.extract.rendered.max-concurrency:2}") int maxConcurrency, @@ -49,6 +56,8 @@ public RenderedHtmlDocumentClient( enabled, pageLoadTimeoutMs, scriptTimeoutMs, + notionPageLoadTimeoutMs, + notionScriptTimeoutMs, waitTimeoutMs, minVisibleTextLength, maxConcurrency, @@ -64,7 +73,7 @@ public RenderedHtmlDocumentClient( long waitTimeoutMs, int minVisibleTextLength ) { - this(enabled, pageLoadTimeoutMs, scriptTimeoutMs, waitTimeoutMs, minVisibleTextLength, 2, 500, null); + this(enabled, pageLoadTimeoutMs, scriptTimeoutMs, 30000, 10000, waitTimeoutMs, minVisibleTextLength, 2, 500, null); } RenderedHtmlDocumentClient( @@ -81,6 +90,34 @@ public RenderedHtmlDocumentClient( enabled, pageLoadTimeoutMs, scriptTimeoutMs, + 30000, + 10000, + waitTimeoutMs, + minVisibleTextLength, + maxConcurrency, + acquireTimeoutMs, + urlValidator + ); + } + + RenderedHtmlDocumentClient( + boolean enabled, + long pageLoadTimeoutMs, + long scriptTimeoutMs, + long notionPageLoadTimeoutMs, + long notionScriptTimeoutMs, + long waitTimeoutMs, + int minVisibleTextLength, + int maxConcurrency, + long acquireTimeoutMs, + ExternalHtmlDocumentClient urlValidator + ) { + this( + enabled, + pageLoadTimeoutMs, + scriptTimeoutMs, + notionPageLoadTimeoutMs, + notionScriptTimeoutMs, waitTimeoutMs, minVisibleTextLength, acquireTimeoutMs, @@ -98,10 +135,38 @@ public RenderedHtmlDocumentClient( long acquireTimeoutMs, ExternalHtmlDocumentClient urlValidator, Semaphore renderPermits + ) { + this( + enabled, + pageLoadTimeoutMs, + scriptTimeoutMs, + 30000, + 10000, + waitTimeoutMs, + minVisibleTextLength, + acquireTimeoutMs, + urlValidator, + renderPermits + ); + } + + RenderedHtmlDocumentClient( + boolean enabled, + long pageLoadTimeoutMs, + long scriptTimeoutMs, + long notionPageLoadTimeoutMs, + long notionScriptTimeoutMs, + long waitTimeoutMs, + int minVisibleTextLength, + long acquireTimeoutMs, + ExternalHtmlDocumentClient urlValidator, + Semaphore renderPermits ) { this.enabled = enabled; this.pageLoadTimeout = Duration.ofMillis(pageLoadTimeoutMs); this.scriptTimeout = Duration.ofMillis(scriptTimeoutMs); + this.notionPageLoadTimeout = Duration.ofMillis(notionPageLoadTimeoutMs); + this.notionScriptTimeout = Duration.ofMillis(notionScriptTimeoutMs); this.waitTimeout = Duration.ofMillis(waitTimeoutMs); this.minVisibleTextLength = minVisibleTextLength; this.acquireTimeout = Duration.ofMillis(Math.max(0L, acquireTimeoutMs)); @@ -119,7 +184,6 @@ public Optional render(String originalUrl) { return Optional.empty(); } - WebDriver driver = null; boolean permitAcquired = false; try { permitAcquired = acquirePermit(safeUrl); @@ -127,15 +191,84 @@ public Optional render(String originalUrl) { return Optional.empty(); } log.info("Rendered HTML fallback started. url={}", safeUrl); - driver = createDriver(); + boolean notionUrl = isNotionUrl(originalUrl); + RenderSettings settings = renderSettings(notionUrl); + int maxAttempts = notionUrl ? 2 : 1; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return renderAttempt(originalUrl, safeUrl, settings); + } catch (TimeoutException e) { + if (attempt >= maxAttempts) { + log.warn( + "Rendered HTML fallback failed. url={}, reason={}, message={}", + safeUrl, + e.getClass().getSimpleName(), + e.getMessage() + ); + return Optional.empty(); + } + log.warn( + "Rendered HTML fallback timed out; retrying with a new driver. url={}, attempt={}, reason={}, message={}", + safeUrl, + attempt, + e.getClass().getSimpleName(), + e.getMessage() + ); + } catch (MaterialException e) { + throw e; + } catch (RuntimeException e) { + if (!isRetryableTimeout(e)) { + throw e; + } + if (attempt >= maxAttempts) { + log.warn( + "Rendered HTML fallback failed. url={}, reason={}, message={}", + safeUrl, + e.getClass().getSimpleName(), + e.getMessage() + ); + return Optional.empty(); + } + log.warn( + "Rendered HTML fallback timed out; retrying with a new driver. url={}, attempt={}, reason={}, message={}", + safeUrl, + attempt, + e.getClass().getSimpleName(), + e.getMessage() + ); + } + } + return Optional.empty(); + } catch (MaterialException e) { + throw e; + } catch (RuntimeException e) { + log.warn( + "Rendered HTML fallback failed. url={}, reason={}, message={}", + safeUrl, + e.getClass().getSimpleName(), + e.getMessage() + ); + return Optional.empty(); + } finally { + if (permitAcquired) { + renderPermits.release(); + } + } + } + + private Optional renderAttempt(String originalUrl, String safeUrl, RenderSettings settings) { + WebDriver driver = null; + try { + driver = createDriver(settings.pageLoadStrategy()); log.debug("Rendered HTML driver created. url={}", safeUrl); - driver.manage().timeouts().pageLoadTimeout(pageLoadTimeout); - driver.manage().timeouts().scriptTimeout(scriptTimeout); + driver.manage().timeouts().pageLoadTimeout(settings.pageLoadTimeout()); + driver.manage().timeouts().scriptTimeout(settings.scriptTimeout()); driver.get(originalUrl); String currentUrl = driver.getCurrentUrl(); if (currentUrl == null || currentUrl.isBlank() || !isAllowedUrl(currentUrl, "final")) { return Optional.empty(); } + validatePublicSourceUrl(currentUrl); log.debug( "Rendered HTML page loaded. url={}, readyState={}, visibleTextLength={}", safeUrl, @@ -147,6 +280,7 @@ public Optional render(String originalUrl) { if (currentUrl == null || currentUrl.isBlank() || !isAllowedUrl(currentUrl, "final")) { return Optional.empty(); } + validatePublicSourceUrl(currentUrl); triggerNotionLazyLoad(driver, currentUrl); String pageSource = driver.getPageSource(); @@ -159,15 +293,8 @@ public Optional render(String originalUrl) { if (pageSource == null || pageSource.isBlank()) { return Optional.empty(); } + validatePublicSourceHtml(pageSource, currentUrl, originalUrl); return Optional.of(new HtmlDocument(originalUrl, pageSource, "text/html")); - } catch (RuntimeException e) { - log.warn( - "Rendered HTML fallback failed. url={}, reason={}, message={}", - safeUrl, - e.getClass().getSimpleName(), - e.getMessage() - ); - return Optional.empty(); } finally { if (driver != null) { try { @@ -181,15 +308,34 @@ public Optional render(String originalUrl) { ); } } - if (permitAcquired) { - renderPermits.release(); - } } } - protected WebDriver createDriver() { + private RenderSettings renderSettings(boolean notionUrl) { + return notionUrl + ? new RenderSettings(PageLoadStrategy.EAGER, notionPageLoadTimeout, notionScriptTimeout) + : new RenderSettings(PageLoadStrategy.NORMAL, pageLoadTimeout, scriptTimeout); + } + + private boolean isRetryableTimeout(RuntimeException exception) { + return exception instanceof TimeoutException || exception instanceof ScriptTimeoutException; + } + + private void validatePublicSourceUrl(String currentUrl) { + if (ProtectedSourceDetector.isAuthenticationUrl(currentUrl)) { + throw new MaterialException(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } + } + + private void validatePublicSourceHtml(String pageSource, String currentUrl, String originalUrl) { + if (ProtectedSourceDetector.isProtectedHtml(pageSource, currentUrl, originalUrl)) { + throw new MaterialException(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } + } + + protected WebDriver createDriver(PageLoadStrategy pageLoadStrategy) { ChromeOptions options = new ChromeOptions(); - options.setPageLoadStrategy(PageLoadStrategy.NORMAL); + options.setPageLoadStrategy(pageLoadStrategy); options.addArguments( "--headless=new", "--disable-gpu", @@ -200,6 +346,13 @@ protected WebDriver createDriver() { return new ChromeDriver(options); } + private record RenderSettings( + PageLoadStrategy pageLoadStrategy, + Duration pageLoadTimeout, + Duration scriptTimeout + ) { + } + private boolean acquirePermit(String safeUrl) { try { boolean acquired = acquireTimeout.isZero() diff --git a/src/test/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClientTest.java b/src/test/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClientTest.java index 86b52e4..809c490 100644 --- a/src/test/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClientTest.java +++ b/src/test/java/com/teaching/backend/domain/material/service/extract/ExternalHtmlDocumentClientTest.java @@ -92,6 +92,22 @@ void failsWhenStatusIs404() throws IOException { assertExtractionFailed(() -> client.fetch(url)); } + @Test + void rejectsUnauthorizedStatusAsSourceAuthRequired() throws IOException { + String url = startServer(401, "text/html", "login required", 0); + ExternalHtmlDocumentClient client = testClient(Duration.ofSeconds(2)); + + assertSourceAuthRequired(() -> client.fetch(url)); + } + + @Test + void treatsForbiddenStatusAsGenericExtractionFailure() throws IOException { + String url = startServer(403, "text/html", "access denied", 0); + ExternalHtmlDocumentClient client = testClient(Duration.ofSeconds(2)); + + assertExtractionFailed(() -> client.fetch(url)); + } + @Test void failsWhenStatusIs500() throws IOException { String url = startServer(500, "text/html", "error", 0); @@ -138,6 +154,53 @@ void failsWhenContentTypeIsPdf() throws IOException { assertExtractionFailed(() -> client.fetch(url)); } + @Test + void rejectsHtmlLoginFormAsSourceAuthRequired() throws IOException { + String url = startServer( + 200, + "text/html", + """ + + Sign in + +
+

Log in

+
+
+ + + """, + 0 + ); + ExternalHtmlDocumentClient client = testClient(Duration.ofSeconds(2)); + + assertSourceAuthRequired(() -> client.fetch(url)); + } + + @Test + void allowsTechnicalDocumentThatMentionsLoginWithoutPasswordForm() throws IOException { + String url = startServer( + 200, + "text/html", + """ + + How to build a login page + +
+ Spring Security login flow and OAuth 회원가입 API design notes for developers. +
+ + + """, + 0 + ); + ExternalHtmlDocumentClient client = testClient(Duration.ofSeconds(2)); + + HtmlDocument result = client.fetch(url); + + assertThat(result.body()).contains("Spring Security login flow"); + } + @Test void failsWhenBodyIsEmpty() throws IOException { String url = startServer(200, "text/html", "", 0); @@ -444,4 +507,11 @@ private void assertExtractionFailed(Runnable action) { .extracting("errorCode") .isEqualTo(MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED); } + + private void assertSourceAuthRequired(Runnable action) { + assertThatThrownBy(action::run) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } } diff --git a/src/test/java/com/teaching/backend/domain/material/service/extract/HtmlMaterialContentExtractorTest.java b/src/test/java/com/teaching/backend/domain/material/service/extract/HtmlMaterialContentExtractorTest.java index 1f690e8..c9d8951 100644 --- a/src/test/java/com/teaching/backend/domain/material/service/extract/HtmlMaterialContentExtractorTest.java +++ b/src/test/java/com/teaching/backend/domain/material/service/extract/HtmlMaterialContentExtractorTest.java @@ -165,6 +165,91 @@ void genericWebExtractorUsesWebFallbackForUnclassifiedPage() { assertThat(result.content()).contains("Landing page copy"); } + @Test + void genericWebExtractorRejectsLoginFormPage() { + ExternalHtmlDocumentClient client = client(""" + + Login + +
+

Sign in

+ +
+ + + """); + GenericWebMaterialContentExtractor extractor = new GenericWebMaterialContentExtractor(client); + + assertThatThrownBy(() -> extractor.extract(URL)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } + + @Test + void genericWebExtractorDoesNotRejectTechnicalDocumentWithLoginWordsOnly() { + ExternalHtmlDocumentClient client = client(""" + + How to build a login page + +
+ Spring Security login flow, OAuth login, and 회원가입 API design content for developers. +
+ + + """); + GenericWebMaterialContentExtractor extractor = new GenericWebMaterialContentExtractor(client); + + ExtractedMaterialContent result = extractor.extract(URL); + + assertThat(result.content()).contains("Spring Security login flow"); + } + + @Test + void genericWebExtractorDoesNotRejectLoginTextWithoutPasswordForm() { + ExternalHtmlDocumentClient client = client(""" + +
Login troubleshooting article content with enough public explanation.
+ + """); + GenericWebMaterialContentExtractor extractor = new GenericWebMaterialContentExtractor(client); + + ExtractedMaterialContent result = extractor.extract(URL); + + assertThat(result.content()).contains("Login troubleshooting"); + } + + @Test + void genericWebExtractorRejectsPrivateNotionAppShell() { + String notionAppUrl = "https://app.notion.com/p/private-page-id"; + ExternalHtmlDocumentClient client = mock(ExternalHtmlDocumentClient.class); + when(client.fetch(notionAppUrl)).thenReturn(new HtmlDocument( + notionAppUrl, + """ + + Notion | Where teams and agents work together + +
+

Where teams and agents work together

+

Notion is the connected workspace for your docs, projects, and knowledge.

+ Log in + Sign up +
+ + + """, + "text/html" + )); + GenericWebMaterialContentExtractor extractor = new GenericWebMaterialContentExtractor(client); + + assertThatThrownBy(() -> extractor.extract(notionAppUrl)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } + @Test void genericWebExtractorUsesRenderedFallbackWhenStaticExtractionIsEmpty() { ExternalHtmlDocumentClient client = client("
short
"); @@ -498,7 +583,7 @@ void cafeExtractorRejectsLoginRequiredPage() { assertThatThrownBy(() -> extractor.extract(URL)) .isInstanceOf(MaterialException.class) .extracting("errorCode") - .isEqualTo(MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED); + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); } @Test @@ -513,6 +598,20 @@ void cafeExtractorTreatsNullBodyAsEmptyContent() { .isEqualTo(MaterialErrorCode.MATERIAL_CONTENT_EMPTY); } + @Test + void cafeValidateDocumentAllowsNullDocument() { + TestCafeMaterialContentExtractor extractor = new TestCafeMaterialContentExtractor(mock(ExternalHtmlDocumentClient.class)); + + extractor.validateForTest(null); + } + + @Test + void notionValidateDocumentAllowsNullDocument() { + TestNotionMaterialContentExtractor extractor = new TestNotionMaterialContentExtractor(mock(ExternalHtmlDocumentClient.class)); + + extractor.validateForTest(null); + } + @Test void cafeExtractorTreatsBlankBodyAsEmptyContent() { ExternalHtmlDocumentClient client = client(""); @@ -532,7 +631,18 @@ void cafeExtractorRejectsAccessLimitedPage() { assertThatThrownBy(() -> extractor.extract(URL)) .isInstanceOf(MaterialException.class) .extracting("errorCode") - .isEqualTo(MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED); + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + } + + @Test + void notionExtractorRejectsAccessDeniedPageAsSourceAuthRequired() { + ExternalHtmlDocumentClient client = client("You do not have access"); + NotionMaterialContentExtractor extractor = new NotionMaterialContentExtractor(client); + + assertThatThrownBy(() -> extractor.extract(URL)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); } @Test @@ -567,7 +677,7 @@ void notionExtractorRejectsPrivatePage() { assertThatThrownBy(() -> extractor.extract(URL)) .isInstanceOf(MaterialException.class) .extracting("errorCode") - .isEqualTo(MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED); + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); } @Test @@ -741,4 +851,26 @@ private ExternalHtmlDocumentClient client(String body) { .thenReturn(new HtmlDocument("https://youtube.com/watch?v=1", body, "text/html")); return client; } + + private static final class TestCafeMaterialContentExtractor extends CafeMaterialContentExtractor { + + private TestCafeMaterialContentExtractor(ExternalHtmlDocumentClient htmlDocumentClient) { + super(htmlDocumentClient); + } + + private void validateForTest(HtmlDocument document) { + validateDocument(document); + } + } + + private static final class TestNotionMaterialContentExtractor extends NotionMaterialContentExtractor { + + private TestNotionMaterialContentExtractor(ExternalHtmlDocumentClient htmlDocumentClient) { + super(htmlDocumentClient); + } + + private void validateForTest(HtmlDocument document) { + validateDocument(document); + } + } } diff --git a/src/test/java/com/teaching/backend/domain/material/service/extract/ProtectedSourceDetectorTest.java b/src/test/java/com/teaching/backend/domain/material/service/extract/ProtectedSourceDetectorTest.java new file mode 100644 index 0000000..416c077 --- /dev/null +++ b/src/test/java/com/teaching/backend/domain/material/service/extract/ProtectedSourceDetectorTest.java @@ -0,0 +1,118 @@ +package com.teaching.backend.domain.material.service.extract; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProtectedSourceDetectorTest { + + @Test + void rejectsPrivateNotionAppShellForAppDocumentUrl() { + String originalUrl = "https://app.notion.com/p/private-page-id"; + String html = """ + + Notion | Where teams and agents work together + +
+

Where teams and agents work together

+

Notion is the connected workspace for your docs, projects, and knowledge.

+ Log in + Sign up +
+ + + """; + + boolean result = ProtectedSourceDetector.isProtectedHtml( + html, + "https://app.notion.com/p/private-page-id", + originalUrl + ); + + assertThat(result).isTrue(); + } + + @Test + void doesNotRejectPublicNotionSiteDocument() { + String html = """ + + Public Notion Page + +
+

BE 학습 정리

+

공개 Notion 문서의 실제 본문 내용입니다.

+
+ + + """; + + boolean result = ProtectedSourceDetector.isProtectedHtml( + html, + "https://example.notion.site/public-page", + "https://example.notion.site/public-page" + ); + + assertThat(result).isFalse(); + } + + @Test + void doesNotRejectGenericWebDocumentMentioningNotionAndLogin() { + String html = """ + + Notion login 설명 + +
+

Notion 사용법

+

Notion login flow와 OAuth 설정을 설명하는 공개 기술 문서입니다.

+
+ + + """; + + boolean result = ProtectedSourceDetector.isProtectedHtml( + html, + "https://example.com/articles/notion-login", + "https://example.com/articles/notion-login" + ); + + assertThat(result).isFalse(); + } + + @Test + void doesNotTreatPublicGithubRepositoryAsAuthenticationUrl() { + boolean result = ProtectedSourceDetector.isAuthenticationUrl("https://github.com/openai/openai-python"); + + assertThat(result).isFalse(); + } + + @Test + void treatsGithubLoginPathAsAuthenticationUrl() { + boolean result = ProtectedSourceDetector.isAuthenticationUrl("https://github.com/login"); + + assertThat(result).isTrue(); + } + + @Test + void doesNotRejectByNotionMarketingTitleAlone() { + String originalUrl = "https://app.notion.com/p/private-page-id"; + String html = """ + + Notion | Where teams and agents work together + +
+

Public article

+

Title alone is not enough to classify this page as protected.

+
+ + + """; + + boolean result = ProtectedSourceDetector.isProtectedHtml( + html, + originalUrl, + originalUrl + ); + + assertThat(result).isFalse(); + } +} diff --git a/src/test/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClientTest.java b/src/test/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClientTest.java index dee7c64..35b2032 100644 --- a/src/test/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClientTest.java +++ b/src/test/java/com/teaching/backend/domain/material/service/extract/RenderedHtmlDocumentClientTest.java @@ -1,22 +1,32 @@ package com.teaching.backend.domain.material.service.extract; import com.teaching.backend.domain.material.exception.MaterialErrorCode; +import com.teaching.backend.domain.material.exception.MaterialException; import org.junit.jupiter.api.Test; import org.openqa.selenium.JavascriptExecutor; +import org.openqa.selenium.PageLoadStrategy; +import org.openqa.selenium.ScriptTimeoutException; +import org.openqa.selenium.TimeoutException; import org.openqa.selenium.WebDriver; import java.net.URI; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.concurrent.Semaphore; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; @@ -150,12 +160,310 @@ void triggersBoundedScrollForNotionRenderedPages() { verify(javascriptExecutor).executeScript("window.scrollTo(0, 0);"); } - private ExternalHtmlDocumentClient validatorAllowing(String url) { + @Test + void usesEagerStrategyAndLongerTimeoutsForNotion() { + String notionUrl = "https://example.notion.site/page"; + WebDriver driver = readableDriver(notionUrl); + WebDriver.Timeouts timeouts = timeouts(driver); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(notionUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient( + permits, + validator, + 10000, + 5000, + 30000, + 10000, + driver + ); + + Optional result = client.render(notionUrl); + + assertThat(result).isPresent(); + assertThat(client.pageLoadStrategies).containsExactly(PageLoadStrategy.EAGER); + verify(timeouts).pageLoadTimeout(Duration.ofSeconds(30)); + verify(timeouts).scriptTimeout(Duration.ofSeconds(10)); + } + + @Test + void keepsNormalStrategyAndDefaultTimeoutsForNonNotion() { + WebDriver driver = readableDriver(URL); + WebDriver.Timeouts timeouts = timeouts(driver); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(URL); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient( + permits, + validator, + 10000, + 5000, + 30000, + 10000, + driver + ); + + Optional result = client.render(URL); + + assertThat(result).isPresent(); + assertThat(client.pageLoadStrategies).containsExactly(PageLoadStrategy.NORMAL); + verify(timeouts).pageLoadTimeout(Duration.ofSeconds(10)); + verify(timeouts).scriptTimeout(Duration.ofSeconds(5)); + } + + @Test + void retriesNotionTimeoutWithNewDriver() { + String notionUrl = "https://example.notion.site/page"; + WebDriver firstDriver = driverFailingOnGet(notionUrl, new TimeoutException("page load timeout")); + WebDriver secondDriver = readableDriver(notionUrl); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(notionUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient( + permits, + validator, + firstDriver, + secondDriver + ); + + Optional result = client.render(notionUrl); + + assertThat(result).isPresent(); + assertThat(client.createDriverCalls).isEqualTo(2); + assertThat(client.pageLoadStrategies).containsExactly(PageLoadStrategy.EAGER, PageLoadStrategy.EAGER); + verify(firstDriver).quit(); + verify(secondDriver).quit(); + assertThat(permits.availablePermits()).isEqualTo(1); + } + + @Test + void retriesNotionScriptTimeoutWithNewDriver() { + String notionUrl = "https://example.notion.site/page"; + WebDriver firstDriver = readableDriver(notionUrl); + JavascriptExecutor javascriptExecutor = (JavascriptExecutor) firstDriver; + when(javascriptExecutor.executeScript("return document.readyState")) + .thenThrow(new ScriptTimeoutException("script timeout")); + WebDriver secondDriver = readableDriver(notionUrl); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(notionUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient( + permits, + validator, + firstDriver, + secondDriver + ); + + Optional result = client.render(notionUrl); + + assertThat(result).isPresent(); + assertThat(client.createDriverCalls).isEqualTo(2); + verify(firstDriver).quit(); + verify(secondDriver).quit(); + } + + @Test + void stopsAfterTwoNotionTimeoutAttempts() { + String notionUrl = "https://example.notion.site/page"; + WebDriver firstDriver = driverFailingOnGet(notionUrl, new TimeoutException("first timeout")); + WebDriver secondDriver = driverFailingOnGet(notionUrl, new TimeoutException("second timeout")); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(notionUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient( + permits, + validator, + firstDriver, + secondDriver + ); + + Optional result = client.render(notionUrl); + + assertThat(result).isEmpty(); + assertThat(client.createDriverCalls).isEqualTo(2); + verify(firstDriver).quit(); + verify(secondDriver).quit(); + } + + @Test + void doesNotRetryNonTimeoutRuntimeException() { + WebDriver driver = driverFailingOnGet(URL, new IllegalStateException("browser failed")); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(URL); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, permits, validator); + + Optional result = client.render(URL); + + assertThat(result).isEmpty(); + assertThat(client.createDriverCalls).isEqualTo(1); + verify(driver).quit(); + } + + @Test + void rejectsAuthenticationFinalUrlWithoutRetry() { + String finalUrl = "https://example.com/login?redirect=/private/article"; + WebDriver driver = readableDriver(finalUrl); + ExternalHtmlDocumentClient validator = validatorAllowing(URL, finalUrl); + Semaphore permits = new Semaphore(1); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, permits, validator); + + assertThatThrownBy(() -> client.render(URL)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + + assertThat(client.createDriverCalls).isEqualTo(1); + verify(driver).quit(); + } + + @Test + void rejectsSigninFinalUrlWithoutRetry() { + String finalUrl = "https://example.com/signin"; + WebDriver driver = readableDriver(finalUrl); + ExternalHtmlDocumentClient validator = validatorAllowing(URL, finalUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, new Semaphore(1), validator); + + assertThatThrownBy(() -> client.render(URL)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + + assertThat(client.createDriverCalls).isEqualTo(1); + verify(driver).quit(); + } + + @Test + void rejectsOAuthFinalUrlWithoutRetry() { + String finalUrl = "https://accounts.google.com/o/oauth2/v2/auth"; + WebDriver driver = readableDriver(finalUrl); + ExternalHtmlDocumentClient validator = validatorAllowing(URL, finalUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, new Semaphore(1), validator); + + assertThatThrownBy(() -> client.render(URL)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + + assertThat(client.createDriverCalls).isEqualTo(1); + verify(driver).quit(); + } + + @Test + void doesNotRejectLoginWordInQueryOnly() { + String urlWithLoginQuery = "https://example.com/article?keyword=login"; + WebDriver driver = readableDriver(urlWithLoginQuery); + ExternalHtmlDocumentClient validator = validatorAllowing(urlWithLoginQuery); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, new Semaphore(1), validator); + + Optional result = client.render(urlWithLoginQuery); + + assertThat(result).isPresent(); + verify(driver).quit(); + } + + @Test + void doesNotRejectAuthWordInArticlePath() { + String articleUrl = "https://example.com/articles/auth"; + WebDriver driver = readableDriver(articleUrl); + ExternalHtmlDocumentClient validator = validatorAllowing(articleUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, new Semaphore(1), validator); + + Optional result = client.render(articleUrl); + + assertThat(result).isPresent(); + verify(driver).quit(); + } + + @Test + void rejectsRenderedLoginFormPage() { + WebDriver driver = readableDriver(URL); + when(driver.getPageSource()).thenReturn(""" + + Login +

Sign in

+ + """); + ExternalHtmlDocumentClient validator = validatorAllowing(URL); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, new Semaphore(1), validator); + + assertThatThrownBy(() -> client.render(URL)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + + assertThat(client.createDriverCalls).isEqualTo(1); + verify(driver).quit(); + } + + @Test + void rejectsPrivateNotionAppShellWithoutRetry() { + String originalUrl = "https://app.notion.com/p/private-page-id"; + WebDriver driver = readableDriver(originalUrl); + when(driver.getPageSource()).thenReturn(""" + + Notion | Where teams and agents work together + +
+

Where teams and agents work together

+

Notion is the connected workspace for your docs, projects, and knowledge.

+ Log in + Sign up +
+ + + """); + ExternalHtmlDocumentClient validator = validatorAllowing(originalUrl); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, new Semaphore(1), validator); + + assertThatThrownBy(() -> client.render(originalUrl)) + .isInstanceOf(MaterialException.class) + .extracting("errorCode") + .isEqualTo(MaterialErrorCode.MATERIAL_SOURCE_AUTH_REQUIRED); + + assertThat(client.createDriverCalls).isEqualTo(1); + verify(driver).quit(); + } + + @Test + void doesNotRetryUnsafeInitialUrl() { + String unsafeUrl = "file:///etc/passwd"; ExternalHtmlDocumentClient validator = mock(ExternalHtmlDocumentClient.class); - when(validator.validateFetchTarget(url)).thenReturn(URI.create(url)); + when(validator.validateFetchTarget(unsafeUrl)).thenThrow(new HtmlFetchException( + MaterialErrorCode.MATERIAL_CONTENT_EXTRACTION_FAILED, + false + )); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(readableDriver(unsafeUrl), new Semaphore(1), validator); + + Optional result = client.render(unsafeUrl); + + assertThat(result).isEmpty(); + assertThat(client.createDriverCalls).isZero(); + } + + @Test + void doesNotRetryTimeoutForNonNotion() { + WebDriver driver = driverFailingOnGet(URL, new TimeoutException("page load timeout")); + Semaphore permits = new Semaphore(1); + ExternalHtmlDocumentClient validator = validatorAllowing(URL); + TestRenderedHtmlDocumentClient client = new TestRenderedHtmlDocumentClient(driver, permits, validator); + + Optional result = client.render(URL); + + assertThat(result).isEmpty(); + assertThat(client.createDriverCalls).isEqualTo(1); + assertThat(client.pageLoadStrategies).containsExactly(PageLoadStrategy.NORMAL); + verify(driver).quit(); + } + + private ExternalHtmlDocumentClient validatorAllowing(String... urls) { + ExternalHtmlDocumentClient validator = mock(ExternalHtmlDocumentClient.class); + for (String url : urls) { + when(validator.validateFetchTarget(url)).thenReturn(URI.create(url)); + } return validator; } + private WebDriver driverFailingOnGet(String url, RuntimeException exception) { + WebDriver driver = readableDriver(url); + doThrow(exception).when(driver).get(url); + return driver; + } + private WebDriver readableDriver(String currentUrl) { WebDriver driver = mock(WebDriver.class, withSettings().extraInterfaces(JavascriptExecutor.class)); WebDriver.Options options = mock(WebDriver.Options.class); @@ -174,9 +482,14 @@ private WebDriver readableDriver(String currentUrl) { return driver; } + private WebDriver.Timeouts timeouts(WebDriver driver) { + return driver.manage().timeouts(); + } + private static final class TestRenderedHtmlDocumentClient extends RenderedHtmlDocumentClient { - private final WebDriver driver; + private final List drivers; + private final List pageLoadStrategies = new ArrayList<>(); private int createDriverCalls; private boolean failDriverCreation; @@ -185,7 +498,7 @@ private TestRenderedHtmlDocumentClient( Semaphore permits, ExternalHtmlDocumentClient validator ) { - this(driver, permits, validator, 0); + this(permits, validator, 0, driver); } private TestRenderedHtmlDocumentClient( @@ -194,17 +507,49 @@ private TestRenderedHtmlDocumentClient( ExternalHtmlDocumentClient validator, long acquireTimeoutMs ) { - super(true, 1000, 1000, 100, 20, acquireTimeoutMs, validator, permits); - this.driver = driver; + this(permits, validator, acquireTimeoutMs, driver); + } + + private TestRenderedHtmlDocumentClient( + Semaphore permits, + ExternalHtmlDocumentClient validator, + WebDriver... drivers + ) { + this(permits, validator, 0, drivers); + } + + private TestRenderedHtmlDocumentClient( + Semaphore permits, + ExternalHtmlDocumentClient validator, + long acquireTimeoutMs, + WebDriver... drivers + ) { + super(true, 1000, 1000, 30000, 10000, 100, 20, acquireTimeoutMs, validator, permits); + this.drivers = Arrays.asList(drivers); + } + + private TestRenderedHtmlDocumentClient( + Semaphore permits, + ExternalHtmlDocumentClient validator, + long pageLoadTimeoutMs, + long scriptTimeoutMs, + long notionPageLoadTimeoutMs, + long notionScriptTimeoutMs, + WebDriver... drivers + ) { + super(true, pageLoadTimeoutMs, scriptTimeoutMs, notionPageLoadTimeoutMs, notionScriptTimeoutMs, + 100, 20, 0, validator, permits); + this.drivers = Arrays.asList(drivers); } @Override - protected WebDriver createDriver() { + protected WebDriver createDriver(PageLoadStrategy pageLoadStrategy) { createDriverCalls++; + pageLoadStrategies.add(pageLoadStrategy); if (failDriverCreation) { throw new IllegalStateException("driver unavailable"); } - return driver; + return drivers.get(createDriverCalls - 1); } } }