|
| 1 | +package manager |
| 2 | + |
| 3 | +import ( |
| 4 | + "path/filepath" |
| 5 | + "testing" |
| 6 | + "time" |
| 7 | + "unicode/utf8" |
| 8 | + |
| 9 | + "github.com/ma111e/downlink/cmd/server/internal/scrapers" |
| 10 | + "github.com/ma111e/downlink/cmd/server/internal/store" |
| 11 | + "github.com/ma111e/downlink/pkg/models" |
| 12 | + |
| 13 | + "gorm.io/datatypes" |
| 14 | +) |
| 15 | + |
| 16 | +// stubScraper returns a fixed set of items and never re-scrapes. |
| 17 | +type stubScraper struct { |
| 18 | + items []models.FeedItem |
| 19 | +} |
| 20 | + |
| 21 | +func (s *stubScraper) Fetch(url string, params map[string]any) ([]models.FeedItem, *scrapers.RawResponse, error) { |
| 22 | + return s.items, &scrapers.RawResponse{}, nil |
| 23 | +} |
| 24 | + |
| 25 | +func (s *stubScraper) ScrapeContent(url string, params map[string]any) (string, error) { |
| 26 | + return "", nil |
| 27 | +} |
| 28 | + |
| 29 | +// FetchFeed should keep an article whose content has invalid UTF-8, storing it |
| 30 | +// with the invalid bytes stripped rather than dropping it as an error. |
| 31 | +func TestFetchFeed_SanitizesInvalidUTF8Content(t *testing.T) { |
| 32 | + db, err := store.New(filepath.Join(t.TempDir(), "test.db")) |
| 33 | + if err != nil { |
| 34 | + t.Fatalf("store.New: %v", err) |
| 35 | + } |
| 36 | + t.Cleanup(func() { _ = db.Close() }) |
| 37 | + |
| 38 | + m := NewFeedManager(db) |
| 39 | + m.RegisterScraper("rss", &stubScraper{items: []models.FeedItem{{ |
| 40 | + Id: "item-1", |
| 41 | + Title: "Café story", |
| 42 | + Link: "https://example.com/a", |
| 43 | + Content: "caf\xe9 bar", // stray Latin-1 0xe9, invalid UTF-8 |
| 44 | + PublishedAt: time.Now(), |
| 45 | + }}}) |
| 46 | + |
| 47 | + // scraping "none" keeps the feed's own content (no re-scrape). |
| 48 | + feed := models.Feed{ |
| 49 | + Id: "feed-1", |
| 50 | + URL: "https://example.com/rss", |
| 51 | + Type: "rss", |
| 52 | + Scraper: datatypes.JSONMap{"scraping": "none"}, |
| 53 | + } |
| 54 | + |
| 55 | + result, err := m.FetchFeed(feed, nil, nil, false, false, 0) |
| 56 | + if err != nil { |
| 57 | + t.Fatalf("FetchFeed: %v", err) |
| 58 | + } |
| 59 | + |
| 60 | + if result.Stored != 1 { |
| 61 | + t.Fatalf("Stored = %d, want 1", result.Stored) |
| 62 | + } |
| 63 | + if len(result.Errors) != 0 { |
| 64 | + t.Fatalf("Errors = %v, want none", result.Errors) |
| 65 | + } |
| 66 | + if len(result.Warnings) != 1 { |
| 67 | + t.Fatalf("Warnings = %v, want 1 sanitize notice", result.Warnings) |
| 68 | + } |
| 69 | + if len(result.StoredArticleIDs) != 1 { |
| 70 | + t.Fatalf("StoredArticleIDs = %v, want 1 id", result.StoredArticleIDs) |
| 71 | + } |
| 72 | + |
| 73 | + art, err := db.GetArticle(result.StoredArticleIDs[0]) |
| 74 | + if err != nil { |
| 75 | + t.Fatalf("GetArticle: %v", err) |
| 76 | + } |
| 77 | + if !utf8.ValidString(art.Content) { |
| 78 | + t.Fatalf("stored content is not valid UTF-8: %q", art.Content) |
| 79 | + } |
| 80 | + if art.Content != "caf bar" { |
| 81 | + t.Fatalf("Content = %q, want %q", art.Content, "caf bar") |
| 82 | + } |
| 83 | +} |
0 commit comments