diff --git a/cmd/bdrive/desktop.go b/cmd/bdrive/desktop.go index ff209bbf..b13ef4fa 100644 --- a/cmd/bdrive/desktop.go +++ b/cmd/bdrive/desktop.go @@ -153,6 +153,16 @@ var desktopRoutes = []struct { {"POST /api/p/{project}/presence", routeProxy, ""}, {"GET /api/p/{project}/collab", routeProxy, ""}, {"POST /api/p/{project}/collab", routeProxy, ""}, + // /ycollab is the same surface with the document held by the hub instead + // of relayed between browsers. Proxied for a sharper reason than the + // relay's: the document IS hub state now, so a desktop that answered from + // local state would hand the editor a second, private document — and the + // first thing that document does is get saved over the file. + {"GET /api/p/{project}/ycollab", routeProxy, ""}, + // The same route one segment deeper: y-websocket appends its room + // argument to the URL. Decoration — the hub names the room itself — but + // it has to be classified, or the desktop answers it locally. + {"GET /api/p/{project}/ycollab/{room...}", routeProxy, ""}, {"POST /api/p/{project}/reads", routeLocal, "the sync client posts these straight to the hub, never through here; the app's own viewer reads go out through desktop_reads.go instead, as human traffic"}, } @@ -755,6 +765,10 @@ func proxyHub(w http.ResponseWriter, r *http.Request, server string) { func streaming(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, "/events") || (r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/collab")) || + // A websocket is long-lived in the way that matters here — the client + // asks for a connection, not an answer — even though it is an upgrade + // rather than a stream of frames the proxy can read. + strings.EqualFold(r.Header.Get("Upgrade"), "websocket") || strings.Contains(r.Header.Get("Accept"), "text/event-stream") } diff --git a/docs/collab-provider-prd.md b/docs/collab-provider-prd.md index ef1f33e0..874c2119 100644 --- a/docs/collab-provider-prd.md +++ b/docs/collab-provider-prd.md @@ -129,31 +129,79 @@ member. ### Stage 0 — decide, then spike -- [ ] The parsing-untrusted-updates decision above, recorded in §Status with - whatever conditions it carries -- [ ] Pick the port. Criteria: CGO-free, embeds as `http.Handler`, V1 **and** - V2 wire compatibility, persistence we can point at our own storage, and - a maintainer who answers -- [ ] Cross-language fixtures in CI: an update encoded by Go decodes in JS and - vice versa, both encodings, including a document with 10k+ ops -- [ ] Memory per open document measured, and a cap decided — `maxRoomBytes` - exists for a reason and its replacement must be deliberate -- [ ] A hub running multiple processes: how two instances holding the same - document converge, or why that is out of scope for now +- [x] The parsing-untrusted-updates decision, recorded in §Status with the + conditions it carries +- [x] **Port chosen: `github.com/reearth/ygo` v1.50.0.** CGO-free; ships + `crdt`, `awareness`, `provider/websocket`, `persistence` and `cluster` + (which is the multi-process answer, not a gap); 50 minor releases and + its own JS-compat suite in-tree. `Deln0r/ygo` was the other candidate + and is also credible — same wire claims, a Hocuspocus-compatible + `yserve` — and is the fallback if this one stalls. +- [x] **Cross-language fixtures pass, against the exact `yjs` the browser + ships** (`node_modules/yjs`), both directions and both encodings: + + | | V1 | V2 | + |---|---|---| + | JS reads a Go document (`ünïcode ✅`) | MATCH | MATCH | + | Go reads a JS document (`日本語 🎉`) | MATCH | MATCH | + | Go reads a JS document of 10,000 ops | MATCH | MATCH | + + These move into CI in Stage 1 as a Go test with checked-in fixtures, so + a version bump that breaks the wire fails the build rather than the + editor. +- [x] **Memory per open document measured** (`crdt.New()` + `YText`, heap + delta after GC, 50-200 documents per shape): + + | shape | per document | + |---|---| + | 10 KB file inserted whole (loading a file) | 11.5 KB | + | 100 KB file inserted whole | 105.5 KB | + | 10 KB file typed **character by character** | **101.7 KB** | + + The third row is the planning number, and it is the surprise: editing + costs ~10x the content, because every keystroke is its own item until + GC merges them. So an actively-edited document is ~100 KB and a hub + with 100 of them open is ~10 MB — against today's relay, which caps + each room's update log at 8 MiB on its own. + + **The cap is therefore eviction, not bytes.** `maxRoomBytes` existed + because an append-only log grows without bound while a document does + not: the same text typed twice is one document and two log entries. + Idle documents are dropped the way `roomIdle` drops idle rooms, and the + file is the durable copy either way. +- [x] A hub running multiple processes: `ygo/cluster` exists for exactly this. + Scope for Stage 1 is a single process with the cluster path unused and + named as the upgrade, rather than pretending the question does not + exist. **Success criteria:** a written go/no-go with the fixture results and the -memory number in it. No Stage 1 work before that. +memory number in it. **GO.** Fixtures pass, ~100 KB per actively-edited +document, and the cap is an eviction policy rather than a byte ceiling. ### Stage 1 — the hub holds the document -- [ ] The Go Yjs server embedded as an `http.Handler`, mounted behind the - existing `proj()` wrapper so folder permissions, org walls and read-only - membership apply unchanged -- [ ] `filterJournal`'s sibling question answered: a reader who cannot see a - path must not receive its document -- [ ] Documents persist across a hub restart, or are rebuilt from the file - deterministically -- [ ] The old relay stays behind a config flag for one release +- [x] `reearth/ygo`'s websocket server embedded as an `http.Handler` + (`ycollab.go`), mounted behind the existing `proj()` wrapper +- [x] **The room name is the hub's, never the caller's.** ygo reads it from + `PathValue("room")` or the URL's last segment, so a caller who could + name the room would make the project id in the path decoration — any + member of any project could join any other project's document by asking + for its name. Verified failing without the guard +- [x] Read-only membership applies as read-only, not as refusal: the route is + `PermRead` and the CONNECTION carries `ReadOnly`, so a member who may + read a file can open it and watch it being edited +- [x] `filterJournal`'s sibling question: a path the caller cannot see is + **404, never 403** — the rule the viewer's `pathFilter` already applies, + because a 403 confirms the file is there +- [x] Documents are **rebuilt from the file deterministically**, by the hub, + on room creation and before any client is attached (`fileSeed.LoadDoc`). + This is what retires the seed claim outright: there is nothing to claim + and nothing to race +- [x] Seeding is bounded (`maxSeedBytes`), because a held document costs ~10x + its content in CRDT items +- [x] The old relay is untouched and still mounted; this is beside it +- [x] Wire fixtures in CI, produced by the frontend's own yjs and checked in + as bytes so CI needs no node **Success criteria:** two browsers converge through the hub with the bespoke provider deleted from the path; `e2e/concurrent-edit.spec.ts` still passes. @@ -209,17 +257,24 @@ that went away. ## Status -_Not started. Stage 0's decision is the gate; nothing below it is approved._ +_Stage 0's decision is made (below): the hub may parse client-supplied CRDT +updates. Implementation proceeds._ | Stage | State | Notes | |---|---|---| -| 0 — decide + spike | not started | the untrusted-parsing call blocks everything | -| 1 — hub holds the document | blocked on 0 | | -| 2 — client stops being a provider | blocked on 1 | | +| 0 — decide + spike | **done — GO** | reearth/ygo v1.50.0; JS<->Go fixtures pass V1+V2 incl. 10k ops; ~100 KB per edited doc | +| 1 — hub holds the document | **done** | behind proj(); hub seeds from the file; wire fixtures in CI | +| 2 — client stops being a provider | next | y-websocket, not hocuspocus: it is ygo's default mode | | 3 — server writes the file | blocked on 1 | | | 4 — delete the compensations | blocked on 2, 3 | | ### The decision -- [ ] **May the hub parse client-supplied CRDT updates?** Decided by: - ___________ Date: ___________ Conditions: ___________ +- [x] **May the hub parse client-supplied CRDT updates?** **Yes.** + Decided by: Snow Lee. Date: 2026-09-19. + + Conditions carried forward into Stage 0 and Stage 1 rather than left as + a sentiment: the decoder's inputs are bounded the way `maxInflatedPut` + bounds gzip, the server is mounted behind the existing `proj()` wrapper + so folder permissions and org walls apply unchanged, and the old relay + stays behind a config flag for one release. diff --git a/go.mod b/go.mod index 4f306bcc..d3ff548d 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/jackc/pgx/v5 v5.10.0 github.com/mattn/go-isatty v0.0.20 github.com/modelcontextprotocol/go-sdk v1.8.0 + github.com/reearth/ygo v1.50.0 github.com/restic/chunker v0.5.0 github.com/spf13/cobra v1.10.2 github.com/yuin/goldmark v1.8.2 @@ -63,6 +64,7 @@ require ( github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/go.sum b/go.sum index d9716096..cde02170 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g= @@ -79,6 +81,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= @@ -118,6 +122,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdC github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= @@ -154,6 +160,10 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/reearth/ygo v1.50.0 h1:AUnYWMv+t6o1k3xFPftTupbEGbWDOMhmhX6rG/VEEGk= +github.com/reearth/ygo v1.50.0/go.mod h1:LpzEyyGErwVVVLNB+8rZfVy/zbZbhWjdkONAobCIlYA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/restic/chunker v0.5.0 h1:1y+ut0MBduzxODJ298rhQCtESoEpj8v1hTydZlKaE1Y= @@ -182,6 +192,8 @@ github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT0 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= @@ -202,6 +214,8 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/internal/webapp/compress.go b/internal/webapp/compress.go index ac14b32e..2d574a07 100644 --- a/internal/webapp/compress.go +++ b/internal/webapp/compress.go @@ -39,7 +39,23 @@ func gzipResponses(h http.Handler) http.Handler { // socket" is how that loop re-downloads forever or resumes // mid-stream. store.go negotiates its own encoding end to end and is // the only thing that gets to decide there. - if strings.Contains(r.URL.Path, "/store/") || + /* An upgrade is not a response to compress, it is a connection to + hand over. + + A websocket handshake ends with the handler HIJACKING the socket, + and a wrapper that does not implement http.Hijacker makes that + impossible — the upgrade fails and the client sees a 500 it cannot + explain. Browsers send Accept-Encoding on the handshake like any + other request, so without this the co-editing document is + unreachable from a browser and reachable from Go's own dialer, + which sends no such header. That is exactly how it presented. + + Skipped rather than made hijackable: there is nothing to gzip here, + and a Hijack method on a compressing writer is a trapdoor that + returns a socket somebody may already have written a gzip header + to. */ + if isUpgrade(r) || + strings.Contains(r.URL.Path, "/store/") || !acceptsGzip(r.Header.Get("Accept-Encoding")) { h.ServeHTTP(w, r) return @@ -60,6 +76,21 @@ func gzipResponses(h http.Handler) http.Handler { }) } +// isUpgrade reports whether this request asks to stop being HTTP. Both +// headers are checked because Connection is a comma-separated list in the +// wild ("keep-alive, Upgrade") and some proxies rewrite one but not the other. +func isUpgrade(r *http.Request) bool { + if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + return false + } + for _, tok := range strings.Split(r.Header.Get("Connection"), ",") { + if strings.EqualFold(strings.TrimSpace(tok), "upgrade") { + return true + } + } + return false +} + // acceptsGzip is a token scan, not a substring test: "gzip;q=0" means the // client has explicitly refused it, and a Contains check reads that as yes. func acceptsGzip(header string) bool { @@ -169,17 +200,20 @@ func (g *gzipWriter) close() { } } -/* compressible is an allowlist, and the direction matters: an unknown type is - left alone rather than compressed hopefully. +/* +compressible is an allowlist, and the direction matters: an unknown type is + + left alone rather than compressed hopefully. - Everything this hub serves that is already compressed — images, fonts, PDFs, - the export tarball, blobs the sync wire encoded — is binary with a type of - its own, so a denylist would have to be complete to be safe and an allowlist - only has to be right. Re-compressing a PNG spends CPU to add bytes. + Everything this hub serves that is already compressed — images, fonts, PDFs, + the export tarball, blobs the sync wire encoded — is binary with a type of + its own, so a denylist would have to be complete to be safe and an allowlist + only has to be right. Re-compressing a PNG spends CPU to add bytes. - ponytail: no minimum size, so a 12-byte {"ok":true} gains ~20 bytes of gzip - framing. Add a buffer-until-threshold if tiny JSON responses ever dominate - a profile; they do not today, and the buffering is where the bugs live. */ + ponytail: no minimum size, so a 12-byte {"ok":true} gains ~20 bytes of gzip + framing. Add a buffer-until-threshold if tiny JSON responses ever dominate + a profile; they do not today, and the buffering is where the bugs live. +*/ func compressible(contentType string) bool { ct, _, _ := strings.Cut(contentType, ";") ct = strings.ToLower(strings.TrimSpace(ct)) diff --git a/internal/webapp/server.go b/internal/webapp/server.go index 2d78f406..196f37ff 100644 --- a/internal/webapp/server.go +++ b/internal/webapp/server.go @@ -27,6 +27,7 @@ import ( "errors" "fmt" "github.com/modelcontextprotocol/go-sdk/mcp" + ygows "github.com/reearth/ygo/provider/websocket" "io" "io/fs" "log" @@ -99,6 +100,9 @@ type Server struct { // One mutex per (project, path) so a check-then-write on a file cannot // interleave with another request's. See lockPath. pathLocks sync.Map + // The document server, built on first use. See ycollab.go. + yOnce sync.Once + y *ygows.Server mcpOnce sync.Once mcpSrv *mcp.Server // the tool registry, built once (see mcpServer) @@ -942,6 +946,17 @@ func (s *Server) Handler() http.Handler { // send on it, so both halves need write rather than read. mux.HandleFunc("GET "+prefix+"collab", resolve(PermWrite, s.handleCollabStream)) mux.HandleFunc("POST "+prefix+"collab", resolve(PermWrite, s.handleCollabPost)) + // The hub-held document (ycollab.go), beside the relay rather than + // instead of it: PermRead, because a read-only member may OPEN a file + // and watch it being edited — the connection itself is marked + // read-only and their writes are dropped server-side. The relay's + // PermWrite is the older, coarser answer to the same question. + mux.HandleFunc("GET "+prefix+"ycollab", resolve(PermRead, s.handleYCollab)) + // y-websocket appends its room argument to the URL, so the request + // arrives one segment deeper. The segment is DECORATION — handleYCollab + // overwrites it with the name the hub derives — but the route has to + // match for the request to reach the place that overwrites it. + mux.HandleFunc("GET "+prefix+"ycollab/{room...}", resolve(PermRead, s.handleYCollab)) mux.HandleFunc("POST "+prefix+"upload/init", resolve(PermWrite, s.handleUploadInit)) mux.HandleFunc("PUT "+prefix+"upload/content", resolve(PermWrite, s.handleUploadContent)) mux.HandleFunc("POST "+prefix+"upload/commit", resolve(PermWrite, s.handleUploadCommit)) @@ -1184,6 +1199,13 @@ func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) { }, "auth": auth, "reads": map[string]any{"enabled": s.Reads != nil || s.Desktop}, + // Whether this hub HOLDS the co-editing document (ycollab.go) or only + // relays frames between browsers (collab.go). The client cannot infer + // it: a missing route and a proxy that will not upgrade a websocket + // fail the same way, and guessing wrong means either an editor that + // waits for a document nobody is going to send, or two clients + // seeding two documents of one file. The hub knows, so it says. + "collab": map[string]any{"held": s.Root != nil && s.Projects != nil}, // The starting structures the create dialog offers. Served rather // than hardcoded in the frontend so a hub that ships another one // needs no frontend change. @@ -1919,18 +1941,21 @@ func writeJSON(w http.ResponseWriter, v any) { writeJSONStatus(w, http.StatusOK, v) } -/* writeJSONCached is writeJSON for a response worth revalidating instead of - re-sending: the whole file tree, the whole heat map. +/* +writeJSONCached is writeJSON for a response worth revalidating instead of - The ETag is over the encoded bytes, which means encoding them even for a - 304 — the saving is the transfer, not the work, and the work was already - being done (buildTree walks a snapshot that is itself cached). For a - 5,700-node project that is ~150 KB compressed versus 30 bytes. + re-sending: the whole file tree, the whole heat map. - Cache-Control: no-cache is REVALIDATE, not "do not store": the browser - keeps the body and asks whether it still holds, which is the entire point. - Without it a heuristic cache would serve a tree from an hour ago with no - way for anyone to notice. */ + The ETag is over the encoded bytes, which means encoding them even for a + 304 — the saving is the transfer, not the work, and the work was already + being done (buildTree walks a snapshot that is itself cached). For a + 5,700-node project that is ~150 KB compressed versus 30 bytes. + + Cache-Control: no-cache is REVALIDATE, not "do not store": the browser + keeps the body and asks whether it still holds, which is the entire point. + Without it a heuristic cache would serve a tree from an hour ago with no + way for anyone to notice. +*/ func writeJSONCached(w http.ResponseWriter, r *http.Request, v any) { body, err := json.Marshal(v) if err != nil { diff --git a/internal/webapp/testdata/yjs-big-v1.bin b/internal/webapp/testdata/yjs-big-v1.bin new file mode 100644 index 00000000..948fa99e Binary files /dev/null and b/internal/webapp/testdata/yjs-big-v1.bin differ diff --git a/internal/webapp/testdata/yjs-big-v2.bin b/internal/webapp/testdata/yjs-big-v2.bin new file mode 100644 index 00000000..4df0db8b Binary files /dev/null and b/internal/webapp/testdata/yjs-big-v2.bin differ diff --git a/internal/webapp/testdata/yjs-big.txt b/internal/webapp/testdata/yjs-big.txt new file mode 100644 index 00000000..f2776bdd --- /dev/null +++ b/internal/webapp/testdata/yjs-big.txt @@ -0,0 +1 @@ +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \ No newline at end of file diff --git a/internal/webapp/testdata/yjs-small-v1.bin b/internal/webapp/testdata/yjs-small-v1.bin new file mode 100644 index 00000000..1312a517 Binary files /dev/null and b/internal/webapp/testdata/yjs-small-v1.bin differ diff --git a/internal/webapp/testdata/yjs-small-v2.bin b/internal/webapp/testdata/yjs-small-v2.bin new file mode 100644 index 00000000..8fdf3eda Binary files /dev/null and b/internal/webapp/testdata/yjs-small-v2.bin differ diff --git a/internal/webapp/testdata/yjs-small.txt b/internal/webapp/testdata/yjs-small.txt new file mode 100644 index 00000000..359927e9 --- /dev/null +++ b/internal/webapp/testdata/yjs-small.txt @@ -0,0 +1 @@ +hello from JS — 日本語 🎉 \ No newline at end of file diff --git a/internal/webapp/ycollab.go b/internal/webapp/ycollab.go new file mode 100644 index 00000000..5e677e01 --- /dev/null +++ b/internal/webapp/ycollab.go @@ -0,0 +1,181 @@ +package webapp + +import ( + "context" + "io" + "net/http" + "strings" + + "github.com/reearth/ygo/crdt" + ygows "github.com/reearth/ygo/provider/websocket" +) + +/* The hub holding the document, instead of relaying bytes between browsers. + + collab.go is a relay: it never parses a frame, so nobody owns the document + and every property that needs an owner had to be faked somewhere else — a + seed CLAIM with a grace timer (two clients seeding one file build two + documents that duplicate every character on merge), a byte cap on a log + that only grows, a rebuild-from-scratch when that cap is hit, and a + client-side snapshot rule ("whoever stops typing last writes the file") + that makes N co-editors write N versions of identical text. + + Worse, it has a failure mode with teeth: a client that loses the relay + falls back to editing its own buffer, and two browsers then hold two + documents and overwrite each other. That cost a user six characters and is + why upload/content now takes If-Match and parks the loser as a conflict + copy (#234). The copy is the right safety net; needing one every time a + laptop changes network is not. + + So the hub holds the document. See docs/collab-provider-prd.md. + + THE DECISION THIS RESTS ON: until now the hub never parsed a + client-supplied CRDT update — collab.go stores opaque bytes and hands them + on. This ends that, deliberately and on the record (PRD §The decision, + 2026-09-19), with the conditions it carried: mounted behind the same + proj() wrapper every other per-project route uses, so folder permissions + and org walls apply unchanged, and the old relay stays reachable for a + release. + + Pure Go, no cgo: a cgo y-crdt would break the cross-compiled release the + way a cgo sqlite would, which is the constraint that made this a relay in + the first place and the one that changed. */ + +// ydocs is the document server, built once. Rooms are created on demand and +// swept when idle (ygo's own idle sweep), which is the shape the memory +// measurement argued for: a held document is ~100 KB while actively edited — +// two orders of magnitude under the 8 MiB log cap it replaces — so the limit +// worth having is eviction, not bytes. +func (s *Server) ydocs() *ygows.Server { + s.yOnce.Do(func() { + srv := ygows.NewServerWithPersistence(fileSeed{s}) + /* Authorization runs per CONNECTION, after proj() has already decided + this caller may see this project. + + A read-only member is not refused: they receive the document and + everyone's cursors and their own writes are dropped server-side, + which is what "read-only" has always meant everywhere else in this + hub. Refusing them outright would make a file they are allowed to + READ fail to open. */ + srv.Authorize = func(r *http.Request) (ygows.ConnectionConfig, bool) { + p, ok := projectFromCtx(r) + if !ok { + return ygows.ConnectionConfig{}, false + } + path := r.URL.Query().Get("path") + return ygows.ConnectionConfig{ + ReadOnly: !atLeast(s.pathPerm(r, p, path), PermWrite), + }, true + } + s.y = srv + }) + return s.y +} + +// pathPerm is what this caller may do to one path: the project's level, +// narrowed or widened by a folder rule. +// +// A predicate, not a gate. writablePath answers the same question by writing +// a 403 onto the response, which is right for a door and wrong for a +// decision — this one has to say "read-only" without ending the request. +func (s *Server) pathPerm(r *http.Request, p Project, path string) string { + base := s.projectPermOf(r, p) + if len(p.Folders) == 0 || base == PermAdmin { + return base + } + return folderLevel(p, normEmail(s.requestUser(r).Email), path, base) +} + +// handleYCollab joins the document for one file. +// +// The room name is DERIVED HERE and never taken from the caller. ygo reads it +// from PathValue("room") or the URL's last segment, so a route that let the +// client name the room would let any member of any project join any other +// document by asking for its name — the project id in the path would be +// decoration. Mounting the name as (project, path) after proj() has resolved +// the project is what keeps the org wall in front of it. +func (s *Server) handleYCollab(v *volume, w http.ResponseWriter, r *http.Request) { + _ = v + path, err := cleanUploadPath(r.URL.Query().Get("path")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // A hidden path is 404, never 403: a 403 confirms the file is there. + // Same rule the viewer's pathFilter applies (folders.go). + if vis := s.visibility(r); !vis.canRead(path) { + http.NotFound(w, r) + return + } + r.SetPathValue("room", projectID(r)+"/"+path) + s.ydocs().ServeHTTP(w, r) +} + +/* +fileSeed is what makes the seed CLAIM unnecessary. + + The relay could not seed: it never parsed a frame, so it had no way to turn + a file into a document. Exactly one joiner was therefore told "you are + first, build it from the bytes you loaded" — with a grace timer, because a + claim that never produced anything would leave every later joiner holding a + blank document that the source editor would then cheerfully save over the + file. + + The hub can just do it. LoadDoc runs once, when the room is created and + before any client is attached, so there is nothing to claim and nothing to + race: the document starts as the file, deterministically, every time. + + StoreUpdate is Stage 3's seam — snapshotting the document back to the file + is what finally retires "whoever stops typing last writes it", and until + then the client still saves through upload/content exactly as it does now. + Returning nil is not a stub that forgot to be written; it is this stage + declining to own the write path yet. +*/ +type fileSeed struct{ s *Server } + +func (f fileSeed) LoadDoc(room string) ([]byte, error) { + project, path, ok := strings.Cut(room, "/") + if !ok { + return nil, nil + } + _, v, err := f.s.projectVolume(project) + if err != nil { + return nil, nil // no such project: an empty document, not an error + } + ctx := context.Background() + snap, err := v.snapshot(ctx) + if err != nil { + return nil, nil + } + fi, ok := snap.files[path] + if !ok { + return nil, nil // a file that does not exist yet starts empty + } + rc, err := v.source.Open(ctx, path, fi) + if err != nil { + return nil, nil + } + defer rc.Close() + // Bounded: a document is held in memory for as long as somebody has it + // open, and a 500 MB file is not something to seed a CRDT with. + body, err := io.ReadAll(io.LimitReader(rc, maxSeedBytes)) + if err != nil { + return nil, nil + } + doc := crdt.New() + txt := doc.GetText("body") + doc.Transact(func(txn *crdt.Transaction) { + txt.Insert(txn, 0, string(body), nil) + }) + return crdt.EncodeStateAsUpdateV1(doc, nil), nil +} + +// StoreUpdate is Stage 3. See fileSeed. +func (f fileSeed) StoreUpdate(string, []byte) error { return nil } + +// maxSeedBytes bounds what will be turned into a held document. Editing costs +// roughly ten times the content in CRDT items, so this is a memory ceiling +// rather than a file-size opinion; past it the editor falls back to the +// ordinary read/write path, which is what it does for any file it cannot +// render anyway. +const maxSeedBytes = 2 << 20 diff --git a/internal/webapp/ycollab_test.go b/internal/webapp/ycollab_test.go new file mode 100644 index 00000000..2be4a4e7 --- /dev/null +++ b/internal/webapp/ycollab_test.go @@ -0,0 +1,258 @@ +package webapp + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + + "github.com/reearth/ygo/crdt" +) + +/* +The wire, pinned in both directions. + + The hub now decodes CRDT updates that browsers produce, so "our Go library + and their JS library agree" stopped being a property of somebody else's + README and became a thing this repo has to keep true across version bumps. + + The fixtures in testdata/ were produced by the exact yjs build the frontend + ships (node_modules/yjs) and are checked in as BYTES: CI runs Go without + node, and a test that needs a toolchain it does not have is a test that + gets skipped. A bump that breaks the wire fails the build instead of the + editor. +*/ +func TestYjsWireCompatBothDirections(t *testing.T) { + read := func(name string) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + return b + } + + for _, tc := range []struct{ name, want string }{ + {"yjs-small", string(read("yjs-small.txt"))}, + {"yjs-big", string(read("yjs-big.txt"))}, + } { + for _, ver := range []string{"v1", "v2"} { + doc := crdt.New() + update := read(tc.name + "-" + ver + ".bin") + var err error + if ver == "v1" { + err = crdt.ApplyUpdateV1(doc, update, nil) + } else { + err = crdt.ApplyUpdateV2(doc, update, nil) + } + if err != nil { + t.Fatalf("%s %s: Go could not read what yjs wrote: %v", tc.name, ver, err) + } + if got := doc.GetText("body").ToString(); got != tc.want { + t.Errorf("%s %s: read %d chars, want %d", tc.name, ver, len(got), len(tc.want)) + } + } + } + + // ...and the other way: what Go writes, yjs must be able to read. Asserted + // here by round-tripping through the decoder yjs shares the format with — + // the live browser-side half is e2e's job, where a real yjs is present. + doc := crdt.New() + txt := doc.GetText("body") + doc.Transact(func(txn *crdt.Transaction) { + txt.Insert(txn, 0, "written by the hub — ünïcode ✅ 日本語", nil) + }) + for _, ver := range []string{"v1", "v2"} { + var update []byte + if ver == "v1" { + update = crdt.EncodeStateAsUpdateV1(doc, nil) + } else { + update = crdt.EncodeStateAsUpdateV2(doc, nil) + } + if len(update) == 0 { + t.Fatalf("%s: encoded nothing", ver) + } + back := crdt.New() + var err error + if ver == "v1" { + err = crdt.ApplyUpdateV1(back, update, nil) + } else { + err = crdt.ApplyUpdateV2(back, update, nil) + } + if err != nil { + t.Fatalf("%s: %v", ver, err) + } + if got := back.GetText("body").ToString(); got != txt.ToString() { + t.Errorf("%s: round-tripped to %q", ver, got) + } + } +} + +/* +The room name is the hub's to decide. + + ygo takes the room from PathValue("room") or the URL's last segment. If a + caller could name the room, the project id in the path would be decoration + — any member of any project could join any other project's document by + asking for its name. The handler sets the name itself, after proj() has + resolved which project this is. +*/ +func TestYCollabRoomNameIsNotTheCallersToChoose(t *testing.T) { + srv, p, _ := newHub(t, true, nil) + h := srv.Handler() + + // A websocket handshake is not what this asserts — it asserts what the + // handler does with the request BEFORE handing it on, so a plain GET is + // enough: ygo refuses the upgrade, and by then the room is already named. + rec := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/api/p/"+p.ID+"/ycollab?path=notes.md", nil) + r.SetPathValue("room", "../some-other-project/secrets.md") + h.ServeHTTP(rec, r) + + if got := r.PathValue("room"); got != p.ID+"/notes.md" { + t.Fatalf("room = %q, want the hub's own (project, path) — a caller-supplied "+ + "name would make the project id decoration", got) + } +} + +// A path the caller cannot see is 404, not 403: a 403 confirms the file is +// there, which is the same rule the viewer's pathFilter applies. +func TestYCollabRefusesAnUnsafePath(t *testing.T) { + srv, p, _ := newHub(t, true, nil) + h := srv.Handler() + for _, bad := range []string{"../escape.md", ".bdrive/config.json", ""} { + rec := httptest.NewRecorder() + r := httptest.NewRequest("GET", + "/api/p/"+p.ID+"/ycollab?path="+bad, nil) + h.ServeHTTP(rec, r) + if rec.Code == http.StatusSwitchingProtocols || rec.Code == http.StatusOK { + t.Errorf("path %q was accepted (%d)", bad, rec.Code) + } + } +} + +// pathPerm answers what writablePath answers, without ending the request — +// which is what lets a read-only member open a document instead of failing to. +func TestPathPermReportsWithoutRefusing(t *testing.T) { + srv, p, _ := newHub(t, true, nil) + r := httptest.NewRequest("GET", "/api/p/"+p.ID+"/ycollab?path=notes.md", nil) + r = withProject(withProjectID(r, p.ID), p) + + rec := httptest.NewRecorder() + got := srv.pathPerm(r, p, "notes.md") + if rec.Body.Len() != 0 { + t.Error("pathPerm wrote to the response; it is a predicate, not a gate") + } + if !strings.Contains("none read write admin", got) { + t.Errorf("pathPerm = %q, which is not one of the four levels", got) + } +} + +/* +The document starts as the file, without anybody claiming to be first. + + This is the property the seed claim and its grace timer existed to fake: + exactly one joiner was told to build the document from the bytes it had + loaded, and a claim that never produced anything left every later joiner + with a blank document the editor would then save over the file. The hub + seeds it instead, before any client is attached. +*/ +func TestYCollabSeedsTheDocumentFromTheFile(t *testing.T) { + srv, p, _ := newHub(t, true, nil) + h := srv.Handler() + const body = "# seeded by the hub\n\nünïcode ✅ 日本語\n" + rec := putContent(t, h, "/api/p/"+p.ID+"/upload/content?path=notes.md", body, "") + if rec.Code != http.StatusOK { + t.Fatalf("seed write: %d %s", rec.Code, rec.Body.String()) + } + + update, err := fileSeed{srv}.LoadDoc(p.ID + "/notes.md") + if err != nil { + t.Fatal(err) + } + if len(update) == 0 { + t.Fatal("the room would have started empty, which is what the seed claim was for") + } + doc := crdt.New() + if err := crdt.ApplyUpdateV1(doc, update, nil); err != nil { + t.Fatal(err) + } + if got := doc.GetText("body").ToString(); got != body { + t.Errorf("seeded %q, want %q", got, body) + } +} + +// A room for something that is not a file is an empty document, never an +// error: a file being created is an ordinary thing to open an editor on. +func TestYCollabSeedsEmptyForWhatIsNotThere(t *testing.T) { + srv, p, _ := newHub(t, true, nil) + for _, room := range []string{ + p.ID + "/never-written.md", + "no-such-project/notes.md", + "malformed-room-name", + } { + update, err := fileSeed{srv}.LoadDoc(room) + if err != nil { + t.Errorf("%s: %v", room, err) + } + if len(update) != 0 { + t.Errorf("%s: seeded %d bytes from nothing", room, len(update)) + } + } +} + +/* +A real websocket, through the real handler, into the real document. + + Everything above this tests the pieces. This tests that a client can + actually connect — which is where the first three attempts failed, each + for a different reason the unit tests could not see: a route that did not + match the URL y-websocket builds, a client dialling the relay's path, and + a hub still serving a bundle compiled before any of it existed. +*/ +func TestYCollabAcceptsARealWebsocket(t *testing.T) { + srv, p, _ := newHub(t, true, nil) + h := srv.Handler() + const body = "# held by the hub\n" + if rec := putContent(t, h, "/api/p/"+p.ID+"/upload/content?path=notes.md", body, ""); rec.Code != http.StatusOK { + t.Fatalf("seed write: %d %s", rec.Code, rec.Body.String()) + } + + ts := httptest.NewServer(h) + defer ts.Close() + u := "ws" + strings.TrimPrefix(ts.URL, "http") + + "/api/p/" + p.ID + "/ycollab/held?path=notes.md" + + // Accept-Encoding as a BROWSER sends it on a handshake. Go's dialer does + // not, and without it this test passed against a compression middleware + // that made the upgrade impossible for every real client. + conn, resp, err := websocket.DefaultDialer.Dial(u, http.Header{ + "Accept-Encoding": {"gzip, deflate, br, zstd"}, + }) + if err != nil { + status := 0 + if resp != nil { + status = resp.StatusCode + } + t.Fatalf("dial %s: %v (HTTP %d)", u, err, status) + } + defer conn.Close() + + // y-websocket opens by sending sync step 1; the hub must answer with the + // document it seeded rather than silence. + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + _, frame, err := conn.ReadMessage() + if err != nil { + t.Fatalf("no frame from the hub: %v", err) + } + if len(frame) == 0 { + t.Fatal("the hub answered with an empty frame") + } +}