Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .bazelignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ pkg
bin
.git
.idea
web
10 changes: 10 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
.git
bazel-*
bin
web/**/.next
web/**/coverage
web/**/dist
web/**/node_modules
web/**/playwright-report
web/**/test-results
web/**/*.tsbuildinfo
41 changes: 41 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,24 @@ jobs:
if: ${{ failure() }}
uses: ./.github/actions/upload-testlogs

web:
name: Web
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: web/.nvmrc
- name: Enable Corepack
run: corepack enable
- name: Install web dependencies
run: make web-install
- name: Run web checks
run: make web-check

# ---------------------------------------------------------------------------
# INTEGRATION TESTS (e2e, gateway, orchestrator)
# ---------------------------------------------------------------------------
Expand All @@ -126,6 +144,27 @@ jobs:
if: ${{ failure() }}
uses: ./.github/actions/upload-testlogs

web-e2e:
name: Web E2E
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
- uses: ./.github/actions/setup
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: web/.nvmrc
- name: Enable Corepack
run: corepack enable
- name: Install web dependencies
run: make web-install
- name: Install Playwright Chromium
run: corepack pnpm@10.17.1 --dir web exec playwright install --with-deps chromium
- name: Run real-stack web E2E
run: make web-e2e-test

gateway-integration-test:
name: Gateway Integration Test
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
Expand Down Expand Up @@ -310,6 +349,8 @@ jobs:
- queue-integration-test
- storage-integration-test
- consumer-integration-test
- web
- web-e2e
- workflow-security
steps:
- name: Fail if any required check did not succeed
Expand Down
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,11 @@ bin/

# Make completion cache
.make_targets_cache
/web/node_modules/
/web/**/node_modules/
/web/**/dist/
/web/**/.next/
/web/**/coverage/
/web/**/playwright-report/
/web/**/test-results/
/web/**/*.tsbuildinfo
40 changes: 32 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ COMPOSE = docker-compose

# SubmitQueue compose files
COMPOSE_FILE = service/submitqueue/docker-compose.yml
WEB_COMPOSE_FILE = service/submitqueue/docker-compose.web.yml
GATEWAY_COMPOSE_FILE = service/submitqueue/gateway/server/docker-compose.yml
ORCHESTRATOR_COMPOSE_FILE = service/submitqueue/orchestrator/server/docker-compose.yml

Expand Down Expand Up @@ -41,6 +42,10 @@ PROTO_PACKAGES = api/base/change api/base/hook api/base/mergestrategy api/base/m
# Set REPO_ROOT for docker-compose
export REPO_ROOT := $(shell pwd)

# Local-only credential for the reference web host. Override it for manual
# testing; production authentication is deliberately outside this demo.
export SUBMITQUEUE_WEB_TOKEN ?= submitqueue-local-demo-token-change-me

# Which provider the demo stack targets, and the only difference between a free
# local run and a live one. Selects a configuration directory rather than a code
# path, so adding a provider is mostly adding a directory — see
Expand Down Expand Up @@ -148,7 +153,7 @@ define assert_clean
fi
endef

.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go help
.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go web-build web-check web-e2e-test web-install web-proto help


build: ## Build all services and examples
Expand Down Expand Up @@ -361,7 +366,7 @@ local-submitqueue-clean: ## Stop the stack and remove its volumes, images, and P
@echo "Cleaning all services and data..."
@# The overlay is named so that volumes it declares — Runway's checkouts —
@# are removed too, rather than surviving as an orphan.
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down -v --rmi local
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down -v --rmi local
@rm -rf "$(SQ_GIT_SANDBOX_DIR)"
@echo "All services, volumes, images, and $(SQ_GIT_SANDBOX_DIR) removed."

Expand Down Expand Up @@ -442,7 +447,7 @@ local-runway-stop: ## Stop Runway service
@echo "Runway services stopped."

local-submitqueue-logs: ## View logs from all running services
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) logs -f
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) logs -f

local-submitqueue-orchestrator-start: build-submitqueue-orchestrator-linux ## Start Orchestrator service locally (Orchestrator + 2 MySQL databases)
@echo "Starting Orchestrator with docker-compose..."
Expand All @@ -466,11 +471,12 @@ local-submitqueue-orchestrator-stop: ## Stop Orchestrator service
local-submitqueue-ps: ## Show running containers and their ports
@echo "Running containers and ports:"
@echo ""
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) ps
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) ps
@echo ""
@echo "📡 Service Endpoints:"
@echo " Gateway gRPC: localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'not running')"
@echo " Orchestrator gRPC: localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'not running')"
@echo " Web UX: http://localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-web-service-1 3000 2>/dev/null | cut -d: -f2 || echo 'not running')"
@echo ""
@echo "🗄️ Database Endpoints:"
@echo " MySQL App: localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'not running')"
Expand All @@ -488,7 +494,7 @@ local-submitqueue-ps: ## Show running containers and their ports

local-submitqueue-restart: build-all-linux ## Restart all services (rebuild and restart)
@echo "Restarting all services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) restart
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) restart
@echo "Services restarted!"
@make local-submitqueue-ps

Expand All @@ -511,13 +517,16 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|
else \
export SQ_CONTAINER_USER=$$(id -u):$$(id -g); \
fi; \
$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) up -d --build --wait
$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) up -d --build --wait
@echo "Applying database schemas..."
@$(MAKE) -s local-init-submitqueue-schemas
@echo ""
@echo "✅ Stack is running against provider '$(PROVIDER)'."
@echo ""
@echo "Gateway gRPC port: $$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
@echo "Web UX: http://localhost:$$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-web-service-1 3000 2>/dev/null | cut -d: -f2 || echo 'unknown')"
@echo "Web username: submitqueue"
@echo "Web token: $(SUBMITQUEUE_WEB_TOKEN)"
@if [ "$(PROVIDER)" = "git" ]; then \
echo "Land target: $(SQ_GIT_SANDBOX_DIR)/sandbox.git"; \
fi
Expand All @@ -527,7 +536,7 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|

local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sandbox; the databases do not survive)
@echo "Stopping SubmitQueue services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@# Both MySQL services mount anonymous volumes, so `down` detaches them and
@# the next `up` creates fresh ones. Saying "data preserved" here would be
@# read as "your requests are still there", which they are not.
Expand All @@ -539,7 +548,7 @@ local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sand

local-stop: ## Stop every local stack — SubmitQueue, Stovepipe, and Runway (keep data)
@echo "Stopping all services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(RUNWAY_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down
@echo "Services stopped. Data volumes preserved."
Expand Down Expand Up @@ -644,6 +653,21 @@ tidy-go: ## Run go mod tidy
@echo "Running go mod tidy..."
@$(BAZEL) run @rules_go//go -- mod tidy -e

web-build: web-install ## Build all web packages and the Next.js host
@corepack pnpm@10.17.1 --dir web build

web-check: ## Run web generation, packaging, lint, type, unit, and build checks
@corepack pnpm@10.17.1 --dir web check

web-e2e-test: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-runway-linux web-install ## Run the real-stack Playwright and axe web test
@web/tool/run-e2e.sh

web-install: ## Install the pinned web workspace dependencies
@corepack pnpm@10.17.1 --dir web install --frozen-lockfile

web-proto: web-install ## Regenerate the TypeScript protobuf API
@corepack pnpm@10.17.1 --dir web api:generate

help: ## Show this help message
@echo "Available targets:"
@echo ""
Expand Down
5 changes: 5 additions & 0 deletions doc/howto/QUICKSTART.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,15 @@ Compose publishes each service on a **random** host port so several stacks can r
✅ Stack is running against provider 'fake'.

Gateway gRPC port: 58537
Web UX: http://localhost:58538
Web username: submitqueue
Web token: submitqueue-local-demo-token-change-me
```

You do not have to note it down. Every command below finds the running stack's port for itself, which matters because Compose picks a fresh one on every start — a number copied from an earlier run is the most common reason a demo command cannot connect. Set `GATEWAY_ADDR=host:port` only to reach a gateway this Makefile did not start.

The Web UX is a read-only reference host for the same gateway. Open the printed URL and use the printed HTTP Basic credentials. It lists receipts from `demo-queue` in a fixed trailing 24-hour window; after `make demo-requests`, select any row to see its current status and retained lifecycle history. The host and its default credential are deliberately local-demo wiring, not a production authentication design. Override the token with `SUBMITQUEUE_WEB_TOKEN=<at-least-32-characters> make local-submitqueue-start` when desired.

## Put traffic through it

```bash
Expand Down
34 changes: 34 additions & 0 deletions service/submitqueue/docker-compose.web.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Compose overlay for the local, read-only SubmitQueue web demo.
#
# The browser talks only to this Next.js host. The host reaches the private
# grpc-go gateway over Compose's internal network using explicit plaintext
# HTTP/2, which is intentionally limited to the local demo.

services:
web-service:
image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-web-service:latest
build:
context: ${REPO_ROOT}
dockerfile: web/service/submitqueue/Dockerfile
ports:
- "3000"
environment:
- PORT=3000
- HOSTNAME=0.0.0.0
- SUBMITQUEUE_GATEWAY_URL=http://gateway-service:8080
- SUBMITQUEUE_GATEWAY_ALLOW_PLAINTEXT=true
- SUBMITQUEUE_WEB_TOKEN=${SUBMITQUEUE_WEB_TOKEN:?set SUBMITQUEUE_WEB_TOKEN to at least 32 characters}
depends_on:
gateway-service:
condition: service_started
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://127.0.0.1:3000/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))",
]
interval: 2s
timeout: 2s
retries: 30
1 change: 1 addition & 0 deletions web/.nvmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
22.20.0
32 changes: 32 additions & 0 deletions web/api/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{
"name": "@submitqueue/api",
"version": "0.1.0",
"private": true,
"type": "module",
"files": [
"dist"
],
"exports": {
"./base/change": {
"types": "./dist/api/base/change/proto/change_pb.d.ts",
"import": "./dist/api/base/change/proto/change_pb.js"
},
"./base/mergestrategy": {
"types": "./dist/api/base/mergestrategy/proto/mergestrategy_pb.d.ts",
"import": "./dist/api/base/mergestrategy/proto/mergestrategy_pb.js"
},
"./submitqueue/gateway": {
"types": "./dist/api/submitqueue/gateway/proto/gateway_pb.d.ts",
"import": "./dist/api/submitqueue/gateway/proto/gateway_pb.js"
}
},
"scripts": {
"build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc --project tsconfig.build.json",
"lint": "tsc --project tsconfig.json",
"test": "vitest run",
"typecheck": "tsc --project tsconfig.json"
},
"dependencies": {
"@bufbuild/protobuf": "2.15.0"
}
}
23 changes: 23 additions & 0 deletions web/api/src/api.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { create } from "@bufbuild/protobuf";
import { describe, expect, it } from "vitest";

import {
ListRequestSchema,
SubmitQueueGateway,
} from "./gen/api/submitqueue/gateway/proto/gateway_pb.js";

describe("generated gateway API", () => {
it("constructs list requests without losing int64 values", () => {
const request = create(ListRequestSchema, {
queue: "demo-queue",
receivedAtOrAfterMs: 1_700_000_000_000n,
receivedBeforeMs: 1_700_086_400_000n,
pageSize: 50,
});

expect(request.queue).toBe("demo-queue");
expect(request.receivedAtOrAfterMs).toBe(1_700_000_000_000n);
expect(request.receivedBeforeMs).toBe(1_700_086_400_000n);
expect(SubmitQueueGateway.methods.some((method) => method.localName === "list")).toBe(true);
});
});
Loading