feat: persist generic AWS resources

changes/unreleased/Fixed-20260930-070954.yaml

@@ -0,0 +1,5 @@
+kind: Fixed
+body: Route unhandled S3 bucket subresources through the CRUD fallback and report failed service resets.
+time: 2026-09-30T07:09:54.167849+09:00
+custom:
+    Issue: "174"

cmd/devcloud/fidelity_test.go

@@ -99,16 +99,9 @@ func TestFidelityManifestCoversRegisteredServices(t *testing.T) {
 // lists.
 func TestFidelityManifestCoversCRUDRegistry(t *testing.T) {
 	for _, id := range plugin.DefaultRegistry.RegisteredServices() {
-		// Registry membership is classifiability, not reachability. The CRUD
-		// registry is built from the model, so it also holds operations for
-		// services whose hand-written provider refuses unknown operations
-		// itself (apigatewayv2, xray) — the engine is never routed to for
-		// those, so "unimplemented" is the truth and this check would be
-		// asserting the opposite. TestFidelityManifestCoverage's `served == 0
-		// && RegisteredOps > 0` guard is what catches wiring that goes missing.
-		if !fidelity.Services[id].EngineWired {
-			continue
-		}
+		// The gateway retries explicit AWS-shaped unimplemented responses through
+		// the CRUD engine, so every registry operation is reachable regardless of
+		// whether a legacy provider returns ErrUnhandledOp itself.
 		for op := range crud.RegisteredOps(id) {
 			tier, ok := fidelity.Lookup(id, op)
 			if !ok {

cmd/devcloud/main.go

@@ -6,10 +6,12 @@ import (
 	"context"
 	"errors"
 	"flag"
+	"fmt"
 	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"syscall"
 	"time"
 
@@ -18,6 +20,7 @@ import (
 	"github.com/skyoo2003/devcloud/internal/gateway"
 	"github.com/skyoo2003/devcloud/internal/plugin"
 	iamsvc "github.com/skyoo2003/devcloud/internal/services/iam"
+	"github.com/skyoo2003/devcloud/internal/shared/crud"
 )
 
 // initOrder lists the services that must come up for DevCloud to be useful, in
@@ -31,6 +34,7 @@ var initOrder = []string{
 
 func main() {
 	cfgPath := flag.String("config", "", "Path to config file (optional; uses ./devcloud.yaml if present, else embedded defaults)")
+	resetData := flag.Bool("reset-data", false, "Delete all local DevCloud data and exit")
 	flag.Parse()
 
 	var (
@@ -61,6 +65,23 @@ func main() {
 	}
 
 	registry := plugin.DefaultRegistry
+	if *resetData {
+		if err := resetAllData(cfg, registry); err != nil {
+			slog.Error("failed to reset local data", "error", err)
+			os.Exit(1)
+		}
+		slog.Info("local DevCloud data reset")
+		return
+	}
+	if err := crud.Open(filepath.Join(cfg.DataDir(), "devcloud.db")); err != nil {
+		slog.Error("failed to open DevCloud data store", "error", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := crud.Close(); err != nil {
+			slog.Error("failed to close DevCloud data store", "error", err)
+		}
+	}()
 
 	// A service's provider comes from the plugin itself, so config resolution is
 	// provider-aware without the config package needing to know which services
@@ -74,10 +95,10 @@ func main() {
 		return plugin.ProviderOf(p)
 	}
 
-	initService := func(name string, fatal bool) {
+	initService := func(name string, fatal bool) error {
 		svcCfg := cfg.ProviderService(providerOf(name), name)
 		if !svcCfg.Enabled {
-			return
+			return nil
 		}
 		pluginCfg := plugin.PluginConfig{
 			DataDir: svcCfg.DataDir,
@@ -89,21 +110,22 @@ func main() {
 				os.Exit(1)
 			}
 			slog.Warn("service init failed", "service", name, "error", err)
-			return
+			return err
 		}
 		slog.Info("service initialized", "service", name)
+		return nil
 	}
 
 	for _, name := range initOrder {
-		initService(name, true)
+		_ = initService(name, true)
 	}
 	// RegisteredServices() is sorted, so the long tail starts in a reproducible
 	// order. Services already brought up above are skipped.
 	for _, name := range registry.RegisteredServices() {
 		if _, ok := registry.Get(name); ok {
 			continue
 		}
-		initService(name, false)
+		_ = initService(name, false)
 	}
 
 	// A services block is authoritative and `enabled` defaults to Go's false,
@@ -129,13 +151,44 @@ func main() {
 	var logCollector *admin.LogCollector
 	var unroutedCollector *admin.UnroutedCollector
 	adminHandler := http.NotFoundHandler()
+	var gw *gateway.Gateway
 	if cfg.Admin.Enabled {
 		logCollector = admin.NewLogCollector(1000)
 		unroutedCollector = admin.NewUnroutedCollector(1000)
-		adminHandler = admin.NewAPI(registry, logCollector, unroutedCollector).Handler()
+		resetter := func(ctx context.Context) (int, error) {
+			if gw == nil {
+				return 0, errors.New("gateway is not initialized")
+			}
+			active := registry.ActiveServices()
+			resetErr := gw.WithMaintenance(func() error {
+				if err := registry.ShutdownAll(ctx); err != nil {
+					return err
+				}
+				if err := crud.Close(); err != nil {
+					return err
+				}
+				if err := resetAllData(cfg, registry); err != nil {
+					return err
+				}
+				if err := crud.Open(filepath.Join(cfg.DataDir(), "devcloud.db")); err != nil {
+					return err
+				}
+				for _, name := range active {
+					if err := initService(name, false); err != nil {
+						return fmt.Errorf("reinitialize %s: %w", name, err)
+					}
+				}
+				return nil
+			})
+			if resetErr != nil {
+				return 0, resetErr
+			}
+			return len(registry.ActiveServices()), nil
+		}
+		adminHandler = admin.NewAPI(registry, logCollector, unroutedCollector, resetter).Handler()
 		slog.Info("admin API enabled")
 	}
-	gw := gateway.New(cfg.Server.Port, registry, adminHandler, logCollector, unroutedCollector)
+	gw = gateway.New(cfg.Server.Port, registry, adminHandler, logCollector, unroutedCollector)
 
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
@@ -166,6 +219,28 @@ func main() {
 	}
 }
 
+// resetAllData removes DevCloud-owned state and every configured AWS service
+// data directory. It is called only by the explicit --reset-data command.
+func resetAllData(cfg *config.Config, registry *plugin.Registry) error {
+	paths := map[string]bool{filepath.Clean(cfg.DataDir()): true}
+	for _, id := range registry.RegisteredServices() {
+		p, ok := registry.Construct(id)
+		if !ok {
+			continue
+		}
+		paths[filepath.Clean(cfg.ProviderService(plugin.ProviderOf(p), id).DataDir)] = true
+	}
+	for path := range paths {
+		if path == "." || path == string(filepath.Separator) {
+			return fmt.Errorf("refusing to reset unsafe data path %q", path)
+		}
+		if err := os.RemoveAll(path); err != nil {
+			return fmt.Errorf("remove %s: %w", path, err)
+		}
+	}
+	return nil
+}
+
 func buildOptions(serviceID string, cfg *config.Config, registry *plugin.Registry) map[string]any {
 	// server_port is passed to every service so URL-building providers
 	// (SQS, ECR, CloudFormation, S3, Lambda, etc.) can construct

docs/configuration.md

@@ -96,6 +96,7 @@ logging:
 | `server.port` | `4747` | HTTP server port |
 | `services.<name>.enabled` | `false` | **Required per entry.** Listing a service is not enough — `enabled: true` still has to be set. |
 | `services.<name>.data_dir` | `./data/<name>` | Data directory for persistent storage |
+| `storage.data_dir` | `./data` | Directory holding DevCloud-wide local state, including generic AWS resources |
 | `admin.enabled` | `false` | Serve the admin REST API at `/devcloud/api/*` |
 | `logging.level` | `info` | `debug`, `info`, `warn`, `error` |
 | `logging.format` | `text` | `text` or `json` |
@@ -149,6 +150,12 @@ block** — one under its forward-compatible name, one under its historical one.
 
 ## Data directories
 
+DevCloud's generic resource fallback persists in SQLite at
+`<storage.data_dir>/devcloud.db`. By default this is `./data/devcloud.db`; an
+explicit `DEVCLOUD_DATA_DIR` overrides it together with all service directories.
+Use `devcloud --reset-data` before startup, or (with `admin.enabled: true`)
+`DELETE /devcloud/api/data`, to remove **all** local DevCloud data.
+
 | Service | Default `data_dir` | Backend | Contents |
 |---------|-------------------|---------|----------|
 | S3 | `./data/s3` | Filesystem + SQLite | Object files, `metadata.db` |

docs/coverage.md

@@ -17,8 +17,8 @@ denominators, because [routing and depth are two targets](#the-target):
 | Tier | Serving target | All registered |
 |---|---|---|
 | `hand-verified` | 4,497 | 4,528 |
-| `auto-crud` | 5,193 | 10,871 |
-| `unimplemented` | 2,717 | 3,802 |
+| `auto-crud` | 5,910 | 11,588 |
+| `unimplemented` | 2,000 | 3,085 |
 | **total known** | **12,407** | **19,201** |
 | **hand-verified share** | **36.2%** | **23.6%** |
 

docs/crud-engine.md

@@ -93,7 +93,9 @@ service from answering for operations it does not model — and it matters most
 - `List*` responses return stored objects; when the real AWS output member is a
   list of *names* rather than structures, an SDK may not populate it.
 - No required-parameter validation, so calls succeed with minimal input.
-- The store is in-memory and per-process — not persisted across restarts.
+- Resources are persisted in the local SQLite database under the configured
+  DevCloud data directory and survive restarts. `--reset-data` or
+  `DELETE /devcloud/api/data` clears all local DevCloud state.
 
 To promote an operation from `auto-crud` to `hand-verified`, implement it as an
 explicit `case` in the service provider, following existing patterns. See

docs/getting-started.md

@@ -8,6 +8,10 @@ docker run -p 4747:4747 ghcr.io/skyoo2003/devcloud:latest
 
 To persist data across restarts, mount a volume:
 
+Generic CRUD resources are persisted by default alongside service data. To
+start from a clean local environment, run `devcloud --reset-data`; with the
+admin API enabled, `DELETE /devcloud/api/data` performs the same full reset.
+
 ```bash
 docker run -p 4747:4747 -v $(pwd)/data:/app/data ghcr.io/skyoo2003/devcloud:latest
 ```

internal/admin/api.go

@@ -18,18 +18,27 @@ type API struct {
 	registry     *plugin.Registry
 	logCollector *LogCollector
 	unrouted     *UnroutedCollector
+	resetData    DataResetter
 }
 
+// DataResetter clears all local DevCloud state and returns the number of
+// reinitialized services. It is injected by the application lifecycle layer.
+type DataResetter func(context.Context) (int, error)
+
 // NewAPI creates a new API. A nil unrouted collector is allowed; the unrouted
 // route then reports an empty result rather than 404ing, so a caller reading the
 // endpoint cannot mistake "not collecting" for "nothing was asked for" — the
 // distinction is visible in maxServiceIds.
-func NewAPI(registry *plugin.Registry, logCollector *LogCollector, unrouted *UnroutedCollector) *API {
-	return &API{
+func NewAPI(registry *plugin.Registry, logCollector *LogCollector, unrouted *UnroutedCollector, resetters ...DataResetter) *API {
+	api := &API{
 		registry:     registry,
 		logCollector: logCollector,
 		unrouted:     unrouted,
 	}
+	if len(resetters) > 0 {
+		api.resetData = resetters[0]
+	}
+	return api
 }
 
 // Handler returns an http.Handler that serves all /devcloud/api/* routes.
@@ -41,10 +50,30 @@ func (d *API) Handler() http.Handler {
 	mux.HandleFunc("/devcloud/api/logs", d.handleLogs)
 	mux.HandleFunc("/devcloud/api/fidelity", d.handleFidelity)
 	mux.HandleFunc("/devcloud/api/unrouted", d.handleUnrouted)
+	mux.HandleFunc("/devcloud/api/data", d.handleData)
 
 	return mux
 }
 
+// handleData deletes all DevCloud-owned local state. It is deliberately an
+// admin-only endpoint and exposes no persistence implementation details.
+func (d *API) handleData(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodDelete {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if d.resetData == nil {
+		http.Error(w, "data reset is unavailable", http.StatusServiceUnavailable)
+		return
+	}
+	active, err := d.resetData(r.Context())
+	if err != nil {
+		http.Error(w, "failed to reset local data", http.StatusInternalServerError)
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"reset": true, "activeServices": active})
+}
+
 // handleUnrouted handles GET /devcloud/api/unrouted.
 //
 // It answers "what did callers ask this DevCloud for that it does not

internal/admin/api_test.go

@@ -176,6 +176,27 @@ func TestAPI_UnroutedRejectsNonGET(t *testing.T) {
 	assert.Equal(t, http.StatusMethodNotAllowed, w.Code)
 }
 
+func TestAPI_DataReset(t *testing.T) {
+	called := false
+	api := NewAPI(plugin.NewRegistry(), NewLogCollector(10), nil,
+		func(context.Context) (int, error) {
+			called = true
+			return 3, nil
+		})
+	w := httptest.NewRecorder()
+	api.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/devcloud/api/data", nil))
+	require.Equal(t, http.StatusOK, w.Code)
+	assert.True(t, called)
+	var body map[string]any
+	require.NoError(t, json.NewDecoder(w.Body).Decode(&body))
+	assert.Equal(t, true, body["reset"])
+	assert.Equal(t, float64(3), body["activeServices"])
+
+	w = httptest.NewRecorder()
+	api.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/devcloud/api/data", nil))
+	assert.Equal(t, http.StatusMethodNotAllowed, w.Code)
+}
+
 // TestAPI_Services registers a mock plugin and verifies the
 // /devcloud/api/services endpoint returns it.
 func TestAPI_Services(t *testing.T) {

internal/codegen/gen_fidelity.go

@@ -84,13 +84,13 @@ func BuildFidelityData(
 	for _, serviceID := range sortedKeys(providers) {
 		provider := providers[serviceID]
 
-		// Reachable, not merely classifiable: an unwired provider serves none of
-		// the registry's operations, so its served set is empty and its
-		// CRUD-shaped operations fall through to unimplemented.
-		var served map[string]bool
-		if provider.EngineWired {
-			served = crudOps[serviceID]
-		}
+		// Every classified operation is reachable through the gateway. Providers
+		// that return ErrUnhandledOp take the original path; legacy providers
+		// that return an AWS-shaped NotImplemented/UnsupportedOperation response
+		// are retried by the gateway only when this registry classifies the
+		// operation. This preserves honest failures for non-CRUD operations while
+		// making the generic CRUD contract independent of provider boilerplate.
+		served := crudOps[serviceID]
 
 		hand := make(map[string]bool, len(provider.Operations))
 		universe := make(map[string]bool)

internal/codegen/gen_fidelity_test.go

@@ -28,21 +28,16 @@ func tierOf(t *testing.T, data FidelityData, serviceID, op string) string {
 	return ""
 }
 
-// TestBuildFidelityDataRequiresEngineWiring is the guard against the manifest
-// overstating coverage.
-//
-// Being in the CRUD registry only means an operation is *classifiable* — the
-// engine is reached at runtime only for providers that return
-// plugin.ErrUnhandledOp. Labelling an operation auto-crud on registry
-// membership alone publishes "this is served" for a provider that refuses it,
-// which is exactly what a freshly scaffolded service would look like.
-func TestBuildFidelityDataRequiresEngineWiring(t *testing.T) {
+// TestBuildFidelityDataUsesGatewayFallback proves that legacy providers which
+// return an AWS-shaped unimplemented response still expose classified CRUD
+// operations through the gateway fallback.
+func TestBuildFidelityDataUsesGatewayFallback(t *testing.T) {
 	modelOps := map[string][]string{
 		"wired":   {"CreateThing"},
 		"unwired": {"CreateThing"},
 	}
-	// Neither service hand-implements anything; the only difference is whether
-	// its dispatch reaches the engine.
+	// Neither service hand-implements anything. The gateway handles both the
+	// sentinel and legacy explicit unimplemented responses.
 	providers := map[string]ProviderScan{
 		"wired":   {EngineWired: true},
 		"unwired": {EngineWired: false},
@@ -54,10 +49,9 @@ func TestBuildFidelityDataRequiresEngineWiring(t *testing.T) {
 
 	data := BuildFidelityData(modelOps, map[string]string{"demo": "json-1.1"}, providers, autoCRUD)
 
-	assert.Equal(t, "AutoCRUD", tierOf(t, data, "wired", "CreateThing"),
-		"an engine-wired provider's CRUD-shaped operation is served, so auto-crud is truthful")
-	assert.Equal(t, "Unimplemented", tierOf(t, data, "unwired", "CreateThing"),
-		"a provider that never reaches the engine serves nothing, whatever the CRUD registry says")
+	assert.Equal(t, "AutoCRUD", tierOf(t, data, "wired", "CreateThing"))
+	assert.Equal(t, "AutoCRUD", tierOf(t, data, "unwired", "CreateThing"),
+		"gateway fallback makes a classified operation reachable without provider boilerplate")
 }
 
 // TestBuildFidelityDataPromotesShortDeclaredOperations covers the operation name