From 1166bdb7c1983f73fb5c0d07836ae55f32aef879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quy=E1=BB=81n?= Date: Sat, 10 Oct 2026 20:32:01 +0700 Subject: [PATCH] feat(engines): add PostgreSQL driver pg_dump/pg_restore/psql inside the local postgres container: whole-db clones stage a transient -Fd archive for pg_restore -j (which cannot read stdin), -only subsets stream a filtered plain-format dump through psql. Includes preflight contract, compose service, docs and unit+integration tests. --- README.md | 32 +- compose.yaml | 23 +- internal/driver/postgres/postgres.go | 452 ++++++++++++++++++ .../postgres/postgres_integration_test.go | 299 ++++++++++++ internal/driver/postgres/postgres_test.go | 201 ++++++++ main.go | 4 +- profiles.example.json | 6 +- 7 files changed, 997 insertions(+), 20 deletions(-) create mode 100644 internal/driver/postgres/postgres.go create mode 100644 internal/driver/postgres/postgres_integration_test.go create mode 100644 internal/driver/postgres/postgres_test.go diff --git a/README.md b/README.md index ab08476..51f9fd4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **Website:** https://phuthuycoding.github.io/dbclone/ -Clone MongoDB and MySQL databases between **any two connections** — staging → local, +Clone MongoDB, MySQL and PostgreSQL databases between **any two connections** — staging → local, prod → local, local → staging, staging → another server — whole databases or just the tables/collections you pick, with parallel streaming, retries and a live progress view in the terminal. @@ -40,13 +40,15 @@ Cloning 3 databases prod → local · pool 8 streams · max 4 streams/db ## Requirements -- Docker, with local MongoDB / MySQL running from the official images (`mongo`, `mysql`). - Their tools run every dump and restore — also when neither side is local. By default the - containers are named `mongodb` and `mysql`; override with `DBCLONE_MONGO_CONTAINER` / - `DBCLONE_MYSQL_CONTAINER`. +- Docker, with local MongoDB / MySQL / PostgreSQL running from the official images + (`mongo`, `mysql`, `postgres`). Their tools run every dump and restore — also when + neither side is local. By default the containers are named `mongodb`, `mysql` and + `postgres`; override with `DBCLONE_MONGO_CONTAINER` / `DBCLONE_MYSQL_CONTAINER` / + `DBCLONE_POSTGRES_CONTAINER`. - The local containers must carry their root credentials in the environment — - `MONGO_INITDB_ROOT_USERNAME` / `MONGO_INITDB_ROOT_PASSWORD` for Mongo and - `MYSQL_ROOT_PASSWORD` for MySQL — which is how the official images are normally configured. + `MONGO_INITDB_ROOT_USERNAME` / `MONGO_INITDB_ROOT_PASSWORD` for Mongo, + `MYSQL_ROOT_PASSWORD` for MySQL and `POSTGRES_PASSWORD` for Postgres — which is how + the official images are normally configured. They are used for the `local` profile; you never type local credentials. - Remote servers must be reachable **from inside** the containers. @@ -77,21 +79,23 @@ or download a binary from [Releases](https://github.com/phuthuycoding/dbclone/re ### Or run everything with Docker Compose -No Go, no binary: the repo's [`compose.yaml`](compose.yaml) starts local MongoDB and MySQL from +No Go, no binary: the repo's [`compose.yaml`](compose.yaml) starts local MongoDB, MySQL and +PostgreSQL from the official images, already carrying the root credentials dbclone expects, and builds the CLI into a small image that talks to them over the Docker socket. ```bash git clone https://github.com/phuthuycoding/dbclone && cd dbclone -docker compose up -d # mongodb + mysql -docker compose run --rm dbclone -check # ✓ Docker, ✓ mongo, ✓ mysql +docker compose up -d # mongodb + mysql + postgres +docker compose run --rm dbclone -check # ✓ Docker, ✓ mongo, ✓ mysql, ✓ postgres docker compose run --rm dbclone # add profiles, pick FROM / TO / databases docker compose run --rm dbclone -from staging -only mongo:shop -yes ``` Profiles persist in the `dbclone-config` volume; logs land in `./logs`. Override ports or -names with `DBCLONE_MONGO_PORT`, `DBCLONE_MYSQL_PORT`, `DBCLONE_MONGO_CONTAINER`, -`DBCLONE_MYSQL_CONTAINER`, and the local root password with `DBCLONE_LOCAL_PASSWORD` +names with `DBCLONE_MONGO_PORT`, `DBCLONE_MYSQL_PORT`, `DBCLONE_POSTGRES_PORT`, +`DBCLONE_MONGO_CONTAINER`, `DBCLONE_MYSQL_CONTAINER`, `DBCLONE_POSTGRES_CONTAINER`, +and the local root password with `DBCLONE_LOCAL_PASSWORD` (default `dbclone`; it only guards the databases on your machine). ## Usage @@ -101,8 +105,8 @@ dbclone # first run: add profiles, then pick FROM, TO and the databas dbclone -setup # add, edit or delete profiles ``` -1. **Profiles** — a profile is a named connection: a MongoDB URI and/or MySQL host, port, user - and password. On first run dbclone checks the local setup and opens the profile manager. +1. **Profiles** — a profile is a named connection: a MongoDB URI and/or MySQL and + PostgreSQL host, port, user and password. On first run dbclone checks the local setup and opens the profile manager. Profiles live in `/dbclone/profiles.json` (mode `0600`; `-config` to use another file). An old `.env.staging` in the current directory is imported once as `staging`. Example: [`profiles.example.json`](profiles.example.json). diff --git a/compose.yaml b/compose.yaml index f70db26..eedaf0b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,11 +1,12 @@ # Everything dbclone needs, with nothing installed on the host but Docker: # -# docker compose up -d # local MongoDB + MySQL from the official images +# docker compose up -d # local MongoDB + MySQL + Postgres from the official images # docker compose run --rm dbclone # the CLI itself, talking to them over the Docker socket # # Ports, container names and the local root password can be overridden from the -# environment (DBCLONE_MONGO_PORT, DBCLONE_MYSQL_PORT, DBCLONE_MONGO_CONTAINER, -# DBCLONE_MYSQL_CONTAINER, DBCLONE_LOCAL_PASSWORD). The password only guards databases +# environment (DBCLONE_MONGO_PORT, DBCLONE_MYSQL_PORT, DBCLONE_POSTGRES_PORT, +# DBCLONE_MONGO_CONTAINER, DBCLONE_MYSQL_CONTAINER, DBCLONE_POSTGRES_CONTAINER, +# DBCLONE_LOCAL_PASSWORD). The password only guards databases # on your own machine; dbclone reads it from the containers and never asks for it. services: @@ -32,6 +33,17 @@ services: - mysql-data:/var/lib/mysql restart: unless-stopped + postgres: + image: postgres:17 + container_name: ${DBCLONE_POSTGRES_CONTAINER:-postgres} + environment: + POSTGRES_PASSWORD: ${DBCLONE_LOCAL_PASSWORD:-dbclone} + ports: + - "${DBCLONE_POSTGRES_PORT:-5432}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + restart: unless-stopped + # Not started by `up`; run it on demand: docker compose run --rm dbclone [flags] dbclone: build: . @@ -40,8 +52,9 @@ services: environment: DBCLONE_MONGO_CONTAINER: ${DBCLONE_MONGO_CONTAINER:-mongodb} DBCLONE_MYSQL_CONTAINER: ${DBCLONE_MYSQL_CONTAINER:-mysql} + DBCLONE_POSTGRES_CONTAINER: ${DBCLONE_POSTGRES_CONTAINER:-postgres} volumes: - - /var/run/docker.sock:/var/run/docker.sock # dbclone runs `docker exec` into the two containers above + - /var/run/docker.sock:/var/run/docker.sock # dbclone runs `docker exec` into the three containers above - dbclone-config:/root/.config/dbclone # profiles.json (mode 0600) survives between runs - ./logs:/work/logs # per-run tool output lands next to this file working_dir: /work @@ -50,8 +63,10 @@ services: depends_on: - mongodb - mysql + - postgres volumes: mongodb-data: mysql-data: + postgres-data: dbclone-config: diff --git a/internal/driver/postgres/postgres.go b/internal/driver/postgres/postgres.go new file mode 100644 index 0000000..e6bf46e --- /dev/null +++ b/internal/driver/postgres/postgres.go @@ -0,0 +1,452 @@ +// Package postgres is the PostgreSQL driver. A whole database is one +// `pg_dump -Fc | pg_restore -j` pipe: pg_restore parallelizes the restore and its +// TOC ordering keeps constraints, indexes and triggers behind the data they need — +// postgres has no non-superuser way to defer foreign keys during load, so the +// ordering must come from the tool. A selected subset (-only) runs a plain-format +// dump through an awk filter that drops the foreign keys pointing at unselected +// objects before psql sees them. All tools run inside the local postgres container +// towards any pair of endpoints. +package postgres + +import ( + "bytes" + "cmp" + "context" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + "time" + + "github.com/phuthuycoding/dbclone/internal/docker" + "github.com/phuthuycoding/dbclone/internal/driver" +) + +const ( + keyHost = "postgres.host" + keyPort = "postgres.port" + keyUser = "postgres.user" + keyPassword = "postgres.password" + + // objectNames is a tab-separated line per object: qualified name, relkind, size. + objectNames = `SELECT n.nspname || '.' || c.relname || E'\t' || c.relkind::text || E'\t' || pg_total_relation_size(c.oid) +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind IN ('r','p','v','m') + AND n.nspname <> 'information_schema' AND n.nspname NOT LIKE 'pg\_%' +ORDER BY 1` + listDBs = `SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres' ORDER BY 1` + // Extensions are the user's job on the target; listing them up front warns early. + listExtensions = `SELECT extname FROM pg_extension WHERE extname <> 'plpgsql' ORDER BY 1` + // $TABLES_IN holds a quoted IN-list of the selected schema.name pairs. + ownedSequences = `SELECT DISTINCT pg_get_serial_sequence(n.nspname || '.' || c.relname, a.attname) +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped +WHERE n.nspname || '.' || c.relname IN ($TABLES_IN)` + // Trigger functions are not selectable objects; they are emitted ahead of the + // post-data stream so user triggers on selected tables can be created. + triggerFunctions = `SELECT pg_get_functiondef(p.oid) || ';' +FROM pg_proc p +JOIN pg_trigger t ON t.tgfoid = p.oid AND NOT t.tgisinternal +JOIN pg_class c ON c.oid = t.tgrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname || '.' || c.relname IN ($TABLES_IN)` + countTriggerFunctions = `SELECT count(*) +FROM pg_trigger t +JOIN pg_class c ON c.oid = t.tgrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE NOT t.tgisinternal AND n.nspname || '.' || c.relname IN ($TABLES_IN)` + // Foreign keys from a selected table to an unselected one cannot be created — + // their statements are filtered out and reported as warnings. + outboundFKs = `SELECT con.conname || E'\t' || n1.nspname || '.' || c1.relname || E'\t' || n2.nspname || '.' || c2.relname +FROM pg_constraint con +JOIN pg_class c1 ON c1.oid = con.conrelid JOIN pg_namespace n1 ON n1.oid = c1.relnamespace +JOIN pg_class c2 ON c2.oid = con.confrelid JOIN pg_namespace n2 ON n2.oid = c2.relnamespace +WHERE con.contype = 'f' AND n1.nspname || '.' || c1.relname IN ($TABLES_IN)` + + // awkFilter drops statements whose REFERENCES target is in $DROP_REFS. Records + // are ';'-terminated statements — safe because the filter only ever sees the + // post-data stream, which holds no COPY bodies. + awkFilter = ` + if (match($0, /FOREIGN[[:space:]]+KEY/) && match($0, /REFERENCES[[:space:]]+[A-Za-z0-9_$.]+/)) { + ref = substr($0, RSTART, RLENGTH) + sub(/^REFERENCES[[:space:]]+/, "", ref) + if (index(DROP_REFS, " " ref " ") > 0) next + } + print` + + psqlFlags = ` -X --no-password -v ON_ERROR_STOP=1 -d"$DB"` +) + +var ( + systemDBs = []string{"postgres", "template0", "template1"} + // Qualified schema.name pairs, lowercase only: anything needing quotes is + // rejected rather than quoted, like mysql's safeName. + safeName = regexp.MustCompile(`^[a-z0-9_$]+(\.[a-z0-9_$]+)*$`) + copyLine = regexp.MustCompile(`COPY ([a-z0-9_$.]+)`) + restData = regexp.MustCompile(`processing data for table "?([a-z0-9_]+)"?(?:\.|"|$)"?([a-z0-9_]+)"?`) + // permanent matches tool errors no retry can fix: authentication, privileges, + // a missing database or relation, and a pg_dump/server version mismatch. + permanent = regexp.MustCompile(`password authentication failed|no pg_hba\.conf entry|permission denied|must be owner|does not exist|server version mismatch`) +) + +type Driver struct{} + +func New() *Driver { return &Driver{} } + +func (*Driver) Name() string { return "postgres" } + +func (*Driver) Fields() []driver.Field { + return []driver.Field{ + {Key: keyHost, Title: "Postgres host", Description: "Leave empty if this profile has no Postgres"}, + {Key: keyPort, Title: "Postgres port", Default: "5432"}, + {Key: keyUser, Title: "Postgres user"}, + {Key: keyPassword, Title: "Postgres password", Secret: true}, + } +} + +func (*Driver) Configured(ep driver.Endpoint) bool { + return ep.Local || (ep.Get(keyHost) != "" && ep.Get(keyUser) != "") +} + +func (*Driver) Address(ep driver.Endpoint) string { + if ep.Local { + return driver.LocalName + } + return strings.ToLower(ep.Get(keyHost)) + ":" + port(ep) +} + +func port(ep driver.Endpoint) string { return cmp.Or(ep.Get(keyPort), "5432") } + +func container() string { return driver.Env("DBCLONE_POSTGRES_CONTAINER", "postgres") } + +func (*Driver) Local() driver.Local { + return driver.Local{ + Container: container(), + ContainerEnv: "DBCLONE_POSTGRES_CONTAINER", + Tools: []string{"pg_dump", "pg_restore", "psql", "awk"}, + // POSTGRES_USER is optional in the image (defaults to postgres), so only the + // password is a hard requirement. + Credentials: []string{"POSTGRES_PASSWORD"}, + RunExample: "docker run -d --name " + container() + " -p 5432:5432 " + + "-e POSTGRES_PASSWORD= postgres:17", + } +} + +// conn returns the prefix that makes a client reach ep — password via PGPASSWORD so +// it never sits on argv — and the env vars it reads, named after prefix. The local +// endpoint logs in as the container's superuser. +func conn(ep driver.Endpoint, prefix string) (pwd, args string, env map[string]string) { + if ep.Local { + return `PGPASSWORD="$POSTGRES_PASSWORD" `, ` -U"${POSTGRES_USER:-postgres}"`, nil + } + return `PGPASSWORD="$` + prefix + `_PASSWORD" `, + ` -h"$` + prefix + `_HOST" -p"$` + prefix + `_PORT" -U"$` + prefix + `_USER"`, + map[string]string{ + prefix + "_HOST": ep.Get(keyHost), + prefix + "_PORT": port(ep), + prefix + "_USER": ep.Get(keyUser), + prefix + "_PASSWORD": ep.Get(keyPassword), + } +} + +func sh(ctx context.Context, script string, stdin bool, env ...map[string]string) *exec.Cmd { + all := map[string]string{} + for _, m := range env { + for k, v := range m { + all[k] = v + } + } + return docker.Sh(ctx, container(), script, stdin, all) +} + +// query runs one SQL statement against the QDB database on ep and returns its rows. +func query(ctx context.Context, ep driver.Endpoint, db, sql string) (string, error) { + pwd, args, env := conn(ep, "EP") + return docker.Output(sh(ctx, pwd+`psql`+args+` -X -A -t --no-password -d"$QDB" -c "$SQL"`, false, env, map[string]string{"QDB": db, "SQL": sql})) +} + +// queryMaint runs against the maintenance database: the image's initial database +// locally, the conventional "postgres" on remote endpoints. +func queryMaint(ctx context.Context, ep driver.Endpoint, sql string) (string, error) { + pwd, args, env := conn(ep, "EP") + db := `"postgres"` + if ep.Local { + db = `"${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"` + } + return docker.Output(sh(ctx, pwd+`psql`+args+` -X -A -t --no-password -d`+db+` -c "$SQL"`, false, env, map[string]string{"SQL": sql})) +} + +func (*Driver) Databases(ctx context.Context, ep driver.Endpoint) ([]string, error) { + out, err := queryMaint(ctx, ep, listDBs) + if err != nil { + return nil, err + } + return docker.SplitLines(out, systemDBs...), nil +} + +func (*Driver) Objects(ctx context.Context, ep driver.Endpoint, db string) ([]driver.Object, error) { + out, err := query(ctx, ep, db, objectNames) + if err != nil { + return nil, err + } + var objs []driver.Object + for _, line := range strings.Split(out, "\n") { + cols := strings.Split(line, "\t") + if len(cols) != 3 { + continue + } + kind := "table" + switch cols[1] { + case "v": + kind = "view" + case "m": + kind = "matview" + } + size, _ := strconv.ParseInt(cols[2], 10, 64) + objs = append(objs, driver.Object{Name: cols[0], Kind: kind, Size: size}) + } + return objs, nil +} + +// Prepare creates the database only when it is missing — a remote target user may +// lack the CREATE privilege on a database that already exists. +func (*Driver) Prepare(ctx context.Context, dst driver.Endpoint, db string, fresh bool) error { + if fresh { + if _, err := queryMaint(ctx, dst, `DROP DATABASE IF EXISTS `+quoteIdent(db)+` WITH (FORCE)`); err != nil { + // WITH (FORCE) needs PG13; on older servers drop the connections by hand. + if _, err := queryMaint(ctx, dst, `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = `+quoteLit(db)); err != nil { + return err + } + if _, err := queryMaint(ctx, dst, `DROP DATABASE IF EXISTS `+quoteIdent(db)); err != nil { + return err + } + } + } + n, err := queryMaint(ctx, dst, `SELECT count(*) FROM pg_database WHERE datname = `+quoteLit(db)) + if err != nil || n != "0" { + return err + } + _, err = queryMaint(ctx, dst, `CREATE DATABASE `+quoteIdent(db)) + return err +} + +func (d *Driver) Plan(ctx context.Context, src, dst driver.Endpoint, db string, workers int, only []string) (driver.Plan, error) { + p := pipe{src: src, dst: dst, db: db} + objs, err := d.Objects(ctx, src, db) + if len(only) == 0 { + var total int64 + var warnings []string + if err != nil { + warnings = append(warnings, "cannot list objects, size estimate unknown: "+err.Error()) + } else { + for _, o := range objs { + total += o.Size + } + } + if exts, e := query(ctx, src, db, listExtensions); e != nil { + warnings = append(warnings, "cannot list extensions: "+e.Error()) + } else if names := docker.SplitLines(exts); len(names) > 0 { + warnings = append(warnings, "extensions must exist on the target: "+strings.Join(names, ", ")) + } + return driver.Plan{Estimate: total, Phases: [][]driver.Stream{{p.whole(workers, total)}}, Warnings: warnings}, nil + } + if err != nil { + return driver.Plan{}, fmt.Errorf("list objects: %w", err) + } + return d.subset(ctx, p, src, db, workers, only, objs) +} + +// subset plans a -only clone in three phases: pre-data + data, then the trigger +// functions the selected tables need, then post-data (indexes, constraints, +// triggers) with foreign keys to unselected objects filtered out. +func (d *Driver) subset(ctx context.Context, p pipe, src driver.Endpoint, db string, _ int, only []string, objs []driver.Object) (driver.Plan, error) { + selected, err := driver.Select(objs, only) + if err != nil { + return driver.Plan{}, err + } + var sel, unsel []string + var total int64 + for _, o := range objs { + if !selected[o.Name] { + unsel = append(unsel, o.Name) + continue + } + if !safeName.MatchString(o.Name) { + return driver.Plan{}, fmt.Errorf("object %q has unsupported characters", o.Name) + } + sel = append(sel, o.Name) + total += o.Size + } + in := inList(sel) + seqs, err := query(ctx, src, db, strings.Replace(ownedSequences, "$TABLES_IN", in, 1)) + if err != nil { + return driver.Plan{}, fmt.Errorf("list sequences: %w", err) + } + tables := tableArgs(sel, docker.SplitLines(seqs)) + vars := map[string]string{ + "TABLES": tables, + "FUNCQ": strings.Replace(triggerFunctions, "$TABLES_IN", in, 1), + "DROP_REFS": " " + strings.Join(unsel, " ") + " ", + } + var warnings []string + fks, err := query(ctx, src, db, strings.Replace(outboundFKs, "$TABLES_IN", in, 1)) + if err != nil { + return driver.Plan{}, fmt.Errorf("list foreign keys: %w", err) + } + for _, line := range strings.Split(fks, "\n") { + cols := strings.Split(line, "\t") + if len(cols) == 3 && !selected[cols[2]] { + warnings = append(warnings, fmt.Sprintf("FK %s on %s -> %s skipped: target object not selected", cols[0], cols[1], cols[2])) + } + } + s1 := p.stream(label(sel), total, 1, + ` -Fp --section=pre-data --section=data --clean --if-exists$TABLES --dbname="$DB"`, + "psql", vars) + s2 := p.stream("indexes/constraints", 0, 1, + ` -Fp --section=post-data --clean --if-exists$TABLES --dbname="$DB"`, + "psql-filtered", vars) + phases := [][]driver.Stream{{s1}} + n, err := query(ctx, src, db, strings.Replace(countTriggerFunctions, "$TABLES_IN", in, 1)) + if err != nil { + return driver.Plan{}, fmt.Errorf("count trigger functions: %w", err) + } + if n != "0" { + phases = append(phases, []driver.Stream{p.funcs(vars)}) + } + return driver.Plan{Estimate: total, Phases: append(phases, []driver.Stream{s2}), Warnings: warnings}, nil +} + +type pipe struct { + src, dst driver.Endpoint + db string +} + +// stream builds one `pg_dump ` from src into the restoreScript on dst, +// both running inside the local container. +func (p pipe) stream(lbl string, size, weight int64, dumpArgs, restoreTool string, vars map[string]string) driver.Stream { + srcPwd, srcArgs, srcEnv := conn(p.src, "SRC") + dstPwd, dstArgs, dstEnv := conn(p.dst, "DST") + vars["DB"] = p.db + var restore string + if restoreTool == "psql-filtered" { + restore = `awk -v RS=';' -v ORS=';' -v DROP_REFS="$DROP_REFS" '{` + awkFilter + `}' | ` + + dstPwd + `psql` + dstArgs + psqlFlags + } else { + restore = dstPwd + `psql` + dstArgs + psqlFlags + } + return driver.Stream{ + Label: lbl, + Size: size, + Weight: weight, + Dump: func(ctx context.Context) *exec.Cmd { + return sh(ctx, srcPwd+`pg_dump`+srcArgs+` --no-password`+dumpArgs, false, srcEnv, vars) + }, + Restore: func(ctx context.Context) *exec.Cmd { return sh(ctx, restore, true, dstEnv, vars) }, + } +} + +// funcs emits the CREATE OR REPLACE FUNCTION statements of the selected tables' +// trigger functions — the only function dependency a post-data stream can hit. +func (p pipe) funcs(vars map[string]string) driver.Stream { + srcPwd, srcArgs, srcEnv := conn(p.src, "SRC") + dstPwd, dstArgs, dstEnv := conn(p.dst, "DST") + vars["DB"] = p.db + return driver.Stream{ + Label: "trigger functions", + Weight: 1, + Dump: func(ctx context.Context) *exec.Cmd { + return sh(ctx, srcPwd+`psql`+srcArgs+` -X -A -t --no-password -d"$DB" -c "$FUNCQ"`, false, srcEnv, vars) + }, + Restore: func(ctx context.Context) *exec.Cmd { + return sh(ctx, dstPwd+`psql`+dstArgs+psqlFlags, true, dstEnv, vars) + }, + } +} + +// whole is the single stream of a full-database clone. pg_restore -j cannot read +// an archive from stdin, so the dump is a serial -Fd directory written to a +// unique path inside the container; the restore first drains stdin (waits for the +// dump to finish), then restores it with -j and removes it. Dump-side -j would +// want snapshot privileges managed hosts often withhold; restore-side -j needs +// none, and it is the slower half anyway. The stream holds `workers` pool slots. +func (p pipe) whole(workers int, size int64) driver.Stream { + workers = max(workers, 1) + srcPwd, srcArgs, srcEnv := conn(p.src, "SRC") + dstPwd, dstArgs, dstEnv := conn(p.dst, "DST") + vars := map[string]string{ + "DB": p.db, + "WORKERS": strconv.Itoa(workers), + "STAGE": "/tmp/dbclone-" + strconv.FormatInt(time.Now().UnixNano(), 36), + } + dump := `rm -rf "$STAGE"; ` + srcPwd + `pg_dump` + srcArgs + + ` --no-password -Fd -f "$STAGE" --dbname="$DB"` + restore := `cat >/dev/null; ` + dstPwd + `pg_restore` + dstArgs + + ` --no-password -d"$DB" -j "$WORKERS" --clean --if-exists --no-owner --no-privileges --verbose "$STAGE"; ` + + `rc=$?; rm -rf "$STAGE"; exit $rc` + return driver.Stream{ + Size: size, + Weight: int64(workers), + Dump: func(ctx context.Context) *exec.Cmd { return sh(ctx, dump, false, srcEnv, vars) }, + Restore: func(ctx context.Context) *exec.Cmd { return sh(ctx, restore, true, dstEnv, vars) }, + } +} + +// tableArgs renders the -t list for the selected objects plus their owned +// sequences. The args travel as one env var expanded unquoted, so names carry no +// quotes — safeName already guarantees they are single words. +func tableArgs(sel, seqs []string) string { + var b strings.Builder + for _, n := range sel { + b.WriteString(" -t " + n) + } + for _, n := range seqs { + if safeName.MatchString(n) { + b.WriteString(" -t " + n) + } + } + return b.String() +} + +func quoteIdent(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } +func quoteLit(s string) string { return `'` + strings.ReplaceAll(s, `'`, `''`) + `'` } + +// inList renders qualified names as a SQL IN list; callers validate the names +// against safeName first. +func inList(names []string) string { + q := make([]string, len(names)) + for i, n := range names { + q[i] = quoteLit(n) + } + return strings.Join(q, ",") +} + +func label(names []string) string { + if len(names) == 1 { + return names[0] + } + return fmt.Sprintf("%s +%d", names[0], len(names)-1) +} + +// StageFromStream reads plain-format chunks (-only path): the last COPY statement +// names the table being loaded. +func (*Driver) StageFromStream(chunk []byte) string { + if !bytes.Contains(chunk, []byte("COPY ")) { + return "" + } + all := copyLine.FindAllSubmatch(chunk, -1) + if len(all) == 0 { + return "" + } + return string(all[len(all)-1][1]) +} + +// StageFromLog reads pg_restore --verbose output ("processing data for table"). +func (*Driver) StageFromLog(line string) string { + if m := restData.FindStringSubmatch(line); m != nil { + return m[1] + "." + m[2] + } + return "" +} + +func (*Driver) Permanent(line string) bool { return permanent.MatchString(line) } diff --git a/internal/driver/postgres/postgres_integration_test.go b/internal/driver/postgres/postgres_integration_test.go new file mode 100644 index 0000000..5d8e614 --- /dev/null +++ b/internal/driver/postgres/postgres_integration_test.go @@ -0,0 +1,299 @@ +// Integration tests against real postgres containers on a private docker network. +// They need Docker and are skipped when it is unavailable; `go test -short` skips +// them too. testPg plays the tools host + `local` endpoint; testPg2 is a remote +// server reachable from inside testPg by its network name. +package postgres + +import ( + "context" + "flag" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/phuthuycoding/dbclone/internal/clone" + "github.com/phuthuycoding/dbclone/internal/driver" + "github.com/phuthuycoding/dbclone/internal/preflight" +) + +const ( + testPg = "dbclone-test-pg" + testPg2 = "dbclone-test-pg2" + testNet = "dbclone-test-net" + testPassword = "testpgpw" + testImage = "postgres:17-alpine" +) + +var dockerOK bool + +func TestMain(m *testing.M) { + flag.Parse() + if testing.Short() { + os.Exit(m.Run()) + } + dockerOK = setup() + code := m.Run() + if dockerOK { + exec.Command("docker", "rm", "-f", testPg, testPg2).Run() + exec.Command("docker", "network", "rm", testNet).Run() + } + os.Exit(code) +} + +// setup starts the two test containers on their own network; false means docker +// is unusable and every integration test skips. +func setup() bool { + if _, err := exec.LookPath("docker"); err != nil { + return false + } + if err := exec.Command("docker", "version", "--format", "{{.Server.Version}}").Run(); err != nil { + return false + } + exec.Command("docker", "rm", "-f", testPg, testPg2).Run() + exec.Command("docker", "network", "rm", testNet).Run() + if out, err := exec.Command("docker", "network", "create", testNet).CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "docker network create: %v\n%s", err, out) + return false + } + for _, name := range []string{testPg, testPg2} { + run := exec.Command("docker", "run", "-d", "--name", name, + "--network", testNet, "-e", "POSTGRES_PASSWORD="+testPassword, testImage) + if out, err := run.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "start %s: %v\n%s", testImage, err, out) + return false + } + } + for range 60 { + if err := exec.Command("docker", "exec", testPg, "pg_isready", "-U", "postgres").Run(); err == nil { + return true + } + time.Sleep(time.Second) + } + return false +} + +func pg(t *testing.T) { + t.Helper() + if !dockerOK { + t.Skip("docker or the test containers are unavailable") + } + t.Setenv("DBCLONE_POSTGRES_CONTAINER", testPg) +} + +// psql runs SQL inside a test container for seeding and verification. +func psql(t *testing.T, container, db, sql string) string { + t.Helper() + cmd := exec.Command("docker", "exec", "-e", "PGPASSWORD="+testPassword, + container, "psql", "-U", "postgres", "-X", "-A", "-t", + "-v", "ON_ERROR_STOP=1", "-d", db, "-c", sql) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("psql on %s %q: %v\n%s", container, sql, err, out) + } + return strings.TrimSpace(string(out)) +} + +func localEP() driver.Endpoint { return driver.Endpoint{Name: driver.LocalName, Local: true} } + +func remoteEP(name, host, port, password string) driver.Endpoint { + return driver.Endpoint{Name: name, Values: map[string]string{ + keyHost: host, keyPort: port, keyUser: "postgres", keyPassword: password, + }} +} + +// remoteSrc is the testPg2 server as a connection profile, reachable by its +// network name from inside the tools container. +func remoteSrc() driver.Endpoint { return remoteEP("src", testPg2, "5432", testPassword) } + +type noopRep struct{} + +func (noopRep) Start(int64) {} +func (noopRep) Add(int64) {} +func (noopRep) Stage(string) {} +func (noopRep) Active(int) {} +func (noopRep) Done() {} +func (noopRep) Fail() {} + +func runClone(t *testing.T, job clone.Job, src, dst driver.Endpoint, fresh bool) clone.Result { + t.Helper() + results := clone.Run(context.Background(), []clone.Job{job}, []clone.Reporter{noopRep{}}, clone.Options{ + Parallel: 4, Workers: 2, Fresh: fresh, Source: src, Target: dst, LogDir: t.TempDir(), + }) + if len(results) != 1 { + t.Fatalf("results = %d, want 1", len(results)) + } + return results[0] +} + +// seedSource builds a database exercising every object kind: two schemas, an FK, +// a sequence default, index, trigger+function, view and materialized view. +func seedSource(t *testing.T, db string) { + t.Helper() + psql(t, testPg2, "postgres", `DROP DATABASE IF EXISTS `+db+` WITH (FORCE)`) + psql(t, testPg2, "postgres", `CREATE DATABASE `+db) + psql(t, testPg2, db, ` +CREATE SCHEMA analytics; +CREATE TABLE public.customers (id serial PRIMARY KEY, name text); +CREATE TABLE public.orders (id serial PRIMARY KEY, customer_id int REFERENCES public.customers(id), note text); +CREATE INDEX orders_note_idx ON public.orders(note); +CREATE SEQUENCE analytics.seq; +CREATE TABLE analytics.events (id int DEFAULT nextval('analytics.seq'), kind text); +CREATE FUNCTION public.bump() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END; $$; +CREATE TRIGGER trg BEFORE INSERT ON public.orders FOR EACH ROW EXECUTE FUNCTION public.bump(); +CREATE VIEW public.v_orders AS SELECT id, note FROM public.orders; +CREATE MATERIALIZED VIEW public.mv_orders AS SELECT count(*) AS c FROM public.orders; +INSERT INTO public.customers(name) VALUES ('a'),('b'); +INSERT INTO public.orders(customer_id,note) VALUES (1,'x'),(2,'y'); +INSERT INTO analytics.events(kind) VALUES ('e1'); +REFRESH MATERIALIZED VIEW public.mv_orders; +`) +} + +func TestPreflightPostgres(t *testing.T) { + pg(t) + report, err := preflight.Run(context.Background(), []driver.Driver{New()}) + if err != nil { + t.Fatalf("preflight: %v", err) + } + if len(report.Ready) != 1 || report.Ready[0].Name() != "postgres" { + t.Fatalf("Ready = %+v, want the postgres driver", report.Ready) + } + t.Setenv("DBCLONE_POSTGRES_CONTAINER", "no-such-container") + report, err = preflight.Run(context.Background(), []driver.Driver{New()}) + if err != nil || len(report.Ready) != 0 || !report.Failed() { + t.Fatalf("bogus container: Ready=%v err=%v, want a failed check", report.Ready, err) + } +} + +func TestCloneWholeDatabase(t *testing.T) { + pg(t) + seedSource(t, "app") + res := runClone(t, clone.Job{Driver: New(), DB: "app"}, remoteSrc(), localEP(), false) + if res.Err != nil { + t.Fatalf("clone failed: %v (log %s)", res.Err, res.Log) + } + checks := map[string]string{ + "orders rows": `SELECT count(*) FROM public.orders`, + "order notes": `SELECT string_agg(note, ',' ORDER BY id) FROM public.orders`, + "customer seq": `SELECT last_value FROM public.customers_id_seq`, + "events seq row": `SELECT id FROM analytics.events WHERE kind='e1'`, + "view": `SELECT count(*) FROM public.v_orders`, + "matview data": `SELECT c FROM public.mv_orders`, + "function exists": `SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='public' AND p.proname='bump'`, + "trigger exists": `SELECT count(*) FROM pg_trigger WHERE tgname='trg' AND NOT tgisinternal`, + "index exists": `SELECT count(*) FROM pg_indexes WHERE schemaname='public' AND indexname='orders_note_idx'`, + "fk exists": `SELECT count(*) FROM pg_constraint WHERE conname='orders_customer_id_fkey'`, + } + want := map[string]string{ + "orders rows": "2", "order notes": "x,y", "customer seq": "2", + "events seq row": "1", "view": "2", "matview data": "2", + "function exists": "1", "trigger exists": "1", "index exists": "1", "fk exists": "1", + } + for name, sql := range checks { + if got := psql(t, testPg, "app", sql); got != want[name] { + t.Errorf("%s = %q, want %q", name, got, want[name]) + } + } + // Re-run: drop-and-recreate must make a second clone clean. + res = runClone(t, clone.Job{Driver: New(), DB: "app"}, remoteSrc(), localEP(), false) + if res.Err != nil { + t.Fatalf("second clone failed: %v", res.Err) + } +} + +func TestCloneSelectedObjects(t *testing.T) { + pg(t) + seedSource(t, "app2") + job := clone.Job{Driver: New(), DB: "app2", Only: []string{"public.orders"}} + res := runClone(t, job, remoteSrc(), localEP(), false) + if res.Err != nil { + t.Fatalf("clone failed: %v", res.Err) + } + if got := psql(t, testPg, "app2", `SELECT count(*) FROM public.orders`); got != "2" { + t.Errorf("orders rows = %q, want 2", got) + } + if got := psql(t, testPg, "app2", `SELECT count(*) FROM pg_indexes WHERE schemaname='public' AND indexname='orders_note_idx'`); got != "1" { + t.Error("index on selected table missing") + } + if got := psql(t, testPg, "app2", `SELECT count(*) FROM pg_trigger WHERE tgname='trg' AND NOT tgisinternal`); got != "1" { + t.Error("trigger on selected table missing (its function was not restored)") + } + if got := psql(t, testPg, "app2", `SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='customers'`); got != "0" { + t.Error("unselected table public.customers was created") + } + if got := psql(t, testPg, "app2", `SELECT count(*) FROM pg_constraint WHERE conname='orders_customer_id_fkey'`); got != "0" { + t.Error("FK to unselected table was created") + } + found := false + for _, w := range res.Warnings { + if strings.Contains(w, "orders_customer_id_fkey") { + found = true + } + } + if !found { + t.Errorf("no warning names the skipped FK: %v", res.Warnings) + } +} + +func TestPrepareFresh(t *testing.T) { + pg(t) + psql(t, testPg, "postgres", `DROP DATABASE IF EXISTS freshdb WITH (FORCE)`) + psql(t, testPg, "postgres", `CREATE DATABASE freshdb`) + psql(t, testPg, "freshdb", `CREATE TABLE keepme (id int)`) + d := New() + dst := localEP() + if err := d.Prepare(context.Background(), dst, "freshdb", false); err != nil { + t.Fatalf("Prepare(fresh=false): %v", err) + } + if got := psql(t, testPg, "freshdb", `SELECT count(*) FROM keepme`); got != "0" { + t.Errorf("fresh=false changed the database (keepme rows = %q)", got) + } + // Hold an idle connection so a plain DROP would hang. + hold := exec.Command("docker", "exec", "-d", testPg, + "psql", "-U", "postgres", "-d", "freshdb", "-c", "SELECT pg_sleep(60)") + if out, err := hold.CombinedOutput(); err != nil { + t.Fatalf("hold connection: %v\n%s", err, out) + } + time.Sleep(time.Second) + if err := d.Prepare(context.Background(), dst, "freshdb", true); err != nil { + t.Fatalf("Prepare(fresh=true) with an open connection: %v", err) + } + if got := psql(t, testPg, "freshdb", `SELECT count(*) FROM information_schema.tables WHERE table_schema='public'`); got != "0" { + t.Errorf("fresh did not drop (tables = %q)", got) + } +} + +func TestCloneRemoteToRemote(t *testing.T) { + pg(t) + seedSource(t, "app3") + src := remoteEP("a", testPg2, "5432", testPassword) + dst := remoteEP("b", testPg, "5432", testPassword) + res := runClone(t, clone.Job{Driver: New(), DB: "app3"}, src, dst, false) + if res.Err != nil { + t.Fatalf("remote-to-remote clone failed: %v", res.Err) + } + if got := psql(t, testPg, "app3", `SELECT count(*) FROM public.orders`); got != "2" { + t.Errorf("orders rows on remote target = %q, want 2", got) + } + if got := psql(t, testPg, "app3", `SELECT count(*) FROM pg_indexes WHERE schemaname='public' AND indexname='orders_note_idx'`); got != "1" { + t.Error("index missing on remote target") + } +} + +func TestPermanentAuthFailure(t *testing.T) { + pg(t) + bad := remoteEP("bad", testPg2, "5432", "wrongpw") + _, err := New().Databases(context.Background(), bad) + if err == nil { + t.Fatal("Databases with a wrong password did not fail") + } + if !New().Permanent(err.Error()) { + t.Errorf("auth failure not classified permanent: %v", err) + } + if strings.Contains(err.Error(), "wrongpw") { + t.Error("password leaked into the error output") + } +} diff --git a/internal/driver/postgres/postgres_test.go b/internal/driver/postgres/postgres_test.go new file mode 100644 index 0000000..924bb0c --- /dev/null +++ b/internal/driver/postgres/postgres_test.go @@ -0,0 +1,201 @@ +package postgres + +import ( + "context" + "os/exec" + "strings" + "testing" + + "github.com/phuthuycoding/dbclone/internal/driver" +) + +func TestLocal(t *testing.T) { + d := New() + l := d.Local() + if l.Container != "postgres" || l.ContainerEnv != "DBCLONE_POSTGRES_CONTAINER" { + t.Fatalf("Local() = %+v, want container postgres overridable by DBCLONE_POSTGRES_CONTAINER", l) + } + for _, tool := range []string{"pg_dump", "pg_restore", "psql"} { + if !strings.Contains(strings.Join(l.Tools, " "), tool) { + t.Errorf("tools %v missing %q", l.Tools, tool) + } + } + if len(l.Credentials) != 1 || l.Credentials[0] != "POSTGRES_PASSWORD" { + t.Errorf("Credentials = %v, want only POSTGRES_PASSWORD (POSTGRES_USER is optional)", l.Credentials) + } + if !strings.Contains(l.RunExample, "postgres:") { + t.Errorf("RunExample %q does not run the official image", l.RunExample) + } + t.Setenv("DBCLONE_POSTGRES_CONTAINER", "mypg") + if got := New().Local().Container; got != "mypg" { + t.Errorf("container = %q, want mypg", got) + } +} + +func TestFieldsConfiguredAddress(t *testing.T) { + d := New() + keys := map[string]driver.Field{} + for _, f := range d.Fields() { + keys[f.Key] = f + } + for _, k := range []string{keyHost, keyPort, keyUser, keyPassword} { + if _, ok := keys[k]; !ok { + t.Fatalf("Fields() missing %q", k) + } + } + if keys[keyPort].Default != "5432" || !keys[keyPassword].Secret { + t.Errorf("port default or password secret wrong: %+v", keys) + } + local := driver.Endpoint{Name: driver.LocalName, Local: true} + if !d.Configured(local) { + t.Error("local endpoint must always be configured") + } + ep := driver.Endpoint{Name: "p", Values: map[string]string{keyHost: "HOST", keyUser: "u"}} + if !d.Configured(ep) { + t.Error("host+user should configure") + } + for _, missing := range []driver.Endpoint{ + {Name: "a", Values: map[string]string{keyUser: "u"}}, + {Name: "b", Values: map[string]string{keyHost: "h"}}, + } { + if d.Configured(missing) { + t.Errorf("Configured(%v) = true, want false", missing.Values) + } + } + if got, want := d.Address(ep), "host:5432"; got != want { + t.Errorf("Address = %q, want %q", got, want) + } + if d.Address(local) != driver.LocalName { + t.Error("local address must be the built-in name") + } +} + +func TestSafeName(t *testing.T) { + for _, ok := range []string{"public.orders", "analytics.events_2", "s.a.b", "t", "sch_ema.t_1"} { + if !safeName.MatchString(ok) { + t.Errorf("safeName(%q) = false, want true", ok) + } + } + for _, bad := range []string{"public.Orders", `weird"name`, "a;b", "a b", ".x", "x.", ""} { + if safeName.MatchString(bad) { + t.Errorf("safeName(%q) = true, want false", bad) + } + } +} + +func TestWholeStream(t *testing.T) { + p := pipe{db: "app"} + s := p.whole(4, 1024) + if s.Weight != 4 || s.Size != 1024 { + t.Fatalf("stream weight/size = %d/%d, want 4/1024", s.Weight, s.Size) + } + dump := script(t, s.Dump(context.Background())) + for _, want := range []string{"pg_dump", "-Fd", `--dbname="$DB"`, `-f "$STAGE"`, `rm -rf "$STAGE"`} { + if !strings.Contains(dump, want) { + t.Errorf("dump %q missing %q", dump, want) + } + } + if strings.Contains(dump, "testpw") { + t.Error("password on dump argv") + } + restore := script(t, s.Restore(context.Background())) + for _, want := range []string{"cat >/dev/null", "pg_restore", `-j "$WORKERS"`, "--clean", "--if-exists", "--no-owner", "--no-privileges", "--verbose", `"$STAGE"`, `rm -rf "$STAGE"`, `-d"$DB"`} { + if !strings.Contains(restore, want) { + t.Errorf("restore %q missing %q", restore, want) + } + } +} + +func TestTableArgs(t *testing.T) { + got := tableArgs([]string{"public.orders", "analytics.events"}, []string{"public.orders_id_seq", `bad"name`}) + want := " -t public.orders -t analytics.events -t public.orders_id_seq" + if got != want { + t.Errorf("tableArgs = %q, want %q", got, want) + } +} + +func TestAwkFilter(t *testing.T) { + if _, err := exec.LookPath("awk"); err != nil { + t.Skip("awk not available") + } + sql := ` +ALTER TABLE ONLY public.orders + ADD CONSTRAINT orders_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES public.customers(id); + +ALTER TABLE ONLY public.orders + ADD CONSTRAINT orders_parent_id_fkey FOREIGN KEY (parent_id) REFERENCES analytics.parents(id); + +CREATE INDEX orders_note_idx ON public.orders USING btree (note); + +CREATE TRIGGER trg BEFORE INSERT ON public.orders FOR EACH ROW EXECUTE FUNCTION public.bump(); +` + cmd := exec.Command("awk", "-v", "RS=;", "-v", "ORS=;", "-v", "DROP_REFS= analytics.parents ", "{"+awkFilter+"}") + cmd.Stdin = strings.NewReader(sql) + out, err := cmd.Output() + if err != nil { + t.Fatalf("awk: %v", err) + } + got := string(out) + if strings.Contains(got, "orders_parent_id_fkey") { + t.Error("FK to unselected analytics.parents was not dropped") + } + for _, want := range []string{"orders_customer_id_fkey", "orders_note_idx", "CREATE TRIGGER trg"} { + if !strings.Contains(got, want) { + t.Errorf("filtered output missing %q:\n%s", want, got) + } + } +} + +func TestPermanent(t *testing.T) { + for _, yes := range []string{ + `psql: error: FATAL: password authentication failed for user "r"`, + `FATAL: no pg_hba.conf entry for host "10.0.0.1"`, + `ERROR: permission denied for table orders`, + `ERROR: must be owner of table orders`, + `FATAL: database "app" does not exist`, + `pg_restore: error: relation "public.orders" does not exist`, + `pg_dump: error: aborting because of server version mismatch`, + } { + if !New().Permanent(yes) { + t.Errorf("Permanent(%q) = false, want true", yes) + } + } + for _, no := range []string{ + `server closed the connection unexpectedly`, + `could not connect to server: Connection refused`, + `pg_restore: error: timeout expired`, + } { + if New().Permanent(no) { + t.Errorf("Permanent(%q) = true, want false", no) + } + } +} + +func TestStageMarkers(t *testing.T) { + d := New() + for in, want := range map[string]string{ + `pg_restore: processing data for table "public"."orders"`: "public.orders", + `pg_restore: processing data for table public.orders`: "public.orders", + `pg_restore: creating TABLE "public.orders"`: "", + } { + if got := d.StageFromLog(in); got != want { + t.Errorf("StageFromLog(%q) = %q, want %q", in, got, want) + } + } + chunk := []byte("\nCOPY public.orders (id, note) FROM stdin;\n1\tx\n") + if got := d.StageFromStream(chunk); got != "public.orders" { + t.Errorf("StageFromStream = %q, want public.orders", got) + } + if got := d.StageFromStream([]byte("\x00\x01binary")); got != "" { + t.Errorf("StageFromStream(binary) = %q, want empty", got) + } +} + +// script extracts the `sh -c` payload of a docker exec command for inspection. +func script(t *testing.T, cmd *exec.Cmd) string { + t.Helper() + if len(cmd.Args) == 0 { + t.Fatal("empty command") + } + return cmd.Args[len(cmd.Args)-1] +} diff --git a/main.go b/main.go index e6faf88..5cbdb68 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ import ( "github.com/phuthuycoding/dbclone/internal/driver" "github.com/phuthuycoding/dbclone/internal/driver/mongo" "github.com/phuthuycoding/dbclone/internal/driver/mysql" + "github.com/phuthuycoding/dbclone/internal/driver/postgres" "github.com/phuthuycoding/dbclone/internal/preflight" "github.com/phuthuycoding/dbclone/internal/ui" ) @@ -45,6 +46,7 @@ var version = "dev" var drivers = []driver.Driver{ mongo.New(), mysql.New(), + postgres.New(), } var ( @@ -53,7 +55,7 @@ var ( to = flag.String("to", "", "target profile (default local; asked when empty and run interactively)") confirm = flag.String("confirm", "", "the target profile name, required to write to a non-local target without the prompt") parallel = flag.Int("j", 8, "global pool: concurrent dump→restore streams across all databases") - workers = flag.Int("w", 4, "max concurrent streams for one database (mongo collections / mysql table groups)") + workers = flag.Int("w", 4, "max concurrent streams for one database (mongo collections / mysql table groups / pg_restore jobs)") only = flag.String("only", "", "skip the picker: comma list of engine:db or engine:db.table, e.g. mongo:shop.orders,mysql:app") yes = flag.Bool("yes", false, "do not ask before overwriting databases on the local target") all = flag.Bool("all", false, "skip the picker and clone every database of the source") diff --git a/profiles.example.json b/profiles.example.json index 74febab..13bddae 100644 --- a/profiles.example.json +++ b/profiles.example.json @@ -7,7 +7,11 @@ "mysql.host": "staging-host", "mysql.port": "3306", "mysql.user": "reader", - "mysql.password": "password" + "mysql.password": "password", + "postgres.host": "staging-host", + "postgres.port": "5432", + "postgres.user": "reader", + "postgres.password": "password" } }, {