From 8e36faff893ef159d579e7d861768452c7fb7095 Mon Sep 17 00:00:00 2001 From: Aravind Date: Tue, 22 Sep 2026 11:27:42 +0530 Subject: [PATCH] Export a database's rows, so a tenant can move between deployments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local work has been stranded on one machine: `make seed` replays seed/fixtures/seed.json, which is generated from krow-demo and has never reflected what is in a database. Agents published through importagents, knowledge ingested, applications screened — none of it had a path to another deployment. `make export-data` reads the database itself and writes replayable SQL. Three things it does deliberately: Tables are ordered topologically from pg_constraint rather than left in pg_dump's own order, which sorts by name and so fails on foreign keys in a way that depends on what the tables are called. Parents always precede children, and Kahn's algorithm breaks ties alphabetically so two runs against one schema produce a byte-identical file. Every statement is INSERT ... ON CONFLICT DO NOTHING. The export can create rows on a target and cannot modify or delete one. That is a property of the generated file, not a rule someone has to remember when they apply it. The file refuses to apply to a schema older than the one it came from. A restore into a half-migrated database half-succeeds, and a partial import is harder to unpick than a failed one. pg_dump 18 wraps its output in the psql meta-commands \restrict and \unrestrict. Dumping per table left the closing one without its opener, which fails with "not currently in restricted mode" and, under --single-transaction, rolls back having inserted nothing — silently, if the caller reads psql's output through a pipe instead of its exit code. Each table's slice is therefore cut at the last line ending in a semicolon, which no line of pg_dump's epilogue does and every generated statement does. sessions and schema_migrations are excluded: sessions are bound to cookies one deployment issued, and a stale schema_migrations row would make the target lie about its own version. Verified against a scratch database migrated to 15: all 25 tables match the source row for row, a second run is a no-op, and the guard refuses a version-10 target. The output is real tenant data — password hashes, personal details — so seed/exports/ is gitignored. It moves over scp, not through git. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 4 + Makefile | 13 ++ scripts/export-local-data.py | 232 +++++++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100755 scripts/export-local-data.py diff --git a/.gitignore b/.gitignore index edb5c7c..fa38fcf 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ krowdb_public_snapshot_*.sql # Filled-in Kubernetes secret (the .example is the committed template) infrastructure/k8s/10-secret.yaml + +# Database exports. Real tenant data — password hashes, personal details. +# Generated by `make export-data`; move it over scp, never through git. +seed/exports/ diff --git a/Makefile b/Makefile index 665e70a..cec5c20 100644 --- a/Makefile +++ b/Makefile @@ -163,6 +163,19 @@ seed-fixture: ## Regenerate seed.json from the frontend seed module seed-fixture-check: ## Fail if seed.json no longer matches the frontend seed module cd "$(CURDIR)/../krow-demo" && npm run seed:check +# Moving a tenant between databases. `seed` replays the frontend's demo +# fixture; this replays what is actually IN a database, so work done through +# the API comes with it. The output is real tenant data and is gitignored — +# carry it to the target over scp, never through git. +.PHONY: export-data +export-data: ## Export this database's rows as replayable SQL: make export-data [OUT=path] + python3 scripts/export-local-data.py + +.PHONY: import-data +import-data: ## Apply an export to a target: make import-data TARGET_URL=postgres://… [IN=path] + @test -n "$(TARGET_URL)" || { echo "import-data: TARGET_URL is required"; exit 1; } + psql "$(TARGET_URL)" --single-transaction -v ON_ERROR_STOP=1 -f "$(or $(IN),seed/exports/local-data.sql)" + .PHONY: gen-resources gen-resources: ## Regenerate domain descriptors from the live schema python3 scripts/gen_resources.py > go-api/internal/domain/resources_gen.go diff --git a/scripts/export-local-data.py b/scripts/export-local-data.py new file mode 100755 index 0000000..51d7f61 --- /dev/null +++ b/scripts/export-local-data.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Export this deployment's data as a replayable SQL file, for moving a whole +tenant from one database to another. + + make export-data # -> seed/exports/local-data.sql + make export-data OUT=/tmp/krow.sql + +This is NOT `make seed`. `seed/fixtures/seed.json` is the frontend's demo +dataset, generated from krow-demo and replayed by cmd/seed; it has never +reflected what is actually in a database. This reads the database itself, so +work done through the API — agents published, knowledge ingested, applications +screened — is carried across. The two are complementary and neither replaces +the other. + +Why a generated SQL file rather than pg_dump straight into the target: + + - pg_dump orders tables by its own rules, not by foreign keys, so a + --data-only restore into a populated schema fails on FK violations in a + way that depends on table names. This sorts topologically from + pg_constraint and dumps one table at a time in that order, so parents are + always inserted before children. + + - Every statement is INSERT ... ON CONFLICT DO NOTHING. Re-running is safe + and a row that already exists on the target is left exactly as it is. + Nothing here can overwrite or delete data on the target — by construction, + not by convention. + + - The file carries a guard preamble that refuses to apply against a schema + older than the one it was taken from, because a restore into a half + migrated database half-succeeds and that is worse than failing. + +Apply it with a transaction, so a failure leaves nothing behind: + + psql "$TARGET_URL" --single-transaction -v ON_ERROR_STOP=1 \ + -f seed/exports/local-data.sql + +The output contains real tenant data — password hashes, session tokens, +personal details. It is written under seed/exports/, which is gitignored. Do +not commit it; move it to the target host over scp or a secrets channel. +""" +import os +import subprocess +import sys +from collections import defaultdict + +# Tables whose contents belong to the machine they were written on, not to the +# tenant. schema_migrations is owned by the migration tool and a stale row +# there would make the target lie about its own version; sessions are bound to +# cookies issued by one deployment and are worthless on another. +EXCLUDED = {"schema_migrations", "sessions"} + +OUT = os.environ.get("OUT") or "seed/exports/local-data.sql" + + +def env(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() + + +def psql_env() -> dict: + """Connection environment for psql/pg_dump, from the same DATABASE_* vars + the Go services read. No connection string is ever built by hand, so a + password cannot end up in a process list.""" + e = dict(os.environ) + e["PGHOST"] = env("DATABASE_HOST", "127.0.0.1") + e["PGPORT"] = env("DATABASE_PORT", "5432") + e["PGDATABASE"] = env("DATABASE_NAME") + e["PGUSER"] = env("DATABASE_USER") + if env("DATABASE_PASSWORD"): + e["PGPASSWORD"] = env("DATABASE_PASSWORD") + e["PGSSLMODE"] = env("DATABASE_SSLMODE", "disable") + return e + + +def query(sql: str, e: dict) -> list[str]: + out = subprocess.run( + ["psql", "-Atq", "-c", sql], env=e, capture_output=True, text=True + ) + if out.returncode != 0: + sys.exit(f"export-local-data: query failed: {out.stderr.strip()}") + return [line for line in out.stdout.splitlines() if line] + + +def topological_order(e: dict) -> list[str]: + """Tables sorted so that every table appears after the tables it references. + + Self-references are ignored: a row pointing at another row of its own table + is an ordering problem inside one INSERT batch, which ON CONFLICT DO + NOTHING plus a second run resolves, not a table ordering problem. + """ + tables = query( + "select table_name from information_schema.tables " + "where table_schema='public' and table_type='BASE TABLE' order by 1", e + ) + tables = [t for t in tables if t not in EXCLUDED] + known = set(tables) + + edges = query( + "select distinct src.relname || '|' || tgt.relname " + "from pg_constraint c " + "join pg_class src on src.oid = c.conrelid " + "join pg_class tgt on tgt.oid = c.confrelid " + "join pg_namespace n on n.oid = src.relnamespace " + "where c.contype = 'f' and n.nspname = 'public' " + "and src.relname <> tgt.relname", e + ) + + deps: dict[str, set[str]] = defaultdict(set) # table -> tables it needs first + dependents: dict[str, set[str]] = defaultdict(set) + for edge in edges: + child, parent = edge.split("|", 1) + if child in known and parent in known: + deps[child].add(parent) + dependents[parent].add(child) + + # Kahn's algorithm, alphabetical among ready tables so the output of two + # runs against the same schema is byte-identical and reviewable in a diff. + ready = sorted(t for t in tables if not deps[t]) + ordered: list[str] = [] + while ready: + table = ready.pop(0) + ordered.append(table) + for child in sorted(dependents[table]): + deps[child].discard(table) + if not deps[child]: + ready.append(child) + ready.sort() + + if len(ordered) != len(tables): + cyclic = sorted(set(tables) - set(ordered)) + sys.exit( + "export-local-data: foreign keys form a cycle across " + f"{', '.join(cyclic)}; no insert order can satisfy them. Make one " + "of those constraints DEFERRABLE, or exclude a table here." + ) + return ordered + + +def dump_table(table: str, e: dict) -> str: + r"""One table's rows as INSERT ... ON CONFLICT DO NOTHING. + + pg_dump is invoked per table because it reorders a multi-table dump + internally and would undo the topological sort. That leaves a preamble and + an epilogue around each table's statements, and both have to go: pg_dump 18 + wraps its output in the psql meta-commands `\restrict` and `\unrestrict`, + so keeping the closing one without its opener makes psql fail with + "not currently in restricted mode" and — under --single-transaction — roll + the whole import back having inserted nothing. + + Neither end is trimmed line by line, because a text column may legitimately + contain a line that looks like a comment or a meta-command. Instead the + slice runs from the first line that starts an INSERT to the last line that + ends a statement. Every generated statement ends in a semicolon and nothing + in pg_dump's epilogue does, so this cannot cut a row's own text short. + """ + out = subprocess.run( + ["pg_dump", "--data-only", "--inserts", "--on-conflict-do-nothing", + "--no-owner", "--no-privileges", "--table", f"public.{table}"], + env=e, capture_output=True, text=True, + ) + if out.returncode != 0: + sys.exit(f"export-local-data: pg_dump {table} failed: {out.stderr.strip()}") + + lines = out.stdout.splitlines() + first = next((i for i, l in enumerate(lines) if l.startswith("INSERT INTO ")), None) + if first is None: + return "" # table is empty + last = max(i for i, l in enumerate(lines) if l.rstrip().endswith(";")) + return "\n".join(lines[first:last + 1]).rstrip() + "\n" + + +def main() -> None: + e = psql_env() + if not e["PGDATABASE"] or not e["PGUSER"]: + sys.exit("export-local-data: DATABASE_NAME and DATABASE_USER are required " + "(source them from .env, or use `make export-data`)") + + version = (query("select version from schema_migrations", e) or ["0"])[0] + dirty = (query("select dirty from schema_migrations", e) or ["f"])[0] + if dirty == "t": + sys.exit(f"export-local-data: source schema is dirty at version {version}; " + "fix it with `make migrate-force VERSION=…` before exporting") + + tables = topological_order(e) + os.makedirs(os.path.dirname(OUT) or ".", exist_ok=True) + + counts: dict[str, int] = {} + with open(OUT, "w") as f: + f.write( + f"-- Krow data export from {e['PGDATABASE']} at schema version {version}.\n" + "-- Generated by scripts/export-local-data.py — do not edit by hand.\n" + "--\n" + "-- Every statement is ON CONFLICT DO NOTHING: applying this can create\n" + "-- rows on the target but can never modify or delete one. Tables are\n" + "-- ordered so parents precede children.\n" + "--\n" + "-- psql \"$TARGET_URL\" --single-transaction -v ON_ERROR_STOP=1 \\\n" + f"-- -f {OUT}\n\n" + "\\set ON_ERROR_STOP on\n\n" + "DO $krow_guard$\n" + "DECLARE target_version bigint;\n" + "BEGIN\n" + " SELECT version INTO target_version FROM public.schema_migrations;\n" + f" IF target_version IS NULL OR target_version < {version} THEN\n" + " RAISE EXCEPTION 'target schema is at version %, but this export was " + f"taken at {version}; run migrations on the target first', target_version;\n" + " END IF;\n" + "END\n" + "$krow_guard$;\n\n" + ) + for table in tables: + body = dump_table(table, e) + counts[table] = body.count("INSERT INTO ") + f.write(f"-- {table} ({counts[table]} rows)\n") + f.write(body if body else "-- (empty)\n") + f.write("\n") + + total = sum(counts.values()) + print(f"exported {e['PGDATABASE']} (schema version {version}) -> {OUT}") + for table in tables: + if counts[table]: + print(f" {table:<22} {counts[table]:>5}") + skipped = [t for t in tables if not counts[t]] + if skipped: + print(f" (empty: {', '.join(skipped)})") + print(f" {'TOTAL':<22} {total:>5} rows") + print("\nThis file contains real tenant data. It is gitignored; move it to the " + "target host over scp, not through git.") + + +if __name__ == "__main__": + main()