From 4aed5f4c9a81428dec779412e4799775a617ca36 Mon Sep 17 00:00:00 2001 From: aaron <> Date: Thu, 14 May 2026 10:46:55 +0800 Subject: [PATCH] no message --- scripts/install_root_db_to_volume.sh | 26 +- scripts/rebuild_volume_db_from_root_db.sh | 326 ++++++++++++++++++++++ 2 files changed, 348 insertions(+), 4 deletions(-) create mode 100755 scripts/rebuild_volume_db_from_root_db.sh diff --git a/scripts/install_root_db_to_volume.sh b/scripts/install_root_db_to_volume.sh index 85f62d4..1de3ade 100755 --- a/scripts/install_root_db_to_volume.sh +++ b/scripts/install_root_db_to_volume.sh @@ -57,10 +57,26 @@ import sqlite3 import sys db = os.environ["HOST_DB"] -conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30) -result = conn.execute("PRAGMA integrity_check").fetchone()[0] -tables = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0] -conn.close() +try: + conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30) + result = conn.execute("PRAGMA integrity_check").fetchone()[0] + tables = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0] + conn.close() +except sqlite3.DatabaseError as exc: + print(f"database check failed for {db}: {exc}", file=sys.stderr) + print("", file=sys.stderr) + print("This file is not safe to install as the live AlphaX DB.", file=sys.stderr) + print("Common causes:", file=sys.stderr) + print(" 1. The DB was copied while SQLite WAL mode was active, but the matching .db-wal file was not copied.", file=sys.stderr) + print(" 2. The upload/copy was interrupted and the DB file is truncated.", file=sys.stderr) + print(" 3. The source path is not the real SQLite database file.", file=sys.stderr) + print("", file=sys.stderr) + print("On the server, check:", file=sys.stderr) + print(" ls -lh altcoin_monitor.db*", file=sys.stderr) + print(" file altcoin_monitor.db", file=sys.stderr) + print("", file=sys.stderr) + print("Best fix: export a clean backup from the old machine/container using SQLite backup, then upload that .db file.", file=sys.stderr) + raise SystemExit(2) if result != "ok": print(f"integrity_check failed for {db}: {result}", file=sys.stderr) raise SystemExit(2) @@ -75,6 +91,8 @@ mkdir -p "$(dirname "$TARGET_DB")" "$BACKUP_DIR" info "source root db: $ROOT_DIR/$ROOT_DB" info "target volume db: $ROOT_DIR/$TARGET_DB" +info "root db candidates:" +ls -lh "$ROOT_DB"* 2>/dev/null || true verify_db "$ROOT_DB" diff --git a/scripts/rebuild_volume_db_from_root_db.sh b/scripts/rebuild_volume_db_from_root_db.sh new file mode 100755 index 0000000..825bab9 --- /dev/null +++ b/scripts/rebuild_volume_db_from_root_db.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Rebuild the AlphaX volume DB from a database file placed in the project root. +# +# This is different from a raw copy: +# - it creates a brand-new clean SQLite DB +# - it recreates schema from the root DB +# - it copies readable table rows into the clean DB +# - it can salvage partial data from a DB that fails integrity_check +# +# Server usage: +# bash scripts/rebuild_volume_db_from_root_db.sh +# +# Defaults: +# source: ./altcoin_monitor.db +# target: ./data/altcoin_monitor.db +# +# Overrides: +# ROOT_DB=./backup.db bash scripts/rebuild_volume_db_from_root_db.sh +# STRICT=1 bash scripts/rebuild_volume_db_from_root_db.sh # fail if any table/row cannot be copied +# RESTART=0 bash scripts/rebuild_volume_db_from_root_db.sh + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +COMPOSE_CMD="${COMPOSE_CMD:-docker compose}" +ROOT_DB="${ROOT_DB:-altcoin_monitor.db}" +TARGET_DB="${TARGET_DB:-data/altcoin_monitor.db}" +BACKUP_DIR="${BACKUP_DIR:-data/backups}" +RESTART="${RESTART:-1}" +STRICT="${STRICT:-0}" +WEB_SERVICE="${WEB_SERVICE:-alphax-web}" +SCHEDULER_SERVICE="${SCHEDULER_SERVICE:-alphax-scheduler}" + +compose() { + ${COMPOSE_CMD} "$@" +} + +info() { + echo "[rebuild-db] $*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +service_exists() { + compose config --services 2>/dev/null | grep -qx "$1" +} + +[ -f "$ROOT_DB" ] || die "root database not found: $ROOT_DIR/$ROOT_DB" +[ -s "$ROOT_DB" ] || die "root database is empty: $ROOT_DIR/$ROOT_DB" +command -v python3 >/dev/null 2>&1 || die "python3 is required on the server" + +mkdir -p "$(dirname "$TARGET_DB")" "$BACKUP_DIR" + +STAMP="$(date +%Y%m%d_%H%M%S)" +TEMP_DB="${BACKUP_DIR}/altcoin_monitor.rebuilt_from_root.${STAMP}.tmp.db" +REPORT_FILE="${BACKUP_DIR}/altcoin_monitor.rebuild_report.${STAMP}.json" + +info "source root db: $ROOT_DIR/$ROOT_DB" +info "target volume db: $ROOT_DIR/$TARGET_DB" +info "root db candidates:" +ls -lh "$ROOT_DB"* 2>/dev/null || true + +if [ "$RESTART" = "1" ]; then + info "stopping compose services before replacing SQLite DB" + if service_exists "$SCHEDULER_SERVICE"; then + compose stop "$SCHEDULER_SERVICE" >/dev/null 2>&1 || true + fi + if service_exists "$WEB_SERVICE"; then + compose stop "$WEB_SERVICE" >/dev/null 2>&1 || true + fi +fi + +info "rebuilding a clean SQLite DB from readable source data" +ROOT_DB="$ROOT_DB" TEMP_DB="$TEMP_DB" REPORT_FILE="$REPORT_FILE" STRICT="$STRICT" python3 - <<'PY' +import json +import os +import sqlite3 +import sys +import traceback + +src_path = os.environ["ROOT_DB"] +dst_path = os.environ["TEMP_DB"] +report_file = os.environ["REPORT_FILE"] +strict = os.environ.get("STRICT") == "1" + +for path in (dst_path, dst_path + "-wal", dst_path + "-shm"): + try: + os.remove(path) + except FileNotFoundError: + pass + +report = { + "source": src_path, + "target_temp": dst_path, + "strict": strict, + "source_integrity": None, + "target_integrity": None, + "tables": [], + "schema_errors": [], + "index_errors": [], + "fatal": None, +} + +def quote_ident(name: str) -> str: + return '"' + str(name).replace('"', '""') + '"' + +def record_table(name, copied=0, skipped=0, status="ok", error=""): + report["tables"].append({ + "table": name, + "copied_rows": int(copied), + "skipped_rows": int(skipped), + "status": status, + "error": str(error)[:2000], + }) + +def write_report(): + with open(report_file, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + +try: + src = sqlite3.connect(f"file:{src_path}?mode=ro", uri=True, timeout=30) + src.row_factory = sqlite3.Row + dst = sqlite3.connect(dst_path, timeout=30) + dst.row_factory = sqlite3.Row + dst.execute("PRAGMA foreign_keys=OFF") + dst.execute("PRAGMA journal_mode=DELETE") + dst.execute("PRAGMA synchronous=OFF") + + try: + report["source_integrity"] = src.execute("PRAGMA integrity_check").fetchone()[0] + except Exception as exc: + report["source_integrity"] = f"failed: {exc}" + + try: + objects = src.execute(""" + SELECT type, name, tbl_name, sql + FROM sqlite_master + WHERE sql IS NOT NULL + AND name NOT LIKE 'sqlite_autoindex%' + ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 WHEN 'trigger' THEN 2 ELSE 3 END, name + """).fetchall() + except Exception as exc: + report["fatal"] = f"cannot read sqlite_master: {exc}" + write_report() + raise SystemExit(report["fatal"]) + + table_objects = [ + dict(row) for row in objects + if row["type"] == "table" and not str(row["name"]).startswith("sqlite_") + ] + late_objects = [ + dict(row) for row in objects + if row["type"] in ("index", "trigger", "view") and not str(row["name"]).startswith("sqlite_") + ] + + for obj in table_objects: + try: + dst.execute(obj["sql"]) + except Exception as exc: + report["schema_errors"].append({"object": obj["name"], "error": str(exc)}) + if strict: + raise + + dst.commit() + + for obj in table_objects: + table = obj["name"] + copied = 0 + skipped = 0 + try: + cols = [row["name"] for row in src.execute(f"PRAGMA table_info({quote_ident(table)})").fetchall()] + if not cols: + record_table(table, copied, skipped, "skipped", "no columns") + continue + col_sql = ", ".join(quote_ident(c) for c in cols) + placeholders = ", ".join("?" for _ in cols) + insert_sql = f"INSERT INTO {quote_ident(table)} ({col_sql}) VALUES ({placeholders})" + + rowid_available = False + try: + src.execute(f"SELECT rowid FROM {quote_ident(table)} LIMIT 1").fetchone() + rowid_available = True + except Exception: + rowid_available = False + + if rowid_available: + bounds = src.execute(f"SELECT min(rowid), max(rowid) FROM {quote_ident(table)}").fetchone() + min_id, max_id = bounds[0], bounds[1] + if min_id is None or max_id is None: + record_table(table, 0, 0, "ok") + continue + chunk = 1000 + current = int(min_id) + max_id = int(max_id) + while current <= max_id: + end = min(current + chunk - 1, max_id) + try: + rows = src.execute( + f"SELECT {col_sql} FROM {quote_ident(table)} WHERE rowid BETWEEN ? AND ? ORDER BY rowid", + (current, end), + ).fetchall() + dst.executemany(insert_sql, [tuple(row[c] for c in cols) for row in rows]) + copied += len(rows) + except Exception as chunk_exc: + for rowid in range(current, end + 1): + try: + row = src.execute( + f"SELECT {col_sql} FROM {quote_ident(table)} WHERE rowid=?", + (rowid,), + ).fetchone() + if row is None: + continue + dst.execute(insert_sql, tuple(row[c] for c in cols)) + copied += 1 + except Exception: + skipped += 1 + if strict and skipped: + raise chunk_exc + current = end + 1 + else: + # Fallback for WITHOUT ROWID tables. If a malformed page breaks the + # full scan, the whole table may be skipped. + rows = src.execute(f"SELECT {col_sql} FROM {quote_ident(table)}").fetchall() + dst.executemany(insert_sql, [tuple(row[c] for c in cols) for row in rows]) + copied += len(rows) + + dst.commit() + status = "partial" if skipped else "ok" + record_table(table, copied, skipped, status) + except Exception as exc: + dst.rollback() + record_table(table, copied, skipped, "failed", exc) + if strict: + raise + + for obj in late_objects: + try: + dst.execute(obj["sql"]) + except Exception as exc: + report["index_errors"].append({"object": obj["name"], "type": obj["type"], "error": str(exc)}) + if strict: + raise + + dst.commit() + + try: + report["target_integrity"] = dst.execute("PRAGMA integrity_check").fetchone()[0] + except Exception as exc: + report["target_integrity"] = f"failed: {exc}" + + dst.close() + src.close() + write_report() + + if report["target_integrity"] != "ok": + raise SystemExit(f"rebuilt target integrity_check failed: {report['target_integrity']}") + if strict: + bad = [t for t in report["tables"] if t["status"] != "ok"] + if bad or report["schema_errors"] or report["index_errors"]: + raise SystemExit("strict rebuild failed; see report") + + print(json.dumps({ + "source_integrity": report["source_integrity"], + "target_integrity": report["target_integrity"], + "tables": len(report["tables"]), + "copied_rows": sum(t["copied_rows"] for t in report["tables"]), + "skipped_rows": sum(t["skipped_rows"] for t in report["tables"]), + "report": report_file, + }, ensure_ascii=False)) +except Exception as exc: + report["fatal"] = f"{exc}\n{traceback.format_exc()}" + write_report() + raise +PY + +if [ -e "$TARGET_DB" ]; then + OLD_BACKUP="${BACKUP_DIR}/altcoin_monitor.volume_before_rebuild.${STAMP}.db" + info "backing up old volume db to $OLD_BACKUP" + cp -p "$TARGET_DB" "$OLD_BACKUP" +fi + +for sidecar in "${TARGET_DB}-wal" "${TARGET_DB}-shm"; do + if [ -e "$sidecar" ]; then + sidecar_backup="${BACKUP_DIR}/$(basename "$sidecar").volume_before_rebuild.${STAMP}" + info "moving old sidecar $sidecar to $sidecar_backup" + mv "$sidecar" "$sidecar_backup" + fi +done + +info "installing rebuilt db into volume path" +mv "$TEMP_DB" "$TARGET_DB" +chmod 664 "$TARGET_DB" || true + +info "final integrity check" +HOST_DB="$TARGET_DB" python3 - <<'PY' +import os +import sqlite3 + +db = os.environ["HOST_DB"] +conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30) +result = conn.execute("PRAGMA integrity_check").fetchone()[0] +tables = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0] +conn.close() +if result != "ok": + raise SystemExit(f"integrity_check failed: {result}") +print(f"integrity_check=ok tables={tables} db={db}") +PY + +if [ "$RESTART" = "1" ]; then + info "starting compose services" + if service_exists "$WEB_SERVICE"; then + compose up -d "$WEB_SERVICE" + fi + if service_exists "$SCHEDULER_SERVICE"; then + compose up -d "$SCHEDULER_SERVICE" + fi +fi + +info "done" +info "system database path: $ROOT_DIR/$TARGET_DB" +info "rebuild report: $ROOT_DIR/$REPORT_FILE"