no message

scripts/install_root_db_to_volume.sh

@@ -57,10 +57,26 @@ import sqlite3
 import sys
 
 db = os.environ["HOST_DB"]
-conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30)
+try:
-result = conn.execute("PRAGMA integrity_check").fetchone()[0]
+    conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30)
-tables = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]
+    result = conn.execute("PRAGMA integrity_check").fetchone()[0]
-conn.close()
+    tables = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]
+    conn.close()
+except sqlite3.DatabaseError as exc:
+    print(f"database check failed for {db}: {exc}", file=sys.stderr)
+    print("", file=sys.stderr)
+    print("This file is not safe to install as the live AlphaX DB.", file=sys.stderr)
+    print("Common causes:", file=sys.stderr)
+    print("  1. The DB was copied while SQLite WAL mode was active, but the matching .db-wal file was not copied.", file=sys.stderr)
+    print("  2. The upload/copy was interrupted and the DB file is truncated.", file=sys.stderr)
+    print("  3. The source path is not the real SQLite database file.", file=sys.stderr)
+    print("", file=sys.stderr)
+    print("On the server, check:", file=sys.stderr)
+    print("  ls -lh altcoin_monitor.db*", file=sys.stderr)
+    print("  file altcoin_monitor.db", file=sys.stderr)
+    print("", file=sys.stderr)
+    print("Best fix: export a clean backup from the old machine/container using SQLite backup, then upload that .db file.", file=sys.stderr)
+    raise SystemExit(2)
 if result != "ok":
     print(f"integrity_check failed for {db}: {result}", file=sys.stderr)
     raise SystemExit(2)
@@ -75,6 +91,8 @@ mkdir -p "$(dirname "$TARGET_DB")" "$BACKUP_DIR"
 
 info "source root db: $ROOT_DIR/$ROOT_DB"
 info "target volume db: $ROOT_DIR/$TARGET_DB"
+info "root db candidates:"
+ls -lh "$ROOT_DB"* 2>/dev/null || true
 
 verify_db "$ROOT_DB"
 

scripts/rebuild_volume_db_from_root_db.sh

@@ -0,0 +1,326 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Rebuild the AlphaX volume DB from a database file placed in the project root.
+#
+# This is different from a raw copy:
+# - it creates a brand-new clean SQLite DB
+# - it recreates schema from the root DB
+# - it copies readable table rows into the clean DB
+# - it can salvage partial data from a DB that fails integrity_check
+#
+# Server usage:
+#   bash scripts/rebuild_volume_db_from_root_db.sh
+#
+# Defaults:
+#   source: ./altcoin_monitor.db
+#   target: ./data/altcoin_monitor.db
+#
+# Overrides:
+#   ROOT_DB=./backup.db bash scripts/rebuild_volume_db_from_root_db.sh
+#   STRICT=1 bash scripts/rebuild_volume_db_from_root_db.sh   # fail if any table/row cannot be copied
+#   RESTART=0 bash scripts/rebuild_volume_db_from_root_db.sh
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+COMPOSE_CMD="${COMPOSE_CMD:-docker compose}"
+ROOT_DB="${ROOT_DB:-altcoin_monitor.db}"
+TARGET_DB="${TARGET_DB:-data/altcoin_monitor.db}"
+BACKUP_DIR="${BACKUP_DIR:-data/backups}"
+RESTART="${RESTART:-1}"
+STRICT="${STRICT:-0}"
+WEB_SERVICE="${WEB_SERVICE:-alphax-web}"
+SCHEDULER_SERVICE="${SCHEDULER_SERVICE:-alphax-scheduler}"
+
+compose() {
+  ${COMPOSE_CMD} "$@"
+}
+
+info() {
+  echo "[rebuild-db] $*"
+}
+
+die() {
+  echo "ERROR: $*" >&2
+  exit 1
+}
+
+service_exists() {
+  compose config --services 2>/dev/null | grep -qx "$1"
+}
+
+[ -f "$ROOT_DB" ] || die "root database not found: $ROOT_DIR/$ROOT_DB"
+[ -s "$ROOT_DB" ] || die "root database is empty: $ROOT_DIR/$ROOT_DB"
+command -v python3 >/dev/null 2>&1 || die "python3 is required on the server"
+
+mkdir -p "$(dirname "$TARGET_DB")" "$BACKUP_DIR"
+
+STAMP="$(date +%Y%m%d_%H%M%S)"
+TEMP_DB="${BACKUP_DIR}/altcoin_monitor.rebuilt_from_root.${STAMP}.tmp.db"
+REPORT_FILE="${BACKUP_DIR}/altcoin_monitor.rebuild_report.${STAMP}.json"
+
+info "source root db: $ROOT_DIR/$ROOT_DB"
+info "target volume db: $ROOT_DIR/$TARGET_DB"
+info "root db candidates:"
+ls -lh "$ROOT_DB"* 2>/dev/null || true
+
+if [ "$RESTART" = "1" ]; then
+  info "stopping compose services before replacing SQLite DB"
+  if service_exists "$SCHEDULER_SERVICE"; then
+    compose stop "$SCHEDULER_SERVICE" >/dev/null 2>&1 || true
+  fi
+  if service_exists "$WEB_SERVICE"; then
+    compose stop "$WEB_SERVICE" >/dev/null 2>&1 || true
+  fi
+fi
+
+info "rebuilding a clean SQLite DB from readable source data"
+ROOT_DB="$ROOT_DB" TEMP_DB="$TEMP_DB" REPORT_FILE="$REPORT_FILE" STRICT="$STRICT" python3 - <<'PY'
+import json
+import os
+import sqlite3
+import sys
+import traceback
+
+src_path = os.environ["ROOT_DB"]
+dst_path = os.environ["TEMP_DB"]
+report_file = os.environ["REPORT_FILE"]
+strict = os.environ.get("STRICT") == "1"
+
+for path in (dst_path, dst_path + "-wal", dst_path + "-shm"):
+    try:
+        os.remove(path)
+    except FileNotFoundError:
+        pass
+
+report = {
+    "source": src_path,
+    "target_temp": dst_path,
+    "strict": strict,
+    "source_integrity": None,
+    "target_integrity": None,
+    "tables": [],
+    "schema_errors": [],
+    "index_errors": [],
+    "fatal": None,
+}
+
+def quote_ident(name: str) -> str:
+    return '"' + str(name).replace('"', '""') + '"'
+
+def record_table(name, copied=0, skipped=0, status="ok", error=""):
+    report["tables"].append({
+        "table": name,
+        "copied_rows": int(copied),
+        "skipped_rows": int(skipped),
+        "status": status,
+        "error": str(error)[:2000],
+    })
+
+def write_report():
+    with open(report_file, "w", encoding="utf-8") as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+
+try:
+    src = sqlite3.connect(f"file:{src_path}?mode=ro", uri=True, timeout=30)
+    src.row_factory = sqlite3.Row
+    dst = sqlite3.connect(dst_path, timeout=30)
+    dst.row_factory = sqlite3.Row
+    dst.execute("PRAGMA foreign_keys=OFF")
+    dst.execute("PRAGMA journal_mode=DELETE")
+    dst.execute("PRAGMA synchronous=OFF")
+
+    try:
+        report["source_integrity"] = src.execute("PRAGMA integrity_check").fetchone()[0]
+    except Exception as exc:
+        report["source_integrity"] = f"failed: {exc}"
+
+    try:
+        objects = src.execute("""
+            SELECT type, name, tbl_name, sql
+            FROM sqlite_master
+            WHERE sql IS NOT NULL
+              AND name NOT LIKE 'sqlite_autoindex%'
+            ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 WHEN 'trigger' THEN 2 ELSE 3 END, name
+        """).fetchall()
+    except Exception as exc:
+        report["fatal"] = f"cannot read sqlite_master: {exc}"
+        write_report()
+        raise SystemExit(report["fatal"])
+
+    table_objects = [
+        dict(row) for row in objects
+        if row["type"] == "table" and not str(row["name"]).startswith("sqlite_")
+    ]
+    late_objects = [
+        dict(row) for row in objects
+        if row["type"] in ("index", "trigger", "view") and not str(row["name"]).startswith("sqlite_")
+    ]
+
+    for obj in table_objects:
+        try:
+            dst.execute(obj["sql"])
+        except Exception as exc:
+            report["schema_errors"].append({"object": obj["name"], "error": str(exc)})
+            if strict:
+                raise
+
+    dst.commit()
+
+    for obj in table_objects:
+        table = obj["name"]
+        copied = 0
+        skipped = 0
+        try:
+            cols = [row["name"] for row in src.execute(f"PRAGMA table_info({quote_ident(table)})").fetchall()]
+            if not cols:
+                record_table(table, copied, skipped, "skipped", "no columns")
+                continue
+            col_sql = ", ".join(quote_ident(c) for c in cols)
+            placeholders = ", ".join("?" for _ in cols)
+            insert_sql = f"INSERT INTO {quote_ident(table)} ({col_sql}) VALUES ({placeholders})"
+
+            rowid_available = False
+            try:
+                src.execute(f"SELECT rowid FROM {quote_ident(table)} LIMIT 1").fetchone()
+                rowid_available = True
+            except Exception:
+                rowid_available = False
+
+            if rowid_available:
+                bounds = src.execute(f"SELECT min(rowid), max(rowid) FROM {quote_ident(table)}").fetchone()
+                min_id, max_id = bounds[0], bounds[1]
+                if min_id is None or max_id is None:
+                    record_table(table, 0, 0, "ok")
+                    continue
+                chunk = 1000
+                current = int(min_id)
+                max_id = int(max_id)
+                while current <= max_id:
+                    end = min(current + chunk - 1, max_id)
+                    try:
+                        rows = src.execute(
+                            f"SELECT {col_sql} FROM {quote_ident(table)} WHERE rowid BETWEEN ? AND ? ORDER BY rowid",
+                            (current, end),
+                        ).fetchall()
+                        dst.executemany(insert_sql, [tuple(row[c] for c in cols) for row in rows])
+                        copied += len(rows)
+                    except Exception as chunk_exc:
+                        for rowid in range(current, end + 1):
+                            try:
+                                row = src.execute(
+                                    f"SELECT {col_sql} FROM {quote_ident(table)} WHERE rowid=?",
+                                    (rowid,),
+                                ).fetchone()
+                                if row is None:
+                                    continue
+                                dst.execute(insert_sql, tuple(row[c] for c in cols))
+                                copied += 1
+                            except Exception:
+                                skipped += 1
+                        if strict and skipped:
+                            raise chunk_exc
+                    current = end + 1
+            else:
+                # Fallback for WITHOUT ROWID tables. If a malformed page breaks the
+                # full scan, the whole table may be skipped.
+                rows = src.execute(f"SELECT {col_sql} FROM {quote_ident(table)}").fetchall()
+                dst.executemany(insert_sql, [tuple(row[c] for c in cols) for row in rows])
+                copied += len(rows)
+
+            dst.commit()
+            status = "partial" if skipped else "ok"
+            record_table(table, copied, skipped, status)
+        except Exception as exc:
+            dst.rollback()
+            record_table(table, copied, skipped, "failed", exc)
+            if strict:
+                raise
+
+    for obj in late_objects:
+        try:
+            dst.execute(obj["sql"])
+        except Exception as exc:
+            report["index_errors"].append({"object": obj["name"], "type": obj["type"], "error": str(exc)})
+            if strict:
+                raise
+
+    dst.commit()
+
+    try:
+        report["target_integrity"] = dst.execute("PRAGMA integrity_check").fetchone()[0]
+    except Exception as exc:
+        report["target_integrity"] = f"failed: {exc}"
+
+    dst.close()
+    src.close()
+    write_report()
+
+    if report["target_integrity"] != "ok":
+        raise SystemExit(f"rebuilt target integrity_check failed: {report['target_integrity']}")
+    if strict:
+        bad = [t for t in report["tables"] if t["status"] != "ok"]
+        if bad or report["schema_errors"] or report["index_errors"]:
+            raise SystemExit("strict rebuild failed; see report")
+
+    print(json.dumps({
+        "source_integrity": report["source_integrity"],
+        "target_integrity": report["target_integrity"],
+        "tables": len(report["tables"]),
+        "copied_rows": sum(t["copied_rows"] for t in report["tables"]),
+        "skipped_rows": sum(t["skipped_rows"] for t in report["tables"]),
+        "report": report_file,
+    }, ensure_ascii=False))
+except Exception as exc:
+    report["fatal"] = f"{exc}\n{traceback.format_exc()}"
+    write_report()
+    raise
+PY
+
+if [ -e "$TARGET_DB" ]; then
+  OLD_BACKUP="${BACKUP_DIR}/altcoin_monitor.volume_before_rebuild.${STAMP}.db"
+  info "backing up old volume db to $OLD_BACKUP"
+  cp -p "$TARGET_DB" "$OLD_BACKUP"
+fi
+
+for sidecar in "${TARGET_DB}-wal" "${TARGET_DB}-shm"; do
+  if [ -e "$sidecar" ]; then
+    sidecar_backup="${BACKUP_DIR}/$(basename "$sidecar").volume_before_rebuild.${STAMP}"
+    info "moving old sidecar $sidecar to $sidecar_backup"
+    mv "$sidecar" "$sidecar_backup"
+  fi
+done
+
+info "installing rebuilt db into volume path"
+mv "$TEMP_DB" "$TARGET_DB"
+chmod 664 "$TARGET_DB" || true
+
+info "final integrity check"
+HOST_DB="$TARGET_DB" python3 - <<'PY'
+import os
+import sqlite3
+
+db = os.environ["HOST_DB"]
+conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30)
+result = conn.execute("PRAGMA integrity_check").fetchone()[0]
+tables = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table'").fetchone()[0]
+conn.close()
+if result != "ok":
+    raise SystemExit(f"integrity_check failed: {result}")
+print(f"integrity_check=ok tables={tables} db={db}")
+PY
+
+if [ "$RESTART" = "1" ]; then
+  info "starting compose services"
+  if service_exists "$WEB_SERVICE"; then
+    compose up -d "$WEB_SERVICE"
+  fi
+  if service_exists "$SCHEDULER_SERVICE"; then
+    compose up -d "$SCHEDULER_SERVICE"
+  fi
+fi
+
+info "done"
+info "system database path: $ROOT_DIR/$TARGET_DB"
+info "rebuild report: $ROOT_DIR/$REPORT_FILE"