diff --git a/.env.example b/.env.example index 0a713de..791a952 100644 --- a/.env.example +++ b/.env.example @@ -47,8 +47,14 @@ ALPHAX_ONCHAIN_TIMEOUT=15 ALPHAX_NODEREAL_ENABLED=1 ALPHAX_NODEREAL_CHAINS=ethereum,bsc ALPHAX_NODEREAL_API_KEY= +# 可选：生产若 onchain_token_map 为空，可用 JSON 数组自举 NodeReal 合约映射。 +# 示例：[{"symbol":"STORJ/USDT","chain":"ethereum","contract_address":"0x...","confidence":95}] +ALPHAX_ONCHAIN_TOKEN_MAPPINGS= ALPHAX_NODEREAL_LOG_BLOCK_LOOKBACK=120 ALPHAX_NODEREAL_MAX_LOGS_PER_TOKEN=25 +ALPHAX_NODEREAL_RAW_TRANSFER_ENABLED=1 +ALPHAX_NODEREAL_RAW_BLOCK_LOOKBACK=1 +ALPHAX_NODEREAL_RAW_MAX_LOGS_PER_CHAIN=30 ALPHAX_ONCHAIN_CANDIDATE_ENABLED=1 ALPHAX_ONCHAIN_CANDIDATE_MIN_SCORE=70 ALPHAX_ONCHAIN_CANDIDATE_MIN_CONFIDENCE=70 diff --git a/README_DOCKER.md b/README_DOCKER.md index a4fde51..22305ea 100644 --- a/README_DOCKER.md +++ b/README_DOCKER.md @@ -8,6 +8,7 @@ - 运行时数据库是 PostgreSQL，compose 内置 `postgres:16` 服务。 - `DATABASE_URL` 是应用唯一运行时数据库连接入口。 - 链上主数据源是 NodeReal；`.env` 中配置 `ALPHAX_NODEREAL_API_KEY` 后，`python -m app.cli onchain` 才会产出 NodeReal 链上事件。 +- 生产若出现 `nodereal_no_mappings`，说明 `onchain_token_map` 没有可用合约映射；可在配置中心 `system/onchain.token_mappings` 或 `.env` 的 `ALPHAX_ONCHAIN_TOKEN_MAPPINGS` 写入 `symbol/chain/contract_address` 种子。 - 调度器以并发子进程运行，并通过业务锁组避免主推荐写入冲突。 - `.dockerignore` 排除了 `data/`、真实 `.env` 和所有 DB 文件，避免把数据库/密钥打进镜像。 diff --git a/app/config/system_config.py b/app/config/system_config.py index fc601eb..378b3d9 100644 --- a/app/config/system_config.py +++ b/app/config/system_config.py @@ -77,8 +77,13 @@ def default_onchain_config(default_chains=("ethereum", "bsc")): "nodereal_enabled": _env_bool("ALPHAX_NODEREAL_ENABLED", True), "nodereal_chains": _env_list("ALPHAX_NODEREAL_CHAINS", ("ethereum", "bsc")), "nodereal_api_key_env": "ALPHAX_NODEREAL_API_KEY", + "token_mappings_env": "ALPHAX_ONCHAIN_TOKEN_MAPPINGS", + "token_mappings": [], "nodereal_log_block_lookback": _env_int("ALPHAX_NODEREAL_LOG_BLOCK_LOOKBACK", 120), "nodereal_max_logs_per_token": _env_int("ALPHAX_NODEREAL_MAX_LOGS_PER_TOKEN", 25), + "nodereal_raw_transfer_enabled": _env_bool("ALPHAX_NODEREAL_RAW_TRANSFER_ENABLED", True), + "nodereal_raw_block_lookback": _env_int("ALPHAX_NODEREAL_RAW_BLOCK_LOOKBACK", 1), + "nodereal_raw_max_logs_per_chain": _env_int("ALPHAX_NODEREAL_RAW_MAX_LOGS_PER_CHAIN", 30), "candidate_enabled": _env_bool("ALPHAX_ONCHAIN_CANDIDATE_ENABLED", True), "candidate_min_score": _env_float("ALPHAX_ONCHAIN_CANDIDATE_MIN_SCORE", 70), "candidate_min_confidence": _env_int("ALPHAX_ONCHAIN_CANDIDATE_MIN_CONFIDENCE", 70), diff --git a/app/db/onchain_db.py b/app/db/onchain_db.py index eb90f67..d7a5c86 100644 --- a/app/db/onchain_db.py +++ b/app/db/onchain_db.py @@ -34,6 +34,7 @@ RAW_EVENT_TYPE_LABELS = { "token_profile_latest": "DEX 新币资料变更", "token_boost_latest": "DEX 付费曝光新增", "token_boost_top": "DEX 付费曝光榜", + "evm_transfer": "EVM 原始转账", } RAW_EVENT_EXPLAINERS = { @@ -52,6 +53,11 @@ RAW_EVENT_EXPLAINERS = { "meaning": "代表平台内关注度较高，需要再看成交量、流动性和是否能映射交易对。", "priority": "medium", }, + "evm_transfer": { + "plain": "NodeReal 捕捉到 EVM 链上的 ERC-20 Transfer 原始日志。", + "meaning": "这代表链上确实有资金转移，但没有完成币种映射前，不能直接进入策略候选。", + "priority": "medium", + }, } POSITIVE_SIGNALS = {"dex_volume_spike", "liquidity_add", "exchange_outflow", "whale_accumulation", "holder_growth", "smart_money_buying"} diff --git a/app/services/onchain_monitor.py b/app/services/onchain_monitor.py index a0fe109..27b8085 100644 --- a/app/services/onchain_monitor.py +++ b/app/services/onchain_monitor.py @@ -107,6 +107,7 @@ def get_onchain_params(): etherscan_env = str(cfg.get("etherscan_api_key_env") or "ALPHAX_ETHERSCAN_API_KEY") helius_env = str(cfg.get("helius_api_key_env") or "ALPHAX_HELIUS_API_KEY") nodereal_env = str(cfg.get("nodereal_api_key_env") or "ALPHAX_NODEREAL_API_KEY") + token_mappings_env = str(cfg.get("token_mappings_env") or "ALPHAX_ONCHAIN_TOKEN_MAPPINGS") etherscan_chains_raw = cfg.get("etherscan_chains") or ["ethereum"] if isinstance(etherscan_chains_raw, str): etherscan_chains = [x.strip().lower() for x in etherscan_chains_raw.split(",") if x.strip()] @@ -121,8 +122,13 @@ def get_onchain_params(): "nodereal_chains": _normalize_chain_list(cfg.get("nodereal_chains") or ("ethereum", "bsc")), "nodereal_api_key": os.getenv(nodereal_env, "").strip(), "nodereal_api_key_env": nodereal_env, + "token_mappings": _load_token_mappings(cfg.get("token_mappings"), os.getenv(token_mappings_env, "")), + "token_mappings_env": token_mappings_env, "nodereal_log_block_lookback": int(cfg.get("nodereal_log_block_lookback") or 120), "nodereal_max_logs_per_token": int(cfg.get("nodereal_max_logs_per_token") or 25), + "nodereal_raw_transfer_enabled": bool(cfg.get("nodereal_raw_transfer_enabled", True)), + "nodereal_raw_block_lookback": int(cfg.get("nodereal_raw_block_lookback") or 1), + "nodereal_raw_max_logs_per_chain": int(cfg.get("nodereal_raw_max_logs_per_chain") or 30), "candidate_enabled": bool(cfg.get("candidate_enabled", True)), "candidate_min_score": float(cfg.get("candidate_min_score") or 70), "candidate_min_confidence": int(cfg.get("candidate_min_confidence") or 70), @@ -152,6 +158,67 @@ def _normalize_chain_list(value): return [str(x).strip().lower() for x in (value or []) if str(x).strip()] +def _load_token_mappings(config_value=None, env_value=""): + items = [] + if isinstance(config_value, list): + items.extend(config_value) + if env_value: + try: + parsed = json.loads(env_value) + if isinstance(parsed, list): + items.extend(parsed) + except Exception: + for part in str(env_value or "").split(","): + bits = [x.strip() for x in part.split(":")] + if len(bits) >= 3: + items.append({"symbol": bits[0], "chain": bits[1], "contract_address": bits[2]}) + normalized = [] + seen = set() + for item in items: + if not isinstance(item, dict): + continue + symbol = normalize_symbol(item.get("symbol")) + chain = str(item.get("chain") or "").lower().strip() + contract = str(item.get("contract_address") or item.get("address") or "").strip() + if not symbol or not chain or not contract: + continue + key = (symbol, chain, contract.lower()) + if key in seen: + continue + seen.add(key) + normalized.append({ + "symbol": symbol, + "chain": chain, + "contract_address": contract, + "source": item.get("source") or "nodereal_seed", + "confidence": int(item.get("confidence") or 95), + "raw": item.get("raw") or {}, + }) + return normalized + + +def seed_configured_token_mappings(cfg=None): + cfg = cfg or get_onchain_params() + seeded = [] + errors = [] + for item in cfg.get("token_mappings") or []: + try: + mapping_id = onchain_db.upsert_token_mapping( + item["symbol"], + item["chain"], + item["contract_address"], + source=item.get("source") or "nodereal_seed", + confidence=item.get("confidence") or 95, + raw=item.get("raw") or {}, + is_active=True, + ) + if mapping_id: + seeded.append(item) + except Exception as exc: + errors.append(f"{item.get('symbol')}:seed_mapping:{str(exc)[:160]}") + return {"seeded": len(seeded), "items": seeded, "errors": errors} + + def _now(): return datetime.now() @@ -782,6 +849,37 @@ def _event_from_nodereal_transfer(log, mapping, cfg=None): } +def _raw_event_from_nodereal_transfer(log, chain): + topics = log.get("topics") or [] + if len(topics) < 3: + return None + contract = str(log.get("address") or "").strip() + tx_hash = str(log.get("transactionHash") or "").strip() + amount_raw = _hex_to_int(log.get("data")) + if not contract or amount_raw <= 0: + return None + from_addr = _topic_to_address(topics[1]) + to_addr = _topic_to_address(topics[2]) + return { + "source": "nodereal", + "chain": str(chain or "").lower(), + "event_type": "evm_transfer", + "token_address": contract, + "symbol_guess": "", + "name": "", + "title": "NodeReal ERC-20 原始转账", + "description": f"合约 {_short_addr(contract)} · {_short_addr(from_addr)} -> {_short_addr(to_addr)}", + "url": _chain_explorer_tx_url(chain, tx_hash), + "amount": amount_raw, + "total_amount": 0, + "importance": min(100, max(1, len(str(amount_raw)) * 4)), + "mapped_symbol": "", + "mapping_status": "unmapped", + "detected_at": _now().isoformat(timespec="seconds"), + "raw": log, + } + + def _metric_from_nodereal_holder_count(holder_count, mapping): symbol = normalize_symbol(mapping.get("symbol")) chain = str(mapping.get("chain") or "").lower() @@ -831,7 +929,9 @@ def fetch_nodereal_events(limit=60): return {"metrics": [], "events": [], "errors": ["nodereal_disabled"]} if not cfg.get("nodereal_api_key"): return {"metrics": [], "events": [], "errors": ["nodereal_api_key_missing"]} + seed_result = seed_configured_token_mappings(cfg) client = _nodereal_client(cfg) + raw_result = fetch_nodereal_raw_events(client=client, cfg=cfg, limit=limit) enabled_chains = set(cfg.get("nodereal_chains") or DEFAULT_CHAINS) all_mappings = get_token_mappings(min_confidence=MIN_MAPPING_CONFIDENCE) chain_mappings = [m for m in all_mappings if str(m.get("chain") or "").lower() in enabled_chains] @@ -845,8 +945,9 @@ def fetch_nodereal_events(limit=60): unsupported_chains.add(chain) metrics = [] events = [] - errors = [] + errors = list(seed_result.get("errors") or []) + list(raw_result.get("errors") or []) diagnostics = { + "seeded_mappings": seed_result.get("seeded", 0), "mapping_total": len(all_mappings), "chain_mapping_total": len(chain_mappings), "supported_mapping_total": len(mappings), @@ -895,12 +996,57 @@ def fetch_nodereal_events(limit=60): except Exception as exc: errors.append(f"{mapping.get('symbol')}:nodereal_logs:{str(exc)[:160]}") if not all_mappings: - errors.append("nodereal_no_mappings") + diagnostics["mapping_note"] = "no_strategy_mappings_raw_events_only" elif not chain_mappings: - errors.append("nodereal_no_enabled_chain_mappings:" + json.dumps(diagnostics, ensure_ascii=False, sort_keys=True)) + diagnostics["mapping_note"] = "no_enabled_chain_mappings_raw_events_only" elif not mappings: - errors.append("nodereal_no_supported_mappings:" + json.dumps(diagnostics, ensure_ascii=False, sort_keys=True)) - return {"metrics": metrics, "events": events, "errors": errors, "diagnostics": diagnostics} + diagnostics["mapping_note"] = "no_supported_mappings_raw_events_only" + return { + "metrics": metrics, + "events": events, + "raw_events": raw_result.get("raw_events") or [], + "errors": errors, + "diagnostics": diagnostics, + } + + +def fetch_nodereal_raw_events(client=None, cfg=None, limit=60): + cfg = cfg or get_onchain_params() + if not cfg.get("nodereal_raw_transfer_enabled", True): + return {"raw_events": [], "errors": []} + client = client or _nodereal_client(cfg) + chains = [c for c in (cfg.get("nodereal_chains") or DEFAULT_CHAINS) if client.supports_chain(c)] + lookback = max(0, min(12, int(cfg.get("nodereal_raw_block_lookback") or 1))) + per_chain = max(1, min(int(cfg.get("nodereal_raw_max_logs_per_chain") or 30), int(limit or 60))) + inserted = [] + errors = [] + for chain in chains: + try: + latest = client.block_number(chain) + if latest <= 0: + continue + logs = client.get_logs( + chain, + { + "fromBlock": hex(max(0, latest - lookback)), + "toBlock": hex(latest), + "topics": [TRANSFER_TOPIC], + }, + ) + raw_items = [] + for log in logs: + if not isinstance(log, dict): + continue + item = _raw_event_from_nodereal_transfer(log, chain) + if item: + raw_items.append(item) + raw_items.sort(key=lambda item: item.get("amount") or 0, reverse=True) + for item in raw_items[:per_chain]: + if insert_onchain_raw_event(item): + inserted.append(item) + except Exception as exc: + errors.append(f"{chain}:nodereal_raw_logs:{str(exc)[:160]}") + return {"raw_events": inserted, "errors": errors} def fetch_etherscan_events(limit=60): @@ -1188,6 +1334,7 @@ def run_once(limit=60): node = fetch_nodereal_events(limit=limit) output["metrics_count"] += len(node.get("metrics") or []) output["events_count"] += len(node.get("events") or []) + output["raw_events_count"] += len(node.get("raw_events") or []) output["errors"].extend(node.get("errors") or []) output["discovered_mappings"] = 0 if output.get("discovered_mappings"): @@ -1236,4 +1383,5 @@ __all__ = [ "ingest_normalized_events", "normalize_dexscreener_pair", "run_once", + "seed_configured_token_mappings", ] diff --git a/tests/test_onchain_tracking.py b/tests/test_onchain_tracking.py index 04b3514..ba5b832 100644 --- a/tests/test_onchain_tracking.py +++ b/tests/test_onchain_tracking.py @@ -320,9 +320,12 @@ def test_nodereal_events_generate_metrics_and_normalized_event(monkeypatch, tmp_ return 1000 def get_logs(self, chain, log_filter): + if "address" not in log_filter: + return [] assert log_filter["address"] == "0xabc" return [ { + "address": "0xabc", "transactionHash": "0xtx", "data": hex(200000 * 10**18), "topics": [ @@ -351,13 +354,102 @@ def test_nodereal_no_supported_mapping_error_has_diagnostics(monkeypatch, tmp_pa monkeypatch.setenv("ALPHAX_NODEREAL_CHAINS", "ethereum,bsc") onchain_db.upsert_token_mapping("SOLX", "solana", "Mint111", source="manual", confidence=95) + class EmptyNodeRealClient: + def supports_chain(self, chain): + return chain in {"ethereum", "bsc"} + + def block_number(self, chain): + return 100 + + def get_logs(self, chain, log_filter): + return [] + + monkeypatch.setattr(onchain_monitor, "_nodereal_client", lambda cfg=None: EmptyNodeRealClient()) + result = onchain_monitor.fetch_nodereal_events(limit=10) assert result["metrics"] == [] assert result["events"] == [] assert result["diagnostics"]["mapping_total"] == 1 assert result["diagnostics"]["chain_mapping_total"] == 0 - assert result["errors"][0].startswith("nodereal_no_enabled_chain_mappings:") + assert result["diagnostics"]["mapping_note"] == "no_enabled_chain_mappings_raw_events_only" + assert result["errors"] == [] + + +def test_nodereal_records_raw_events_without_strategy_mappings(monkeypatch, tmp_path): + _temp_db(monkeypatch, tmp_path) + monkeypatch.setenv("ALPHAX_NODEREAL_API_KEY", "test-key") + monkeypatch.setenv("ALPHAX_NODEREAL_CHAINS", "ethereum") + + class RawNodeRealClient: + def supports_chain(self, chain): + return chain == "ethereum" + + def block_number(self, chain): + return 1000 + + def get_logs(self, chain, log_filter): + assert "address" not in log_filter + return [ + { + "address": "0xabc", + "transactionHash": "0xrawtx", + "data": hex(987654321), + "topics": [ + onchain_monitor.TRANSFER_TOPIC, + "0x0000000000000000000000001111111111111111111111111111111111111111", + "0x0000000000000000000000002222222222222222222222222222222222222222", + ], + } + ] + + monkeypatch.setattr(onchain_monitor, "_nodereal_client", lambda cfg=None: RawNodeRealClient()) + + result = onchain_monitor.fetch_nodereal_events(limit=10) + + assert result["errors"] == [] + assert result["events"] == [] + assert len(result["raw_events"]) == 1 + assert result["diagnostics"]["mapping_note"] == "no_strategy_mappings_raw_events_only" + raw = onchain_db.list_onchain_raw_events(hours=50000) + assert raw["total"] == 1 + assert raw["items"][0]["source"] == "nodereal" + assert raw["items"][0]["mapping_status"] == "unmapped" + assert raw["items"][0]["event_type"] == "evm_transfer" + + +def test_nodereal_seeds_configured_token_mappings_from_env(monkeypatch, tmp_path): + _temp_db(monkeypatch, tmp_path) + monkeypatch.setenv("ALPHAX_NODEREAL_API_KEY", "test-key") + monkeypatch.setenv( + "ALPHAX_ONCHAIN_TOKEN_MAPPINGS", + '[{"symbol":"ENVX/USDT","chain":"ethereum","contract_address":"0xabc","confidence":96}]', + ) + + class EmptyNodeRealClient: + def supports_chain(self, chain): + return chain == "ethereum" + + def token_holder_count(self, chain, contract): + return 0 + + def block_number(self, chain): + return 100 + + def get_logs(self, chain, log_filter): + if "address" not in log_filter: + return [] + return [] + + monkeypatch.setattr(onchain_monitor, "_nodereal_client", lambda cfg=None: EmptyNodeRealClient()) + + result = onchain_monitor.fetch_nodereal_events(limit=10) + + assert result["errors"] == [] + assert result["diagnostics"]["seeded_mappings"] == 1 + mappings = onchain_db.get_token_mappings("ENVX/USDT") + assert len(mappings) == 1 + assert mappings[0]["contract_address"] == "0xabc" def test_legacy_helius_is_disabled_by_default(monkeypatch, tmp_path):