113 lines
5.1 KiB
Python
113 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
|
"""显式演练新 PG：停止 laptop 主库服务，验证代理自动切换，再回切。需 --run。"""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import shlex
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location('pg_runner', ROOT / 'run.py')
|
|
runner = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(runner)
|
|
|
|
|
|
def command(args, **kw):
|
|
return subprocess.run(args, capture_output=True, text=True, timeout=35, **kw)
|
|
|
|
|
|
def wait_until(fn, description, timeout=100):
|
|
end = time.monotonic() + timeout
|
|
while time.monotonic() < end:
|
|
try:
|
|
if fn():
|
|
print(description, flush=True)
|
|
return
|
|
except (OSError, ValueError, subprocess.TimeoutExpired):
|
|
pass
|
|
time.sleep(2)
|
|
raise RuntimeError(description + ' timeout')
|
|
|
|
|
|
def role(ip):
|
|
with urllib.request.urlopen('http://' + ip + ':8008/patroni', timeout=3) as r:
|
|
return json.load(r)['role']
|
|
|
|
|
|
def main():
|
|
if sys.argv[1:] != ['--run']:
|
|
raise SystemExit('需显式 --run；会停止生产 laptop PG 服务并在验收后回切。')
|
|
address = 'https://bao.ad.ddupan.top:8200'
|
|
token = runner.spiffe_token(address)
|
|
database = 'homelab_ha_probe_' + str(int(time.time()))
|
|
try:
|
|
request = urllib.request.Request(address + '/v1/kv/data/infra/postgresql/ayatori/prod',
|
|
headers={'X-Vault-Token': token})
|
|
with urllib.request.urlopen(request, timeout=10) as r:
|
|
credentials = json.load(r)['data']['data']
|
|
env = dict(os.environ, PGHOST='pg-prod.ad.ddupan.top', PGHOSTADDR='192.168.10.2',
|
|
PGPORT='5432', PGDATABASE='postgres', PGUSER=credentials['username'],
|
|
PGPASSWORD=credentials['password'], PGSSLMODE='verify-full',
|
|
PGSSLROOTCERT=str(ROOT / 'ayatori/ca.crt'), PGCONNECT_TIMEOUT='3')
|
|
|
|
def sql(query, db='postgres'):
|
|
result = command(['/usr/lib/postgresql/18/bin/psql', '-X', '-At', '-v', 'ON_ERROR_STOP=1'],
|
|
input=query, env=dict(env, PGDATABASE=db))
|
|
if result.returncode:
|
|
raise RuntimeError('SQL 验证失败: ' + result.stderr)
|
|
return result.stdout.strip()
|
|
|
|
def via(ip):
|
|
try:
|
|
return sql("SELECT host(inet_server_addr()) || ':' || "
|
|
"(NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off')::text") == ip + ':true'
|
|
except RuntimeError:
|
|
return False
|
|
|
|
def rows(ip, count):
|
|
args = ['sudo', '-n', '-u', 'pgprod', '/usr/lib/postgresql/18/bin/psql', '-X', '-At',
|
|
'-h', '/run/homelab-postgresql-prod', '-d', database, '-c', 'SELECT count(*) FROM probe']
|
|
if ip != '192.168.10.127':
|
|
args = ['ssh', 'root@' + ip, shlex.join(args)]
|
|
return command(args).stdout.strip() == str(count)
|
|
|
|
assert role('192.168.10.127') in ('master', 'primary')
|
|
assert role('10.60.0.20') == 'replica'
|
|
assert via('192.168.10.127')
|
|
sql('CREATE DATABASE ' + database)
|
|
print('探针数据库:', database, flush=True)
|
|
sql('CREATE TABLE probe(id integer PRIMARY KEY); INSERT INTO probe VALUES(1)', database)
|
|
wait_until(lambda: rows('10.60.0.20', 1), '初始数据已复制')
|
|
started = time.monotonic()
|
|
result = command(['sudo', '-n', 'systemctl', 'stop', 'homelab-postgresql-prod'])
|
|
if result.returncode:
|
|
raise RuntimeError('停止主库服务失败')
|
|
wait_until(lambda: via('10.60.0.20'), '代理已自动切到 standby')
|
|
sql('INSERT INTO probe VALUES(2)', database)
|
|
print('自动恢复写入秒数:', round(time.monotonic() - started, 1), flush=True)
|
|
command(['sudo', '-n', 'systemctl', 'start', 'homelab-postgresql-prod'], check=True)
|
|
wait_until(lambda: role('192.168.10.127') == 'replica' and rows('192.168.10.127', 2),
|
|
'旧主重新作为 replica 加入，数据一致')
|
|
result = command(['sudo', '-n', '-u', 'pgprod', '/opt/homelab-patroni/bin/patronictl',
|
|
'-c', '/etc/homelab-postgresql/prod/patroni.yml', 'switchover',
|
|
'--leader', 'pg-pve1', '--candidate', 'pg-laptop', '--force'])
|
|
if result.returncode:
|
|
raise RuntimeError('计划回切失败: ' + result.stderr)
|
|
wait_until(lambda: via('192.168.10.127'), '已计划回切 laptop')
|
|
sql('INSERT INTO probe VALUES(3)', database)
|
|
wait_until(lambda: role('10.60.0.20') == 'replica' and rows('10.60.0.20', 3), '回切后复制正常')
|
|
sql('DROP DATABASE ' + database)
|
|
print('PASS；专属探针数据库已删除', flush=True)
|
|
finally:
|
|
# 即使演练失败，也确保原主服务重新运行；Patroni 自行决定角色，不强行提升。
|
|
command(['sudo', '-n', 'systemctl', 'start', 'homelab-postgresql-prod'])
|
|
runner.bao_request(address, 'auth/token/revoke-self', {}, token)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|