#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ ipass-agent — iPass 容器远程更新控制面 跑在宿主机上,暴露一个 HTTP 端口,让远端(AI / 人)可以自助完成: docker pull → 备份数据 → 留回滚标签 → 删旧容器 → 按旧容器复刻端口/挂载/网络/env 起新容器 → 自检 单文件、只依赖 Python 标准库 + `docker` CLI。**不需要** Flask / requests。 -------------------------------------------------------------------------------- 用法 -------------------------------------------------------------------------------- python3 ipass-agent.py --check # 只探测(不起服务),打印容器/端口/数据目录 python3 ipass-agent.py # 前台起服务,默认 0.0.0.0:18099 python3 ~/ipass/ipass-agent.py --bg # 后台起服务,日志 ~/ipass/ipass-agent.log python3 ~/ipass/ipass-agent.py --stop # 停掉后台服务 python3 ~/ipass/ipass-agent.py --port 18099 --name ipass --data-dir ~/ipass/data -------------------------------------------------------------------------------- 鉴权 -------------------------------------------------------------------------------- 除 /health 外所有端点都要请求头: Authorization: Bearer token 取值顺序:--token > 环境变量 IPASS_AGENT_TOKEN > ~/ipass/.ipass-agent-token (文件不存在时随机生成并写入,权限 600) -------------------------------------------------------------------------------- 受管容器(限定爆炸半径)—— 重要 -------------------------------------------------------------------------------- 本服务等价 docker 权限,但**只操作一个容器**(下称「受管容器」): ① `--name` 显式指定者优先;② 否则自动探测到的 ipass 容器(结果缓存一次)。 两者都没有时退化为主机名前缀白名单(`ipass*`,见 MANAGED_PREFIX)。 update / restart / rollback 若带 `name` 且**不等于**受管容器 → 直接 403, 不建任务、更不会碰那个容器;`GET /containers` 会标明哪个是 managed。 (同一台机器上往往还跑着别的 nginx / 业务容器,绝不能误伤。) -------------------------------------------------------------------------------- 端点一览 -------------------------------------------------------------------------------- GET /health 存活 + 版本(免鉴权) GET / 服务信息 + 端点清单 GET /check 探测(dry-run):容器 / 端口 / 挂载 / 数据摘要 GET /status 当前容器状态 + 镜像 digest + 数据摘要 + 宿主探活 GET /containers 所有容器(标注哪个是受管容器) GET /logs?tail=200 受管容器的日志 GET /data?file=users 数据文件摘要(api_key 已脱敏) GET /jobs?limit=5 最近任务列表 GET /jobs/?offset=N 任务详情 + 增量日志 POST /pull {"image":...} 拉镜像(异步任务) POST /update {"image":...,"skip_pull":...} 完整更新(异步任务) POST /rollback {"tag":...,"restore_backup":...} 回滚到指定 tag(异步任务); restore_backup=update 返回的备份 tar 路径时,回滚前把 数据目录/审计目录恢复成备份内容(发版验证失败的自动回滚用) POST /restart 只重启容器(同步) `name` 只接受受管容器(见上);传其它容器名 → 403。 任务(job)在同一时刻**只允许一个**在跑,避免并发删容器互相踩。 """ import argparse import hmac import http.client import json import os import random import re import shutil import ssl import string import subprocess import sys import tarfile import threading import time from http.server import BaseHTTPRequestHandler, HTTPServer from socketserver import ThreadingMixIn from urllib.parse import urlparse, parse_qs # ------------------------------------------------------------------ 常量 SERVICE = "ipass-agent" VERSION = "1.0.4" DEFAULT_PORT = 18099 DEFAULT_IMAGE = ("crpi-yscq976uwza88qap.cn-shenzhen.personal.cr.aliyuncs.com/" "ssw188/nginx:ipass-1.0.0") DEFAULT_CONTAINER = "ipass" CONTAINER_DATA = "/usr/local/nginx/data" # 容器内数据目录 CONTAINER_AUDIT = "/usr/local/nginx/audit" # 容器内审计目录(= data 的兄弟目录, # 宿主侧跟随数据目录推导:…/ipass/audit) # agent 自身的运行文件统一在 ~/ipass/ 下,跟 data/backup/audit 同根目录; # 不再散在 $HOME 根目录被 `ls ~` 一眼看到一堆。 INSTALL_DIR = os.path.join(os.path.expanduser("~"), "ipass") TOKEN_FILE = os.path.join(INSTALL_DIR, ".ipass-agent-token") # 600 PID_FILE = os.path.join(INSTALL_DIR, "ipass-agent.pid") LOG_FILE = os.path.join(INSTALL_DIR, "ipass-agent.log") MAIN_CPORT = "18098" # 容器内主端口(宿主映射到它) NGINX_BIN = "/usr/local/nginx/sbin/nginx" ADMIN_HOST = "mdm.mayishangshu.cn" # 探活用的 Host 头 # 1.0.17 起 update 不再在本地留 `ipass-gateway:rollback-*` 镜像标签。镜像在阿里云私仓 # 永远保留,需要时 `docker pull <完整镜像名>` 即可——本地磁盘不再被一堆历史镜像撑大。 # 回滚入口仍保留(`op_rollback` + agent.sh `rollback `),但**必须显式传 tag**。 JOB_KEEP = 20 PULL_TIMEOUT = 1800 # Docker 自己注入的 env,复刻容器时不能照抄 DOCKER_BUILTIN_ENV = { "PATH", "HOSTNAME", "HOME", "NGINX_VERSION", "NJS_VERSION", "PKG_RELEASE", "NJS_RELEASE", "SSL_CERT_FILE", "PWD", "TERM", } # ------------------------------------------------------------------ 基础工具 def now_ts(): return int(time.time()) def ts_tag(): return time.strftime("%Y%m%d-%H%M%S") def find_docker(): p = shutil.which("docker") if p: return p for c in ("/usr/bin/docker", "/usr/local/bin/docker", "/snap/bin/docker"): if os.path.exists(c): return c return "docker" DOCKER = find_docker() def docker(args, timeout=120, stdin_data=None): """执行 docker 命令,返回 (rc, 合并输出文本)。""" try: p = subprocess.run( [DOCKER] + list(args), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=stdin_data, timeout=timeout, ) return p.returncode, p.stdout.decode("utf-8", "replace") except subprocess.TimeoutExpired: return 124, "command timeout after %ss: docker %s" % (timeout, " ".join(args)) except Exception as e: # noqa: BLE001 return 127, "docker 调用失败: %s" % e def sh(args, timeout=120): try: p = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) return p.returncode, p.stdout.decode("utf-8", "replace") except Exception as e: # noqa: BLE001 return 127, str(e) def clean_lines(text): """docker pull 的进度条会刷 \\r,清一下再进日志。""" out = [] for raw in text.replace("\r", "\n").split("\n"): s = raw.rstrip() if s: out.append(s) return out def http_probe(port, path="/api/admin/users", host=ADMIN_HOST, timeout=5): """从宿主机探活容器暴露的端口。返回 HTTP 状态码,0 表示连不上。""" if not port: return 0 try: ctx = ssl._create_unverified_context() conn = http.client.HTTPSConnection("127.0.0.1", int(port), timeout=timeout, context=ctx) conn.request("GET", path, headers={"Host": host}) r = conn.getresponse() r.read() conn.close() return r.status except Exception: # noqa: BLE001 return 0 # ------------------------------------------------------------------ 容器探测 def list_containers(): """列出所有容器:[(name, image, status)]""" rc, out = docker(["ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.Status}}"]) if rc != 0: return [] res = [] for line in out.splitlines(): parts = line.split("\t") if len(parts) >= 3: res.append((parts[0], parts[1], parts[2])) elif len(parts) == 2: res.append((parts[0], parts[1], "")) return res def relevant_containers(exclude=None): """只挑可能跟 ipass 相关的容器,过滤掉 k8s_* / 其它项目的噪音。""" out = [] for n, i, s in list_containers(): if exclude and n == exclude: continue blob = (n + " " + (i or "")).lower() if "ipass" in blob or "nginx" in blob: out.append({"name": n, "image": i, "status": s}) return out[:8] def detect_container(prefer=None): """找 ipass 容器:prefer 命中优先,其次名字/镜像含 ipass。""" conts = list_containers() if not conts: return None if prefer: for n, _i, _s in conts: if n == prefer: return n hits = [n for n, img, _s in conts if "ipass" in n.lower() or "ipass" in (img or "").lower()] if len(hits) == 1: return hits[0] if hits: # 多个候选时优先「在跑的」,再优先名字最短的 running = [n for n, _i, s in conts if n in hits and s.lower().startswith("up")] pool = running or hits return sorted(pool, key=len)[0] return None # ------------------------------------------------- 受管容器(限定爆炸半径) # # 本服务等价 docker 权限,而同一台机器上通常还跑着**别的** nginx / 业务容器 # (实测 .11 上就有 invoicehelper-nginx、d15-nginx-1 …)。所有破坏性操作 # (update / restart / rollback)**只允许作用于「受管容器」**: # ① 显式 `--name` 指定的那个;② 否则自动探测到的 ipass 容器(结果缓存一次)。 # 两者都没有时退化为主机名前缀白名单(`ipass*`)。 # ⚠️ 不要改成「信任调用方传进来的 name」—— 那等于把删/起容器的能力开到全机器。 MANAGED_PREFIX = "ipass" def managed_container(): """受管容器名(缓存)。显式 --name 优先,否则自动探测一次。""" if BACK.get("name"): return BACK["name"] if BACK.get("_managed") is None: BACK["_managed"] = detect_container(None) return BACK["_managed"] def ensure_managed(name): """把目标容器收敛到受管容器;不合法抛 PermissionError(HTTP 层 → 403)。""" managed = managed_container() if not name: if managed: return managed raise PermissionError("未发现受管容器(容器名应以 %r 开头),拒绝操作" % MANAGED_PREFIX) if managed: if name != managed: raise PermissionError( "拒绝操作容器 %s:本控制面只管 %s(避免误伤同机其它容器)" % (name, managed)) return name if not name.lower().startswith(MANAGED_PREFIX): raise PermissionError("拒绝操作容器 %s:只允许 %s* 开头的受管容器" % (name, MANAGED_PREFIX)) return name def inspect_one(name): rc, out = docker(["inspect", name]) if rc != 0: return None try: arr = json.loads(out) return arr[0] if arr else None except Exception: # noqa: BLE001 return None def image_repo_digest(image): rc, out = docker(["image", "inspect", "--format", "{{json .RepoDigests}}", image]) if rc != 0: return None try: arr = json.loads(out.strip() or "[]") return arr[0] if arr else None except Exception: # noqa: BLE001 return None def image_exists(image): rc, _ = docker(["image", "inspect", image]) return rc == 0 def image_id(image): """镜像内容 id(`sha256:…`)。 用于「同一个 tag 被覆盖推送」时判重 —— 那是常态(如 `ipass-1.0.0` 迭代), 只比 tag 名会把「内容已变」误判成「相同」,于是悄悄跳过回滚标签。 """ rc, out = docker(["image", "inspect", "--format", "{{.Id}}", image]) return out.strip() if rc == 0 else "" # ------------------------------------------------------------------ 从 inspect 复刻启动参数 def extract_ports(cfg, main_port_override=None): """返回 ([docker run 端口参数], 宿主主端口)。 注意:`docker inspect` 里同一个映射会带 IPv4 与 IPv6 两条, 不去重就会生成重复的 -p → 「address already in use」。 """ pb = (cfg.get("HostConfig") or {}).get("PortBindings") or {} seen, args, main = set(), [], None for cport, binds in pb.items(): cnum = cport.split("/")[0] for b in binds or []: hip = (b.get("HostIp") or "").strip() hp = (b.get("HostPort") or "").strip() if not hp: continue if main_port_override and cnum == MAIN_CPORT: hp = str(main_port_override) if hip in ("", "0.0.0.0", "::"): key = "%s:%s" % (hp, cnum) else: key = "%s:%s:%s" % (hip, hp, cnum) if key in seen: continue seen.add(key) args += ["-p", key] if cnum == MAIN_CPORT: main = hp if main is None and args: main = args[1].split(":")[0] return args, main def extract_mounts(cfg): """返回 (数据目录源, [其余挂载参数])。""" data_src, extra = None, [] for m in cfg.get("Mounts") or []: dest = m.get("Destination") or "" if dest == CONTAINER_DATA: data_src = m.get("Source") or m.get("Name") continue if dest == CONTAINER_AUDIT: # 审计挂载不进 extra:每次重建都从数据目录**重新推导**(见 do_update), # 避免旧容器的审计挂载与新推导的重复 → 「mount point already mounted」 continue if m.get("Type") == "volume": spec = "%s:%s" % (m.get("Name"), dest) else: spec = "%s:%s" % (m.get("Source"), dest) if m.get("RW") is False: spec += ":ro" extra += ["-v", spec] return data_src, extra def extract_env(cfg, image): """只复刻「用户自己加的」env(剔除 Docker/镜像自带的)。""" env = (cfg.get("Config") or {}).get("Env") or [] base = set(DOCKER_BUILTIN_ENV) if image: rc, out = docker(["image", "inspect", "--format", "{{json .Config.Env}}", image]) if rc == 0: try: base |= set(json.loads(out.strip() or "[]")) except Exception: # noqa: BLE001 pass return [e for e in env if e.split("=", 1)[0] not in base] def extract_net(cfg): mode = (cfg.get("HostConfig") or {}).get("NetworkMode") or "" if mode in ("", "default", "bridge"): return [] return ["--network", mode] def extract_restart(cfg): rp = (cfg.get("HostConfig") or {}).get("RestartPolicy") or {} return rp.get("Name") or "" # ------------------------------------------------------------------ 数据摘要 / 备份 def read_json(path): try: with open(path, "r", encoding="utf-8") as f: return json.load(f) except Exception: # noqa: BLE001 return None def data_summary(data_dir): if not data_dir: return {"error": "未知数据目录"} users = read_json(os.path.join(data_dir, "users.json")) domains = read_json(os.path.join(data_dir, "domains.json")) ups = read_json(os.path.join(data_dir, "llm-upstreams.json")) return { "dir": data_dir, "writable": os.access(data_dir, os.W_OK) if os.path.isdir(data_dir) else False, "users": len(users) if isinstance(users, dict) else None, "domains": len(domains) if isinstance(domains, list) else None, "upstreams": len(ups) if isinstance(ups, dict) else None, } def mask_key(k): if not k: return "" return k[:10] + "…" if len(k) > 12 else "***" def data_view(data_dir, which): fname = {"users": "users.json", "domains": "domains.json", "upstreams": "llm-upstreams.json"}.get(which) if not fname: return None raw = read_json(os.path.join(data_dir, fname)) if raw is None: return None if which == "users" and isinstance(raw, dict): out = {} for name, u in raw.items(): out[name] = { "role": u.get("role"), "display_name": u.get("display_name", ""), "api_key": mask_key(u.get("api_key")), "allowed_models": u.get("allowed_models"), "llm_rpm": u.get("llm_rpm"), "llm_daily": u.get("llm_daily"), "temp_grant_until": u.get("temp_grant_until"), "has_password": bool(u.get("pwd_hash")), } return out if which == "upstreams" and isinstance(raw, dict): out = {} for slot, u in raw.items(): out[slot] = { "base": u.get("base"), "enabled": u.get("enabled"), "models": u.get("models"), "note": u.get("note", ""), "has_auth": bool(u.get("auth")), } return out return raw def backup_data(data_dir, job): if not data_dir or not os.path.isdir(data_dir): jlog(job, " ! 数据目录不存在,跳过备份:%s" % data_dir) return None # 备份统一收到 ~/ipass/backup/ 下:不再散在 $HOME 根目录被 `ls ~` 一眼看到一堆 # `ipass-data-backup-20260914-*.tar.gz` 文件。命名沿用 `ipass-data-.tar.gz` # 保留 `ipass-data-` 前缀便于识别时间序列。 dest_dir = os.path.join(os.path.expanduser("~"), "ipass", "backup") try: os.makedirs(dest_dir, exist_ok=True) except Exception as e: # noqa: BLE001 jlog(job, " ! 创建备份目录失败(继续,但请留意):%s" % e) return None dest = os.path.join(dest_dir, "ipass-data-%s.tar.gz" % ts_tag()) try: with tarfile.open(dest, "w:gz") as tf: tf.add(data_dir, arcname="data") # 审计是 data 的兄弟目录(容器内 /usr/local/nginx/audit,宿主 …/ipass/audit) # 1.0.18+ 的对话审计落这里;缺了它备份就丢了"谁在什么时候调过什么"的证据 audit_src = os.path.join(os.path.dirname(data_dir.rstrip("/")), "audit") if os.path.isdir(audit_src): tf.add(audit_src, arcname="audit") jlog(job, " + 审计目录 %s" % audit_src) size = os.path.getsize(dest) jlog(job, " 已备份 %s(%.1f KB)" % (dest, size / 1024.0)) return dest except Exception as e: # noqa: BLE001 jlog(job, " ! 备份失败(继续,但请留意):%s" % e) return None def check_backup(backup): """备份 tar 的路径校验(不发 IO):限定在 ~/ipass/backup/ 下,防被拿来读任意文件。""" if not backup: raise RuntimeError("restore_backup 为空") backup = os.path.abspath(os.path.expanduser(str(backup))) allow_root = os.path.join(os.path.expanduser("~"), "ipass", "backup") if not backup.startswith(allow_root + os.sep): raise RuntimeError("拒绝恢复非备份目录下的文件:%s" % backup) if not os.path.exists(backup): raise RuntimeError("备份文件不存在:%s(数据未恢复!)" % backup) return backup def restore_data(data_src, backup, job): """把数据目录(含 audit)恢复成备份 tar 时刻的内容。 用于**发版验证失败后的自动回滚**:新(故障)版本可能已把污染写穿进 shm→data (1.0.44 这类改冷却/路由逻辑的版本尤其可能),只回滚镜像不回数据,旧版本启动 会读到新形状的数据。调用方必须**先停容器**再进来(worker 周期落盘会在恢复后 又把旧内存态写回来)。 安全:备份路径限定在 ~/ipass/backup/ 下(防被拿来读任意 tar);原地数据不删, 改名为 `.broken-` 留在旁边,出问题还能人工对比。 """ if not backup: raise RuntimeError("restore_backup 为空") backup = os.path.abspath(os.path.expanduser(str(backup))) allow_root = os.path.join(os.path.expanduser("~"), "ipass", "backup") if not backup.startswith(allow_root + os.sep): raise RuntimeError("拒绝恢复非备份目录下的文件:%s" % backup) if not os.path.exists(backup): raise RuntimeError("备份文件不存在:%s(数据未恢复!)" % backup) parent = os.path.dirname(data_src.rstrip("/")) data_src = data_src.rstrip("/") ts = ts_tag() staging = os.path.join(parent, ".ipass-restore-%s" % ts) os.makedirs(staging) try: with tarfile.open(backup, "r:gz") as tf: members = [m for m in tf.getmembers() if m.name.split("/")[0] in ("data", "audit")] tf.extractall(staging, members=members) # data:现目录改名留证,staging/data 顶上 cur = data_src if os.path.isdir(cur): broken = "%s.broken-%s" % (cur.rstrip("/"), ts) os.rename(cur, broken) jlog(job, " 现数据目录留证:%s" % broken) os.renames(os.path.join(staging, "data"), cur) os.chmod(cur, 0o777) # 与常规一致:容器内 worker 是 nobody,宿主属主不可靠 jlog(job, " ✓ 数据目录已恢复:%s(来自 %s)" % (cur, os.path.basename(backup))) # audit:备份里有才恢复(1.0.7 前的备份没有该成员) staged_audit = os.path.join(staging, "audit") live_audit = os.path.join(parent, "audit") if os.path.isdir(staged_audit): if os.path.isdir(live_audit): os.rename(live_audit, "%s.broken-%s" % (live_audit, ts)) os.renames(staged_audit, live_audit) os.chmod(live_audit, 0o777) jlog(job, " ✓ 审计目录已恢复:%s" % live_audit) summ = data_summary(cur) jlog(job, " 恢复后摘要:账号 %s / 域名 %s / 上游 %s" % (summ.get("users"), summ.get("domains"), summ.get("upstreams"))) finally: shutil.rmtree(staging, ignore_errors=True) # ------------------------------------------------------------------ 任务(job) JOBS = {} JOBS_ORDER = [] JOBS_LOCK = threading.Lock() ACTIVE = threading.Lock() def jlog(job, msg): line = "[%s] %s" % (time.strftime("%H:%M:%S"), msg) with JOBS_LOCK: job["log"].append(line) if not BACK["quiet"]: print(line, flush=True) def new_job(kind, params): jid = "%s-%s" % (ts_tag(), "".join(random.choice(string.hexdigits[:16]) for _ in range(4))) job = {"id": jid, "kind": kind, "state": "running", "rc": None, "log": [], "created": now_ts(), "finished": None, "params": params, "result": None} with JOBS_LOCK: JOBS[jid] = job JOBS_ORDER.append(jid) while len(JOBS_ORDER) > JOB_KEEP: old = JOBS_ORDER.pop(0) JOBS.pop(old, None) return job def start_job(kind, fn, params): """起一个异步任务。同时只允许一个在跑,避免并发删容器。""" if not ACTIVE.acquire(blocking=False): return None, "已有任务在执行中(同一时刻只允许一个变更任务),稍后再试" job = new_job(kind, params) def runner(): try: job["result"] = fn(job, params) job["state"] = "done" job["rc"] = 0 except Exception as e: # noqa: BLE001 jlog(job, "✗ 失败:%s" % e) job["state"] = "failed" job["rc"] = 1 finally: job["finished"] = now_ts() ACTIVE.release() t = threading.Thread(target=runner) t.daemon = True t.start() return job, None # ------------------------------------------------------------------ 业务动作 def op_pull(job, p): image = p.get("image") or DEFAULT_IMAGE user, pwd = p.get("registry_user"), p.get("registry_pass") if user and pwd: registry = image.split("/")[0] jlog(job, "→ docker login %s" % registry) rc, out = docker(["login", registry, "-u", user, "--password-stdin"], timeout=120, stdin_data=pwd.encode()) jlog(job, " rc=%s" % rc) for l in clean_lines(out): jlog(job, " " + l) jlog(job, "→ docker pull %s" % image) rc, out = docker(["pull", image], timeout=PULL_TIMEOUT) for l in clean_lines(out): jlog(job, " " + l) if rc != 0: raise RuntimeError("docker pull 失败 rc=%s" % rc) jlog(job, "收到镜像 digest:%s" % (image_repo_digest(image) or "(未知)")) return {"image": image, "digest": image_repo_digest(image)} def op_update(job, p): # 受管容器约束。HTTP 层已同步校验过一次(非法 name 直接 403、不建任务), # 这里是纵深防御 —— op_rollback 内部也走 op_update,不能只靠入口把关。 name = ensure_managed(p.get("name")) image = p.get("image") or DEFAULT_IMAGE skip_pull = bool(p.get("skip_pull")) data_override = p.get("data_dir") host_port = p.get("host_port") job["container"] = name cfg = inspect_one(name) if cfg is None: raise RuntimeError("容器 %s 不存在" % name) old_image = (cfg.get("Config") or {}).get("Image") or "" # ---- 计划 data_src, extra_mounts = extract_mounts(cfg) if data_override: data_src = os.path.expanduser(str(data_override)) port_args, main_port = extract_ports(cfg, host_port) net_args = extract_net(cfg) envs = extract_env(cfg, old_image) if old_image else [] restart = extract_restart(cfg) or "always" if "host" in net_args: port_args, main_port = [], main_port or "88" if not data_src: raise RuntimeError("旧容器没挂 %s,拒绝更新(会丢数据)。用 DATA_DIR/--data-dir 指定" % CONTAINER_DATA) job["plan"] = { "container": name, "old_image": old_image, "new_image": image, "ports": port_args, "main_port": main_port, "data_dir": data_src, "extra_mounts": extra_mounts, "network": net_args, "restart": restart, "env": [e.split("=", 1)[0] for e in envs], } jlog(job, "== 更新计划 ==") jlog(job, " 容器 %s" % name) jlog(job, " 旧镜像 %s" % (old_image or "(未知)")) jlog(job, " 新镜像 %s" % image) jlog(job, " 端口 %s" % (" ".join(port_args) or "(无)")) jlog(job, " 数据目录 %s -> %s" % (data_src, CONTAINER_DATA)) if extra_mounts: jlog(job, " 附加挂载 %s" % " ".join(extra_mounts)) if net_args: jlog(job, " 网络 %s" % " ".join(net_args)) if envs: jlog(job, " 附加环境 %s" % ", ".join(e.split("=", 1)[0] for e in envs)) # ---- 1 拉镜像 if skip_pull: jlog(job, "\n1/7 跳过拉取(skip_pull=1),校验本地镜像…") if not image_exists(image): raise RuntimeError("本地没有镜像 %s,无法跳过拉取" % image) jlog(job, " 本地已有 %s" % image) else: jlog(job, "\n1/7 拉取镜像") op_pull(job, {"image": image, "registry_user": p.get("registry_user"), "registry_pass": p.get("registry_pass")}) # ---- 2 备份数据 jlog(job, "\n2/6 备份数据") job["backup"] = backup_data(data_src, job) # ---- 3 删旧容器 jlog(job, "\n4/7 停止并删除旧容器") docker(["stop", "-t", "10", name], timeout=60) rc, out = docker(["rm", "-f", name], timeout=60) if rc != 0: raise RuntimeError("删除旧容器失败:%s" % out.strip()) jlog(job, " 已删除 %s" % name) # ---- 5 起新容器 jlog(job, "\n4/6 起新容器") argv = ["run", "-d", "--name", name] if restart: argv += ["--restart", restart] argv += port_args argv += ["-v", "%s:%s" % (data_src, CONTAINER_DATA)] # 审计挂载:宿主侧 = 数据目录的兄弟目录 audit/(docs/llm/AUDIT_LOG.md)。 # 容器重建不丢审计档案;目录不存在就先建。 # ⚠️ worker 在容器里跑 nobody,宿主目录首建属主是 root(agent 是 root 跑的), # nobody 写不动 → 日志刷 `[audit] 建日期目录失败 Permission denied`、整个审计废掉。 # 解决:建完目录**显式 chmod 777**,与 data 目录保持一致(worker 还要建日期子目录)。 # 用 777 而非按 uid chown:宿主侧的 uid 与容器内 nobody 的 uid 不一定一致 # (镜像内 nobody=65534,宿主侧 root 直接 mkdir 后 uid=0),按 uid 改要么拖数字 # 要么拖 docker 进程解析。777 简单可读,且 audit 是「append-only JSON 元数据」、 # 没有「拒绝别人改我的审计」语义。 try: audit_src = os.path.join(os.path.dirname(data_src.rstrip("/")), "audit") os.makedirs(audit_src, exist_ok=True) os.chmod(audit_src, 0o777) argv += ["-v", "%s:%s" % (audit_src, CONTAINER_AUDIT)] jlog(job, " 审计目录 %s -> %s" % (audit_src, CONTAINER_AUDIT)) except OSError as e: jlog(job, " ! 审计目录建不了(%s),本容器不带审计挂载" % e) argv += extra_mounts for e in envs: argv += ["-e", e] argv += net_args argv += [image] rc, out = docker(argv, timeout=120) if rc != 0: jlog(job, " ! 启动失败:%s" % out.strip()) jlog(job, " 回滚命令:") if rollback_tag: jlog(job, " docker rm -f %s && docker run -d --name %s --restart %s %s " "-v %s:%s %s" % (name, name, restart, " ".join(port_args), data_src, CONTAINER_DATA, rollback_tag)) raise RuntimeError("新容器启动失败 rc=%s" % rc) jlog(job, " 新容器已启动(%s)" % out.strip()[:12]) time.sleep(2) # ---- 6 容器内自检 jlog(job, "\n6/7 容器内自检") rc, out = docker(["exec", name, NGINX_BIN, "-t"]) jlog(job, " nginx -t rc=%s %s" % (rc, (out.strip().splitlines() or [""])[-1])) if rc != 0: jlog(job, " ! 配置语法不过,请立刻回滚") wtest = ("[ -w {d} ] && touch {d}/.wtest && mv {d}/.wtest /tmp/.ipass-wtest " "&& echo WRITABLE || echo NOT_WRITABLE").format(d=CONTAINER_DATA) rc, out = docker(["exec", "-u", "nobody", name, "sh", "-c", wtest]) writable = "WRITABLE" in out jlog(job, " worker(nobody) 数据目录可写:%s" % ("是" if writable else "否 !!")) if not writable: jlog(job, " ! 不可写 → 改账号/域名会静默失败,检查 %s 属主" % data_src) # 审计目录可写(审计是旁路功能:不可写只降级、不阻断转发) wtest_a = "[ -w {d} ] && echo WRITABLE || echo NOT_WRITABLE".format(d=CONTAINER_AUDIT) rc, out = docker(["exec", "-u", "nobody", name, "sh", "-c", wtest_a]) jlog(job, " worker(nobody) 审计目录可写:%s" % ("是" if "WRITABLE" in out else "否(审计功能将停用)")) # ---- 6 数据 + 宿主探活 jlog(job, "\n6/6 数据与探活") summ = data_summary(data_src) jlog(job, " 账号 %s / 域名 %s / 上游 %s" % (summ.get("users"), summ.get("domains"), summ.get("upstreams"))) code = http_probe(main_port) if code: jlog(job, " https://127.0.0.1:%s 可达(HTTP %s;401=模块正常只是没会话)" % (main_port, code)) else: jlog(job, " ! 宿主 %s 无响应(容器内正常则检查防火墙/端口映射)" % main_port) jlog(job, "\n✔ 更新完成") return {"container": name, "image": image, "main_port": main_port, "data_dir": data_src, "backup": job.get("backup"), "rollback_tag": job.get("rollback_tag"), "probe": code, "data": summ} def op_rollback(job, p): # tag / image 两个键都认:release 自动回滚双写两者(旧调用方只填 image, # 新调用方填 tag)——参数键位不一致曾让 1.1.1 的自动回滚 job 直接失败。 tag = p.get("tag") or p.get("image") # 1.0.17 起 update 不再在本地留 `ipass-gateway:rollback-*` 镜像标签(理由: # 镜像动辄 100 MB+,本地磁盘会越积越多;阿里云私仓永远保留所有 tag,需要时 # `docker pull` 即可)。所以回滚**必须显式传 tag**,否则直接报错——避免误 # 用「之前回滚过一次的旧镜像」导致回滚到非预期版本。 if not tag: raise RuntimeError( "rollback 需要显式传 tag(1.0.17 起 update 不在本地留 rollback-* 镜像;" "请从阿里云私仓拉指定 tag 后再 rollback,例如:" "`bash deploy/agent.sh rollback ipass-1.0.15`)" ) # 三种形态都收:完整镜像名(release 自动回滚传这个)、`ipass-x.y.z` 短 tag、 # 裸 `x.y.z`。短形态用受管容器当前镜像的仓库段补全——私仓前缀在 .11 现场 # 取现值最可靠,别让调用方背地址。 if "/" not in tag: short = tag if tag.startswith("ipass-") else "ipass-" + tag name = ensure_managed(p.get("name")) rc, out = docker(["inspect", "-f", "{{.Config.Image}}", name], timeout=30) cur = out.strip() if rc == 0 else "" if cur and ":" in cur.rsplit("/", 1)[-1]: cur = cur.rsplit(":", 1)[0] if not cur: raise RuntimeError("短 tag %s 无法补全:读不到受管容器镜像引用" % short) tag = "%s:%s" % (cur, short) jlog(job, "短 tag 补全为 %s" % tag) if not image_exists(tag): # 自动补齐:私仓永远保留所有 tag。发版失败触发的**自动**回滚必须能自愈, # 不能停在「本地恰好没拉过旧镜像」上(人工场景报错让人决定)。 jlog(job, "本地没有 %s,先拉取(回滚不该卡在缺镜像上)" % tag) op_pull(job, {"image": tag, "registry_user": p.get("registry_user"), "registry_pass": p.get("registry_pass")}) if not image_exists(tag): raise RuntimeError("拉取后仍找不到镜像:%s" % tag) # 数据回滚:发版验证失败时,data/audit 里可能已被新(故障)版本写穿污染。 # restore_backup = 本次 update 返回的备份 tar 路径 → 恢复到备份时刻。 backup = p.get("restore_backup") if backup: name = ensure_managed(p.get("name")) cfg = inspect_one(name) if cfg is None: raise RuntimeError("容器 %s 不存在,无法定位数据目录来恢复" % name) data_src, _ = extract_mounts(cfg) # 与 op_update 同口径:显式 data_dir 优先于容器挂载 if p.get("data_dir"): data_src = os.path.expanduser(str(p["data_dir"])) if not data_src: raise RuntimeError("受管容器没挂 %s,无处恢复" % CONTAINER_DATA) # ⚠️ **停容器前**预检备份(路径合法 + 文件存在 + tar 可读): # stop 是手工停止,restart policy 不会救回来——若停在「容器已停、备份却坏了」, # 线上就一直空跑到人工介入,比不回滚还糟。 check_backup(backup) with tarfile.open(backup, "r:gz") as tf: # 提前解析一遍,坏包在这里就炸 names = {m.name.split("/")[0] for m in tf.getmembers()[:50]} if "data" not in names: raise RuntimeError("备份里没有 data/ 成员,拒绝用它恢复:%s" % backup) # 先 stop 不 rm:worker 周期落盘会在恢复后又把(污染)统计写回 data; # 不删容器是留给 op_update 里的 inspect 复刻参数用(stop 后 rm 照常成功)。 jlog(job, "→ 停止容器(恢复数据前冻结写入)") docker(["stop", "-t", "10", name], timeout=60) restore_data(data_src, backup, job) jlog(job, "回滚到 %s\n" % tag) params = {"image": tag, "skip_pull": True} for k in ("name", "data_dir", "host_port"): if p.get(k): params[k] = p[k] return op_update(job, params) # ------------------------------------------------------------------ 配置摘要(dry-run) def build_check(name=None, data_dir=None, host_port=None): cname = name or detect_container(None) if not cname: return {"ok": False, "error": "未发现候选容器", "containers": relevant_containers(), "docker": DOCKER} cfg = inspect_one(cname) if cfg is None: return {"ok": False, "error": "容器不存在:%s" % cname} data_src, extra = extract_mounts(cfg) if data_dir: data_src = os.path.expanduser(data_dir) port_args, main = extract_ports(cfg, host_port) return { "ok": True, "container": cname, "running": (cfg.get("State") or {}).get("Status") == "running", "image": (cfg.get("Config") or {}).get("Image"), "ports": port_args, "main_port": main, "data_dir": data_src, "extra_mounts": extra, "network": extract_net(cfg), "restart": extract_restart(cfg), "default_image": DEFAULT_IMAGE, } def build_status(name=None): cname = name or detect_container(None) if not cname: return {"ok": False, "error": "未发现候选容器"} cfg = inspect_one(cname) if cfg is None: return {"ok": False, "error": "容器不存在:%s" % cname} st = cfg.get("State") or {} data_src, _ = extract_mounts(cfg) _pa, main = extract_ports(cfg) image = (cfg.get("Config") or {}).get("Image") return { "ok": True, "service": SERVICE, "version": VERSION, "container": cname, "exists": True, "running": st.get("Status") == "running", "state": st.get("Status"), "started_at": st.get("StartedAt"), "image": image, "image_id": (cfg.get("Image") or "")[:19], "digest": image_repo_digest(image) if image else None, "main_port": main, "data_dir": data_src, "data": data_summary(data_src), "probe": {"code": http_probe(main), "url": "https://127.0.0.1:%s/api/admin/users" % main}, "other_containers": relevant_containers(cname), } # ------------------------------------------------------------------ HTTP 层 BACK = {"quiet": False, "name": None, "data_dir": None, "token": "", "_managed": None} class Handler(BaseHTTPRequestHandler): server_version = "%s/%s" % (SERVICE, VERSION) protocol_version = "HTTP/1.1" # -------- 基础设施 def log_message(self, fmt, *a): # noqa: A003 if not BACK["quiet"]: sys.stderr.write("[http] %s - %s\n" % (self.address_string(), fmt % a)) def _send(self, code, payload): body = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") self.send_response(code) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def _auth(self): if self.path.split("?")[0] == "/health": return True got = self.headers.get("Authorization") or "" if got.lower().startswith("bearer "): got = got[7:].strip() if not got: got = self.headers.get("X-Auth-Token") or "" return bool(BACK["token"]) and hmac.compare_digest(got, BACK["token"]) def _read_body(self): """读请求体。 **鉴权失败时也必须读掉**:HTTP/1.1 是长连接,body 留在 socket 里会被 当成下一个请求的请求行,导致响应串包。 """ n = int(self.headers.get("Content-Length") or 0) if n <= 0: return {} if n > (1 << 20): return {"__oversize__": n} try: return json.loads(self.rfile.read(n).decode("utf-8")) or {} except Exception: # noqa: BLE001 return {} def _qs(self): return parse_qs(urlparse(self.path).query) # -------- 路由 def do_GET(self): # noqa: N802 self._dispatch("GET") def do_POST(self): # noqa: N802 self._dispatch("POST") def _dispatch(self, method): path = urlparse(self.path).path.rstrip("/") or "/" self._payload = self._read_body() if not self._auth(): return self._send(401, {"error": "unauthorized", "hint": "Authorization: Bearer "}) try: if method == "GET": return self._get(path) return self._post(path) except PermissionError as e: return self._send(403, {"error": str(e)}) except Exception as e: # noqa: BLE001 return self._send(500, {"error": str(e)}) def _get(self, path): qs = self._qs() if path == "/health": return self._send(200, {"ok": True, "service": SERVICE, "version": VERSION, "time": now_ts()}) if path == "/": return self._send(200, { "service": SERVICE, "version": VERSION, "docker": DOCKER, "container": managed_container(), "note": "破坏性操作(update/restart/rollback)只作用于 container 这个受管容器", "default_image": DEFAULT_IMAGE, "endpoints": [ "GET /health", "GET /check", "GET /status", "GET /containers", "GET /logs?tail=200", "GET /data?file=users|domains|upstreams", "GET /jobs?limit=5", "GET /jobs/?offset=N", "POST /pull {image}", "POST /update {image,skip_pull,name,data_dir,host_port}", "POST /rollback {tag,restore_backup?}", "POST /restart", ], }) if path == "/check": return self._send(200, build_check(BACK["name"], BACK["data_dir"], None)) if path == "/status": return self._send(200, build_status(BACK["name"])) if path == "/containers": managed = managed_container() return self._send(200, { "managed": managed, "note": "只有 managed 这个容器会被本服务更新/重启/回滚", "containers": [ {"name": n, "image": i, "status": s, "managed": n == managed} for n, i, s in list_containers()]}) if path == "/logs": name = BACK["name"] or detect_container(None) if not name: return self._send(404, {"error": "未发现容器"}) tail = (qs.get("tail") or ["200"])[0] if not str(tail).isdigit(): tail = "200" rc, out = docker(["logs", "--tail", str(tail), name], timeout=60) return self._send(200, {"container": name, "rc": rc, "lines": len(out.splitlines()), "log": out}) if path == "/data": which = (qs.get("file") or ["users"])[0] data_dir = BACK["data_dir"] if not data_dir: cname = BACK["name"] or detect_container(None) cfg = inspect_one(cname) if cname else None if cfg: data_dir, _ = extract_mounts(cfg) if not data_dir: return self._send(404, {"error": "未知数据目录"}) view = data_view(data_dir, which) if view is None: return self._send(404, {"error": "读取失败或文件不存在:%s" % which}) return self._send(200, {"file": which, "dir": data_dir, "data": view}) if path == "/jobs": limit = int((qs.get("limit") or ["5"])[0]) with JOBS_LOCK: ids = list(reversed(JOBS_ORDER))[:limit] return self._send(200, {"jobs": [ {"id": i, "kind": JOBS[i]["kind"], "state": JOBS[i]["state"], "created": JOBS[i]["created"], "finished": JOBS[i]["finished"]} for i in ids]}) if path.startswith("/jobs/"): jid = path.split("/", 2)[2] with JOBS_LOCK: job = JOBS.get(jid) if not job: return self._send(404, {"error": "任务不存在:%s" % jid}) off = int((qs.get("offset") or ["0"])[0]) lines = job["log"][off:] return self._send(200, { "id": jid, "kind": job["kind"], "state": job["state"], "rc": job["rc"], "created": job["created"], "finished": job["finished"], "next_offset": len(job["log"]), "params": job["params"], "result": job.get("result"), "container": job.get("container"), "plan": job.get("plan"), "log": "\n".join(lines)}) return self._send(404, {"error": "未知端点:%s" % path}) def _post(self, path): p = self._payload if path == "/pull": job, err = start_job("pull", op_pull, p) return self._send(202 if job else 409, {"job": job["id"]} if job else {"error": err}) if path == "/update": p.setdefault("data_dir", BACK["data_dir"]) p["name"] = ensure_managed(p.get("name")) # 非法 → 403,不建任务 job, err = start_job("update", op_update, p) return self._send(202 if job else 409, {"job": job["id"]} if job else {"error": err}) if path == "/rollback": p.setdefault("data_dir", BACK["data_dir"]) p["name"] = ensure_managed(p.get("name")) job, err = start_job("rollback", op_rollback, p) return self._send(202 if job else 409, {"job": job["id"]} if job else {"error": err}) if path == "/restart": name = ensure_managed(p.get("name")) # 非法 → 403 rc, out = docker(["restart", name], timeout=120) code = http_probe(build_check(name).get("main_port")) return self._send(200 if rc == 0 else 500, {"container": name, "rc": rc, "log": out.strip(), "probe": code}) return self._send(404, {"error": "未知端点:%s" % path}) class ThreadingServer(ThreadingMixIn, HTTPServer): daemon_threads = True allow_reuse_address = True # ------------------------------------------------------------------ token def load_token(cli_token): if cli_token: return cli_token env = os.environ.get("IPASS_AGENT_TOKEN") if env: return env path = TOKEN_FILE if os.path.exists(path): try: with open(path, "r", encoding="utf-8") as f: t = f.read().strip() if t: return t except Exception: # noqa: BLE001 pass t = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(40)) try: with open(path, "w", encoding="utf-8") as f: f.write(t + "\n") os.chmod(path, 0o600) except Exception: # noqa: BLE001 pass return t # ------------------------------------------------------------------ main def daemonize(logfile, pidfile): if os.fork() > 0: os._exit(0) os.setsid() if os.fork() > 0: os._exit(0) os.chdir("/") fd = os.open(logfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) os.dup2(fd, 1) os.dup2(fd, 2) os.dup2(os.open(os.devnull, os.O_RDONLY), 0) try: with open(pidfile, "w") as f: f.write(str(os.getpid()) + "\n") except Exception: # noqa: BLE001 pass def do_stop(pidfile): if not os.path.exists(pidfile): print("没有 pid 文件 %s,服务可能没在跑" % pidfile) return 1 with open(pidfile) as f: pid = int((f.read().strip() or "0")) try: os.kill(pid, 15) print("已发送 SIGTERM 给 pid %s" % pid) except ProcessLookupError: print("pid %s 已不存在" % pid) try: os.unlink(pidfile) except Exception: # noqa: BLE001 pass return 0 def main(): ap = argparse.ArgumentParser(description="iPass 容器远程更新控制面") ap.add_argument("--port", type=int, default=DEFAULT_PORT) ap.add_argument("--bind", default="0.0.0.0") ap.add_argument("--name", help="受管容器名(缺省自动探测 ipass* 容器;" "破坏性操作只作用于它)") ap.add_argument("--data-dir", help="数据目录(缺省从容器挂载读)") ap.add_argument("--image", help="默认更新目标镜像") ap.add_argument("--token", help="访问令牌(缺省用环境变量/~/ipass/.ipass-agent-token)") ap.add_argument("--check", action="store_true", help="只探测并打印结果,不起服务") ap.add_argument("--bg", action="store_true", help="后台运行") ap.add_argument("--stop", action="store_true", help="停止后台服务") ap.add_argument("--quiet", action="store_true", help="不打印访问日志") a = ap.parse_args() if a.image: globals()["DEFAULT_IMAGE"] = a.image BACK["quiet"] = a.quiet BACK["name"] = a.name BACK["data_dir"] = a.data_dir BACK["token"] = load_token(a.token) pidfile = PID_FILE logfile = LOG_FILE if a.stop: return do_stop(pidfile) if a.check: print(json.dumps(build_check(a.name, a.data_dir, None), ensure_ascii=False, indent=2)) return 0 if a.bg: if os.path.exists(pidfile): try: with open(pidfile) as f: pid = int(f.read().strip()) os.kill(pid, 0) print("已在运行(pid %s)。先 --stop 再启动。" % pid) return 1 except Exception: # noqa: BLE001 pass daemonize(logfile, pidfile) BACK["quiet"] = True print("[%s] %s %s 启动于 %s:%s" % (time.strftime("%F %T"), SERVICE, VERSION, a.bind, a.port), flush=True) print("token: %s" % BACK["token"], flush=True) srv = ThreadingServer((a.bind, a.port), Handler) cname = detect_container(a.name) if not a.bg: print("%s %s" % (SERVICE, VERSION)) print(" 监听 %s:%s" % (a.bind, a.port)) print(" token %s" % BACK["token"]) print(" 容器 %s" % (cname or "(未发现,可用 --name 指定)")) print(" 镜像 %s" % DEFAULT_IMAGE) print(" 日志 %s" % (logfile if a.bg else "(stdout)")) try: srv.serve_forever() except KeyboardInterrupt: print("\n退出") finally: srv.server_close() return 0 if __name__ == "__main__": sys.exit(main())