修改太多懒得写了

This commit is contained in:
handsomeAq 2026-08-05 16:33:27 +08:00
parent 2c1f06c84c
commit 52a67967b9
20 changed files with 765 additions and 92 deletions

View File

@ -1,17 +1,24 @@
"""后台管理接口:任务/Agent/日志/手动管控。""" """后台管理接口:任务/Agent/日志/手动管控。"""
from typing import Optional from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Request from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request
from app.constants import TaskStatus from app.constants import TaskStatus
from app.models.schemas import AgentInfo, LogEntry, TaskInfo from app.models.schemas import AgentInfo, LogEntry, TaskInfo
from app.repository.log_repo import LogRepo from app.repository.log_repo import LogRepo
from app.constants import AgentStatus
from app.services.agent_service import AgentService from app.services.agent_service import AgentService
from app.services.security import require_admin
from app.services.task_service import TaskService from app.services.task_service import TaskService
router = APIRouter() router = APIRouter()
def require_admin_dep(x_admin_auth: str = Header(..., alias="X-Admin-Auth")) -> None:
"""后台管理接口鉴权:校验 X-Admin-Auth 头与 ADMIN_AUTH 一致。"""
require_admin(x_admin_auth)
def get_task_service(request: Request) -> TaskService: def get_task_service(request: Request) -> TaskService:
return TaskService(request.app.state.redis) return TaskService(request.app.state.redis)
@ -30,12 +37,17 @@ async def list_tasks(
status: Optional[TaskStatus] = None, status: Optional[TaskStatus] = None,
limit: int = 100, limit: int = 100,
svc: TaskService = Depends(get_task_service), svc: TaskService = Depends(get_task_service),
_admin: None = Depends(require_admin_dep),
): ):
return await svc.list(status=status, limit=limit) return await svc.list(status=status, limit=limit)
@router.get("/tasks/{request_id}", response_model=TaskInfo, summary="任务详情") @router.get("/tasks/{request_id}", response_model=TaskInfo, summary="任务详情")
async def task_detail(request_id: str, svc: TaskService = Depends(get_task_service)): async def task_detail(
request_id: str,
svc: TaskService = Depends(get_task_service),
_admin: None = Depends(require_admin_dep),
):
task = await svc.get(request_id) task = await svc.get(request_id)
if not task: if not task:
raise HTTPException(status_code=404, detail="task not found") raise HTTPException(status_code=404, detail="task not found")
@ -43,14 +55,22 @@ async def task_detail(request_id: str, svc: TaskService = Depends(get_task_servi
@router.post("/tasks/{request_id}/cancel", summary="取消任务") @router.post("/tasks/{request_id}/cancel", summary="取消任务")
async def admin_cancel(request_id: str, svc: TaskService = Depends(get_task_service)): async def admin_cancel(
request_id: str,
svc: TaskService = Depends(get_task_service),
_admin: None = Depends(require_admin_dep),
):
if not await svc.cancel(request_id): if not await svc.cancel(request_id):
raise HTTPException(status_code=400, detail="cannot cancel task") raise HTTPException(status_code=400, detail="cannot cancel task")
return {"ok": True} return {"ok": True}
@router.post("/tasks/{request_id}/reset", summary="重置任务状态") @router.post("/tasks/{request_id}/reset", summary="重置任务状态")
async def admin_reset(request_id: str, svc: TaskService = Depends(get_task_service)): async def admin_reset(
request_id: str,
svc: TaskService = Depends(get_task_service),
_admin: None = Depends(require_admin_dep),
):
if not await svc.reset(request_id): if not await svc.reset(request_id):
raise HTTPException(status_code=400, detail="cannot reset task") raise HTTPException(status_code=400, detail="cannot reset task")
return {"ok": True} return {"ok": True}
@ -58,17 +78,48 @@ async def admin_reset(request_id: str, svc: TaskService = Depends(get_task_servi
# ---------- Agent 管理 ---------- # ---------- Agent 管理 ----------
@router.get("/agents", response_model=list[AgentInfo], summary="Agent 列表") @router.get("/agents", response_model=list[AgentInfo], summary="Agent 列表")
async def list_agents(svc: AgentService = Depends(get_agent_service)): async def list_agents(
svc: AgentService = Depends(get_agent_service),
_admin: None = Depends(require_admin_dep),
):
return await svc.all() return await svc.all()
@router.post("/agents/{agent_id}/offline", summary="下线 Agent") @router.post("/agents/{agent_id}/unavailable", summary="置 Agent 不可用")
async def agent_offline(agent_id: str, svc: AgentService = Depends(get_agent_service)): async def agent_unavailable(
if not await svc.take_offline(agent_id): agent_id: str,
svc: AgentService = Depends(get_agent_service),
_admin: None = Depends(require_admin_dep),
):
if not await svc.set_unavailable(agent_id):
raise HTTPException(status_code=404, detail="agent not found") raise HTTPException(status_code=404, detail="agent not found")
return {"ok": True} return {"ok": True}
@router.post("/agents/{agent_id}/available", summary="置 Agent 可用")
async def agent_available(
agent_id: str,
svc: AgentService = Depends(get_agent_service),
_admin: None = Depends(require_admin_dep),
):
if not await svc.set_available(agent_id):
raise HTTPException(status_code=404, detail="agent not found")
return {"ok": True}
@router.post("/agents/{agent_id}/priority", response_model=AgentInfo, summary="设置 Agent 优先级")
async def agent_priority(
agent_id: str,
priority: int = Body(..., ge=1, le=5, description="优先级 1-5越小越先分配"),
svc: AgentService = Depends(get_agent_service),
_admin: None = Depends(require_admin_dep),
):
agent = await svc.set_priority(agent_id, priority)
if not agent:
raise HTTPException(status_code=404, detail="agent not found")
return agent
# ---------- 日志审计 ---------- # ---------- 日志审计 ----------
@router.get("/logs", response_model=list[LogEntry], summary="日志审计") @router.get("/logs", response_model=list[LogEntry], summary="日志审计")
async def logs( async def logs(
@ -76,6 +127,7 @@ async def logs(
request_id: Optional[str] = None, request_id: Optional[str] = None,
agent_id: Optional[str] = None, agent_id: Optional[str] = None,
repo: LogRepo = Depends(get_log_repo), repo: LogRepo = Depends(get_log_repo),
_admin: None = Depends(require_admin_dep),
): ):
return await repo.list(limit=limit, request_id=request_id, agent_id=agent_id) return await repo.list(limit=limit, request_id=request_id, agent_id=agent_id)
@ -85,6 +137,7 @@ async def logs(
async def overview( async def overview(
task_svc: TaskService = Depends(get_task_service), task_svc: TaskService = Depends(get_task_service),
agent_svc: AgentService = Depends(get_agent_service), agent_svc: AgentService = Depends(get_agent_service),
_admin: None = Depends(require_admin_dep),
): ):
tasks = await task_svc.list(limit=1000) tasks = await task_svc.list(limit=1000)
agents = await agent_svc.all() agents = await agent_svc.all()
@ -98,6 +151,9 @@ async def overview(
"task_success": status_count.get(TaskStatus.SUCCESS.value, 0), "task_success": status_count.get(TaskStatus.SUCCESS.value, 0),
"task_failed": status_count.get(TaskStatus.FAILED.value, 0), "task_failed": status_count.get(TaskStatus.FAILED.value, 0),
"agent_total": len(agents), "agent_total": len(agents),
"agent_online": sum(1 for a in agents if a.status.value == "online"), "agent_ready": sum(1 for a in agents if a.status == AgentStatus.READY),
"agent_offline": sum(1 for a in agents if a.status.value == "offline"), "agent_processing": sum(1 for a in agents if a.status == AgentStatus.PROCESSING),
"agent_stopping": sum(1 for a in agents if a.status == AgentStatus.STOPPING),
"agent_offline": sum(1 for a in agents if a.status == AgentStatus.OFFLINE),
"agent_unavailable": sum(1 for a in agents if a.status == AgentStatus.UNAVAILABLE),
} }

View File

@ -27,6 +27,7 @@ async def register(body: AgentRegister, svc: AgentService = Depends(get_agent_se
async def unregister(body: dict, svc: AgentService = Depends(get_agent_service)): async def unregister(body: dict, svc: AgentService = Depends(get_agent_service)):
from fastapi import HTTPException from fastapi import HTTPException
require_auth(body.get("auth", ""))
agent_id = body.get("agent_id") agent_id = body.get("agent_id")
if not agent_id: if not agent_id:
raise HTTPException(status_code=400, detail="agent_id required") raise HTTPException(status_code=400, detail="agent_id required")
@ -39,6 +40,7 @@ async def unregister(body: dict, svc: AgentService = Depends(get_agent_service))
async def heartbeat(body: AgentHeartbeat, svc: AgentService = Depends(get_agent_service)): async def heartbeat(body: AgentHeartbeat, svc: AgentService = Depends(get_agent_service)):
from fastapi import HTTPException from fastapi import HTTPException
require_auth(body.auth)
if not await svc.heartbeat(body): if not await svc.heartbeat(body):
raise HTTPException(status_code=404, detail="agent not registered") raise HTTPException(status_code=404, detail="agent not registered")
return {"ok": True} return {"ok": True}
@ -48,6 +50,7 @@ async def heartbeat(body: AgentHeartbeat, svc: AgentService = Depends(get_agent_
async def result(body: TaskResult, relay: RelayService = Depends(get_relay_service)): async def result(body: TaskResult, relay: RelayService = Depends(get_relay_service)):
from fastapi import HTTPException from fastapi import HTTPException
require_auth(body.auth)
if not await relay.on_result(body): if not await relay.on_result(body):
raise HTTPException(status_code=400, detail="result rejected") raise HTTPException(status_code=400, detail="result rejected")
return {"ok": True} return {"ok": True}

View File

@ -70,9 +70,14 @@ async def submit_task(body: TaskSubmit, svc: TaskService = Depends(get_task_serv
@router.get("/tasks/{request_id}", response_model=TaskInfo, summary="查询任务") @router.get("/tasks/{request_id}", response_model=TaskInfo, summary="查询任务")
async def get_task(request_id: str, svc: TaskService = Depends(get_task_service)): async def get_task(
request_id: str,
auth: str = "",
svc: TaskService = Depends(get_task_service),
):
from fastapi import HTTPException from fastapi import HTTPException
require_auth(auth)
task = await svc.get(request_id) task = await svc.get(request_id)
if not task: if not task:
raise HTTPException(status_code=404, detail="task not found") raise HTTPException(status_code=404, detail="task not found")
@ -80,9 +85,14 @@ async def get_task(request_id: str, svc: TaskService = Depends(get_task_service)
@router.post("/tasks/{request_id}/cancel", summary="取消任务") @router.post("/tasks/{request_id}/cancel", summary="取消任务")
async def cancel_task(request_id: str, svc: TaskService = Depends(get_task_service)): async def cancel_task(
request_id: str,
auth: str = "",
svc: TaskService = Depends(get_task_service),
):
from fastapi import HTTPException from fastapi import HTTPException
require_auth(auth)
if not await svc.cancel(request_id): if not await svc.cancel(request_id):
raise HTTPException(status_code=400, detail="cannot cancel task") raise HTTPException(status_code=400, detail="cannot cancel task")
return {"ok": True, "request_id": request_id} return {"ok": True, "request_id": request_id}

View File

@ -42,6 +42,9 @@ class Settings(BaseSettings):
# 认证CLI 提交任务 / Agent 注册时必须携带的密码 # 认证CLI 提交任务 / Agent 注册时必须携带的密码
gateway_auth: str = "dev-gateway-auth" gateway_auth: str = "dev-gateway-auth"
# 认证Admin 后台管理接口必须携带的密码(与协议层 gateway_auth 分离)
admin_auth: str = "dev-admin-auth"
# 分布式锁相关 # 分布式锁相关
lock_timeout: int = 30 lock_timeout: int = 30

View File

@ -10,8 +10,11 @@ class TaskStatus(str, Enum):
class AgentStatus(str, Enum): class AgentStatus(str, Enum):
ONLINE = "online" OFFLINE = "offline" # 离线:心跳失联(超时)
OFFLINE = "offline" UNAVAILABLE = "unavailable" # 不可用:后台手动禁用(心跳不复活)
READY = "ready" # 就绪(注册后、空闲时),调度只分配此状态
PROCESSING = "processing" # 处理中(调度选中并下发后)
STOPPING = "stopping" # 正在停止(收到取消指令后)
# ---- 任务池键 ---- # ---- 任务池键 ----

View File

@ -53,9 +53,11 @@ class AgentRegister(BaseModel):
agent_tags: list[str] = Field(default_factory=list) agent_tags: list[str] = Field(default_factory=list)
max_concurrent: int = Field(default=1, ge=1) max_concurrent: int = Field(default=1, ge=1)
current_load: int = Field(default=0, ge=0) current_load: int = Field(default=0, ge=0)
priority: int = Field(default=3, ge=1, le=5, description="调度优先级,越小越先分配")
class AgentHeartbeat(BaseModel): class AgentHeartbeat(BaseModel):
auth: str = Field(..., description="网关认证密码,需与 GATEWAY_AUTH 一致")
agent_id: str agent_id: str
current_load: int = Field(default=0, ge=0) current_load: int = Field(default=0, ge=0)
@ -66,13 +68,15 @@ class AgentInfo(BaseModel):
agent_tags: list[str] = Field(default_factory=list) agent_tags: list[str] = Field(default_factory=list)
max_concurrent: int = 1 max_concurrent: int = 1
current_load: int = 0 current_load: int = 0
priority: int = Field(default=3, ge=1, le=5, description="调度优先级,越小越先分配")
last_heartbeat: float = Field(default_factory=time.time) last_heartbeat: float = Field(default_factory=time.time)
status: AgentStatus = AgentStatus.ONLINE status: AgentStatus = AgentStatus.READY
create_time: float = Field(default_factory=time.time) create_time: float = Field(default_factory=time.time)
# ---------- 结果回传 ---------- # ---------- 结果回传 ----------
class TaskResult(BaseModel): class TaskResult(BaseModel):
auth: str = Field(..., description="网关认证密码,需与 GATEWAY_AUTH 一致")
request_id: str request_id: str
agent_id: str agent_id: str
status: TaskStatus = TaskStatus.SUCCESS status: TaskStatus = TaskStatus.SUCCESS

View File

@ -37,6 +37,7 @@ class AgentRepo:
"agent_tags": ",".join(agent.agent_tags), "agent_tags": ",".join(agent.agent_tags),
"max_concurrent": str(agent.max_concurrent), "max_concurrent": str(agent.max_concurrent),
"current_load": str(agent.current_load), "current_load": str(agent.current_load),
"priority": str(agent.priority),
"last_heartbeat": str(agent.last_heartbeat), "last_heartbeat": str(agent.last_heartbeat),
"status": agent.status.value, "status": agent.status.value,
"create_time": str(agent.create_time), "create_time": str(agent.create_time),
@ -51,23 +52,86 @@ class AgentRepo:
if not await self.redis.exists(key): if not await self.redis.exists(key):
return False return False
await self.redis.hset(key, "status", status.value) await self.redis.hset(key, "status", status.value)
if status == AgentStatus.OFFLINE: # 离线(心跳失联)与不可用均退出心跳 ZSet避免被误判/干扰
if status in (AgentStatus.OFFLINE, AgentStatus.UNAVAILABLE):
await self.redis.zrem(C.AGENT_HEARTBEAT_ZSET, agent_id) await self.redis.zrem(C.AGENT_HEARTBEAT_ZSET, agent_id)
return True return True
async def set_unavailable(self, agent_id: str) -> bool:
"""后台手动置为不可用unavailable心跳不会复活。"""
key = self._info_key(agent_id)
if not await self.redis.exists(key):
return False
await self.redis.hset(key, "status", AgentStatus.UNAVAILABLE.value)
await self.redis.zrem(C.AGENT_HEARTBEAT_ZSET, agent_id)
return True
async def set_available(self, agent_id: str) -> bool:
"""后台手动恢复可用ready清除不可用状态。"""
key = self._info_key(agent_id)
if not await self.redis.exists(key):
return False
await self.redis.hset(key, "status", AgentStatus.READY.value)
# 恢复就绪时补齐心跳时间戳,避免被误判为超时
await self.redis.zadd(C.AGENT_HEARTBEAT_ZSET, {agent_id: time.time()})
return True
async def mark_idle(self, agent_id: str) -> bool:
"""任务结束/取消后标记 Agent 空闲。
agent 已被手动置为不可用unavailable保持不可用不复活
否则置回 ready
"""
key = self._info_key(agent_id)
if not await self.redis.exists(key):
return False
cur = await self.get(agent_id)
if cur is None:
return False
if cur.status == AgentStatus.UNAVAILABLE:
return True
await self.redis.hset(key, "status", AgentStatus.READY.value)
return True
async def beat(self, agent_id: str, current_load: int) -> bool: async def beat(self, agent_id: str, current_load: int) -> bool:
"""更新心跳时间与负载。""" """更新心跳时间与负载,按状态机规则流转状态。
- 不可用unavailable保持不可用不因心跳复活
- 离线offline心跳失联恢复 ready
- ready/processing/stopping保持原状态
"""
key = self._info_key(agent_id) key = self._info_key(agent_id)
if not await self.redis.exists(key): if not await self.redis.exists(key):
return False return False
now = time.time() now = time.time()
cur = await self.get(agent_id)
if cur is None:
return False
if cur.status == AgentStatus.UNAVAILABLE:
status = AgentStatus.UNAVAILABLE
elif cur.status == AgentStatus.OFFLINE:
status = AgentStatus.READY
else:
status = cur.status
await self.redis.hset( await self.redis.hset(
key, key,
mapping={"last_heartbeat": str(now), "current_load": str(current_load), "status": AgentStatus.ONLINE.value}, mapping={
"last_heartbeat": str(now),
"current_load": str(current_load),
"status": status.value,
},
) )
await self.redis.zadd(C.AGENT_HEARTBEAT_ZSET, {agent_id: now}) await self.redis.zadd(C.AGENT_HEARTBEAT_ZSET, {agent_id: now})
return True return True
async def update_priority(self, agent_id: str, priority: int) -> bool:
"""更新 Agent 调度优先级。"""
key = self._info_key(agent_id)
if not await self.redis.exists(key):
return False
await self.redis.hset(key, "priority", str(priority))
return True
async def adjust_load(self, agent_id: str, delta: int) -> bool: async def adjust_load(self, agent_id: str, delta: int) -> bool:
"""原子调整 Agent 负载delta 可为正/负),负载下限为 0。""" """原子调整 Agent 负载delta 可为正/负),负载下限为 0。"""
key = self._info_key(agent_id) key = self._info_key(agent_id)
@ -90,13 +154,16 @@ class AgentRepo:
agent_tags=[t for t in raw.get("agent_tags", "").split(",") if t], agent_tags=[t for t in raw.get("agent_tags", "").split(",") if t],
max_concurrent=int(raw.get("max_concurrent", 1) or 1), max_concurrent=int(raw.get("max_concurrent", 1) or 1),
current_load=int(raw.get("current_load", 0) or 0), current_load=int(raw.get("current_load", 0) or 0),
priority=int(raw.get("priority", 3) or 3),
last_heartbeat=float(raw.get("last_heartbeat", 0) or 0), last_heartbeat=float(raw.get("last_heartbeat", 0) or 0),
status=AgentStatus(raw.get("status", AgentStatus.ONLINE.value)), status=AgentStatus(raw.get("status", AgentStatus.READY.value)),
create_time=float(raw.get("create_time", 0) or 0), create_time=float(raw.get("create_time", 0) or 0),
) )
async def online(self) -> list[AgentInfo]: async def online(self) -> list[AgentInfo]:
return [a for a in await self.all() if a.status == AgentStatus.ONLINE] # 可参与调度的状态ready空闲与 processing忙碌但仍有并发容量
# offline心跳失联、unavailable手动禁用与 stopping正在停止不参与调度。
return [a for a in await self.all() if a.status in (AgentStatus.READY, AgentStatus.PROCESSING)]
async def all(self) -> list[AgentInfo]: async def all(self) -> list[AgentInfo]:
ids = list(await self.redis.smembers(C.AGENT_ALL_SET)) ids = list(await self.redis.smembers(C.AGENT_ALL_SET))
@ -108,7 +175,10 @@ class AgentRepo:
return out return out
async def by_tags(self, tags: list[str]) -> list[AgentInfo]: async def by_tags(self, tags: list[str]) -> list[AgentInfo]:
"""按标签索引取 Agent 池(取所有标签的交集,无标签则返回全部在线)。""" """按标签索引取候选 Agent 池(取所有标签的交集,无标签则返回全部可调度)。
只返回可调度的 agentready/processingoffline/unavailable/stopping 被排除
"""
if not tags: if not tags:
return await self.online() return await self.online()
keys = [self._tag_key(t) for t in tags] keys = [self._tag_key(t) for t in tags]
@ -121,7 +191,7 @@ class AgentRepo:
out = [] out = []
for aid in ids: for aid in ids:
a = await self.get(aid) a = await self.get(aid)
if a and a.status == AgentStatus.ONLINE: if a and a.status in (AgentStatus.READY, AgentStatus.PROCESSING):
out.append(a) out.append(a)
return out return out

View File

@ -23,7 +23,8 @@ class AgentService:
agent_tags=reg.agent_tags, agent_tags=reg.agent_tags,
max_concurrent=reg.max_concurrent, max_concurrent=reg.max_concurrent,
current_load=reg.current_load, current_load=reg.current_load,
status=AgentStatus.ONLINE, priority=reg.priority,
status=AgentStatus.READY,
create_time=time.time(), create_time=time.time(),
last_heartbeat=time.time(), last_heartbeat=time.time(),
) )
@ -54,19 +55,46 @@ class AgentService:
async def get(self, agent_id: str) -> AgentInfo | None: async def get(self, agent_id: str) -> AgentInfo | None:
return await self.repo.get(agent_id) return await self.repo.get(agent_id)
async def take_offline(self, agent_id: str) -> bool: async def set_unavailable(self, agent_id: str) -> bool:
"""手动下线 Agent""" """后台手动置为「不可用」unavailable心跳不会复活"""
a = await self.repo.get(agent_id) a = await self.repo.get(agent_id)
if not a: if not a:
return False return False
await self.repo.update_status(agent_id, AgentStatus.OFFLINE) await self.repo.set_unavailable(agent_id)
logger.info("agent taken offline agent_id=%s", agent_id) logger.info("agent set unavailable agent_id=%s", agent_id)
return True return True
async def set_available(self, agent_id: str) -> bool:
"""后台手动恢复「可用」ready清除不可用状态。"""
a = await self.repo.get(agent_id)
if not a:
return False
await self.repo.set_available(agent_id)
logger.info("agent set available agent_id=%s", agent_id)
return True
async def set_priority(self, agent_id: str, priority: int) -> AgentInfo | None:
"""设置 Agent 调度优先级1-5越小越先分配"""
if priority < 1 or priority > 5:
raise ValueError("priority must be in [1, 5]")
ok = await self.repo.update_priority(agent_id, priority)
if not ok:
return None
logger.info("agent priority updated agent_id=%s priority=%s", agent_id, priority)
return await self.repo.get(agent_id)
async def purge_stale(self, timeout: float) -> list[str]: async def purge_stale(self, timeout: float) -> list[str]:
"""剔除超时未心跳的 Agent。""" """剔除超时未心跳的 Agent非 unavailable 的置为 offline心跳失联
手动置为不可用的 agent 不参与心跳计时已从心跳 ZSet 移除保持 unavailable
"""
stale = await self.repo.stale_agents(timeout) stale = await self.repo.stale_agents(timeout)
purged = []
for aid in stale: for aid in stale:
a = await self.repo.get(aid)
if a and a.status == AgentStatus.UNAVAILABLE:
continue
await self.repo.update_status(aid, AgentStatus.OFFLINE) await self.repo.update_status(aid, AgentStatus.OFFLINE)
logger.info("agent purged (stale) agent_id=%s", aid) logger.info("agent purged (stale) agent_id=%s", aid)
return stale purged.append(aid)
return purged

View File

@ -60,6 +60,24 @@ class TaskNotifier:
logger.info("notify task done session=%s request=%s status=%s", logger.info("notify task done session=%s request=%s status=%s",
task.cli_session_id, task.request_id, task.status.value) task.cli_session_id, task.request_id, task.status.value)
async def publish_task_stop(self, cli_session_id: str, request_id: str) -> None:
"""取消任务时向 agent 下发停止指令(复用同一 CLI 会话通道event=task_stop
允许未携带 cli_session_id 时跳过此时无法通过 SSE 通知只能等待 agent 超时
"""
if not cli_session_id:
return
payload = json.dumps(
{
"event": "task_stop",
"request_id": request_id,
},
ensure_ascii=False,
)
channel = self.channel(cli_session_id)
await self.redis.publish(channel, payload)
logger.info("notify task stop session=%s request=%s", cli_session_id, request_id)
# ---------- 消费 ---------- # ---------- 消费 ----------
async def replay(self, cli_session_id: str) -> list[str]: async def replay(self, cli_session_id: str) -> list[str]:
"""返回该会话的历史完成事件(新 → 旧),供 SSE 连接回放。""" """返回该会话的历史完成事件(新 → 旧),供 SSE 连接回放。"""

View File

@ -5,7 +5,7 @@ import httpx
import redis.asyncio as aioredis import redis.asyncio as aioredis
from app.config import settings from app.config import settings
from app.constants import TaskStatus from app.constants import AgentStatus, TaskStatus
from app.models.schemas import TaskResult from app.models.schemas import TaskResult
from app.repository.agent_repo import AgentRepo from app.repository.agent_repo import AgentRepo
from app.repository.task_repo import TaskRepo from app.repository.task_repo import TaskRepo
@ -35,11 +35,16 @@ class RelayService:
if not agent.endpoint: if not agent.endpoint:
logger.warning("relay dispatch failed: endpoint empty agent=%s request=%s", agent_id, request_id) logger.warning("relay dispatch failed: endpoint empty agent=%s request=%s", agent_id, request_id)
return False return False
task = await self.task_repo.get(request_id)
url = f"{agent.endpoint.rstrip('/')}/tasks/{request_id}" url = f"{agent.endpoint.rstrip('/')}/tasks/{request_id}"
body = { body = {
"auth": settings.gateway_auth, "auth": settings.gateway_auth,
"request_id": request_id, "request_id": request_id,
"payload": payload, "payload": payload,
"cli_session_id": getattr(task, "cli_session_id", None) if task else None,
"task_type": getattr(task, "task_type", None) if task else None,
"task_tags": getattr(task, "task_tags", None) if task else None,
"timeout": getattr(task, "timeout", None) if task else None,
} }
try: try:
async with httpx.AsyncClient(timeout=5.0) as client: async with httpx.AsyncClient(timeout=5.0) as client:
@ -57,8 +62,14 @@ class RelayService:
logger.info("relay dispatch ok agent=%s request=%s url=%s", agent_id, request_id, url) logger.info("relay dispatch ok agent=%s request=%s url=%s", agent_id, request_id, url)
return True return True
_TERMINAL = {TaskStatus.SUCCESS, TaskStatus.FAILED}
async def on_result(self, result: TaskResult) -> bool: async def on_result(self, result: TaskResult) -> bool:
"""反向Agent 回传结果,更新任务状态并记录日志。""" """反向Agent 回传结果,更新任务状态并记录日志。
终态保护任务已处于终态如已取消置 failed忽略本次任务状态覆盖
但匹配的 Agent 仍会释放负载并置回 ready
"""
task = await self.task_repo.get(result.request_id) task = await self.task_repo.get(result.request_id)
if not task: if not task:
logger.warning("result for unknown task request=%s", result.request_id) logger.warning("result for unknown task request=%s", result.request_id)
@ -67,6 +78,8 @@ class RelayService:
logger.warning("result agent mismatch request=%s expected=%s got=%s", logger.warning("result agent mismatch request=%s expected=%s got=%s",
result.request_id, task.agent_id, result.agent_id) result.request_id, task.agent_id, result.agent_id)
return False return False
already_terminal = task.status in self._TERMINAL
if not already_terminal:
await self.task_repo.update( await self.task_repo.update(
result.request_id, result.request_id,
status=result.status, status=result.status,
@ -78,9 +91,12 @@ class RelayService:
# 释放 Agent 算力(任务确实绑定在该 Agent 时才释放) # 释放 Agent 算力(任务确实绑定在该 Agent 时才释放)
if task.agent_id == result.agent_id: if task.agent_id == result.agent_id:
await self.agent_repo.adjust_load(result.agent_id, -1) await self.agent_repo.adjust_load(result.agent_id, -1)
# Agent 空闲,置回就绪(若已被手动置为不可用则保持不可用,不会复活)
if task.agent_id:
await self.agent_repo.mark_idle(task.agent_id)
await self._log(result.agent_id, result.request_id, "result", f"result received status={result.status.value}") await self._log(result.agent_id, result.request_id, "result", f"result received status={result.status.value}")
# 任务到达终态,通知对应 CLI 会话(若携带了 cli_session_id # 任务到达终态,通知对应 CLI 会话(若携带了 cli_session_id
if task.cli_session_id: if not already_terminal and task.cli_session_id:
finished = await self.task_repo.get(result.request_id) finished = await self.task_repo.get(result.request_id)
if finished: if finished:
await TaskNotifier(self.redis).publish_task_done(finished) await TaskNotifier(self.redis).publish_task_done(finished)

View File

@ -3,7 +3,7 @@ import logging
import redis.asyncio as aioredis import redis.asyncio as aioredis
from app.constants import TaskStatus from app.constants import AgentStatus, TaskStatus
from app.models.schemas import AgentInfo from app.models.schemas import AgentInfo
from app.repository.agent_repo import AgentRepo from app.repository.agent_repo import AgentRepo
from app.repository.task_repo import TaskRepo from app.repository.task_repo import TaskRepo
@ -26,14 +26,14 @@ class Scheduler:
self.relay = relay or RelayService(redis) self.relay = relay or RelayService(redis)
async def pick_candidate(self, task_tags: list[str]) -> AgentInfo | None: async def pick_candidate(self, task_tags: list[str]) -> AgentInfo | None:
"""按规则挑选最优 Agent标签匹配→负载过滤→最低负载""" """按规则挑选最优 Agent标签匹配→负载过滤→优先级最小且空闲"""
candidates = await self.agent_repo.by_tags(task_tags) candidates = await self.agent_repo.by_tags(task_tags)
# 负载过滤:剔除满载 # 负载过滤:剔除满载
candidates = [a for a in candidates if a.current_load < a.max_concurrent] candidates = [a for a in candidates if a.current_load < a.max_concurrent]
if not candidates: if not candidates:
return None return None
# 最优:负载最低、在线时间最长(稳定) # 最优:优先级最小(越先分配),同优先级取负载最低、在线时间最长(稳定)
candidates.sort(key=lambda a: (a.current_load, -a.create_time)) candidates.sort(key=lambda a: (a.priority, a.current_load, -a.create_time))
return candidates[0] return candidates[0]
async def dispatch(self, request_id: str) -> bool: async def dispatch(self, request_id: str) -> bool:
@ -52,15 +52,18 @@ class Scheduler:
progress=0, progress=0,
) )
await self.task_repo.mark_running(request_id) await self.task_repo.mark_running(request_id)
# 标记 Agent 为处理中
await self.agent_repo.update_status(agent.agent_id, AgentStatus.PROCESSING)
# 更新 Agent 负载 # 更新 Agent 负载
await self.agent_repo.adjust_load(agent.agent_id, 1) await self.agent_repo.adjust_load(agent.agent_id, 1)
# 真实推送 payload 到 Agent 端点 # 真实推送 payload 到 Agent 端点
pushed = await self.relay.dispatch_command(agent.agent_id, request_id, task.payload or {}) pushed = await self.relay.dispatch_command(agent.agent_id, request_id, task.payload or {})
if not pushed: if not pushed:
# 推送失败:回退 pending 并释放负载,等待下次调度(避免重复扣负载) # 推送失败:回退 pending 并恢复 agent 就绪、释放负载,等待下次调度(避免重复扣负载)
await self.task_repo.update(request_id, status=TaskStatus.PENDING, agent_id="", progress=0) await self.task_repo.update(request_id, status=TaskStatus.PENDING, agent_id="", progress=0)
await self.task_repo.mark_pending(request_id) await self.task_repo.mark_pending(request_id)
await self.agent_repo.adjust_load(agent.agent_id, -1) await self.agent_repo.adjust_load(agent.agent_id, -1)
await self.agent_repo.update_status(agent.agent_id, AgentStatus.READY)
logger.warning("task dispatch push failed, reverted request_id=%s agent=%s", request_id, agent.agent_id) logger.warning("task dispatch push failed, reverted request_id=%s agent=%s", request_id, agent.agent_id)
return False return False
logger.info("task dispatched request_id=%s -> agent=%s", request_id, agent.agent_id) logger.info("task dispatched request_id=%s -> agent=%s", request_id, agent.agent_id)

View File

@ -1,10 +1,23 @@
"""认证校验CLI / Agent 请求必须携带与 GATEWAY_AUTH 一致的密码。""" """认证校验CLI / Agent 请求携带 GATEWAY_AUTH后台管理请求携带 ADMIN_AUTH。"""
import secrets
from fastapi import HTTPException from fastapi import HTTPException
from app.config import settings from app.config import settings
def _constant_time_eq(a: str, b: str) -> bool:
"""常量时间字符串比较,避免时序侧信道。"""
return secrets.compare_digest((a or "").encode("utf-8"), (b or "").encode("utf-8"))
def require_auth(auth: str) -> None: def require_auth(auth: str) -> None:
"""auth 不匹配时抛出 401。""" """协议层鉴权auth 必须与 GATEWAY_AUTH 一致,否则 401。"""
if auth != settings.gateway_auth: if not _constant_time_eq(auth, settings.gateway_auth):
raise HTTPException(status_code=401, detail="invalid auth") raise HTTPException(status_code=401, detail="invalid auth")
def require_admin(auth: str) -> None:
"""后台管理鉴权auth 必须与 ADMIN_AUTH 一致,否则 401。"""
if not _constant_time_eq(auth, settings.admin_auth):
raise HTTPException(status_code=401, detail="invalid admin auth")

View File

@ -5,7 +5,7 @@ import time
import redis.asyncio as aioredis import redis.asyncio as aioredis
from app.config import settings from app.config import settings
from app.constants import TaskStatus from app.constants import AgentStatus, TaskStatus
from app.models.schemas import TaskInfo, TaskSubmit from app.models.schemas import TaskInfo, TaskSubmit
from app.repository.agent_repo import AgentRepo from app.repository.agent_repo import AgentRepo
from app.repository.task_repo import TaskRepo from app.repository.task_repo import TaskRepo
@ -53,15 +53,26 @@ class TaskService:
return await self.repo.list(status=status, limit=limit) return await self.repo.list(status=status, limit=limit)
async def cancel(self, request_id: str) -> bool: async def cancel(self, request_id: str) -> bool:
"""取消任务(仅 pending/running 可取消)。""" """取消任务(仅 pending/running 可取消)。
- running 且绑定 agent agent stopping并通过同一 cli_session_id
SSE 通道下发 task_stop 停止指令agent 停止后回传结果 on_result 终态
保护将其置回 ready
- pending无需下发直接标记失败
"""
task = await self.repo.get(request_id) task = await self.repo.get(request_id)
if not task: if not task:
return False return False
if task.status in (TaskStatus.SUCCESS, TaskStatus.FAILED): if task.status in (TaskStatus.SUCCESS, TaskStatus.FAILED):
return False return False
was_running = task.status == TaskStatus.RUNNING
await self._release_if_running(task) await self._release_if_running(task)
await self.repo.update(request_id, status=TaskStatus.FAILED, error_info="cancelled by user") await self.repo.update(request_id, status=TaskStatus.FAILED, error_info="cancelled by user")
await self.repo.mark_finished(request_id) await self.repo.mark_finished(request_id)
if was_running and task.agent_id:
# 通知 agent 停止当前任务
await self.agent_repo.update_status(task.agent_id, AgentStatus.STOPPING)
await TaskNotifier(self.redis).publish_task_stop(task.cli_session_id, request_id)
if task.cli_session_id: if task.cli_session_id:
finished = await self.repo.get(request_id) finished = await self.repo.get(request_id)
if finished: if finished:
@ -88,6 +99,8 @@ class TaskService:
if not task: if not task:
continue continue
if task.timeout and (now - task.create_time) > task.timeout: if task.timeout and (now - task.create_time) > task.timeout:
if task.status == TaskStatus.RUNNING and task.agent_id:
await self.agent_repo.mark_idle(task.agent_id)
await self._release_if_running(task) await self._release_if_running(task)
await self.repo.update(rid, status=TaskStatus.FAILED, error_info="timeout") await self.repo.update(rid, status=TaskStatus.FAILED, error_info="timeout")
await self.repo.mark_finished(rid) await self.repo.mark_finished(rid)

View File

@ -16,7 +16,20 @@ async def test_upsert_and_get(agent_repo):
got = await agent_repo.get("a1") got = await agent_repo.get("a1")
assert got is not None assert got is not None
assert got.agent_tags == ["compile", "test"] assert got.agent_tags == ["compile", "test"]
assert got.status == AgentStatus.ONLINE assert got.status == AgentStatus.READY
assert got.priority == 3 # 默认优先级
@pytest.mark.asyncio
async def test_priority_roundtrip_and_update(agent_repo):
a = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"], priority=1)
await agent_repo.upsert(a)
assert (await agent_repo.get("a1")).priority == 1
assert await agent_repo.update_priority("a1", 5) is True
assert (await agent_repo.get("a1")).priority == 5
assert await agent_repo.update_priority("missing", 2) is False
@pytest.mark.asyncio @pytest.mark.asyncio
@ -46,6 +59,99 @@ async def test_by_tags(agent_repo):
assert ids == {"a2"} assert ids == {"a2"}
@pytest.mark.asyncio
async def test_by_tags_excludes_offline_unavailable_stopping(agent_repo):
"""调度候选排除 offline/unavailable/stopping允许 ready/processingprocessing 仍有并发容量)。"""
a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"], max_concurrent=2)
a2 = AgentInfo(agent_id="a2", endpoint="e2", agent_tags=["compile"], max_concurrent=2)
a3 = AgentInfo(agent_id="a3", endpoint="e3", agent_tags=["compile"], max_concurrent=2)
a4 = AgentInfo(agent_id="a4", endpoint="e4", agent_tags=["compile"], max_concurrent=2)
a5 = AgentInfo(agent_id="a5", endpoint="e5", agent_tags=["compile"], max_concurrent=2)
await agent_repo.upsert(a1)
await agent_repo.upsert(a2)
await agent_repo.upsert(a3)
await agent_repo.upsert(a4)
await agent_repo.upsert(a5)
await agent_repo.update_status("a2", AgentStatus.OFFLINE) # 离线(心跳失联)不可调度
await agent_repo.set_unavailable("a3") # 手动不可用不可调度
await agent_repo.update_status("a4", AgentStatus.STOPPING) # 停止中不可调度
await agent_repo.update_status("a5", AgentStatus.PROCESSING) # 处理中但还有容量,可调度
ids = {a.agent_id for a in await agent_repo.by_tags(["compile"])}
assert ids == {"a1", "a5"}
@pytest.mark.asyncio
async def test_unavailable_heartbeat_not_revive(agent_repo):
"""手动置不可用unavailable即使发心跳也不复活。"""
a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"], max_concurrent=2)
await agent_repo.upsert(a1)
await agent_repo.set_unavailable("a1")
assert (await agent_repo.get("a1")).status == AgentStatus.UNAVAILABLE
# 心跳不应复活
await agent_repo.beat("a1", 0)
assert (await agent_repo.get("a1")).status == AgentStatus.UNAVAILABLE
# 恢复可用
await agent_repo.set_available("a1")
assert (await agent_repo.get("a1")).status == AgentStatus.READY
@pytest.mark.asyncio
async def test_unavailable_distinct_from_offline(agent_repo):
"""离线(心跳失联)与不可用(手动禁用)是两个独立状态。"""
a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"], max_concurrent=2)
a2 = AgentInfo(agent_id="a2", endpoint="e2", agent_tags=["compile"], max_concurrent=2)
await agent_repo.upsert(a1)
await agent_repo.upsert(a2)
# a1 心跳失联→离线a2 手动禁用→不可用
await agent_repo.update_status("a1", AgentStatus.OFFLINE)
await agent_repo.set_unavailable("a2")
assert (await agent_repo.get("a1")).status == AgentStatus.OFFLINE
assert (await agent_repo.get("a2")).status == AgentStatus.UNAVAILABLE
# 发心跳:离线的 a1 恢复 ready不可用的 a2 保持 unavailable
await agent_repo.beat("a1", 0)
await agent_repo.beat("a2", 0)
assert (await agent_repo.get("a1")).status == AgentStatus.READY
assert (await agent_repo.get("a2")).status == AgentStatus.UNAVAILABLE
@pytest.mark.asyncio
async def test_mark_idle_keeps_unavailable(agent_repo):
"""任务结束后 mark_idleunavailable 的 agent 保持不可用,其余置回 ready。"""
a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"], max_concurrent=2)
a2 = AgentInfo(agent_id="a2", endpoint="e2", agent_tags=["compile"], max_concurrent=2)
await agent_repo.upsert(a1)
await agent_repo.upsert(a2)
await agent_repo.set_unavailable("a1")
await agent_repo.update_status("a2", AgentStatus.PROCESSING)
await agent_repo.mark_idle("a1")
await agent_repo.mark_idle("a2")
assert (await agent_repo.get("a1")).status == AgentStatus.UNAVAILABLE
assert (await agent_repo.get("a2")).status == AgentStatus.READY
@pytest.mark.asyncio
async def test_heartbeat_recovers_stale_offline(agent_repo):
"""超时离线(非手动)的 agent 在发心跳后恢复为 ready。"""
a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"], max_concurrent=2)
await agent_repo.upsert(a1)
await agent_repo.update_status("a1", AgentStatus.OFFLINE)
assert (await agent_repo.get("a1")).status == AgentStatus.OFFLINE
await agent_repo.beat("a1", 0)
assert (await agent_repo.get("a1")).status == AgentStatus.READY
@pytest.mark.asyncio
async def test_heartbeat_keeps_busy_states(agent_repo):
"""processing/stopping 状态不因心跳改变。"""
for st in (AgentStatus.PROCESSING, AgentStatus.STOPPING):
a = AgentInfo(agent_id=f"a-{st.value}", endpoint="e1", agent_tags=["compile"], max_concurrent=2)
await agent_repo.upsert(a)
await agent_repo.update_status(a.agent_id, st)
await agent_repo.beat(a.agent_id, 1)
assert (await agent_repo.get(a.agent_id)).status == st
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_remove(agent_repo): async def test_remove(agent_repo):
a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"]) a1 = AgentInfo(agent_id="a1", endpoint="e1", agent_tags=["compile"])

View File

@ -8,6 +8,7 @@ from app.config import settings
from app.main import app from app.main import app
AUTH = settings.gateway_auth AUTH = settings.gateway_auth
ADMIN_AUTH = settings.admin_auth
@pytest_asyncio.fixture @pytest_asyncio.fixture
@ -36,7 +37,7 @@ async def test_submit_and_query_task(client):
assert data["status"] == "pending" assert data["status"] == "pending"
rid = data["request_id"] rid = data["request_id"]
r2 = await client.get(f"/api/cli/tasks/{rid}") r2 = await client.get(f"/api/cli/tasks/{rid}", params={"auth": AUTH})
assert r2.status_code == 200 assert r2.status_code == 200
assert r2.json()["request_id"] == rid assert r2.json()["request_id"] == rid
@ -48,7 +49,7 @@ async def test_agent_register_heartbeat_result(client):
assert r.status_code == 200 assert r.status_code == 200
# 心跳 # 心跳
r = await client.post("/api/agent/heartbeat", json={"agent_id": "a1", "current_load": 0}) r = await client.post("/api/agent/heartbeat", json={"auth": AUTH, "agent_id": "a1", "current_load": 0})
assert r.status_code == 200 assert r.status_code == 200
# 提交任务并等待调度(手动触发一次调度) # 提交任务并等待调度(手动触发一次调度)
@ -58,20 +59,44 @@ async def test_agent_register_heartbeat_result(client):
await Scheduler(app.state.redis).dispatch_pending(10) await Scheduler(app.state.redis).dispatch_pending(10)
# 回传结果 # 回传结果
r = await client.post("/api/agent/result", json={"request_id": rid, "agent_id": "a1", "status": "success", "progress": 100, "result": {"ok": True}}) r = await client.post("/api/agent/result", json={"auth": AUTH, "request_id": rid, "agent_id": "a1", "status": "success", "progress": 100, "result": {"ok": True}})
assert r.status_code == 200 assert r.status_code == 200
detail = await client.get(f"/api/cli/tasks/{rid}") detail = await client.get(f"/api/cli/tasks/{rid}", params={"auth": AUTH})
assert detail.json()["status"] == "success" assert detail.json()["status"] == "success"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_admin_overview(client): async def test_admin_overview(client):
r = await client.get("/api/admin/overview") r = await client.get("/api/admin/overview", headers={"X-Admin-Auth": ADMIN_AUTH})
assert r.status_code == 200 assert r.status_code == 200
assert "task_total" in r.json() assert "task_total" in r.json()
@pytest.mark.asyncio
async def test_admin_rejects_bad_auth(client):
r = await client.get("/api/admin/overview", headers={"X-Admin-Auth": "wrong-admin"})
assert r.status_code == 401
@pytest.mark.asyncio
async def test_admin_requires_auth(client):
r = await client.get("/api/admin/overview")
assert r.status_code == 422 # 缺 X-Admin-Auth header
@pytest.mark.asyncio
async def test_agent_heartbeat_rejects_bad_auth(client):
r = await client.post("/api/agent/heartbeat", json={"auth": "wrong", "agent_id": "x1", "current_load": 0})
assert r.status_code == 401
@pytest.mark.asyncio
async def test_cli_query_rejects_bad_auth(client):
r = await client.get("/api/cli/tasks/whatever", params={"auth": "wrong"})
assert r.status_code == 401
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_submit_task_rejects_bad_auth(client): async def test_submit_task_rejects_bad_auth(client):
r = await client.post("/api/cli/tasks", json={"auth": "wrong-password", "task_type": "compile"}) r = await client.post("/api/cli/tasks", json={"auth": "wrong-password", "task_type": "compile"})

View File

@ -1,13 +1,27 @@
"""调度服务单元测试。""" """调度服务单元测试。"""
import pytest import pytest
from app.constants import TaskStatus from app.constants import AgentStatus, TaskStatus
from app.models.schemas import AgentInfo, TaskInfo from app.models.schemas import AgentInfo, TaskInfo
from app.repository.agent_repo import AgentRepo from app.repository.agent_repo import AgentRepo
from app.repository.task_repo import TaskRepo from app.repository.task_repo import TaskRepo
from app.services.scheduler import Scheduler from app.services.scheduler import Scheduler
class _StubRelay:
"""模拟 relay不发起真实网络请求直接返回推送成功。"""
def __init__(self, ok: bool = True):
self.ok = ok
async def dispatch_command(self, agent_id: str, request_id: str, payload: dict) -> bool:
return self.ok
def _sched(redis, task_repo, agent_repo):
return Scheduler(redis, task_repo, agent_repo, relay=_StubRelay())
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_dispatch_binds_min_load_agent(redis): async def test_dispatch_binds_min_load_agent(redis):
task_repo = TaskRepo(redis) task_repo = TaskRepo(redis)
@ -19,7 +33,7 @@ async def test_dispatch_binds_min_load_agent(redis):
await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"])) await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"]))
sched = Scheduler(redis, task_repo, agent_repo) sched = _sched(redis, task_repo, agent_repo)
ok = await sched.dispatch("t1") ok = await sched.dispatch("t1")
assert ok is True assert ok is True
@ -29,6 +43,35 @@ async def test_dispatch_binds_min_load_agent(redis):
assert "t1" in await task_repo.running_tasks() assert "t1" in await task_repo.running_tasks()
@pytest.mark.asyncio
async def test_pick_candidate_lowest_priority(redis):
agent_repo = AgentRepo(redis)
# 三个 compile Agentb1 负载最低但优先级高(最不被优先)b2 负载高但优先级低(最优先)
await agent_repo.upsert(AgentInfo(agent_id="b1", endpoint="e1", agent_tags=["compile"], max_concurrent=2, current_load=0, priority=5))
await agent_repo.upsert(AgentInfo(agent_id="b2", endpoint="e2", agent_tags=["compile"], max_concurrent=2, current_load=1, priority=1))
await agent_repo.upsert(AgentInfo(agent_id="b3", endpoint="e3", agent_tags=["compile"], max_concurrent=2, current_load=0, priority=3))
sched = Scheduler(redis)
picked = await sched.pick_candidate(["compile"])
assert picked is not None
assert picked.agent_id == "b2" # 优先级最小优先分配
@pytest.mark.asyncio
async def test_pick_candidate_same_priority_min_load(redis):
agent_repo = AgentRepo(redis)
# 同优先级(默认3)时,负载更低者优先
await agent_repo.upsert(AgentInfo(agent_id="b1", endpoint="e1", agent_tags=["compile"], max_concurrent=2, current_load=1))
await agent_repo.upsert(AgentInfo(agent_id="b2", endpoint="e2", agent_tags=["compile"], max_concurrent=2, current_load=0))
sched = Scheduler(redis)
picked = await sched.pick_candidate(["compile"])
assert picked is not None
assert picked.agent_id == "b2"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_dispatch_no_agent_stays_pending(redis): async def test_dispatch_no_agent_stays_pending(redis):
task_repo = TaskRepo(redis) task_repo = TaskRepo(redis)
@ -62,6 +105,70 @@ async def test_dispatch_pending(redis):
await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"])) await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"]))
await task_repo.create(TaskInfo(request_id="t2", task_type="compile", task_tags=["compile"])) await task_repo.create(TaskInfo(request_id="t2", task_type="compile", task_tags=["compile"]))
sched = Scheduler(redis, task_repo, agent_repo) sched = _sched(redis, task_repo, agent_repo)
n = await sched.dispatch_pending(10) n = await sched.dispatch_pending(10)
assert n == 2 assert n == 2
@pytest.mark.asyncio
async def test_dispatch_excludes_offline_unavailable_stopping(redis):
"""调度排除 offline心跳失联、unavailable手动不可用与 stopping 的 agent。"""
task_repo = TaskRepo(redis)
agent_repo = AgentRepo(redis)
# b1 手动不可用b2 正在停止b3 离线b4 就绪
await agent_repo.upsert(AgentInfo(agent_id="b1", endpoint="e1", agent_tags=["compile"], max_concurrent=2))
await agent_repo.upsert(AgentInfo(agent_id="b2", endpoint="e2", agent_tags=["compile"], max_concurrent=2))
await agent_repo.upsert(AgentInfo(agent_id="b3", endpoint="e3", agent_tags=["compile"], max_concurrent=2))
await agent_repo.upsert(AgentInfo(agent_id="b4", endpoint="e4", agent_tags=["compile"], max_concurrent=2))
await agent_repo.set_unavailable("b1")
await agent_repo.update_status("b2", AgentStatus.STOPPING)
await agent_repo.update_status("b3", AgentStatus.OFFLINE)
await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"]))
sched = _sched(redis, task_repo, agent_repo)
ok = await sched.dispatch("t1")
assert ok is True
task = await task_repo.get("t1")
assert task.agent_id == "b4"
@pytest.mark.asyncio
async def test_dispatch_processing_agent_with_capacity(redis):
"""processing 的 agent 若还有并发容量,仍可接收新任务。"""
task_repo = TaskRepo(redis)
agent_repo = AgentRepo(redis)
await agent_repo.upsert(AgentInfo(agent_id="b1", endpoint="e1", agent_tags=["compile"], max_concurrent=2))
await agent_repo.update_status("b1", AgentStatus.PROCESSING)
await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"]))
sched = _sched(redis, task_repo, agent_repo)
assert await sched.dispatch("t1") is True
assert (await task_repo.get("t1")).agent_id == "b1"
@pytest.mark.asyncio
async def test_dispatch_marks_agent_processing(redis):
"""调度下发成功后agent 状态流转为 processing。"""
task_repo = TaskRepo(redis)
agent_repo = AgentRepo(redis)
await agent_repo.upsert(AgentInfo(agent_id="b1", endpoint="e1", agent_tags=["compile"], max_concurrent=2))
await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"]))
sched = _sched(redis, task_repo, agent_repo)
assert await sched.dispatch("t1") is True
assert (await agent_repo.get("b1")).status == AgentStatus.PROCESSING
@pytest.mark.asyncio
async def test_dispatch_push_fail_reverts_agent_to_ready(redis):
"""推送失败时任务回退 pendingagent 释放负载并回到 ready。"""
task_repo = TaskRepo(redis)
agent_repo = AgentRepo(redis)
await agent_repo.upsert(AgentInfo(agent_id="b1", endpoint="e1", agent_tags=["compile"], max_concurrent=2))
await task_repo.create(TaskInfo(request_id="t1", task_type="compile", task_tags=["compile"]))
sched = Scheduler(redis, task_repo, agent_repo, relay=_StubRelay(ok=False))
assert await sched.dispatch("t1") is False
task = await task_repo.get("t1")
assert task.status == TaskStatus.PENDING
assert not task.agent_id
a = await agent_repo.get("b1")
assert a.status == AgentStatus.READY
assert a.current_load == 0

View File

@ -1,14 +1,43 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onMounted, onUnmounted, ref } from 'vue' import { computed, onMounted, onUnmounted, ref } from 'vue'
import { useRoute } from 'vue-router' import { useRoute } from 'vue-router'
import { api } from './api' import { api, getAdminAuth, setAdminAuth } from './api'
import type { Overview } from './api' import type { Overview } from './api'
const route = useRoute() const route = useRoute()
const overview = ref<Overview | null>(null) const overview = ref<Overview | null>(null)
const isCollapse = ref(false) const isCollapse = ref(false)
const authed = ref(!!getAdminAuth())
const loginInput = ref('')
const loginError = ref('')
const verifying = ref(false)
let timer: number | undefined let timer: number | undefined
function onAuthExpired() {
authed.value = false
loginInput.value = ''
}
async function doLogin() {
const auth = loginInput.value.trim()
if (!auth) {
loginError.value = '请输入管理密钥'
return
}
verifying.value = true
loginError.value = ''
try {
setAdminAuth(auth)
await api.overview()
authed.value = true
} catch {
setAdminAuth('')
loginError.value = '密钥校验失败,请重试'
} finally {
verifying.value = false
}
}
const title = computed(() => (route.meta.title as string) || '') const title = computed(() => (route.meta.title as string) || '')
const menus = [ const menus = [
@ -27,17 +56,51 @@ async function refresh() {
} }
onMounted(() => { onMounted(() => {
refresh() window.addEventListener('gw-auth-expired', onAuthExpired)
if (authed.value) refresh()
timer = window.setInterval(refresh, 5000) timer = window.setInterval(refresh, 5000)
}) })
onUnmounted(() => { onUnmounted(() => {
window.removeEventListener('gw-auth-expired', onAuthExpired)
if (timer) window.clearInterval(timer) if (timer) window.clearInterval(timer)
}) })
</script> </script>
<template> <template>
<div class="flex h-full"> <div v-if="!authed" class="flex items-center justify-center h-full bg-slate-100">
<div class="w-96 bg-white rounded-2xl shadow-lg p-8">
<div class="flex flex-col items-center mb-6">
<div
class="w-14 h-14 rounded-xl bg-gradient-to-br from-[#1e63e8] to-[#3b82f6] flex items-center justify-center font-bold text-white text-2xl mb-3"
>
A
</div>
<div class="text-lg font-semibold text-slate-800">A2A 智能网关</div>
<div class="text-xs text-slate-400 mt-1">运维控制台 · 请输入管理密钥</div>
</div>
<el-input
v-model="loginInput"
type="password"
placeholder="ADMIN_AUTH 管理密钥"
show-password
size="large"
@keyup.enter="doLogin"
/>
<p v-if="loginError" class="text-xs text-red-500 mt-2">{{ loginError }}</p>
<el-button
type="primary"
size="large"
class="w-full mt-4"
:loading="verifying"
@click="doLogin"
>
进入控制台
</el-button>
</div>
</div>
<div v-else class="flex h-full">
<!-- 可折叠侧边菜单 --> <!-- 可折叠侧边菜单 -->
<aside <aside
class="flex flex-col shrink-0 h-full bg-[#0f172a] transition-all duration-200 overflow-hidden" class="flex flex-col shrink-0 h-full bg-[#0f172a] transition-all duration-200 overflow-hidden"
@ -99,8 +162,8 @@ onUnmounted(() => {
<div class="flex items-center gap-5 text-sm shrink-0"> <div class="flex items-center gap-5 text-sm shrink-0">
<div class="flex items-center gap-1.5"> <div class="flex items-center gap-1.5">
<span class="badge-dot bg-[#10b981] pulse"></span> <span class="badge-dot bg-[#10b981] pulse"></span>
<span class="text-slate-600">在线 Agent</span> <span class="text-slate-600">就绪 Agent</span>
<span class="font-bold text-[#10b981]">{{ overview?.agent_online ?? '-' }}</span> <span class="font-bold text-[#10b981]">{{ overview?.agent_ready ?? '-' }}</span>
</div> </div>
<div class="flex items-center gap-1.5"> <div class="flex items-center gap-1.5">
<span class="badge-dot bg-[#1e63e8]"></span> <span class="badge-dot bg-[#1e63e8]"></span>

View File

@ -1,11 +1,31 @@
const BASE = '/api/admin' const BASE = '/api/admin'
const ADMIN_AUTH_KEY = 'gw_admin_auth'
export function getAdminAuth(): string {
return localStorage.getItem(ADMIN_AUTH_KEY) || ''
}
export function setAdminAuth(auth: string): void {
localStorage.setItem(ADMIN_AUTH_KEY, auth)
}
export function clearAdminAuth(): void {
localStorage.removeItem(ADMIN_AUTH_KEY)
}
async function request<T>(path: string, options: RequestInit = {}): Promise<T> { async function request<T>(path: string, options: RequestInit = {}): Promise<T> {
const res = await fetch(`${BASE}${path}`, { const res = await fetch(`${BASE}${path}`, {
headers: { 'Content-Type': 'application/json' }, headers: {
'Content-Type': 'application/json',
'X-Admin-Auth': getAdminAuth(),
},
...options, ...options,
}) })
if (!res.ok) { if (!res.ok) {
if (res.status === 401) {
clearAdminAuth()
window.dispatchEvent(new CustomEvent('gw-auth-expired'))
}
const text = await res.text() const text = await res.text()
throw new Error(text || `HTTP ${res.status}`) throw new Error(text || `HTTP ${res.status}`)
} }
@ -31,8 +51,9 @@ export interface AgentInfo {
agent_tags: string[] agent_tags: string[]
max_concurrent: number max_concurrent: number
current_load: number current_load: number
priority: number
last_heartbeat: number last_heartbeat: number
status: 'online' | 'offline' status: 'offline' | 'unavailable' | 'ready' | 'processing' | 'stopping'
create_time: number create_time: number
} }
@ -53,8 +74,11 @@ export interface Overview {
task_success: number task_success: number
task_failed: number task_failed: number
agent_total: number agent_total: number
agent_online: number agent_ready: number
agent_processing: number
agent_stopping: number
agent_offline: number agent_offline: number
agent_unavailable: number
} }
export const api = { export const api = {
@ -66,7 +90,13 @@ export const api = {
cancelTask: (id: string) => request<{ ok: boolean }>(`/tasks/${id}/cancel`, { method: 'POST' }), cancelTask: (id: string) => request<{ ok: boolean }>(`/tasks/${id}/cancel`, { method: 'POST' }),
resetTask: (id: string) => request<{ ok: boolean }>(`/tasks/${id}/reset`, { method: 'POST' }), resetTask: (id: string) => request<{ ok: boolean }>(`/tasks/${id}/reset`, { method: 'POST' }),
listAgents: () => request<AgentInfo[]>('/agents'), listAgents: () => request<AgentInfo[]>('/agents'),
offlineAgent: (id: string) => request<{ ok: boolean }>(`/agents/${id}/offline`, { method: 'POST' }), unavailableAgent: (id: string) => request<{ ok: boolean }>(`/agents/${id}/unavailable`, { method: 'POST' }),
availableAgent: (id: string) => request<{ ok: boolean }>(`/agents/${id}/available`, { method: 'POST' }),
setAgentPriority: (id: string, priority: number) =>
request<AgentInfo>(`/agents/${id}/priority`, {
method: 'POST',
body: JSON.stringify(priority),
}),
listLogs: (params?: { request_id?: string; agent_id?: string }) => { listLogs: (params?: { request_id?: string; agent_id?: string }) => {
const q = new URLSearchParams() const q = new URLSearchParams()
if (params?.request_id) q.set('request_id', params.request_id) if (params?.request_id) q.set('request_id', params.request_id)

View File

@ -6,6 +6,18 @@ const agents = ref<AgentInfo[]>([])
const loading = ref(false) const loading = ref(false)
let timer: number | undefined let timer: number | undefined
const STATUS_META: Record<string, { label: string; tag: 'success' | 'warning' | 'info' | 'danger'; color: string; dot: string }> = {
ready: { label: '就绪', tag: 'success', color: '#10b981', dot: 'bg-[#10b981] pulse' },
processing: { label: '处理中', tag: 'warning', color: '#f59e0b', dot: 'bg-[#f59e0b]' },
stopping: { label: '正在停止', tag: 'info', color: '#3b82f6', dot: 'bg-[#3b82f6]' },
offline: { label: '离线', tag: 'danger', color: '#ef4444', dot: 'bg-[#ef4444]' },
unavailable: { label: '不可用', tag: 'danger', color: '#9ca3af', dot: 'bg-[#9ca3af]' },
}
function meta(a: AgentInfo) {
return STATUS_META[a.status] || STATUS_META.offline
}
async function load() { async function load() {
loading.value = true loading.value = true
try { try {
@ -26,6 +38,34 @@ function loadPct(a: AgentInfo) {
return a.max_concurrent ? Math.round((a.current_load / a.max_concurrent) * 100) : 0 return a.max_concurrent ? Math.round((a.current_load / a.max_concurrent) * 100) : 0
} }
async function setPriority(a: AgentInfo, priority: number) {
try {
await api.setAgentPriority(a.agent_id, priority)
a.priority = priority
} catch (e) {
console.error(e)
load()
}
}
async function setUnavailable(a: AgentInfo) {
try {
await api.unavailableAgent(a.agent_id)
await load()
} catch (e) {
console.error(e)
}
}
async function setAvailable(a: AgentInfo) {
try {
await api.availableAgent(a.agent_id)
await load()
} catch (e) {
console.error(e)
}
}
onMounted(() => { onMounted(() => {
load() load()
timer = window.setInterval(load, 5000) timer = window.setInterval(load, 5000)
@ -39,7 +79,13 @@ onUnmounted(() => {
<div class="space-y-5 fade-in"> <div class="space-y-5 fade-in">
<div class="flex items-center justify-between"> <div class="flex items-center justify-between">
<h2 class="font-semibold">Agent 池状态</h2> <h2 class="font-semibold">Agent 池状态</h2>
<span class="text-sm text-slate-500">在线 {{ agents.filter((a) => a.status === 'online').length }} / {{ agents.length }}</span> <span class="text-sm text-slate-500">
就绪 {{ agents.filter((a) => a.status === 'ready').length }} /
处理中 {{ agents.filter((a) => a.status === 'processing').length }} /
停止 {{ agents.filter((a) => a.status === 'stopping').length }} /
不可用 {{ agents.filter((a) => a.status === 'unavailable').length }} /
离线 {{ agents.filter((a) => a.status === 'offline').length }} / {{ agents.length }}
</span>
</div> </div>
<div v-if="loading && agents.length === 0" class="page-card text-center text-slate-400 py-16">加载中...</div> <div v-if="loading && agents.length === 0" class="page-card text-center text-slate-400 py-16">加载中...</div>
@ -48,19 +94,14 @@ onUnmounted(() => {
</div> </div>
<div class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4"> <div class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4">
<div <div v-for="a in agents" :key="a.agent_id" class="stat-card" :class="{ 'opacity-70': a.status === 'offline' || a.status === 'unavailable' }">
v-for="a in agents"
:key="a.agent_id"
class="stat-card"
:class="{ 'ring-2 ring-[#ef4444]': a.status === 'offline' }"
>
<div class="flex items-start justify-between"> <div class="flex items-start justify-between">
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
<el-icon :size="18" :color="a.status === 'online' ? '#10b981' : '#ef4444'"><Cpu /></el-icon> <el-icon :size="18" :color="meta(a).color"><Cpu /></el-icon>
<span class="font-mono font-semibold">{{ a.agent_id }}</span> <span class="font-mono font-semibold">{{ a.agent_id }}</span>
</div> </div>
<el-tag :type="a.status === 'online' ? 'success' : 'danger'" effect="light" size="small"> <el-tag :type="meta(a).tag" effect="light" size="small">
{{ a.status === 'online' ? '在线' : '离线' }} {{ meta(a).label }}
</el-tag> </el-tag>
</div> </div>
@ -70,6 +111,19 @@ onUnmounted(() => {
<el-tag v-for="t in a.agent_tags" :key="t" size="small" type="info" effect="plain">{{ t }}</el-tag> <el-tag v-for="t in a.agent_tags" :key="t" size="small" type="info" effect="plain">{{ t }}</el-tag>
</div> </div>
<div class="mt-3 flex items-center justify-between">
<span class="text-xs text-slate-500">调度优先级</span>
<el-select
:model-value="a.priority"
size="small"
style="width: 90px"
@update:model-value="(v: number) => setPriority(a, v)"
>
<el-option v-for="p in [1, 2, 3, 4, 5]" :key="p" :label="`P${p}`" :value="p" />
</el-select>
</div>
<div class="mt-1 text-[11px] text-slate-400">数值越小越优先分配任务</div>
<div class="mt-4"> <div class="mt-4">
<div class="flex justify-between text-xs text-slate-500 mb-1"> <div class="flex justify-between text-xs text-slate-500 mb-1">
<span>负载 {{ a.current_load }}/{{ a.max_concurrent }}</span> <span>负载 {{ a.current_load }}/{{ a.max_concurrent }}</span>
@ -83,9 +137,30 @@ onUnmounted(() => {
</div> </div>
<div class="mt-3 text-xs text-slate-400 flex items-center gap-1"> <div class="mt-3 text-xs text-slate-400 flex items-center gap-1">
<span class="badge-dot" :class="a.status === 'online' ? 'bg-[#10b981] pulse' : 'bg-[#ef4444]'"></span> <span class="badge-dot" :class="meta(a).dot"></span>
心跳 {{ fmtTime(a.last_heartbeat) }} · 注册 {{ fmtTime(a.create_time) }} 心跳 {{ fmtTime(a.last_heartbeat) }} · 注册 {{ fmtTime(a.create_time) }}
</div> </div>
<div class="mt-3 flex items-center gap-2">
<el-button
v-if="a.status === 'ready' || a.status === 'processing' || a.status === 'stopping'"
size="small"
type="danger"
plain
@click="setUnavailable(a)"
>
置为不可用
</el-button>
<el-button
v-else
size="small"
type="success"
plain
@click="setAvailable(a)"
>
置为可用
</el-button>
</div>
</div> </div>
</div> </div>
</div> </div>

View File

@ -42,17 +42,36 @@ async function resetTask(id: string) {
} }
} }
async function offlineAgent(id: string) { async function unavailableAgent(id: string) {
try { try {
await ElMessageBox.confirm(`确认下线 Agent ${id} `, '下线 Agent', { type: 'warning' }) await ElMessageBox.confirm(`确认将 Agent ${id} 置为不可用?`, '置为不可用', { type: 'warning' })
await api.offlineAgent(id) await api.unavailableAgent(id)
ElMessage.success('Agent 已下线') ElMessage.success('Agent 已置为不可用')
load() load()
} catch (e) { } catch (e) {
if (e !== 'cancel') console.error(e) if (e !== 'cancel') console.error(e)
} }
} }
async function availableAgent(id: string) {
try {
await ElMessageBox.confirm(`确认将 Agent ${id} 置为可用?`, '置为可用', { type: 'warning' })
await api.availableAgent(id)
ElMessage.success('Agent 已置为可用')
load()
} catch (e) {
if (e !== 'cancel') console.error(e)
}
}
function statusLabel(s: string) {
return ({ ready: '就绪', processing: '处理中', stopping: '正在停止', offline: '离线', unavailable: '不可用' } as Record<string, string>)[s] || s
}
function statusType(s: string): 'success' | 'warning' | 'info' | 'danger' {
return ({ ready: 'success', processing: 'warning', stopping: 'info', offline: 'danger', unavailable: 'danger' } as Record<string, 'success' | 'warning' | 'info' | 'danger'>)[s] || 'info'
}
onMounted(load) onMounted(load)
</script> </script>
@ -85,26 +104,34 @@ onMounted(load)
</div> </div>
<div class="page-card"> <div class="page-card">
<h2 class="font-semibold mb-4">下线 Agent</h2> <h2 class="font-semibold mb-4">Agent 状态管理</h2>
<el-table :data="agents" v-loading="loading" stripe> <el-table :data="agents" v-loading="loading" stripe>
<el-table-column label="AgentID" prop="agent_id" min-width="180" /> <el-table-column label="AgentID" prop="agent_id" min-width="180" />
<el-table-column label="地址" prop="endpoint" min-width="160" show-overflow-tooltip /> <el-table-column label="地址" prop="endpoint" min-width="160" show-overflow-tooltip />
<el-table-column label="状态" width="100"> <el-table-column label="状态" width="120">
<template #default="{ row }"> <template #default="{ row }">
<el-tag :type="row.status === 'online' ? 'success' : 'danger'" effect="light"> <el-tag :type="statusType(row.status)" effect="light">
{{ row.status === 'online' ? '在线' : '离线' }} {{ statusLabel(row.status) }}
</el-tag> </el-tag>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="操作" width="160"> <el-table-column label="操作" width="130">
<template #default="{ row }"> <template #default="{ row }">
<el-button <el-button
v-if="row.status === 'ready' || row.status === 'processing' || row.status === 'stopping'"
link link
type="danger" type="danger"
:disabled="row.status !== 'online'" @click="unavailableAgent(row.agent_id)"
@click="offlineAgent(row.agent_id)"
> >
下线 置为不可用
</el-button>
<el-button
v-else
link
type="success"
@click="availableAgent(row.agent_id)"
>
置为可用
</el-button> </el-button>
</template> </template>
</el-table-column> </el-table-column>