pre release updates

This commit is contained in:
seniorswe
2025-09-19 22:31:58 -04:00
committed by seniorswe
parent 58bebcb785
commit 4b75e4811b
30 changed files with 1212 additions and 36 deletions

View File

@@ -192,6 +192,29 @@ Quick go-live checklist
Optional: run `bash scripts/smoke.sh` (uses `BASE_URL`, `STARTUP_ADMIN_EMAIL`, `STARTUP_ADMIN_PASSWORD`).
## Demo Data Seeder
- Populate the platform with realistic random data (users, APIs, endpoints, roles, groups, tokens, subscriptions, logs, protos) for UI exploration.
Run from repo root:
```
python backend-services/scripts/seed_demo_data.py --users 40 --apis 15 --endpoints 6 --protos 6 --logs 1500
```
Flags:
- `--users` count (default 30)
- `--apis` count (default 12)
- `--endpoints` per-API (default 5)
- `--groups` extra groups (default 6)
- `--protos` proto files (default 5)
- `--logs` log lines to append (default 1000)
- `--seed` RNG seed for reproducibility
Notes:
- Works with memory-only or MongoDB modes. Metrics (for the dashboard) are in-memory and will reset on server restart; re-run the seeder to repopulate.
- Admin user is preserved; additional roles/groups are added if missing.
- Proto files are written under `backend-services/proto/` (no external tooling required to view in UI).
## License Information
The contents of this repository are property of doorman.so.

View File

@@ -0,0 +1,6 @@
# Netscape HTTP Cookie File
# https://curl.se/docs/http-cookies.html
# This file was generated by libcurl! Edit at your own risk.
#HttpOnly_.localhost TRUE / FALSE 1758337273 access_token_cookie eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsInJvbGUiOiJhZG1pbiIsImV4cCI6MTc1ODMzNzI3MywianRpIjoiNGY1NGU0MDUtN2Q4Mi00NDE5LTg1NmQtNDUzYWUyMmNhMjNhIiwiYWNjZXNzZXMiOnsidWlfYWNjZXNzIjp0cnVlLCJtYW5hZ2VfdXNlcnMiOnRydWUsIm1hbmFnZV9hcGlzIjp0cnVlLCJtYW5hZ2VfZW5kcG9pbnRzIjp0cnVlLCJtYW5hZ2VfZ3JvdXBzIjp0cnVlLCJtYW5hZ2Vfcm9sZXMiOnRydWUsIm1hbmFnZV9yb3V0aW5ncyI6dHJ1ZSwibWFuYWdlX2dhdGV3YXkiOnRydWUsIm1hbmFnZV9zdWJzY3JpcHRpb25zIjp0cnVlLCJtYW5hZ2Vfc2VjdXJpdHkiOnRydWUsImV4cG9ydF9sb2dzIjp0cnVlLCJ2aWV3X2xvZ3MiOnRydWV9fQ.oQSyvnu4raM2X0C18H6Evr-_UgZ8rGbAHrtQNGxJTcI
.localhost TRUE / FALSE 1758337273 csrf_token 17996394-05ab-4cb2-a86b-ecc3b72f071d

View File

@@ -38,6 +38,7 @@ from routes.dashboard_routes import dashboard_router
from routes.memory_routes import memory_router
from routes.security_routes import security_router
from routes.token_routes import token_router
from routes.demo_routes import demo_router
from routes.monitor_routes import monitor_router
from utils.security_settings_util import load_settings, start_auto_save_task, stop_auto_save_task, get_cached_settings
from utils.memory_dump_util import dump_memory_to_file, restore_memory_from_file, find_latest_dump_path
@@ -423,6 +424,7 @@ doorman.include_router(memory_router, prefix="/platform", tags=["Memory"])
doorman.include_router(security_router, prefix="/platform", tags=["Security"])
doorman.include_router(monitor_router, prefix="/platform", tags=["Monitor"])
doorman.include_router(token_router, prefix="/platform/token", tags=["Token"])
doorman.include_router(demo_router, prefix="/platform/demo", tags=["Demo"])
def start():
if os.path.exists(PID_FILE):

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package alerts_qlrsje_v3;
service AlertsQlrsjeService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package alerts_utz_v1;
service AlertsUtzService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package alerts_xupo_v1;
service AlertsXupoService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package crypto_xhyhse_v1;
service CryptoXhyhseService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package customers_jdqwhv_v2;
service CustomersJdqwhvService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package inventory_pscsl_v1;
service InventoryPscslService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package inventory_qhq_v3;
service InventoryQhqService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package inventory_srxncz_v3;
service InventorySrxnczService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package inventory_zwrd_v2;
service InventoryZwrdService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package metrics_cfhe_v1;
service MetricsCfheService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package metrics_jtfb_v2;
service MetricsJtfbService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package news_pvu_v1;
service NewsPvuService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package orders_owypz_v2;
service OrdersOwypzService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package payments_fgabj_v3;
service PaymentsFgabjService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package payments_vxzhs_v2;
service PaymentsVxzhsService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package recommendations_adury_v2;
service RecommendationsAduryService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package weather_itjwo_v3;
service WeatherItjwoService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,16 @@
syntax = "proto3";
package weather_lgf_v3;
service WeatherLgfService {
rpc GetStatus (StatusRequest) returns (StatusReply) {}
}
message StatusRequest {
string id = 1;
}
message StatusReply {
string status = 1;
string message = 2;
}

View File

@@ -0,0 +1,53 @@
"""
Protected demo seeding routes for populating the running server with dummy data.
Only available to users with 'manage_gateway' OR 'manage_tokens'.
"""
from fastapi import APIRouter, Request
from models.response_model import ResponseModel
from utils.response_util import respond_rest
from utils.role_util import platform_role_required_bool
from utils.auth_util import auth_required
from utils.demo_seed_util import run_seed
import uuid
import time
import logging
demo_router = APIRouter()
logger = logging.getLogger("doorman.gateway")
@demo_router.post("/seed",
description="Seed the running server with demo data",
response_model=ResponseModel
)
async def demo_seed(request: Request,
users: int = 40,
apis: int = 15,
endpoints: int = 6,
groups: int = 8,
protos: int = 6,
logs: int = 1500,
seed: int | None = None):
request_id = str(uuid.uuid4())
start_time = time.time() * 1000
try:
payload = await auth_required(request)
username = payload.get("sub")
logger.info(f"{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}")
logger.info(f"{request_id} | Endpoint: {request.method} {str(request.url.path)}")
if not (await platform_role_required_bool(username, 'manage_gateway') or await platform_role_required_bool(username, 'manage_tokens')):
return respond_rest(ResponseModel(
status_code=403,
error_code='DEMO001',
error_message='Permission denied to run seeder'
))
res = run_seed(users=users, apis=apis, endpoints=endpoints, groups=groups, protos=protos, logs=logs, seed=seed)
return respond_rest(ResponseModel(status_code=200, response=res, message='Seed completed'))
except Exception as e:
logger.error(f"{request_id} | Demo seed error: {str(e)}", exc_info=True)
return respond_rest(ResponseModel(status_code=500, error_code='DEMO999', error_message='Failed to seed demo data'))
finally:
end_time = time.time() * 1000
logger.info(f"{request_id} | Total time: {str(end_time - start_time)}ms")

View File

@@ -24,6 +24,71 @@ token_router = APIRouter()
logger = logging.getLogger("doorman.gateway")
@token_router.get("/defs",
description="List token definitions",
response_model=ResponseModel,
responses={
200: {"description": "Successful Response"}
}
)
async def list_token_definitions(request: Request, page: int = 1, page_size: int = 50):
request_id = str(uuid.uuid4())
start_time = time.time() * 1000
try:
payload = await auth_required(request)
username = payload.get("sub")
logger.info(f"{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}")
logger.info(f"{request_id} | Endpoint: {request.method} {str(request.url.path)}")
if not await platform_role_required_bool(username, 'manage_tokens'):
return respond_rest(ResponseModel(
status_code=403,
error_code='TKN002',
error_message='Unable to retrieve tokens'
))
return respond_rest(await TokenService.list_token_defs(page, page_size, request_id))
except Exception as e:
logger.critical(f"{request_id} | Unexpected error: {str(e)}", exc_info=True)
return process_response(ResponseModel(
status_code=500,
response_headers={"request_id": request_id},
error_code="GTW999",
error_message="An unexpected error occurred"
).dict(), "rest")
finally:
end_time = time.time() * 1000
logger.info(f"{request_id} | Total time: {str(end_time - start_time)}ms")
@token_router.get("/defs/{api_token_group}",
description="Get a token definition",
response_model=ResponseModel,
)
async def get_token_definition(api_token_group: str, request: Request):
request_id = str(uuid.uuid4())
start_time = time.time() * 1000
try:
payload = await auth_required(request)
username = payload.get("sub")
logger.info(f"{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}")
logger.info(f"{request_id} | Endpoint: {request.method} {str(request.url.path)}")
if not await platform_role_required_bool(username, 'manage_tokens'):
return respond_rest(ResponseModel(
status_code=403,
error_code='TKN002',
error_message='Unable to retrieve tokens'
))
return respond_rest(await TokenService.get_token_def(api_token_group, request_id))
except Exception as e:
logger.critical(f"{request_id} | Unexpected error: {str(e)}", exc_info=True)
return process_response(ResponseModel(
status_code=500,
response_headers={"request_id": request_id},
error_code="GTW999",
error_message="An unexpected error occurred"
).dict(), "rest")
finally:
end_time = time.time() * 1000
logger.info(f"{request_id} | Total time: {str(end_time - start_time)}ms")
@token_router.post("",
description="Create a token",
response_model=ResponseModel,

View File

@@ -192,6 +192,68 @@ class TokenService:
error_message='Database error occurred while deleting token'
).dict()
@staticmethod
async def list_token_defs(page: int, page_size: int, request_id):
"""List token definitions (masked), paginated."""
logger.info(request_id + " | Listing token definitions")
try:
cursor = token_def_collection.find({}).sort('api_token_group', 1)
# In-memory collections support chaining skip/limit; pymongo cursor also does
if page and page_size:
cursor = cursor.skip(max((page - 1), 0) * page_size).limit(page_size)
items = []
for doc in cursor:
if doc.get('_id'):
del doc['_id']
items.append({
'api_token_group': doc.get('api_token_group'),
'api_key_header': doc.get('api_key_header'),
# do not return key; just whether present
'api_key_present': bool(doc.get('api_key')),
'token_tiers': doc.get('token_tiers', []),
})
return ResponseModel(
status_code=200,
response={'items': items}
).dict()
except PyMongoError as e:
logger.error(request_id + f" | Token list failed with database error: {str(e)}")
return ResponseModel(
status_code=500,
error_code='TKN020',
error_message='Database error occurred while listing tokens'
).dict()
@staticmethod
async def get_token_def(api_token_group: str, request_id):
"""Get a single token definition (masked)."""
logger.info(request_id + " | Getting token definition")
try:
doc = token_def_collection.find_one({'api_token_group': api_token_group})
if not doc:
return ResponseModel(
status_code=404,
error_code='TKN021',
error_message='Token definition not found'
).dict()
if doc.get('_id'):
del doc['_id']
# mask key presence only
masked = {
'api_token_group': doc.get('api_token_group'),
'api_key_header': doc.get('api_key_header'),
'api_key_present': bool(doc.get('api_key')),
'token_tiers': doc.get('token_tiers', []),
}
return ResponseModel(status_code=200, response=masked).dict()
except PyMongoError as e:
logger.error(request_id + f" | Token fetch failed with database error: {str(e)}")
return ResponseModel(
status_code=500,
error_code='TKN022',
error_message='Database error occurred while retrieving token'
).dict()
@staticmethod
async def add_tokens(username: str, data: UserTokenModel, request_id):
"""Add or update a user's token balances for one or more groups."""

View File

@@ -0,0 +1,320 @@
from __future__ import annotations
import os
import uuid
import random
import string
from datetime import datetime, timedelta
from utils import password_util
from utils.database import (
api_collection,
endpoint_collection,
group_collection,
role_collection,
subscriptions_collection,
token_def_collection,
user_token_collection,
user_collection,
)
from utils.metrics_util import metrics_store, MinuteBucket
from utils.token_util import encrypt_value
def _rand_choice(seq):
return random.choice(seq)
def _rand_word(min_len=4, max_len=10) -> str:
length = random.randint(min_len, max_len)
return ''.join(random.choices(string.ascii_lowercase, k=length))
def _rand_name() -> str:
firsts = ["alex","casey","morgan","sam","taylor","riley","jamie","jordan","drew","quinn","kyle","parker","blake","devon"]
lasts = ["lee","kim","patel","garcia","nguyen","williams","brown","davis","miller","wilson","moore","taylor","thomas"]
return f"{_rand_choice(firsts)}.{_rand_choice(lasts)}"
def _rand_domain() -> str:
return _rand_choice(["example.com","acme.io","contoso.net","demo.dev"])
def _rand_password() -> str:
upp = _rand_choice(string.ascii_uppercase)
low = ''.join(random.choices(string.ascii_lowercase, k=8))
dig = ''.join(random.choices(string.digits, k=4))
spc = _rand_choice('!@#$%^&*()-_=+[]{};:,.<>?/')
tail = ''.join(random.choices(string.ascii_letters + string.digits, k=6))
raw = upp + low + dig + spc + tail
return ''.join(random.sample(raw, len(raw)))
def ensure_roles() -> list[str]:
roles = [("developer", dict(manage_apis=True, manage_endpoints=True, manage_subscriptions=True, manage_tokens=True, view_logs=True)),
("analyst", dict(view_logs=True, export_logs=True)),
("viewer", dict(view_logs=True)),
("ops", dict(manage_gateway=True, view_logs=True, export_logs=True, manage_security=True))]
created = []
for role_name, extra in roles:
if not role_collection.find_one({"role_name": role_name}):
doc = {"role_name": role_name, "role_description": f"{role_name} role"}
doc.update({k: bool(v) for k, v in extra.items()})
role_collection.insert_one(doc)
created.append(role_name)
return ["admin", *created]
def seed_groups(n: int, api_keys: list[str]) -> list[str]:
names = []
for i in range(n):
gname = f"team-{_rand_word(3,6)}-{i}"
if group_collection.find_one({"group_name": gname}):
names.append(gname)
continue
access = sorted(set(random.sample(api_keys, k=min(len(api_keys), random.randint(1, max(1, len(api_keys)//3))))) ) if api_keys else []
group_collection.insert_one({"group_name": gname, "group_description": f"Auto group {gname}", "api_access": access})
names.append(gname)
for base in ("ALL", "admin"):
if not group_collection.find_one({"group_name": base}):
group_collection.insert_one({"group_name": base, "group_description": f"{base} group", "api_access": []})
if base not in names:
names.append(base)
return names
def seed_users(n: int, roles: list[str], groups: list[str]) -> list[str]:
usernames = []
for i in range(n):
uname = f"{_rand_name()}_{i}"
email = f"{uname.replace('.', '_')}@{_rand_domain()}"
if user_collection.find_one({"username": uname}):
usernames.append(uname)
continue
hashed = password_util.hash_password(_rand_password())
ugrps = sorted(set(random.sample(groups, k=min(len(groups), random.randint(1, min(3, max(1, len(groups))))))))
role = _rand_choice(roles)
user_collection.insert_one({
"username": uname,
"email": email,
"password": hashed,
"role": role,
"groups": ugrps,
"rate_limit_duration": random.randint(100, 10000),
"rate_limit_duration_type": _rand_choice(["minute","hour","day"]),
"throttle_duration": random.randint(1000, 100000),
"throttle_duration_type": _rand_choice(["second","minute"]),
"throttle_wait_duration": random.randint(100, 10000),
"throttle_wait_duration_type": _rand_choice(["seconds","minutes"]),
"custom_attributes": {"dept": _rand_choice(["sales","eng","support","ops"])},
"active": True,
"ui_access": _rand_choice([True, False])
})
usernames.append(uname)
return usernames
def seed_apis(n: int, roles: list[str], groups: list[str]) -> list[tuple[str,str]]:
pairs = []
for i in range(n):
name = _rand_choice(["customers","orders","billing","weather","news","crypto","search","inventory","shipping","payments","alerts","metrics","recommendations"]) + f"-{_rand_word(3,6)}"
ver = _rand_choice(["v1","v2","v3"])
if api_collection.find_one({"api_name": name, "api_version": ver}):
pairs.append((name, ver))
continue
api_id = str(uuid.uuid4())
doc = {
"api_name": name,
"api_version": ver,
"api_description": f"Auto API {name}/{ver}",
"api_allowed_roles": sorted(set(random.sample(roles, k=min(len(roles), random.randint(1, min(3, len(roles))))))),
"api_allowed_groups": sorted(set(random.sample(groups, k=min(len(groups), random.randint(1, min(5, len(groups))))))),
"api_servers": [f"http://localhost:{8000+random.randint(0,999)}"],
"api_type": "REST",
"api_allowed_retry_count": random.randint(0, 3),
"api_id": api_id,
"api_path": f"/{name}/{ver}",
}
api_collection.insert_one(doc)
pairs.append((name, ver))
return pairs
def seed_endpoints(apis: list[tuple[str,str]], per_api: int) -> None:
methods = ["GET","POST","PUT","DELETE","PATCH"]
bases = ["/status","/health","/items","/items/{id}","/search","/reports","/export","/metrics","/list","/detail/{id}"]
for (name, ver) in apis:
created = set()
for _ in range(per_api):
m = _rand_choice(methods)
u = _rand_choice(bases)
key = (m, u)
if key in created:
continue
created.add(key)
if endpoint_collection.find_one({"api_name": name, "api_version": ver, "endpoint_method": m, "endpoint_uri": u}):
continue
endpoint_collection.insert_one({
"api_name": name,
"api_version": ver,
"endpoint_method": m,
"endpoint_uri": u,
"endpoint_description": f"{m} {u} for {name}",
"api_id": api_collection.find_one({"api_name": name, "api_version": ver}).get("api_id"),
"endpoint_id": str(uuid.uuid4()),
})
def seed_tokens() -> list[str]:
groups = ["ai-basic","ai-pro","maps-basic","maps-pro","news-tier","weather-tier"]
tiers_catalog = [
{"tier_name": "basic", "tokens": 100, "input_limit": 100, "output_limit": 100, "reset_frequency": "monthly"},
{"tier_name": "pro", "tokens": 1000, "input_limit": 500, "output_limit": 500, "reset_frequency": "monthly"},
{"tier_name": "enterprise", "tokens": 10000, "input_limit": 2000, "output_limit": 2000, "reset_frequency": "monthly"},
]
created = []
for g in groups:
if token_def_collection.find_one({"api_token_group": g}):
created.append(g)
continue
tiers = random.sample(tiers_catalog, k=random.randint(1, 3))
token_def_collection.insert_one({
"api_token_group": g,
"api_key": encrypt_value(uuid.uuid4().hex),
"api_key_header": _rand_choice(["x-api-key","authorization","x-token"]),
"token_tiers": tiers,
})
created.append(g)
return created
def seed_user_tokens(usernames: list[str], token_groups: list[str]) -> None:
pick_users = random.sample(usernames, k=min(len(usernames), max(1, len(usernames)//2))) if usernames else []
for u in pick_users:
users_tokens = {}
for g in random.sample(token_groups, k=random.randint(1, min(3, len(token_groups)))):
users_tokens[g] = {
"tier_name": _rand_choice(["basic","pro","enterprise"]),
"available_tokens": random.randint(10, 10000),
"reset_date": (datetime.utcnow() + timedelta(days=random.randint(1, 30))).strftime("%Y-%m-%d"),
"user_api_key": encrypt_value(uuid.uuid4().hex),
}
existing = user_token_collection.find_one({"username": u})
if existing:
user_token_collection.update_one({"username": u}, {"$set": {"users_tokens": users_tokens}})
else:
user_token_collection.insert_one({"username": u, "users_tokens": users_tokens})
def seed_subscriptions(usernames: list[str], apis: list[tuple[str,str]]) -> None:
api_keys = [f"{a}/{v}" for a, v in apis]
for u in usernames:
subs = sorted(set(random.sample(api_keys, k=random.randint(1, min(5, len(api_keys))))) ) if api_keys else []
existing = subscriptions_collection.find_one({"username": u})
if existing:
subscriptions_collection.update_one({"username": u}, {"$set": {"apis": subs}})
else:
subscriptions_collection.insert_one({"username": u, "apis": subs})
def seed_logs(n: int, usernames: list[str], apis: list[tuple[str,str]]) -> None:
base_dir = os.path.dirname(os.path.abspath(__file__))
base_dir = os.path.abspath(os.path.join(base_dir, '..'))
logs_dir = os.path.join(base_dir, "logs")
os.makedirs(logs_dir, exist_ok=True)
log_path = os.path.join(logs_dir, "doorman.log")
methods = ["GET","POST","PUT","DELETE","PATCH"]
uris = ["/status","/list","/items","/items/123","/search?q=test","/export","/metrics"]
now = datetime.now()
with open(log_path, "a", encoding="utf-8") as lf:
for _ in range(n):
api = _rand_choice(apis) if apis else ("demo","v1")
method = _rand_choice(methods)
uri = _rand_choice(uris)
ts = (now - timedelta(seconds=random.randint(0, 3600))).strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]
rid = str(uuid.uuid4())
user = _rand_choice(usernames) if usernames else "admin"
port = random.randint(10000, 65000)
msg = f"{rid} | Username: {user} | From: 127.0.0.1:{port} | Endpoint: {method} /{api[0]}/{api[1]}{uri} | Total time: {random.randint(5,500)}ms"
lf.write(f"{ts} - doorman.gateway - INFO - {msg}\n")
def seed_protos(n: int, apis: list[tuple[str,str]]) -> None:
base_dir = os.path.dirname(os.path.abspath(__file__))
base_dir = os.path.abspath(os.path.join(base_dir, '..'))
proto_dir = os.path.join(base_dir, "proto")
gen_dir = os.path.join(base_dir, "generated")
os.makedirs(proto_dir, exist_ok=True)
os.makedirs(gen_dir, exist_ok=True)
picked = random.sample(apis, k=min(n, len(apis))) if apis else []
for name, ver in picked:
key = f"{name}_{ver}".replace('-', '_')
svc = ''.join([p.capitalize() for p in name.split('-')])
content = f'''syntax = "proto3";
package {key};
service {svc}Service {{
rpc GetStatus (StatusRequest) returns (StatusReply) {{}}
}}
message StatusRequest {{
string id = 1;
}}
message StatusReply {{
string status = 1;
string message = 2;
}}
'''
path = os.path.join(proto_dir, f"{key}.proto")
with open(path, "w", encoding="utf-8") as f:
f.write(content)
def seed_metrics(usernames: list[str], apis: list[tuple[str,str]], minutes: int = 400) -> None:
now = datetime.utcnow()
for i in range(minutes, 0, -1):
minute_start = int(((now - timedelta(minutes=i)).timestamp()) // 60) * 60
b = MinuteBucket(start_ts=minute_start)
count = random.randint(0, 50)
for _ in range(count):
dur = random.uniform(10, 400)
status = _rand_choice([200,200,200,201,204,400,401,403,404,500])
b.add(dur, status)
metrics_store.total_requests += 1
metrics_store.total_ms += dur
metrics_store.status_counts[status] += 1
u = _rand_choice(usernames) if usernames else None
if u:
metrics_store.username_counts[u] += 1
if apis:
metrics_store.api_counts[f"rest:{_rand_choice(apis)[0]}"] += 1
metrics_store._buckets.append(b)
def run_seed(users=30, apis=12, endpoints=5, groups=6, protos=5, logs=1000, seed=None):
if seed is not None:
random.seed(seed)
roles = ensure_roles()
api_pairs = seed_apis(apis, roles, ["ALL","admin"])
group_names = seed_groups(groups, [f"{a}/{v}" for a, v in api_pairs])
usernames = seed_users(users, roles, group_names)
seed_endpoints(api_pairs, endpoints)
token_groups = seed_tokens()
seed_user_tokens(usernames, token_groups)
seed_subscriptions(usernames, api_pairs)
seed_logs(logs, usernames, api_pairs)
seed_protos(protos, api_pairs)
seed_metrics(usernames, api_pairs)
return {
'users': user_collection.count_documents({}),
'apis': api_collection.count_documents({}),
'endpoints': endpoint_collection.count_documents({}),
'groups': group_collection.count_documents({}),
'roles': role_collection.count_documents({}),
'subscriptions': subscriptions_collection.count_documents({}),
'token_defs': token_def_collection.count_documents({}),
'user_tokens': user_token_collection.count_documents({}),
}

View File

@@ -0,0 +1,127 @@
'use client'
import React, { useEffect, useState } from 'react'
import Link from 'next/link'
import { useParams, useRouter } from 'next/navigation'
import Layout from '@/components/Layout'
import { SERVER_URL } from '@/utils/config'
import { getJson, putJson, delJson } from '@/utils/api'
import { ProtectedRoute } from '@/components/ProtectedRoute'
import ConfirmModal from '@/components/ConfirmModal'
export default function EditTokenDefPage() {
const params = useParams<{ group: string }>()
const router = useRouter()
const group = decodeURIComponent(params.group)
const [api_token_group] = useState(group)
const [api_key_header, setHeader] = useState('x-api-key')
const [api_key, setKey] = useState('')
const [tokenTiersText, setTiersText] = useState('[]')
const [working, setWorking] = useState(false)
const [error, setError] = useState<string | null>(null)
const [success, setSuccess] = useState<string | null>(null)
const [showDelete, setShowDelete] = useState(false)
const [confirmText, setConfirmText] = useState('')
useEffect(() => { load() }, [])
const load = async () => {
try {
setError(null)
const res = await getJson<any>(`${SERVER_URL}/platform/token/defs/${encodeURIComponent(group)}`)
const data = res?.response || res
setHeader(data.api_key_header || 'x-api-key')
setTiersText(JSON.stringify(data.token_tiers || [], null, 2))
} catch (e:any) {
setError(e?.message || 'Failed to load token definition')
}
}
const update = async () => {
setWorking(true); setError(null); setSuccess(null)
try {
const tiers = JSON.parse(tokenTiersText || '[]')
const payload: any = { api_token_group, api_key_header, token_tiers: tiers }
if (api_key) payload.api_key = api_key // set only if provided
await putJson(`${SERVER_URL}/platform/token/${encodeURIComponent(api_token_group)}`, payload)
setSuccess('Token definition updated')
setKey('')
} catch (e:any) {
setError(e?.message || 'Failed to update token definition')
} finally {
setWorking(false)
}
}
const onDelete = async () => {
if (confirmText !== api_token_group) return
setWorking(true)
try {
await delJson(`${SERVER_URL}/platform/token/${encodeURIComponent(api_token_group)}`)
router.push('/token-defs')
} catch (e:any) {
setError(e?.message || 'Failed to delete token definition')
} finally {
setWorking(false)
}
}
return (
<ProtectedRoute requiredPermission="manage_tokens">
<Layout>
<div className="space-y-6">
<div className="page-header">
<div>
<h1 className="page-title">Edit Token Definition</h1>
<p className="text-gray-600 dark:text-gray-400 mt-1">{api_token_group}</p>
</div>
<Link href="/token-defs" className="btn btn-secondary">Back to Token Definitions</Link>
</div>
<div className="card">
<div className="p-6 space-y-4">
{error && <div className="text-sm text-error-600">{error}</div>}
{success && <div className="text-sm text-success-600">{success}</div>}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label className="block text-sm font-medium">API Token Group</label>
<input className="input" value={api_token_group} readOnly />
</div>
<div>
<label className="block text-sm font-medium">API Key Header</label>
<input className="input" value={api_key_header} onChange={e => setHeader(e.target.value)} />
</div>
<div className="md:col-span-2">
<label className="block text-sm font-medium">API Key (leave blank to keep existing)</label>
<input className="input" value={api_key} onChange={e => setKey(e.target.value)} placeholder="sk_live_..." />
</div>
<div className="md:col-span-2">
<label className="block text-sm font-medium">Token Tiers (JSON)</label>
<textarea className="input font-mono text-xs h-48" value={tokenTiersText} onChange={e => setTiersText(e.target.value)} />
</div>
</div>
<div className="flex gap-2">
<button onClick={update} disabled={working} className="btn btn-primary">{working ? 'Saving...' : 'Save Changes'}</button>
<button onClick={() => setShowDelete(true)} className="btn btn-error">Delete</button>
<Link href="/token-defs" className="btn btn-ghost">Cancel</Link>
</div>
</div>
</div>
<ConfirmModal
open={showDelete}
title="Delete Token Definition"
message={<div>
Type <span className="font-mono">{api_token_group}</span> to confirm deletion.
<div className="mt-3"><input className="input w-full" value={confirmText} onChange={e => setConfirmText(e.target.value)} /></div>
</div>}
confirmLabel={working ? 'Deleting...' : 'Delete'}
cancelLabel="Cancel"
onCancel={() => setShowDelete(false)}
onConfirm={onDelete}
/>
</div>
</Layout>
</ProtectedRoute>
)
}

View File

@@ -0,0 +1,78 @@
'use client'
import React, { useState } from 'react'
import Link from 'next/link'
import Layout from '@/components/Layout'
import { SERVER_URL } from '@/utils/config'
import { postJson } from '@/utils/api'
import { ProtectedRoute } from '@/components/ProtectedRoute'
export default function AddTokenDefPage() {
const [api_token_group, setGroup] = useState('')
const [api_key_header, setHeader] = useState('x-api-key')
const [api_key, setKey] = useState('')
const [tokenTiersText, setTiersText] = useState('[\n {"tier_name":"basic","tokens":100,"input_limit":150,"output_limit":150,"reset_frequency":"monthly"}\n]')
const [working, setWorking] = useState(false)
const [error, setError] = useState<string | null>(null)
const [success, setSuccess] = useState<string | null>(null)
const create = async () => {
setWorking(true); setError(null); setSuccess(null)
try {
if (!api_token_group.trim()) throw new Error('Group is required')
const tiers = JSON.parse(tokenTiersText || '[]')
await postJson(`${SERVER_URL}/platform/token`, { api_token_group: api_token_group.trim(), api_key, api_key_header, token_tiers: tiers })
setSuccess('Token definition created')
} catch (e: any) {
setError(e?.message || 'Failed to create token definition')
} finally {
setWorking(false)
}
}
return (
<ProtectedRoute requiredPermission="manage_tokens">
<Layout>
<div className="space-y-6">
<div className="page-header">
<div>
<h1 className="page-title">Add Token Definition</h1>
<p className="text-gray-600 dark:text-gray-400 mt-1">Create a token group and tiers</p>
</div>
<Link href="/token-defs" className="btn btn-secondary">Back to Token Definitions</Link>
</div>
<div className="card">
<div className="p-6 space-y-4">
{error && <div className="text-sm text-error-600">{error}</div>}
{success && <div className="text-sm text-success-600">{success}</div>}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label className="block text-sm font-medium">API Token Group</label>
<input className="input" value={api_token_group} onChange={e => setGroup(e.target.value)} placeholder="ai-basic" />
</div>
<div>
<label className="block text-sm font-medium">API Key Header</label>
<input className="input" value={api_key_header} onChange={e => setHeader(e.target.value)} placeholder="x-api-key" />
</div>
<div className="md:col-span-2">
<label className="block text-sm font-medium">API Key</label>
<input className="input" value={api_key} onChange={e => setKey(e.target.value)} placeholder="sk_live_..." />
</div>
<div className="md:col-span-2">
<label className="block text-sm font-medium">Token Tiers (JSON)</label>
<textarea className="input font-mono text-xs h-48" value={tokenTiersText} onChange={e => setTiersText(e.target.value)} />
</div>
</div>
<div className="flex gap-2">
<button onClick={create} disabled={working} className="btn btn-primary">{working ? 'Saving...' : 'Create'}</button>
<Link href="/token-defs" className="btn btn-ghost">Cancel</Link>
</div>
</div>
</div>
</div>
</Layout>
</ProtectedRoute>
)
}

View File

@@ -0,0 +1,179 @@
'use client'
import React, { useEffect, useState } from 'react'
import Link from 'next/link'
import Layout from '@/components/Layout'
import Pagination from '@/components/Pagination'
import { SERVER_URL } from '@/utils/config'
import { getJson } from '@/utils/api'
import { ProtectedRoute } from '@/components/ProtectedRoute'
interface TokenDefItem {
api_token_group: string
api_key_header: string
api_key_present: boolean
token_tiers: Array<{ tier_name: string; tokens: number; input_limit: number; output_limit: number; reset_frequency: string }>
}
export default function TokenDefsPage() {
const [items, setItems] = useState<TokenDefItem[]>([])
const [allItems, setAllItems] = useState<TokenDefItem[]>([])
const [loading, setLoading] = useState(true)
const [error, setError] = useState<string | null>(null)
const [search, setSearch] = useState('')
const [page, setPage] = useState(1)
const [pageSize, setPageSize] = useState(10)
const [hasNext, setHasNext] = useState(false)
useEffect(() => { fetchDefs() }, [page, pageSize])
const fetchDefs = async () => {
try {
setLoading(true); setError(null)
const res = await getJson<any>(`${SERVER_URL}/platform/token/defs?page=${page}&page_size=${pageSize}`)
const list = Array.isArray(res) ? res : (res.items || res.response?.items || [])
setItems(list)
setAllItems(list)
setHasNext(list.length === pageSize)
} catch (e: any) {
setError(e?.message || 'Failed to load token definitions')
setItems([])
setAllItems([])
setHasNext(false)
} finally {
setLoading(false)
}
}
const onSearch = (e: React.FormEvent) => {
e.preventDefault()
if (!search.trim()) { setItems(allItems); return }
const s = search.toLowerCase()
setItems(allItems.filter(it => (
it.api_token_group.toLowerCase().includes(s) ||
(it.api_key_header || '').toLowerCase().includes(s) ||
(it.token_tiers || []).some(t => t.tier_name.toLowerCase().includes(s))
)))
}
return (
<ProtectedRoute requiredPermission="manage_tokens">
<Layout>
<div className="space-y-6">
{/* Header */}
<div className="page-header">
<div>
<h1 className="page-title">Token Definitions</h1>
<p className="text-gray-600 dark:text-gray-400 mt-1">Define API token groups and tiers</p>
</div>
<Link href="/token-defs/add" className="btn btn-primary">
<svg className="h-4 w-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
Add Token Definition
</Link>
</div>
{/* Search */}
<div className="card">
<form onSubmit={onSearch} className="flex-1">
<div className="relative">
<svg className="absolute left-3 top-1/2 transform -translate-y-1/2 h-4 w-4 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z" />
</svg>
<input type="text" className="search-input" placeholder="Search by group, header, or tier..." value={search} onChange={(e) => setSearch(e.target.value)} />
</div>
</form>
</div>
{/* Error */}
{error && (
<div className="rounded-lg bg-error-50 border border-error-200 p-4 dark:bg-error-900/20 dark:border-error-800">
<div className="flex">
<svg className="h-5 w-5 text-error-400 dark:text-error-500 mt-0.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 8v4m0 4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
<div className="ml-3">
<p className="text-sm text-error-700 dark:text-error-300">{error}</p>
</div>
</div>
</div>
)}
{/* Loading */}
{loading ? (
<div className="card"><div className="flex items-center justify-center py-12"><div className="text-center"><div className="spinner mx-auto mb-4"></div><p className="text-gray-600 dark:text-gray-400">Loading token definitions...</p></div></div></div>
) : (
<div className="card">
<div className="overflow-x-auto">
<table className="table">
<thead>
<tr>
<th>Group</th>
<th>Header</th>
<th>Tiers</th>
<th>API Key</th>
<th className="w-40">Actions</th>
</tr>
</thead>
<tbody>
{items.map((it) => (
<tr key={it.api_token_group}>
<td><span className="font-medium">{it.api_token_group}</span></td>
<td><span className="badge badge-gray">{it.api_key_header || '-'}</span></td>
<td>
{(it.token_tiers || []).length > 0 ? (
<div className="flex flex-wrap gap-1">
{(it.token_tiers || []).slice(0, 3).map((t, idx) => (
<span key={idx} className="badge badge-primary">{t.tier_name}</span>
))}
{(it.token_tiers || []).length > 3 && <span className="text-xs text-gray-500">+{(it.token_tiers || []).length - 3}</span>}
</div>
) : <span className="text-gray-500 text-sm">None</span>}
</td>
<td>
{it.api_key_present ? (
<span className="badge badge-success">Configured</span>
) : (
<span className="badge badge-warning">Missing</span>
)}
</td>
<td>
<div className="flex items-center gap-2">
<Link href={`/token-defs/${encodeURIComponent(it.api_token_group)}`} className="btn btn-secondary btn-sm">
Edit
</Link>
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
<Pagination
page={page}
pageSize={pageSize}
onPageChange={setPage}
onPageSizeChange={setPageSize}
hasNext={hasNext}
/>
{items.length === 0 && !loading && (
<div className="text-center py-12">
<div className="h-16 w-16 mx-auto mb-4 rounded-full bg-gray-100 dark:bg-gray-800 flex items-center justify-center">
<svg className="h-8 w-8 text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 8c-1.657 0-3 1.343-3 3 0 2.239 3 5 3 5s3-2.761 3-5c0-1.657-1.343-3-3-3z M12 13a2 2 0 110-4 2 2 0 010 4z" />
</svg>
</div>
<h3 className="text-lg font-medium text-gray-900 dark:text-white mb-2">No token definitions</h3>
<p className="text-gray-600 dark:text-gray-400 mb-4">Create a token definition to get started.</p>
<Link href="/token-defs/add" className="btn btn-primary">Add Token Definition</Link>
</div>
)}
</div>
)}
</div>
</Layout>
</ProtectedRoute>
)
}

View File

@@ -175,34 +175,16 @@ export default function TokensPage() {
</div>
</div>
{/* Token Definition */}
{/* Token Definitions CTA */}
<div className="card">
<div className="card-header"><h3 className="card-title">Token Definition</h3></div>
<div className="p-6 space-y-4">
{form.error && <div className="text-sm text-error-600">{form.error}</div>}
{form.success && <div className="text-sm text-success-600">{form.success}</div>}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label className="block text-sm font-medium">API Token Group</label>
<input className="input" value={form.api_token_group} onChange={e => setForm(f => ({ ...f, api_token_group: e.target.value }))} placeholder="ai-group-1" />
</div>
<div>
<label className="block text-sm font-medium">API Key Header</label>
<input className="input" value={form.api_key_header} onChange={e => setForm(f => ({ ...f, api_key_header: e.target.value }))} placeholder="x-api-key" />
</div>
<div className="md:col-span-2">
<label className="block text-sm font-medium">API Key</label>
<input className="input" value={form.api_key} onChange={e => setForm(f => ({ ...f, api_key: e.target.value }))} placeholder="sk_live_xxx" />
</div>
<div className="md:col-span-2">
<label className="block text-sm font-medium">Token Tiers (JSON)</label>
<textarea className="input font-mono text-xs h-40" value={form.token_tiers_text} onChange={e => setForm(f => ({ ...f, token_tiers_text: e.target.value }))} />
</div>
<div className="p-6 flex flex-col sm:flex-row items-start sm:items-center justify-between gap-4">
<div>
<h3 className="card-title">Token Definitions</h3>
<p className="text-gray-600 dark:text-gray-400">Create and manage token groups, headers, and tiers</p>
</div>
<div className="flex gap-2">
<button onClick={handleCreate} disabled={form.working} className="btn btn-primary">Create</button>
<button onClick={handleUpdate} disabled={form.working} className="btn btn-secondary">Update</button>
<button onClick={openDeleteModal} disabled={form.working} className="btn btn-error">Delete</button>
<a href="/token-defs" className="btn btn-secondary">View Definitions</a>
<a href="/token-defs/add" className="btn btn-primary">Add Definition</a>
</div>
</div>
</div>
@@ -278,17 +260,7 @@ export default function TokensPage() {
</div>
</div>
<ConfirmModal
open={showDeleteModal}
title="Delete Token Definition"
message={<>
This action cannot be undone. This will permanently delete the token definition for group "{form.api_token_group.trim()}".
</>}
confirmLabel={form.working ? 'Deleting...' : 'Delete'}
cancelLabel="Cancel"
onCancel={() => setShowDeleteModal(false)}
onConfirm={handleDeleteConfirm}
/>
{/* No delete modal needed on this page anymore */}
</Layout>
</ProtectedRoute>
)

View File

@@ -27,6 +27,7 @@ const menuItems: MenuItem[] = [
{ label: 'Logging', href: '/logging', icon: 'M9 5H7a2 2 0 00-2 2v10a2 2 0 002 2h8a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2', permission: 'view_logs' },
{ label: 'Monitor', href: '/monitor', icon: 'M9 19v-6a2 2 0 00-2-2H5a2 2 0 00-2 2v6a2 2 0 002 2h2a2 2 0 002-2zm0 0V9a2 2 0 012-2h2a2 2 0 012 2v10m-6 0a2 2 0 002 2h2a2 2 0 002-2m0 0V5a2 2 0 012-2h2a2 2 0 012 2v14a2 2 0 01-2 2h-2a2 2 0 01-2-2z', permission: 'manage_gateway' },
{ label: 'Tokens', href: '/tokens', icon: 'M12 8c-1.657 0-3 1.343-3 3 0 2.239 3 5 3 5s3-2.761 3-5c0-1.657-1.343-3-3-3z M12 13a2 2 0 110-4 2 2 0 010 4z', permission: 'manage_tokens' },
{ label: 'Token Definitions', href: '/token-defs', icon: 'M5 13l4 4L19 7', permission: 'manage_tokens' },
{ label: 'Subscriptions', href: '/subscriptions', icon: 'M3 5h12M9 3v2m-6 4h12M9 9v2m-6 4h12m-6 0v2', permission: 'manage_subscriptions' },
{ label: 'Authorizations', href: '/authorizations', icon: 'M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z', permission: 'manage_subscriptions' },
{ label: 'Auth Control', href: '/auth-admin', icon: 'M5 13l4 4L19 7', permission: 'manage_auth' },