"""Validate committed exposure contracts and resolve the dev-box edge routes.""" import argparse import hashlib import ipaddress import json from pathlib import Path import sys from jsonschema import Draft202012Validator import yaml if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from platform_gitops.serving import domain as serving_domain ROOT = Path(__file__).resolve().parents[4] # The route chart's catalogue, its development environment values and the mesh pins (decision 0040). ROUTES = ROOT / 'gitops/manifests/routes' CATALOGUE = ROUTES / 'base/services.yaml' SCHEMA = ROUTES / 'base/schema.json' ENVIRONMENT = ROUTES / 'overlays/devbox/values.yaml' ISTIO_PINS = ROOT / 'devops/ansible/roles/istio/files/pins.yaml' # A dedicated loopback listener denies every management route, even through SSH. WORKLOAD_PORT = 8444 # Review a new public surface here as well as in the catalogue. A generic backend # field must never turn databases, administration ports or predictors into routes. EXPOSURES = { 'mcp-gateway': ('mcp', 'mcp-gateway', 8082, 'http', 'public', 'http'), 'mcp-discovery': ('mcp', 'mcp-gateway', 8082, 'http', 'public', 'http'), 'mcp-broker': ('mcp', 'mcp-broker', 8083, 'http', 'public', 'http'), 'web': ('pharos', 'web', 8080, 'http', 'public', 'http'), 'api': ('pharos', 'api', 8080, 'http', 'public', 'http'), 'api-auth': ('pharos', 'api', 8080, 'http', 'public', 'http'), 'api-health': ('pharos', 'api', 8080, 'http', 'public', 'http'), 'platform': ('pharos', 'web', 8080, 'http', 'public', 'http'), 'site-api-run-logs': ('pharos', 'site-api', 8443, 'https', 'public', 'http'), 'site-api-commands': ('pharos', 'site-api', 8443, 'https', 'public', 'http'), 'site-api-ssh-access': ('pharos', 'site-api', 8443, 'https', 'public', 'http'), 'site-api-executions': ('pharos', 'site-api', 8443, 'https', 'public', 'http'), 'site-api-repositories': ('pharos', 'site-api', 8443, 'https', 'public', 'http'), 'aai': ('pharos-aai', 'keycloak', 8080, 'http', 'public', 'http'), 'ceph-dashboard': ('rook-ceph', 'rook-ceph-mgr-dashboard', 8443, 'https', 'management', 'http'), 'argocd': ('argocd', 'argocd-server', 80, 'http', 'management', 'http'), 'gitlab': ('gitlab', 'gitlab-webservice-default', 8181, 'http', 'public', 'http'), 'waldur': ('waldur', 'waldur-homeport', 80, 'http', 'public', 'http'), 'waldur-api': ('waldur', 'waldur-mastermind-api', 80, 'http', 'public', 'http'), 'waldur-api-auth': ('waldur', 'waldur-mastermind-api', 80, 'http', 'public', 'http'), 'waldur-static': ('waldur', 'waldur-mastermind-api', 80, 'http', 'public', 'http'), 'waldur-health': ('waldur', 'waldur-mastermind-api', 80, 'http', 'public', 'http'), 'waldur-admin': ('waldur', 'waldur-mastermind-api', 80, 'http', 'management', 'http'), 'themelio-compat': ('themelio-compat', 'web', 80, 'http', 'public', 'http'), 'zammad': ('zammad', 'zammad-nginx', 8080, 'http', 'management', 'http'), 'open-webui': ('llm', 'open-webui', 80, 'http', 'public', 'http'), # Only the member inference paths of the gateway: its management routes are never an exposure. 'llm-api': ('llm', 'litellm', 4000, 'http', 'public', 'http'), 'llm-api-chat': ('llm', 'litellm', 4000, 'http', 'public', 'http'), 'llm-api-completions': ('llm', 'litellm', 4000, 'http', 'public', 'http'), 'llm-api-embeddings': ('llm', 'litellm', 4000, 'http', 'public', 'http'), 'llm-api-responses': ('llm', 'litellm', 4000, 'http', 'public', 'http'), 'llm-api-messages': ('llm', 'litellm', 4000, 'http', 'public', 'http'), # Pending: delivered with the kubeflow-core component. The browser route reaches the dashboard # the external authorisation check protects; the login endpoints reach the proxy itself. 'kubeflow': ('kubeflow', 'dashboard', 80, 'http', 'public', 'http'), 'kubeflow-login': ('oauth2-proxy', 'oauth2-proxy', 4180, 'http', 'public', 'http'), 'kubeflow-jupyter': ('kubeflow', 'jupyter-web-app-service', 80, 'http', 'public', 'http'), } # These routes share host-wide external authorisation policies. Each must exclude the login # endpoints so that a second policy cannot intercept the proxy's own responses. AUTH = {'kubeflow': {'provider': 'oauth2-proxy', 'notPaths': ['/oauth2/'], 'identityHeaders': ['kubeflow-userid', 'x-auth-request-user']}, 'kubeflow-jupyter': {'provider': 'oauth2-proxy', 'notPaths': ['/oauth2/'], 'identityHeaders': ['kubeflow-userid', 'x-auth-request-user']}} # The CA Secret in the gateway namespace, SNI and verified name for each HTTPS backend. Ceph's leaf # has no SAN, so its certificate is pinned by fingerprint instead of a verified name. BACKEND_TLS = { 'ceph-dashboard': {'trust': 'pharos-ceph-trust', 'sni': 'ceph-dashboard'}, 'site-api-run-logs': {'trust': 'pharos-site-api-trust', 'sni': 'site-api.pharos.svc', 'subjectAltName': 'site-api.pharos.svc'}, 'site-api-commands': {'trust': 'pharos-site-api-trust', 'sni': 'site-api.pharos.svc', 'subjectAltName': 'site-api.pharos.svc'}, 'site-api-ssh-access': {'trust': 'pharos-site-api-trust', 'sni': 'site-api.pharos.svc', 'subjectAltName': 'site-api.pharos.svc'}, 'site-api-executions': {'trust': 'pharos-site-api-trust', 'sni': 'site-api.pharos.svc', 'subjectAltName': 'site-api.pharos.svc'}, 'site-api-repositories': {'trust': 'pharos-site-api-trust', 'sni': 'site-api.pharos.svc', 'subjectAltName': 'site-api.pharos.svc'}, } PROFILES = {'mcp', 'pharos', 'platform', 'aai', 'storage', 'argocd', 'gitlab', 'waldur', 'themelio-compat', 'pharos-site', 'llm-chat', 'llm-api', 'zammad', 'kserve', 'kubeflow', # A dev-box serving profile: it selects KServe and the model delivery and publishes no route of its own. 'llm-inference'} LEGACY_NODE_PORTS = {'aai': 30080} SURFACES = { 'mcp-gateway': ('mcp.pharos', '/mcp', 'mcp', False), 'mcp-discovery': ('mcp.pharos', '/.well-known/oauth-protected-resource/mcp', 'mcp', False), 'mcp-broker': ('mcp.pharos', '/', 'mcp', False), 'web': ('web.pharos', '/', 'pharos', True), 'api': ('api.pharos', '/api/v1/', 'pharos', False), 'api-auth': ('api.pharos', '/auth/', 'pharos', False), 'api-health': ('api.pharos', '/healthz', 'pharos', False), 'platform': ('platform.pharos', '/', 'platform', True), 'site-api-run-logs': ('site.pharos', '/v1/site/commands/run-logs/', 'pharos-site', False), 'site-api-commands': ('site.pharos', '/v1/site/commands/', 'pharos-site', False), 'site-api-ssh-access': ('site.pharos', '/v1/site/ssh-access/', 'pharos-site', False), 'site-api-executions': ('site.pharos', '/v1/site/executions/', 'pharos-site', False), 'site-api-repositories': ('site.pharos', '/v1/site/repositories/', 'pharos-site', False), 'aai': ('pharos', '/auth/', 'aai', False), 'ceph-dashboard': ('ceph.pharos', '/', 'storage', False), 'argocd': ('argo.pharos', '/', 'argocd', True), 'gitlab': ('gitlab.pharos', '/', 'gitlab', False), 'waldur': ('waldur.pharos', '/', 'waldur', False), 'waldur-api': ('waldur.pharos', '/api/', 'waldur', False), 'waldur-api-auth': ('waldur.pharos', '/api-auth/', 'waldur', False), 'waldur-static': ('waldur.pharos', '/static/', 'waldur', False), 'waldur-health': ('waldur.pharos', '/health-check/', 'waldur', False), 'waldur-admin': ('waldur.pharos', '/admin/', 'waldur', False), 'themelio-compat': ('themelio-compat.pharos', '/', 'themelio-compat', True), 'zammad': ('zammad.pharos', '/', 'zammad', True), 'open-webui': ('open-webui.pharos', '/', 'llm-chat', True), 'llm-api': ('llm-api.pharos', '/v1/models', 'llm-api', False), 'llm-api-chat': ('llm-api.pharos', '/v1/chat/completions', 'llm-api', False), 'llm-api-completions': ('llm-api.pharos', '/v1/completions', 'llm-api', False), 'llm-api-embeddings': ('llm-api.pharos', '/v1/embeddings', 'llm-api', False), 'llm-api-responses': ('llm-api.pharos', '/v1/responses', 'llm-api', False), 'llm-api-messages': ('llm-api.pharos', '/v1/messages', 'llm-api', False), 'kubeflow': ('kubeflow.pharos', '/', 'kubeflow', True), 'kubeflow-login': ('kubeflow.pharos', '/oauth2/', 'kubeflow', False), 'kubeflow-jupyter': ('kubeflow.pharos', '/jupyter/', 'kubeflow', False), } class UniqueLoader(yaml.SafeLoader): """Reject duplicate YAML keys instead of silently discarding exposure policy.""" def unique_mapping(loader, node): result = {} for key_node, value_node in node.value: key = loader.construct_object(key_node) if not isinstance(key, str) or key in result: raise ValueError('Catalogue YAML needs unique string keys') result[key] = loader.construct_object(value_node) return result UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, unique_mapping) def load(path): return yaml.load(Path(path).read_text(), Loader=UniqueLoader) def validate(catalogue, environment): schema = json.loads(SCHEMA.read_text()) Draft202012Validator.check_schema(schema) for kind, value in (('catalogue', catalogue), ('environment', environment)): validator = Draft202012Validator({**schema, '$ref': '#/$defs/' + kind}) errors = sorted(validator.iter_errors(value), key=lambda error: str(error.path)) if errors: error = errors[0] raise ValueError(f'{kind}.{".".join(map(str, error.path))}: {error.message}') services = catalogue['services'] if set(environment['routes']) != set(services): raise ValueError('Environment routes must explicitly cover every catalogued service') for gateway in catalogue['gateways'].values(): if gateway['namespace'] not in catalogue['namespaces']: raise ValueError('Every gateway namespace must be declared') seen = set() for name, route in services.items(): backend = route['backend'] if backend['namespace'] not in catalogue['namespaces']: raise ValueError('Every backend namespace must be declared') if route['gateway'] not in catalogue['gateways']: raise ValueError('Every route must select a known gateway') key = (route['host'], route['path']) if key in seen: raise ValueError('Each host+path must be unique') seen.add(key) exposure = (*(backend[field] for field in ('namespace', 'service', 'port', 'protocol')), route['access'], route['transport']) if exposure != EXPOSURES.get(name): raise ValueError('Backend/access/transport is not an approved exposure: ' + name) if backend.get('tls') != BACKEND_TLS.get(name): raise ValueError('Backend TLS trust is not approved: ' + name) if route.get('auth') != AUTH.get(name): raise ValueError('External authorisation is not an approved exposure: ' + name) if tuple(route[field] for field in ('host', 'path', 'profile', 'websocket')) != SURFACES[name]: raise ValueError('Host/path/profile/websocket is not an approved surface: ' + name) if route.get('git') != ({'service': 'gitlab'} if name == 'themelio-compat' else None): raise ValueError('Git forwarding is restricted to retained Themelio and the shared GitLab') binding = environment['routes'][name] if environment['routing']['mode'] == 'nodeports' and binding['enabled'] and (binding.get('node_port') != LEGACY_NODE_PORTS.get(name) or name not in LEGACY_NODE_PORTS): raise ValueError('A1 supports only the existing browser NodePorts: ' + name) for cidr in environment['access']['management']: ipaddress.ip_network(cidr, strict=True) for profile, rule in (environment['access'].get('services') or {}).items(): for cidr in rule['cidrs']: ipaddress.ip_network(cidr, strict=True) # A public prefix opens part of a profile that is otherwise source-restricted, so it must sit under one of # that profile's own routes at `/`; it never reaches another profile's paths. roots = [route for route in catalogue['services'].values() if route['profile'] == profile and route['path'] == '/'] if rule.get('public_paths') and not roots: raise ValueError('A public path needs a route of its profile at /: ' + profile) hosts = environment.get('hosts', {}) if len(set(hosts.values())) != len(hosts): raise ValueError('Public service hostnames must remain distinct') if environment.get('serving'): if environment['routing']['mode'] != 'istio': raise ValueError('Serving requires the shared gateway') reserved = [*hosts.values(), *environment.get('browser_hosts', {}).values()] if environment['domain']: reserved += [route['host'] + '.' + environment['domain'] for route in services.values()] serving_domain(environment['serving']['domain'], reserved) return catalogue, environment def extensions(catalogue, environment, values): """Validate a laboratory's own routes: its own hosts and namespace, served only through the gateway.""" validator = Draft202012Validator({**json.loads(SCHEMA.read_text()), '$ref': '#/$defs/extension'}) # Every catalogue host stays the platform's, whichever profiles are active. reserved = {'pharos', *(route['host'] for route in catalogue['services'].values())} names, hosts, seen = set(), {}, set() for value in values: errors = sorted(validator.iter_errors(value), key=lambda error: str(error.path)) if errors: raise ValueError(f'extension.{".".join(map(str, errors[0].path))}: {errors[0].message}') name = value['name'] if name in names: raise ValueError('Each extension needs a unique name: ' + name) names.add(name) if value['namespace'] in catalogue['namespaces']: raise ValueError('An extension needs its own namespace: ' + name) if environment['routing']['mode'] != 'istio': raise ValueError('Extensions are served only through the gateway: ' + name) for route in value['routes'].values(): if route['host'] in reserved: raise ValueError('An extension cannot serve a platform host: ' + route['host']) if hosts.setdefault(route['host'], name) != name: raise ValueError('Extensions cannot share a host: ' + route['host']) if (route['host'], route['path']) in seen: raise ValueError('Each host+path must be unique') seen.add((route['host'], route['path'])) return sorted(values, key=lambda value: value['name']) def host(environment, alias): if environment['edge']['mode'] == 'public-cert': value = environment.get('hosts', {}).get(alias) if not value: raise ValueError('Public routing needs an approved hostname for ' + alias) return value return alias + '.' + environment['domain'] def resolve(catalogue, environment, profiles=(), registered=()): validate(catalogue, environment) if environment['edge']['mode'] == 'public-cert' and registered: raise ValueError('Public site routes cannot inherit development laboratory registrations') added = extensions(catalogue, environment, registered) active = set(environment['profiles']) | set(profiles) if active - PROFILES: raise ValueError('Unknown edge profile') servers = {} for name, route in catalogue['services'].items(): binding = environment['routes'][name] if not binding['enabled'] or route['profile'] not in active: continue hostname = host(environment, route['host']) port = environment['routing']['node_port'] if environment['routing']['mode'] == 'istio' else binding['node_port'] browser = environment.get('browser_hosts', {}).get(route['host']) rule = (environment['access'].get('services') or {}).get(route['profile']) resolved = {**route, 'name': name, 'node_port': port, **({'browser_host': browser} if browser else {}), **({'sources': list(rule['cidrs'])} if rule and route['access'] == 'public' else {})} variants = [resolved] if rule and route['path'] == '/' and route['access'] == 'public': # The profile's public prefixes keep the root route's backend and settings without its source rule. variants += [{key: value for key, value in resolved.items() if key != 'sources'} | {'name': name + '-public' + str(index), 'path': path} for index, path in enumerate(rule.get('public_paths') or [])] for value in variants: servers.setdefault(hostname, []).append(value) if browser: servers.setdefault(browser, []).append(value) for extension in added: for name, route in extension['routes'].items(): servers.setdefault(route['host'] + '.' + environment['domain'], []).append({ **route, 'name': extension['name'] + '-' + name, 'node_port': environment['routing']['node_port'], 'backend': {**route['backend'], 'namespace': extension['namespace']}}) if environment.get('serving'): suffix = environment['serving']['domain'] serving_domain(suffix, servers) servers['*.' + suffix] = [{ 'name': 'run-serving', 'path': '/', 'access': 'public', 'transport': 'http', 'websocket': True, 'timeout': '120s', 'body': {'limit': '8m', 'buffering': False}, 'backend': {'namespace': 'pharos', 'service': 'api', 'port': 8080, 'protocol': 'http'}, 'node_port': environment['routing']['node_port']}] # Longest prefix wins in Nginx; deterministic ordering makes reviews clearer. for routes in servers.values(): routes.sort(key=lambda route: (-len(route['path']), route['name'])) aliases = set(environment.get('browser_hosts', {}).values()) & servers.keys() chart_digest = hashlib.sha256() chart = ROOT / 'gitops/manifests/routes/chart' for path in sorted(chart.rglob('*')): if path.is_file(): chart_digest.update(str(path.relative_to(chart)).encode()) chart_digest.update(path.read_bytes()) return {'port': environment['edge']['port'], 'workload_port': WORKLOAD_PORT, 'mode': environment['edge']['mode'], 'block_indexing': environment['edge'].get('block_indexing', False), 'chart_checksum': chart_digest.hexdigest(), 'routing': environment['routing'], 'profiles': sorted(active), 'extensions': [extension['name'] for extension in added], 'browser_aliases': sorted(aliases), 'servers': servers, 'management_cidrs': environment['access']['management'], 'certificate_names': sorted(servers) if environment['edge']['mode'] == 'public-cert' else sorted({'pharos.' + environment['domain'], '*.pharos.' + environment['domain'], *aliases, *(['*.' + environment['serving']['domain']] if environment.get('serving') else []), *(route['host'] + '.' + environment['domain'] for route in catalogue['services'].values()), *(route['host'] + '.' + environment['domain'] for extension in added for route in extension['routes'].values())})} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--catalogue', type=Path, default=CATALOGUE) parser.add_argument('--environment', type=Path, default=ENVIRONMENT) parser.add_argument('--profile', action='append', default=[]) # The edge passes the extensions registered on the host, and its caller's, as one JSON list. parser.add_argument('--extensions', type=argparse.FileType(), help='a JSON list of extensions, or - for stdin') args = parser.parse_args() registered = json.load(args.extensions) if args.extensions else [] print(json.dumps(resolve(load(args.catalogue), load(args.environment), args.profile, registered))) if __name__ == '__main__': main()