Coverage for lunar_toolkit/cli/serve.py: 79%

1"""serve - the web app, Range-capable Parquet / image serving, a PDS image relay, status and jobs.""" 

2 

3from __future__ import annotations 

4 

5import argparse 

6import hmac 

7import http.server 

8import ipaddress 

9import json 

10import math 

11import secrets 

12import shutil 

13import socket 

14import threading 

15import time 

16import urllib.error 

17import urllib.parse 

18import urllib.request 

19import webbrowser 

20from dataclasses import dataclass 

21from http.client import HTTPMessage 

22from pathlib import Path 

23from typing import IO, Any 

24 

25from ._jobs import TERMINAL, JobContext, JobManager, submit 

26from ._status import StatusCache, StatusDirs 

27 

28COMMAND = "serve" 

29 

30_STATIC_DIR = Path(__file__).parent.parent / "static" 

31_INDEX = _STATIC_DIR / "index.html" 

32_BASEMAPS = _STATIC_DIR / "basemaps" 

33 

34_IMAGE_TYPES = { 

35 ".tif": "image/tiff", 

36 ".tiff": "image/tiff", 

37 ".png": "image/png", 

38 ".jpg": "image/jpeg", 

39 ".jpeg": "image/jpeg", 

40} 

41# Image hosts the page may fetch through /remote: pds.mcp.nasa.gov drops its CORS header 

42# whenever the request carries extra headers (browsers always send `priority`). 

43_RELAY_HOSTS = ( 

44 "pds.mcp.nasa.gov", 

45 "pds.lroc.im-ldi.com", 

46 "pds.shadowcam.im-ldi.com", 

47 "data.darts.isas.jaxa.jp", 

48 "pgda.gsfc.nasa.gov", 

49) 

50# Hosts relayed only under these path prefixes (PGDA: the LOLA DEMs for the map's contours) 

51_RELAY_PATHS = {"pgda.gsfc.nasa.gov": "/data/LOLA_20mpp/"} 

52_BASEMAP_TYPES = {**_IMAGE_TYPES, ".json": "application/json"} 

53_MAX_LISTED_IMAGES = 2000 

54RELAY_MAX = 6 

55RELAY_WAIT = 30.0 

56MAX_BODY = 64 * 1024 

57SSE_KEEPALIVE = 15.0 

58SSE_MIN_INTERVAL = 0.25 

59_LOCAL_NAMES = ("localhost", "127.0.0.1", "::1") 

60 

61 

62def add_server_args(p: argparse.ArgumentParser, host: str, port: int) -> None: 

63 """Options shared by ``lunar serve`` and ``lunar app``.""" 

64 p.add_argument( 

65 "--port", 

66 type=int, 

67 default=port, 

68 metavar="PORT", 

69 help=f"Port to listen on (default: {port or 'any free'}).", 

70 ) 

71 p.add_argument( 

72 "--host", 

73 default=host, 

74 metavar="HOST", 

75 help=f"Address to bind (default: {host}); 0.0.0.0 serves the network (read-only there).", 

76 ) 

77 p.add_argument( 

78 "--data-dir", 

79 default=None, 

80 metavar="DIR", 

81 help="Directory containing parquet files. Defaults to LUNAR_DATA_DIR or the user data dir.", 

82 ) 

83 p.add_argument( 

84 "--images", 

85 default=None, 

86 metavar="DIR", 

87 help="Also serve the GeoTIFF / PNG / JPEG files under DIR (with Range) for the Image view.", 

88 ) 

89 p.add_argument( 

90 "--runs-dir", 

91 default=None, 

92 metavar="DIR", 

93 help="SfS runs listed in the Settings panel (default: LUNAR_RUNS_DIR).", 

94 ) 

95 p.add_argument( 

96 "--admin-token", 

97 default=None, 

98 metavar="TOKEN", 

99 help="Let remote clients with this token (Authorization: Bearer TOKEN or ?token=TOKEN) " 

100 "start jobs and see local paths; 'auto' generates one.", 

101 ) 

102 

103 

104def add_parser(sub: argparse._SubParsersAction) -> None: 

105 p = sub.add_parser( 

106 COMMAND, 

107 help="Serve the Lunar Toolkit web app with local parquet files.", 

108 formatter_class=argparse.RawDescriptionHelpFormatter, 

109 epilog=""" 

110Examples 

111-------- 

112 lunar serve 

113 lunar serve --port 9000 

114 lunar serve --data-dir /mnt/lroc-data 

115 lunar serve --images ~/sfs_runs # browse local GeoTIFF / COG results in the Image view 

116 lunar serve --host 0.0.0.0 # share on the network; remote clients are read-only 

117 lunar serve --host 0.0.0.0 --admin-token auto # ... and print a token for remote admin 

118 

119Admin (the Settings panel: build indexes, download, see local paths) is allowed from this 

120machine (loopback) and for requests carrying --admin-token; everyone else can browse, 

121search and view. Use `lunar app` for a desktop window. 

122""", 

123 ) 

124 add_server_args(p, "localhost", 8080) 

125 p.add_argument("--no-browser", action="store_true", help="Don't open a browser tab automatically.") 

126 

127 

128def _byte_range(header: str, size: int) -> tuple[int, int] | tuple[()] | None: 

129 """``(start, length)`` for a single ``bytes=`` Range, ``()`` without one, None if unsatisfiable.""" 

130 if not header.startswith("bytes="): 

131 return () 

132 first, _, last = header[6:].split(",")[0].strip().partition("-") 

133 try: 

134 if not first: # suffix range: the last N bytes 

135 n = int(last) 

136 start, end = max(size - n, 0), size - 1 

137 else: 

138 start = int(first) 

139 end = min(int(last), size - 1) if last else size - 1 

140 except ValueError: 

141 return None 

142 if start >= size or end < start: 

143 return None 

144 return start, end - start + 1 

145 

146 

147def _image_path(root: Path, rel: str) -> Path | None: 

148 """The image file ``rel`` (URL-quoted, relative) under ``root``, or None if it escapes or isn't one.""" 

149 parts = urllib.parse.unquote(rel).replace("\\", "/").split("/") 

150 if not rel or any(not part or part.startswith(".") for part in parts): 

151 return None 

152 base = root.resolve() 

153 path = base.joinpath(*parts).resolve() 

154 if not path.is_relative_to(base) or path.suffix.lower() not in _IMAGE_TYPES or not path.is_file(): 

155 return None 

156 return path 

157 

158 

159def _list_images(root: Path) -> list[str]: 

160 """Relative POSIX paths of the image files under ``root`` (hidden entries skipped).""" 

161 base = root.resolve() 

162 out = [] 

163 for path in sorted(base.rglob("*")): 

164 rel = path.relative_to(base) 

165 if any(part.startswith(".") for part in rel.parts) or path.suffix.lower() not in _IMAGE_TYPES: 

166 continue 

167 if path.is_file() and path.resolve().is_relative_to(base): 

168 out.append(rel.as_posix()) 

169 if len(out) >= _MAX_LISTED_IMAGES: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 break 

171 return out 

172 

173 

174def _report_path(runs_dir: Path, name: str) -> Path | None: 

175 """``<runs_dir>/<name>/report/report.html`` if *name* is a plain run dir name and it exists.""" 

176 name = urllib.parse.unquote(name) 

177 if not name or "/" in name or "\\" in name or name.startswith("."): 

178 return None 

179 base = runs_dir.resolve() 

180 path = (base / name / "report" / "report.html").resolve() 

181 return path if path.is_relative_to(base) and path.is_file() else None 

182 

183 

184def _relay_allowed(url: str) -> bool: 

185 """Only https image files on the PDS / JAXA image hosts are relayed.""" 

186 try: 

187 u = urllib.parse.urlsplit(url) 

188 port = u.port 

189 except ValueError: 

190 return False 

191 return ( 

192 u.scheme == "https" 

193 and u.hostname in _RELAY_HOSTS 

194 and port is None 

195 and not u.username 

196 and Path(u.path).suffix.lower() in _IMAGE_TYPES 

197 and u.path.startswith(_RELAY_PATHS.get(u.hostname, "/")) 

198 ) 

199 

200 

201class _RelayRedirect(urllib.request.HTTPRedirectHandler): 

202 def redirect_request( 

203 self, 

204 req: urllib.request.Request, 

205 fp: IO[bytes], 

206 code: int, 

207 msg: str, 

208 headers: HTTPMessage, 

209 newurl: str, 

210 ) -> urllib.request.Request | None: 

211 if not _relay_allowed(newurl): 

212 raise urllib.error.HTTPError(newurl, 403, "redirect outside the relay hosts", headers, fp) 

213 return super().redirect_request(req, fp, code, msg, headers, newurl) 

214 

215 

216_relay_opener = urllib.request.build_opener(_RelayRedirect) 

217 

218 

219def is_loopback(addr: str) -> bool: 

220 """Whether client address *addr* is this machine (IPv4, IPv6 or IPv4-mapped IPv6).""" 

221 try: 

222 ip = ipaddress.ip_address(addr.split("%", 1)[0]) 

223 except ValueError: 

224 return False 

225 if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None: 

226 ip = ip.ipv4_mapped 

227 return ip.is_loopback 

228 

229 

230def _local_host_header(host: str) -> bool: 

231 """A ``Host`` header naming this machine (guards loopback admin against DNS rebinding).""" 

232 try: 

233 name = urllib.parse.urlsplit("//" + host).hostname or "" 

234 except ValueError: 

235 return False 

236 return name in _LOCAL_NAMES or name.endswith(".localhost") 

237 

238 

239@dataclass 

240class ServerConfig: 

241 """What a `LunarServer` serves and where its jobs write.""" 

242 

243 data_dir: Path 

244 explicit_dir: Path | None = None 

245 images_dir: Path | None = None 

246 runs_dir: Path | None = None 

247 admin_token: str | None = None 

248 cache_dir: Path | None = None 

249 download_dir: Path | None = None 

250 reference_dir: Path | None = None 

251 

252 def __post_init__(self) -> None: 

253 from ..constants import DEFAULT_CACHE_DIR 

254 from ..reference import REFERENCE_CACHE_DIR 

255 

256 self.cache_dir = self.cache_dir or DEFAULT_CACHE_DIR 

257 self.download_dir = self.download_dir or self.cache_dir / "downloads" 

258 self.reference_dir = self.reference_dir or REFERENCE_CACHE_DIR 

259 

260 

261class LunarServer(http.server.ThreadingHTTPServer): 

262 """The HTTP server with its job manager, status cache and relay limit.""" 

263 

264 daemon_threads = True 

265 request_queue_size = 128 

266 

267 def __init__(self, address: tuple[str, int], cfg: ServerConfig, workers: int = 2) -> None: 

268 self.cfg = cfg 

269 assert cfg.cache_dir and cfg.download_dir and cfg.reference_dir 

270 dirs = StatusDirs(cfg.data_dir, cfg.cache_dir, cfg.download_dir, cfg.reference_dir, cfg.runs_dir) 

271 self.status = StatusCache(dirs) 

272 self.jobs = JobManager(workers, on_finish=lambda job: self.status.invalidate()) 

273 self.job_context = JobContext(cfg.data_dir, cfg.download_dir, cfg.reference_dir) 

274 self.relay_slots = threading.BoundedSemaphore(RELAY_MAX) 

275 self.sdem_members: dict[int, list[dict[str, Any]]] = {} 

276 if ":" in address[0]: 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true

277 self.address_family = socket.AF_INET6 

278 super().__init__(address, LunarHandler) 

279 

280 def close(self) -> None: 

281 """Cancel jobs, stop serving and close the socket.""" 

282 self.jobs.close() 

283 self.shutdown() 

284 self.server_close() 

285 

286 

287class LunarHandler(http.server.BaseHTTPRequestHandler): 

288 # Keep-alive: DuckDB-WASM and the image viewer issue thousands of small Range requests 

289 protocol_version = "HTTP/1.1" 

290 server: LunarServer 

291 

292 def client_ip(self) -> str: 

293 return self.client_address[0] 

294 

295 def _query(self) -> dict[str, list[str]]: 

296 return urllib.parse.parse_qs(self.path.partition("?")[2]) 

297 

298 def _token(self) -> str | None: 

299 auth = self.headers.get("Authorization", "") 

300 if auth.lower().startswith("bearer "): 

301 return auth[7:].strip() 

302 return self._query().get("token", [None])[0] 

303 

304 def is_admin(self) -> bool: 

305 """Loopback clients (with a local ``Host``) and holders of the admin token.""" 

306 token, given = self.server.cfg.admin_token, self._token() 

307 if token and given and hmac.compare_digest(given.encode(), token.encode()): 

308 return True 

309 return is_loopback(self.client_ip()) and _local_host_header(self.headers.get("Host", "")) 

310 

311 def do_GET(self) -> None: 

312 self._route(body=True) 

313 

314 def do_HEAD(self) -> None: 

315 self._route(body=False) 

316 

317 def do_POST(self) -> None: 

318 path = self.path.partition("?")[0] 

319 if path != "/api/jobs": 319 ↛ 320line 319 didn't jump to line 320 because the condition on line 319 was never true

320 self._error(404, "not found") 

321 return 

322 if not self._check_admin(write=True): 

323 return 

324 if "application/json" not in self.headers.get("Content-Type", ""): 

325 self._error(415, "expected application/json") 

326 return 

327 try: 

328 n = int(self.headers.get("Content-Length", "0")) 

329 except ValueError: 

330 n = -1 

331 if not 0 < n <= MAX_BODY: 331 ↛ 332line 331 didn't jump to line 332 because the condition on line 331 was never true

332 self._error(400 if n <= 0 else 413, "bad request body") 

333 return 

334 try: 

335 payload = json.loads(self.rfile.read(n)) 

336 job = submit(self.server.jobs, payload, self.server.job_context) 

337 except ValueError as exc: # also json.JSONDecodeError 

338 self._error(409 if "already" in str(exc) else 400, str(exc)) 

339 return 

340 except RuntimeError as exc: 

341 self._error(503, str(exc)) 

342 return 

343 self._json(job.to_dict(), code=201) 

344 

345 def do_DELETE(self) -> None: 

346 path = self.path.partition("?")[0] 

347 parts = path.strip("/").split("/") 

348 if len(parts) != 3 or parts[:2] != ["api", "jobs"]: 348 ↛ 349line 348 didn't jump to line 349 because the condition on line 348 was never true

349 self._error(404, "not found") 

350 return 

351 if not self._check_admin(write=True): 

352 return 

353 job = self.server.jobs.cancel(parts[2]) 

354 if job is None: 

355 self._error(404, "no such job") 

356 return 

357 self._json(job.to_dict()) 

358 

359 def do_OPTIONS(self) -> None: 

360 self._error(405, "method not allowed") 

361 

362 def _check_admin(self, write: bool = False) -> bool: 

363 if not self.is_admin(): 

364 self._error(403, "read-only client: admin needs localhost or the --admin-token") 

365 return False 

366 origin = self.headers.get("Origin") 

367 if write and origin and urllib.parse.urlsplit(origin).netloc != self.headers.get("Host", ""): 

368 self._error(403, "cross-origin request") 

369 return False 

370 return True 

371 

372 def _route(self, body: bool) -> None: 

373 path, _, query = self.path.partition("?") 

374 cfg = self.server.cfg 

375 if path in ("/", "/index.html"): 

376 self._serve(_INDEX, "text/html; charset=utf-8", body) 

377 elif path == "/api/config": 

378 from ..constants import APP_VERSION 

379 

380 self._json( 

381 { 

382 "relay": list(_RELAY_HOSTS), 

383 "images": cfg.images_dir is not None, 

384 "admin": self.is_admin(), 

385 "jobs": True, 

386 "runs": cfg.runs_dir is not None, 

387 "version": APP_VERSION, 

388 }, 

389 body, 

390 ) 

391 elif path.startswith("/basemaps/"): 

392 name = path[len("/basemaps/") :] 

393 ctype = _BASEMAP_TYPES.get(Path(name).suffix.lower()) 

394 if not ctype or "/" in name or name.startswith("."): 

395 self._send_code(404) 

396 return 

397 self._serve(_BASEMAPS / name, ctype, body) 

398 elif path == "/api/images": 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true

399 self._json(_list_images(cfg.images_dir) if cfg.images_dir else [], body) 

400 elif path.startswith("/images/") and cfg.images_dir is not None: 400 ↛ 401line 400 didn't jump to line 401 because the condition on line 400 was never true

401 img = _image_path(cfg.images_dir, path[len("/images/") :]) 

402 if img is None: 

403 self._send_code(404) 

404 return 

405 self._serve(img, _IMAGE_TYPES[img.suffix.lower()], body) 

406 elif path == "/remote": 

407 self._relay(urllib.parse.parse_qs(query).get("url", [""])[0], body) 

408 elif path.startswith("/data/"): 

409 from ..constants import find_parquet 

410 

411 name = path[len("/data/") :] 

412 if not name or "/" in name or name.startswith("."): 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true

413 self._send_code(404) 

414 return 

415 self._serve(find_parquet(name, cfg.data_dir), "application/octet-stream", body) 

416 elif path == "/api/status": 

417 refresh = self._query().get("refresh", ["0"])[0] == "1" 

418 self._json(self.server.status.get(self.is_admin(), refresh=refresh), body) 

419 elif path == "/api/regions": 419 ↛ 420line 419 didn't jump to line 420 because the condition on line 419 was never true

420 self._json(_regions(), body) 

421 elif path == "/api/sdem/members": 

422 if self._check_admin(): 422 ↛ 423line 422 didn't jump to line 423 because the condition on line 422 was never true

423 self._sdem_members(self._query().get("region", [""])[0], body) 

424 elif path.startswith("/api/jobs"): 

425 self._jobs_get(path, body) 

426 elif path.startswith("/runs/") and path.endswith("/report.html") and cfg.runs_dir is not None: 426 ↛ 435line 426 didn't jump to line 435 because the condition on line 426 was always true

427 if not self._check_admin(): 

428 return 

429 report = _report_path(cfg.runs_dir, path[len("/runs/") : -len("/report.html")]) 

430 if report is None: 430 ↛ 431line 430 didn't jump to line 431 because the condition on line 430 was never true

431 self._send_code(404) 

432 return 

433 self._serve(report, "text/html; charset=utf-8", body) 

434 else: 

435 self._send_code(404) 

436 

437 def _jobs_get(self, path: str, body: bool) -> None: 

438 admin = self.is_admin() 

439 parts = path.strip("/").split("/")[2:] 

440 jobs = self.server.jobs 

441 if not parts: 

442 self._json([j.to_dict(admin) for j in jobs.jobs()], body) 

443 elif parts == ["events"]: 

444 self._events(None, admin) 

445 elif len(parts) in (1, 2) and parts[1:] in ([], ["events"]): 445 ↛ 454line 445 didn't jump to line 454 because the condition on line 445 was always true

446 job = jobs.get(parts[0]) 

447 if job is None: 447 ↛ 448line 447 didn't jump to line 448 because the condition on line 447 was never true

448 self._error(404, "no such job") 

449 elif len(parts) == 2: 

450 self._events(parts[0], admin) 

451 else: 

452 self._json(job.to_dict(admin), body) 

453 else: 

454 self._error(404, "not found") 

455 

456 def _events(self, jid: str | None, admin: bool) -> None: 

457 """Server-Sent Events: ``event: jobs`` (all jobs) or ``event: job`` (one) on every change.""" 

458 jobs = self.server.jobs 

459 self.send_response(200) 

460 self._common_headers("text/event-stream") 

461 self.send_header("Cache-Control", "no-cache") 

462 self.end_headers() 

463 self.close_connection = True 

464 version = -1 

465 data: Any 

466 try: 

467 while not jobs.closed: 

468 new = jobs.wait(version, SSE_KEEPALIVE) 

469 if new == version: 469 ↛ 470line 469 didn't jump to line 470 because the condition on line 469 was never true

470 self.wfile.write(b": keepalive\n\n") 

471 self.wfile.flush() 

472 continue 

473 version = new 

474 if jid is None: 

475 event, data = "jobs", [j.to_dict(admin) for j in jobs.jobs()] 

476 final = False 

477 else: 

478 job = jobs.get(jid) 

479 if job is None: 479 ↛ 480line 479 didn't jump to line 480 because the condition on line 479 was never true

480 break 

481 event, data, final = "job", job.to_dict(admin), job.status in TERMINAL 

482 self.wfile.write(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode()) 

483 self.wfile.flush() 

484 if final: 

485 break 

486 time.sleep(SSE_MIN_INTERVAL) 

487 except (BrokenPipeError, ConnectionResetError, OSError): 

488 pass 

489 

490 def _sdem_members(self, region: str, body: bool) -> None: 

491 from .. import reference as ref 

492 

493 try: 

494 reg = ref.get_region(region) 

495 except KeyError as exc: 

496 self._error(400, str(exc.args[0] if exc.args else exc)) 

497 return 

498 cache = self.server.sdem_members 

499 if reg.id not in cache: 

500 try: 

501 df = ref.sdem_members(reg) 

502 except OSError as exc: 

503 self._error(502, f"could not read the zip directory: {exc}") 

504 return 

505 cache[reg.id] = [{"name": str(r["name"]), "size": int(r["size"])} for r in df.to_dict("records")] 

506 self._json({"region": reg.id, "zip": reg.zenodo_file, "members": cache[reg.id]}, body) 

507 

508 def _serve(self, path: Path, ctype: str, body: bool = True) -> None: 

509 if not path.exists(): 

510 self._send_code(404) 

511 return 

512 size = path.stat().st_size 

513 span = _byte_range(self.headers.get("Range", ""), size) 

514 if span is None: 514 ↛ 515line 514 didn't jump to line 515 because the condition on line 514 was never true

515 self._send_code(416) 

516 return 

517 start, length = span if span else (0, size) 

518 self.send_response(206 if span else 200) 

519 self._common_headers(ctype) 

520 self.send_header("Content-Length", str(length)) 

521 if span: 

522 self.send_header("Content-Range", f"bytes {start}-{start + length - 1}/{size}") 

523 self.end_headers() 

524 if body: 524 ↛ exitline 524 didn't return from function '_serve' because the condition on line 524 was always true

525 self._send_bytes(path, start, length) 

526 

527 def _json(self, obj: object, body: bool = True, code: int = 200) -> None: 

528 data = json.dumps(obj).encode() 

529 self.send_response(code) 

530 self._common_headers("application/json") 

531 self.send_header("Cache-Control", "no-store") 

532 self.send_header("Content-Length", str(len(data))) 

533 self.end_headers() 

534 if body: 534 ↛ exitline 534 didn't return from function '_json' because the condition on line 534 was always true

535 self.wfile.write(data) 

536 

537 def _error(self, code: int, message: str) -> None: 

538 self._json({"error": message}, self.command != "HEAD", code) 

539 

540 def _relay(self, url: str, body: bool) -> None: 

541 """Forward one Range request, through the downloader's per-host slots, pacing and cooldown.""" 

542 from .. import downloader 

543 

544 if not _relay_allowed(url): 544 ↛ 545line 544 didn't jump to line 545 because the condition on line 544 was never true

545 self._send_code(403) 

546 return 

547 host = downloader._host(url) 

548 wait = host.cooldown_until - time.time() 

549 if wait > 0: 

550 self._send_code(429, {"Retry-After": str(math.ceil(wait))}) 

551 return 

552 if not self.server.relay_slots.acquire(timeout=RELAY_WAIT): 

553 self._send_code(503, {"Retry-After": "2"}) 

554 return 

555 try: 

556 if not host.slots.acquire(timeout=RELAY_WAIT): 556 ↛ 557line 556 didn't jump to line 557 because the condition on line 556 was never true

557 self._send_code(503, {"Retry-After": "2"}) 

558 return 

559 try: 

560 host.pace() 

561 self._relay_once(url, body, host) 

562 finally: 

563 host.slots.release() 

564 finally: 

565 self.server.relay_slots.release() 

566 

567 def _relay_once(self, url: str, body: bool, host: Any) -> None: 

568 from .. import downloader 

569 

570 headers = {"User-Agent": downloader._USER_AGENT} 

571 if self.headers.get("Range"): 

572 headers["Range"] = self.headers["Range"] 

573 req = urllib.request.Request(url, headers=headers, method="GET" if body else "HEAD") 

574 started = False 

575 try: 

576 with _relay_opener.open(req, timeout=60) as resp: 

577 self.send_response(resp.status) 

578 started = True 

579 self._common_headers(resp.headers.get("Content-Type", "application/octet-stream")) 

580 for name in ("Content-Length", "Content-Range", "ETag", "Last-Modified"): 

581 if resp.headers.get(name): 

582 self.send_header(name, resp.headers[name]) 

583 if not resp.headers.get("Content-Length"): 583 ↛ 584line 583 didn't jump to line 584 because the condition on line 583 was never true

584 self.close_connection = True 

585 self.end_headers() 

586 if body: 586 ↛ exitline 586 didn't jump to the function exit

587 shutil.copyfileobj(resp, self.wfile, 65536) 

588 except urllib.error.HTTPError as e: 

589 if e.code in downloader.THROTTLE_CODES: 589 ↛ 594line 589 didn't jump to line 594 because the condition on line 589 was always true

590 wait = downloader._retry_after(e) or 60.0 

591 host.cool(wait) 

592 if not started: 592 ↛ exitline 592 didn't return from function '_relay_once' because the condition on line 592 was always true

593 self._send_code(429, {"Retry-After": str(math.ceil(wait))}) 

594 elif not started: 

595 self._send_code(e.code) 

596 except OSError: # upstream unreachable / timed out, or the browser went away 

597 self.close_connection = True 

598 if not started: 

599 self._send_code(502) 

600 

601 def _common_headers(self, ctype: str) -> None: 

602 self.send_header("Content-Type", ctype) 

603 self.send_header("Accept-Ranges", "bytes") 

604 # Cross-origin isolation for DuckDB-WASM; `credentialless` (not require-corp) so 

605 # browse images from hosts without CORP/CORS (JAXA DARTS) still load in <img>. 

606 self.send_header("Cross-Origin-Opener-Policy", "same-origin") 

607 self.send_header("Cross-Origin-Embedder-Policy", "credentialless") 

608 

609 def _send_bytes(self, path: Path, offset: int, length: int) -> None: 

610 with open(path, "rb") as f: 

611 f.seek(offset) 

612 remaining = length 

613 while remaining > 0: 

614 buf = f.read(min(65536, remaining)) 

615 if not buf: 615 ↛ 616line 615 didn't jump to line 616 because the condition on line 615 was never true

616 break 

617 try: 

618 self.wfile.write(buf) 

619 except (BrokenPipeError, ConnectionResetError): # the browser aborted the read 

620 self.close_connection = True 

621 return 

622 remaining -= len(buf) 

623 

624 def _send_code(self, code: int, headers: dict[str, str] | None = None) -> None: 

625 self.send_response(code) 

626 for k, v in (headers or {}).items(): 

627 self.send_header(k, v) 

628 self.send_header("Content-Length", "0") 

629 self.end_headers() 

630 

631 def log_message(self, fmt: str, *args: object) -> None: # noqa: ARG002 

632 pass # suppress per-request noise; server start/stop is printed below 

633 

634 

635def _regions() -> dict[str, Any]: 

636 from .. import reference as ref 

637 

638 return { 

639 "regions": [ 

640 { 

641 "id": r.id, 

642 "name": r.name, 

643 "pgda_site": r.pgda_site, 

644 "zenodo_file": r.zenodo_file, 

645 "zenodo_size": r.zenodo_size, 

646 } 

647 for r in ref.REGIONS 

648 ], 

649 "pgda_kinds": list(ref.PGDA_KINDS), 

650 } 

651 

652 

653def make_server(args: argparse.Namespace) -> tuple[LunarServer, ServerConfig]: 

654 """Bind a `LunarServer` from the shared options (ValueError for a bad option).""" 

655 import os 

656 

657 from ..config import RUNS_DIR_ENV 

658 from ..constants import get_data_dir 

659 

660 explicit = Path(args.data_dir).expanduser() if args.data_dir else None 

661 images = Path(args.images).expanduser() if args.images else None 

662 if images is not None and not images.is_dir(): 662 ↛ 663line 662 didn't jump to line 663 because the condition on line 662 was never true

663 raise ValueError(f"--images: not a directory: {images}") 

664 runs = args.runs_dir or os.environ.get(RUNS_DIR_ENV) 

665 token = args.admin_token 

666 if token == "auto": 

667 token = secrets.token_urlsafe(18) 

668 elif token is not None and len(token) < 12: 

669 raise ValueError("--admin-token: use at least 12 characters (or 'auto')") 

670 cfg = ServerConfig( 

671 data_dir=explicit or get_data_dir(), 

672 explicit_dir=explicit, 

673 images_dir=images, 

674 runs_dir=Path(runs).expanduser() if runs else None, 

675 admin_token=token, 

676 ) 

677 return LunarServer((args.host, args.port), cfg), cfg 

678 

679 

680def server_url(server: LunarServer, host: str) -> str: