From f2c71870b119fa4fb4c8094883a6411050acba54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 5 May 2026 15:20:38 +1200 Subject: [PATCH 1/4] plan: frond-server Plan for a new QUIC server binary for canopy: separate frond-server crate speaking a custom application protocol over raw QUIC, ALPN bes.canopy/1, bare-SPKI mTLS reusing the existing device_keys table, and QUIC-LB plaintext CIDs for AWS NLB QUIC-passthrough horizontal scaling. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/plans/frond-server.md | 175 +++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 docs/plans/frond-server.md diff --git a/docs/plans/frond-server.md b/docs/plans/frond-server.md new file mode 100644 index 0000000..2e88b80 --- /dev/null +++ b/docs/plans/frond-server.md @@ -0,0 +1,175 @@ +# frond-server: QUIC server for canopy + +A new internet-facing server that speaks a custom application protocol over raw QUIC, deployed alongside the existing public/private servers and sharing the same Postgres. + +## Why + +We need a custom application protocol over QUIC for devices. Two architectural facts force this into a separate binary rather than an extension of public-server: + +- The Envoy/Envoy-Gateway fabric in front of the cluster cannot proxy WebTransport or raw QUIC usefully (verified May 2026: Envoy issues #41981, #42221, #40229; no other production-viable proxy implements WT either). So QUIC has to terminate in the application. +- AWS NLB shipped QUIC passthrough mode with QUIC-LB Plaintext-CID routing (Nov 2025). With the AWS Load Balancer Controller injecting `AWS_LBC_QUIC_SERVER_ID`, we can horizontally scale a quinn-based server while preserving connection migration — without joining the Cilium-on-EKS adventure. + +This server is not a replacement for public-server. It is a sibling, with its own deployment, its own port, its own auth path. + +## What + +A new binary `frond-server` that: + +- Listens on a configurable UDP port for QUIC (default proposal: 4433). +- Negotiates a single ALPN: `bes.canopy/1`. +- Authenticates clients with mTLS using **bare SPKI** (RFC 7250 raw public keys). For now: a single hardcoded allowed client SPKI (the contents of `identity.pub.pem` at the repo root). Wiring this to `device_keys` / `Device::from_key` is deferred — call out as a TODO. +- Generates QUIC connection IDs in QUIC-LB Plaintext format, embedding the per-pod 8-byte server ID from `AWS_LBC_QUIC_SERVER_ID` (base64-decoded) when present, falling back to a random ID for local dev. Single code path, env-var-as-parameter. +- Speaks the application protocol on top of QUIC streams. **Protocol semantics (messages, framing, multiplexing) are deliberately out of scope for this plan and will be covered by a follow-up.** + +## Crate layout + +A new workspace member `crates/frond-server/`. Reuses `commons-errors`, `commons-types`, and `commons-servers::health` for the HTTP health sidecar (Phase 5). No `database` dependency until the SPKI lookup is wired up post-MVP. + +Sketch of `Cargo.toml`: + +```toml +[package] +publish = false +name = "frond-server" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0-or-later" + +[[bin]] +name = "frond-server" +required-features = ["cli"] + +[dependencies] +quinn = "0.11" +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +rustls-pemfile = "2" +clap = { workspace = true, features = ["derive", "env"], optional = true } +commons-errors = { path = "../commons-errors" } +commons-servers = { path = "../commons-servers" } +commons-types = { path = "../commons-types" } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } +tracing.workspace = true +lloggs = { workspace = true, optional = true, features = ["miette-7"] } +miette = { workspace = true, optional = true, features = ["fancy"] } +serde = { workspace = true, features = ["derive"] } +base64 = "0.22" + +[features] +default = ["cli"] +cli = ["dep:clap", "dep:lloggs", "dep:miette"] +``` + +## Phases + +Each phase is a commit (or a small contiguous group of commits, per "commit as you write"). Plan→implement→unplan when all phases land. + +### Phase 0 — META_LOG → CANOPY_LOG rename + +Repo-wide chore preceding frond-server work, since the new binary should use `CANOPY_LOG` from day one and we want a single rename rather than a mixed naming scheme. + +- Replace `META_LOG` with `CANOPY_LOG` across the workspace (public-server, private-server, jobs, anywhere else it appears). +- Update Helm/manifest hand-off notes if any are tracked here. +- Single commit `chore: rename META_LOG to CANOPY_LOG`. + +### Phase 1 — scaffold + +- Add `crates/frond-server/` to workspace `members`. +- Empty `lib.rs`, minimal `main.rs` that parses args (port, bind, logging via `CANOPY_LOG`) and exits. +- `just watch-frond` recipe analogous to `watch-public`. +- `cargo check` passes. + +### Phase 2 — QUIC listener with ALPN, throwaway identity + +- Wire `quinn::Endpoint::server` with a freshly-generated server keypair at process start. **TODO marker:** persist this in the database and load on startup; deferred to a follow-up plan. +- ALPN: `bes.canopy/1` only (reject everything else). +- Bind: same shape as public-server (`PORT` env, `BIND_ADDRESS` env, default `7899`, IPv6 localhost in dev). +- Accept loop: log peer, ALPN, then close. +- Smoke-test integration test in `tests/connect.rs` using a quinn client in-process. + +### Phase 3 — QUIC-LB Plaintext CID generator + +- Implement a custom `quinn_proto::ConnectionIdGenerator` that emits 16-byte CIDs in QUIC-LB Plaintext format: + - byte 0: config rotation byte per draft-ietf-quic-load-balancers §5.2 + - bytes 1–8: 8-byte server ID + - bytes 9–15: 7 random bytes (matches AWS LBC's `nonce_length_bytes: 7`) +- Read `AWS_LBC_QUIC_SERVER_ID` env var, base64-decode, expect 8 bytes. +- Fallback: 8 random bytes at startup if env var absent. Same code path either way. +- Unit tests: encoding shape, server-ID extraction round-trip, random fallback. +- **Pre-spike before coding:** (a) check whether a `quic-lb` quinn-ecosystem crate already exists; if so, use it; (b) read draft-ietf-quic-load-balancers §5.2 to nail down the exact bit layout of byte 0 (config rotation bits + length encoding), since that's the load-bearing detail AWS will decode against. + +### Phase 4 — bare SPKI mTLS + +- Configure rustls with `RawPublicKey` certificate type (rustls 0.23+). +- **Server identity:** generated fresh at process start (Phase 2 already does this). TODO marker for database-backed persistence. +- **Client verification:** + - Custom `ClientCertVerifier` extracting the client SPKI bytes. + - For now: a single hardcoded allowed SPKI, embedded as a `const` in the source — the contents of `identity.pub.pem` from the repo root, decoded to its raw SPKI byte form at compile time (or `LazyLock` at runtime if compile-time decoding is awkward). + - Reject any client whose SPKI doesn't match. No auto-create, no DB lookup. + - **TODO:** swap to `Device::from_key` lookup once the database wiring lands. The client-cert-verifier seam is the only place that changes. + +### Phase 5 — graceful shutdown + HTTP health sidecar + +- SIGTERM handler: stop accepting new connections, signal connection close to peers, `Endpoint::wait_idle` with a deadline (default 60s, `SHUTDOWN_GRACE_SECONDS` env). +- Sibling HTTP/1 listener on a separate port (`HEALTH_PORT` env, default TBD — pick something near 7899 that doesn't clash, e.g. 7900) running `commons_servers::health::routes()` — `/livez` and `/healthz` already exist. NLB's HTTP/TCP target group health-checks point at this port. This is required because UDP target groups don't support UDP-level checks. + +### Phase 6 — observability + +- `tracing` spans per connection: peer IP, device_id, ALPN, CID prefix. +- A few counters: active connections, accepted, rejected, handshake errors. +- Match the lloggs/miette setup public-server uses (`PreArgs::parse_with_env`). + +### Phase 7 — release plumbing + +- `release.toml` mirroring public-server's. +- Whatever CI step builds release binaries needs to be updated; assumed external to this repo (confirm). + +### Phase 8 — application protocol [DEFERRED] + +Wire-protocol semantics for `bes.canopy/1` are explicitly out of scope here. Once frond-server can accept-auth-and-disconnect cleanly, a separate plan defines what messages flow over the streams. + +## Deployment requirements (out-of-repo) + +Deployment manifests don't live in this repo, but the contract frond-server expects from the cluster is: + +- **NLB QUIC listener** on the chosen port: + - `service.beta.kubernetes.io/aws-load-balancer-type: external` + - `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip` + - `service.beta.kubernetes.io/aws-load-balancer-quic-enabled-ports: ""` + - NLB has **no security groups** (AWS QUIC limitation). + - **No IPv6 target group** (AWS QUIC limitation). IPv4 only on the data path. +- **CID injection:** + - Namespace label `elbv2.k8s.aws/quic-server-id-inject=enabled`. + - Pod annotation `service.beta.kubernetes.io/aws-load-balancer-quic-enabled-containers: frond-server`. + - LBC version must support server-ID injection — confirm cluster's LBC version before deployment. +- **HTTP health target group** registered against the sidecar port, checking `/livez`. +- `terminationGracePeriodSeconds: 90` (drain window > Phase 5 grace). +- `PodDisruptionBudget` with `maxUnavailable: 1`. +- QUIC v1 only — quinn defaults match. + +## Local dev + +- `just watch-frond` runs the binary against `127.0.0.1:7899` (or `[::1]`) with a freshly-generated server keypair each restart (no persistence yet). +- `tests/connect.rs` integration test: spin up server + quinn client in-process, complete handshake using the hardcoded `identity.pub.pem` SPKI as the client identity, exchange a ping. +- No `AWS_LBC_QUIC_SERVER_ID` → random 8-byte server ID. Behaviour identical to prod, just routes-to-itself. + +## Settled decisions + +- **QUIC port:** 7899. +- **Health port:** TBD near 7899 (e.g. 7900); HTTP-style with `/livez`/`/healthz` via `commons_servers::health::routes()`. +- **Unknown SPKIs are rejected.** No auto-create. (Public-server may shift to the same posture in a separate change.) +- **Client allowlist:** single hardcoded SPKI from `identity.pub.pem` for now, deferred to database-backed lookup. +- **Server identity:** generated fresh at process start, no persistence yet, marked TODO for database-backed storage. +- **Log env var:** `CANOPY_LOG` everywhere (Phase 0 renames the existing `META_LOG` usages too). + +## Risks + +- `quinn_proto::ConnectionIdGenerator` may constrain CID shape in ways that don't match the QUIC-LB plaintext format. Mitigation: spike Phase 3 first — if quinn restricts us, escalate before doing more work. +- rustls `RawPublicKey` support is recent; the `ClientCertVerifier` API may not surface raw SPKI bytes cleanly. Mitigation: prototype Phase 4 with a no-op verifier, layer auth on top once the wire path works. +- AWS may change the QUIC-LB draft version it tracks. The risk is small (AWS docs say "stable for several months") but the encoding is the contract. + +## Out of scope + +- HTTP/3, WebTransport. +- Application protocol design (separate plan). +- Migrating any existing public-server traffic to QUIC. +- Multi-region / multi-cluster QUIC steering. From 27058c15d07313ef0ff970869dbd11963cfc0c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 5 May 2026 15:34:19 +1200 Subject: [PATCH 2/4] chore: rename META_LOG to CANOPY_LOG --- crates/private-server/src/main.rs | 2 +- crates/public-server/src/main.rs | 2 +- private-web/e2e/fixture.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/private-server/src/main.rs b/crates/private-server/src/main.rs index 43dde9f..5dc64ed 100644 --- a/crates/private-server/src/main.rs +++ b/crates/private-server/src/main.rs @@ -22,7 +22,7 @@ async fn main() -> miette::Result<()> { use lloggs::PreArgs; use private_server::state::AppState; - let mut _guard = PreArgs::parse_with_env("META_LOG").setup()?; + let mut _guard = PreArgs::parse_with_env("CANOPY_LOG").setup()?; let args = Args::parse(); if _guard.is_none() { _guard = Some(args.logging.setup(|v| match v { diff --git a/crates/public-server/src/main.rs b/crates/public-server/src/main.rs index ed48653..14383e7 100644 --- a/crates/public-server/src/main.rs +++ b/crates/public-server/src/main.rs @@ -23,7 +23,7 @@ struct Args { #[tokio::main] async fn main() -> miette::Result<()> { - let mut _guard = PreArgs::parse_with_env("META_LOG").setup()?; + let mut _guard = PreArgs::parse_with_env("CANOPY_LOG").setup()?; let args = Args::parse(); if _guard.is_none() { _guard = Some(args.logging.setup(|v| match v { diff --git a/private-web/e2e/fixture.ts b/private-web/e2e/fixture.ts index c46f7cd..77444d2 100644 --- a/private-web/e2e/fixture.ts +++ b/private-web/e2e/fixture.ts @@ -184,7 +184,7 @@ export async function startStack(opts: StartOptions = {}): Promise ...process.env, DATABASE_URL: databaseUrl, BIND_ADDRESS: `127.0.0.1:${apiPort}`, - META_LOG: process.env.META_LOG ?? "private_server=info,warn", + CANOPY_LOG: process.env.CANOPY_LOG ?? "private_server=info,warn", }, }); pipeOutput(api, "api", silent); From 88c86c38aca83c1d55bbcef7e67f56f8bf3a9cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 5 May 2026 15:35:39 +1200 Subject: [PATCH 3/4] feat(frond): scaffold frond-server crate --- Cargo.lock | 11 ++++++++++ Cargo.toml | 1 + crates/frond-server/Cargo.toml | 26 ++++++++++++++++++++++++ crates/frond-server/src/lib.rs | 4 ++++ crates/frond-server/src/main.rs | 36 +++++++++++++++++++++++++++++++++ justfile | 4 ++++ 6 files changed, 82 insertions(+) create mode 100644 crates/frond-server/Cargo.toml create mode 100644 crates/frond-server/src/lib.rs create mode 100644 crates/frond-server/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 693d1ea..9d81d76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1346,6 +1346,17 @@ dependencies = [ "postgres-types", ] +[[package]] +name = "frond-server" +version = "0.1.0" +dependencies = [ + "clap", + "lloggs", + "miette", + "tokio", + "tracing", +] + [[package]] name = "fs_extra" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index e08585c..f646ba2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/commons-tests", "crates/commons-types", "crates/database", + "crates/frond-server", "crates/jobs", "crates/private-server", "crates/public-server", diff --git a/crates/frond-server/Cargo.toml b/crates/frond-server/Cargo.toml new file mode 100644 index 0000000..b685f8c --- /dev/null +++ b/crates/frond-server/Cargo.toml @@ -0,0 +1,26 @@ +[package] +publish = false +name = "frond-server" +version = "0.1.0" +edition = "2024" +resolver = "3" +license = "GPL-3.0-or-later" +authors = [ + "Félix Saparelli ", + "BES Developers ", +] + +[[bin]] +name = "frond-server" +required-features = ["cli"] + +[dependencies] +clap = { workspace = true, optional = true, features = ["derive", "env"] } +lloggs = { workspace = true, optional = true, features = ["miette-7"] } +miette = { workspace = true, optional = true, features = ["fancy"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } +tracing.workspace = true + +[features] +default = ["cli"] +cli = ["dep:clap", "dep:lloggs", "dep:miette"] diff --git a/crates/frond-server/src/lib.rs b/crates/frond-server/src/lib.rs new file mode 100644 index 0000000..4bcb06b --- /dev/null +++ b/crates/frond-server/src/lib.rs @@ -0,0 +1,4 @@ +// frond-server: a QUIC endpoint for canopy speaking the bes.canopy/1 protocol. +// +// Currently a scaffold. Real listener wiring lands in subsequent phases of +// `docs/plans/frond-server.md`. diff --git a/crates/frond-server/src/main.rs b/crates/frond-server/src/main.rs new file mode 100644 index 0000000..42ea015 --- /dev/null +++ b/crates/frond-server/src/main.rs @@ -0,0 +1,36 @@ +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; + +use clap::Parser; +use lloggs::{LoggingArgs, PreArgs}; + +#[derive(Debug, Parser)] +struct Args { + #[command(flatten)] + logging: LoggingArgs, + + #[arg(long, short, default_value = "7899", env = "PORT")] + port: u16, + + #[arg(long, env = "BIND_ADDRESS", conflicts_with = "port")] + bind: Option, +} + +#[tokio::main] +async fn main() -> miette::Result<()> { + let mut _guard = PreArgs::parse_with_env("CANOPY_LOG").setup()?; + let args = Args::parse(); + if _guard.is_none() { + _guard = Some(args.logging.setup(|v| match v { + 0 => "info", + 1 => "debug", + _ => "trace", + })?); + } + + let addr = args + .bind + .unwrap_or_else(|| SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, args.port, 0, 0))); + + tracing::info!(%addr, "frond-server scaffold; QUIC listener wiring lands in Phase 2"); + Ok(()) +} diff --git a/justfile b/justfile index d9f6b35..709a658 100644 --- a/justfile +++ b/justfile @@ -30,6 +30,10 @@ build-image: watch-public: watchexec -I -w crates -- cargo run --bin public-server +# Run the frond (QUIC) server and reload on change +watch-frond: + watchexec -I -w crates -- cargo run --bin frond-server + # Rebuild the private-server binary on source change (pair with watch-private-api) watch-private-build: watchexec -I -w crates -- cargo build --bin private-server From d4b3d234a31dfb56634411b5103971ba5bdc4a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 5 May 2026 15:48:23 +1200 Subject: [PATCH 4/4] feat(frond): bind QUIC listener with bes.canopy/1 ALPN Generates an ephemeral Ed25519 keypair, builds rustls config with RFC 7250 raw-public-key support on both server and client cert paths, and binds a quinn endpoint that accepts connections on the negotiated bes.canopy/1 ALPN. Phase 2 stub: each accepted connection is logged and immediately closed; real stream handling lands later. Server identity is in-memory only with a TODO marker for database-backed persistence. Client verification is permissive (accepts any well-formed RPK); Phase 4 swaps in an allowlist verifier pinned to identity.pub.pem. Includes an integration smoke-test that connects with a quinn client using ephemeral RPK identity and asserts the ALPN is negotiated. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 223 ++++++++++++++++++++++++++- crates/frond-server/Cargo.toml | 11 +- crates/frond-server/src/keys.rs | 40 +++++ crates/frond-server/src/lib.rs | 13 +- crates/frond-server/src/main.rs | 11 +- crates/frond-server/src/server.rs | 58 +++++++ crates/frond-server/src/tls.rs | 111 +++++++++++++ crates/frond-server/tests/connect.rs | 143 +++++++++++++++++ 8 files changed, 602 insertions(+), 8 deletions(-) create mode 100644 crates/frond-server/src/keys.rs create mode 100644 crates/frond-server/src/server.rs create mode 100644 crates/frond-server/src/tls.rs create mode 100644 crates/frond-server/tests/connect.rs diff --git a/Cargo.lock b/Cargo.lock index 9d81d76..5286662 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -501,6 +501,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -895,6 +901,33 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.21.3" @@ -1185,6 +1218,31 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -1280,6 +1338,24 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fastbloom" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +dependencies = [ + "getrandom 0.3.4", + "libm", + "rand 0.9.4", + "siphasher", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1351,8 +1427,15 @@ name = "frond-server" version = "0.1.0" dependencies = [ "clap", + "ed25519-dalek", "lloggs", "miette", + "quinn", + "rand_core 0.6.4", + "rustls", + "rustls-pki-types", + "sha2 0.11.0", + "subtle", "tokio", "tracing", ] @@ -2025,6 +2108,22 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2034,7 +2133,7 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys", + "jni-sys 0.4.1", "log", "simd_cesu8", "thiserror 2.0.18", @@ -2055,6 +2154,15 @@ dependencies = [ "syn", ] +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + [[package]] name = "jni-sys" version = "0.4.1" @@ -2708,6 +2816,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -2968,6 +3086,7 @@ checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", + "fastbloom", "getrandom 0.3.4", "lru-slab", "rand 0.9.4", @@ -2975,6 +3094,7 @@ dependencies = [ "rustc-hash", "rustls", "rustls-pki-types", + "rustls-platform-verifier 0.6.2", "slab", "thiserror 2.0.18", "tinyvec", @@ -3211,7 +3331,7 @@ dependencies = [ "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.0", "serde", "serde_json", "sync_wrapper", @@ -3389,6 +3509,7 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3417,6 +3538,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -3425,7 +3567,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls", @@ -3652,6 +3794,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.9" @@ -4708,6 +4859,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -4744,6 +4904,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -4777,6 +4952,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4789,6 +4970,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4801,6 +4988,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4825,6 +5018,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4837,6 +5036,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4849,6 +5054,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4861,6 +5072,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/crates/frond-server/Cargo.toml b/crates/frond-server/Cargo.toml index b685f8c..f756267 100644 --- a/crates/frond-server/Cargo.toml +++ b/crates/frond-server/Cargo.toml @@ -16,11 +16,18 @@ required-features = ["cli"] [dependencies] clap = { workspace = true, optional = true, features = ["derive", "env"] } +ed25519-dalek = { version = "2.2.0", features = ["pkcs8", "rand_core"] } lloggs = { workspace = true, optional = true, features = ["miette-7"] } -miette = { workspace = true, optional = true, features = ["fancy"] } +miette = { workspace = true, features = ["fancy"] } +quinn = "0.11.9" +rand_core = { version = "0.6", features = ["getrandom"] } +rustls = { version = "0.23.40", default-features = false, features = ["ring", "std"] } +rustls-pki-types = "1.14.1" +sha2 = "0.11.0" +subtle = "2.6.1" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } tracing.workspace = true [features] default = ["cli"] -cli = ["dep:clap", "dep:lloggs", "dep:miette"] +cli = ["dep:clap", "dep:lloggs"] diff --git a/crates/frond-server/src/keys.rs b/crates/frond-server/src/keys.rs new file mode 100644 index 0000000..bcc8d4c --- /dev/null +++ b/crates/frond-server/src/keys.rs @@ -0,0 +1,40 @@ +use ed25519_dalek::SigningKey; +use rand_core::OsRng; +use sha2::{Digest, Sha256}; + +/// Generate a fresh Ed25519 keypair in memory. +/// +/// TODO: persist this in the database and load on startup so that the +/// server's SPKI is stable across restarts. +pub fn generate_ephemeral() -> SigningKey { + SigningKey::generate(&mut OsRng) +} + +/// Build the SubjectPublicKeyInfo (SPKI) DER encoding for an Ed25519 key. +/// +/// ```text +/// SEQUENCE { +/// SEQUENCE { OID 1.3.101.112 } -- Ed25519 +/// BIT STRING { 0x00 || 32-byte key } +/// } +/// ``` +pub fn spki_der(key: &SigningKey) -> Vec { + const PREFIX: [u8; 12] = [ + 0x30, 0x2a, // SEQUENCE 42 bytes + 0x30, 0x05, // SEQUENCE 5 bytes + 0x06, 0x03, 0x2b, 0x65, 0x70, // OID 1.3.101.112 + 0x03, 0x21, 0x00, // BIT STRING 33 bytes, 0 unused bits + ]; + let mut out = Vec::with_capacity(44); + out.extend_from_slice(&PREFIX); + out.extend_from_slice(key.verifying_key().as_bytes()); + out +} + +/// SHA-256 fingerprint of a byte slice as lowercase hex. +pub fn fingerprint(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} diff --git a/crates/frond-server/src/lib.rs b/crates/frond-server/src/lib.rs index 4bcb06b..98997e1 100644 --- a/crates/frond-server/src/lib.rs +++ b/crates/frond-server/src/lib.rs @@ -1,4 +1,13 @@ // frond-server: a QUIC endpoint for canopy speaking the bes.canopy/1 protocol. // -// Currently a scaffold. Real listener wiring lands in subsequent phases of -// `docs/plans/frond-server.md`. +// See `docs/plans/frond-server.md` for the staged build-out. + +pub mod keys; +pub mod server; +pub mod tls; + +pub use server::{accept_loop, bind}; + +/// ALPN negotiated for frond-server QUIC connections. Bumping this string is +/// the lever for incompatible protocol revisions. +pub const ALPN: &[u8] = b"bes.canopy/1"; diff --git a/crates/frond-server/src/main.rs b/crates/frond-server/src/main.rs index 42ea015..7650d5c 100644 --- a/crates/frond-server/src/main.rs +++ b/crates/frond-server/src/main.rs @@ -2,6 +2,7 @@ use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use clap::Parser; use lloggs::{LoggingArgs, PreArgs}; +use miette::IntoDiagnostic; #[derive(Debug, Parser)] struct Args { @@ -27,10 +28,18 @@ async fn main() -> miette::Result<()> { })?); } + rustls::crypto::ring::default_provider() + .install_default() + .expect("ring crypto provider already installed"); + let addr = args .bind .unwrap_or_else(|| SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, args.port, 0, 0))); - tracing::info!(%addr, "frond-server scaffold; QUIC listener wiring lands in Phase 2"); + let endpoint = frond_server::bind(addr)?; + let local = endpoint.local_addr().into_diagnostic()?; + tracing::info!(%local, "frond-server listening"); + + frond_server::accept_loop(endpoint).await; Ok(()) } diff --git a/crates/frond-server/src/server.rs b/crates/frond-server/src/server.rs new file mode 100644 index 0000000..f22acee --- /dev/null +++ b/crates/frond-server/src/server.rs @@ -0,0 +1,58 @@ +use std::{net::SocketAddr, sync::Arc}; + +use miette::{Context, IntoDiagnostic, Result}; +use quinn::Endpoint; + +use crate::{keys, tls}; + +/// Bind a fresh frond-server endpoint on `addr`. +/// +/// Generates an ephemeral Ed25519 keypair, builds the rustls + quinn config, +/// and returns the listening endpoint. The caller drives the accept loop. +pub fn bind(addr: SocketAddr) -> Result { + let key = keys::generate_ephemeral(); + let spki = keys::spki_der(&key); + let fp = keys::fingerprint(&spki); + tracing::info!(server_fingerprint = %fp, "frond-server identity (ephemeral; TODO: persist)"); + + let tls = tls::build_server_config(&key, spki) + .map_err(|e| miette::miette!("building TLS config: {e}"))?; + let quic = quinn::crypto::rustls::QuicServerConfig::try_from(tls) + .into_diagnostic() + .wrap_err("converting rustls config to quinn QuicServerConfig")?; + + let server_config = quinn::ServerConfig::with_crypto(Arc::new(quic)); + Endpoint::server(server_config, addr) + .into_diagnostic() + .wrap_err_with(|| format!("binding QUIC endpoint on {addr}")) +} + +/// Run the accept loop until the endpoint is closed. +/// +/// Phase 2 stub: each accepted connection is logged and immediately closed. +/// Real stream handling lands in a later phase. +pub async fn accept_loop(endpoint: Endpoint) { + while let Some(incoming) = endpoint.accept().await { + tokio::spawn(handle_incoming(incoming)); + } +} + +async fn handle_incoming(incoming: quinn::Incoming) { + let peer = incoming.remote_address(); + match incoming.await { + Ok(conn) => { + let alpn = conn + .handshake_data() + .and_then(|d| { + d.downcast::() + .ok() + .and_then(|d| d.protocol.clone()) + }) + .map(|p| String::from_utf8_lossy(&p).into_owned()) + .unwrap_or_default(); + tracing::info!(%peer, %alpn, "accepted connection (Phase 2 stub: closing)"); + conn.close(0u32.into(), b"phase 2 stub"); + } + Err(e) => tracing::warn!(%peer, "incoming connection failed: {e}"), + } +} diff --git a/crates/frond-server/src/tls.rs b/crates/frond-server/src/tls.rs new file mode 100644 index 0000000..8e342c1 --- /dev/null +++ b/crates/frond-server/src/tls.rs @@ -0,0 +1,111 @@ +use std::sync::Arc; + +use ed25519_dalek::{SigningKey, pkcs8::EncodePrivateKey}; +use rustls::{ + DigitallySignedStruct, DistinguishedName, ServerConfig, SignatureScheme, + client::danger::HandshakeSignatureValid, + server::{ + AlwaysResolvesServerRawPublicKeys, + danger::{ClientCertVerified, ClientCertVerifier}, + }, + sign::CertifiedKey, +}; +use rustls_pki_types::{ + CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, SubjectPublicKeyInfoDer, UnixTime, +}; + +use crate::ALPN; + +/// Build the rustls server config used by frond-server. +/// +/// `key` is the server's signing key (paired with `spki`, its DER-encoded +/// SubjectPublicKeyInfo). The resulting config: +/// +/// - Presents `spki` as a raw public key (RFC 7250) instead of an X.509 cert. +/// - Requires clients to do the same. +/// - Negotiates only the `bes.canopy/1` ALPN. +/// - Uses [`PermissiveClientVerifier`] so any well-formed client RPK is +/// accepted. Phase 4 swaps this for an allowlist verifier. +pub fn build_server_config( + key: &SigningKey, + spki: Vec, +) -> Result> { + let pkcs8 = key.to_pkcs8_der()?; + let private_key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(pkcs8.as_bytes().to_vec())); + let signing_key = rustls::crypto::ring::sign::any_supported_type(&private_key)?; + + let cert = CertificateDer::from(spki); + let certified_key = Arc::new(CertifiedKey::new(vec![cert], signing_key)); + let resolver = Arc::new(AlwaysResolvesServerRawPublicKeys::new(certified_key)); + + let client_verifier: Arc = Arc::new(PermissiveClientVerifier); + + let mut tls = ServerConfig::builder() + .with_client_cert_verifier(client_verifier) + .with_cert_resolver(resolver); + + tls.alpn_protocols = vec![ALPN.to_vec()]; + Ok(tls) +} + +/// Phase 2 stand-in: accepts any well-formed client raw public key without +/// allowlist or DB lookup. Phase 4 of `docs/plans/frond-server.md` replaces +/// this with a verifier that pins a specific SPKI from `identity.pub.pem`, +/// and a later phase swaps in a `device_keys` lookup. +#[derive(Debug)] +pub struct PermissiveClientVerifier; + +impl ClientCertVerifier for PermissiveClientVerifier { + fn root_hint_subjects(&self) -> &[DistinguishedName] { + &[] + } + + fn verify_client_cert( + &self, + end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _now: UnixTime, + ) -> Result { + let fp = crate::keys::fingerprint(end_entity.as_ref()); + tracing::debug!(client_fingerprint = %fp, "permissive verifier accepted client"); + Ok(ClientCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &rustls::crypto::ring::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature_with_raw_key( + message, + &SubjectPublicKeyInfoDer::from(cert.as_ref()), + dss, + &rustls::crypto::ring::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + + fn requires_raw_public_keys(&self) -> bool { + true + } +} diff --git a/crates/frond-server/tests/connect.rs b/crates/frond-server/tests/connect.rs new file mode 100644 index 0000000..5d049d9 --- /dev/null +++ b/crates/frond-server/tests/connect.rs @@ -0,0 +1,143 @@ +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, + time::Duration, +}; + +use ed25519_dalek::pkcs8::EncodePrivateKey; +use frond_server::{ALPN, keys}; +use rustls::{ + ClientConfig as TlsClientConfig, DigitallySignedStruct, SignatureScheme, + client::{ + AlwaysResolvesClientRawPublicKeys, + danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + }, + sign::CertifiedKey, +}; +use rustls_pki_types::{ + CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName, SubjectPublicKeyInfoDer, + UnixTime, +}; + +fn install_provider() { + use std::sync::Once; + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + rustls::crypto::ring::default_provider() + .install_default() + .expect("install ring crypto provider"); + }); +} + +#[derive(Debug)] +struct AcceptAnyServer; + +impl ServerCertVerifier for AcceptAnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &rustls::crypto::ring::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature_with_raw_key( + message, + &SubjectPublicKeyInfoDer::from(cert.as_ref()), + dss, + &rustls::crypto::ring::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + + fn requires_raw_public_keys(&self) -> bool { + true + } +} + +fn build_test_client_config() -> TlsClientConfig { + let key = keys::generate_ephemeral(); + let spki = keys::spki_der(&key); + let pkcs8 = key.to_pkcs8_der().unwrap(); + let private_key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(pkcs8.as_bytes().to_vec())); + let signing = rustls::crypto::ring::sign::any_supported_type(&private_key).unwrap(); + let cert = CertificateDer::from(spki); + let certified = Arc::new(CertifiedKey::new(vec![cert], signing)); + let resolver = Arc::new(AlwaysResolvesClientRawPublicKeys::new(certified)); + + let mut tls = TlsClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServer)) + .with_client_cert_resolver(resolver); + tls.alpn_protocols = vec![ALPN.to_vec()]; + tls +} + +#[tokio::test(flavor = "multi_thread")] +async fn handshake_negotiates_bes_canopy_1() { + install_provider(); + + let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0)); + let server_endpoint = frond_server::bind(bind_addr).expect("bind server"); + let server_addr = server_endpoint.local_addr().expect("local addr"); + tokio::spawn(frond_server::accept_loop(server_endpoint)); + + let tls = build_test_client_config(); + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); + let client_cfg = quinn::ClientConfig::new(Arc::new(quic)); + let mut client = quinn::Endpoint::client(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::UNSPECIFIED, + 0, + ))) + .expect("client endpoint"); + client.set_default_client_config(client_cfg); + + let conn = tokio::time::timeout( + Duration::from_secs(5), + client.connect(server_addr, "frond").expect("connect call"), + ) + .await + .expect("connect timeout") + .expect("handshake"); + + let handshake = conn.handshake_data().expect("handshake data"); + let data = handshake + .downcast::() + .expect("rustls handshake data"); + assert_eq!( + data.protocol.as_deref(), + Some(ALPN), + "server should negotiate the bes.canopy/1 ALPN" + ); + + conn.close(0u32.into(), b"test done"); + client.wait_idle().await; +}