diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..96a21aa --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.github +target +.runtime +*.log +tests +verification +docs +*.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cf5199..993c501 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,13 @@ jobs: run: ./scripts/verify.sh all --profile ci - name: Shellcheck - run: shellcheck scripts/verify.sh scripts/lib/*.sh scripts/phases/*.sh - continue-on-error: true + run: shellcheck --shell=bash scripts/verify.sh scripts/lib/*.sh scripts/phases/*.sh start.sh stop.sh install.sh + + - name: Cargo audit + run: cargo install cargo-audit --locked && cargo audit + + - name: Cargo deny + run: cargo install cargo-deny --locked && cargo deny check release-build: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index bff24b8..2c3277d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,10 +89,10 @@ dependencies = [ "axum-macros", "bytes", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.10.1", + "hyper", "hyper-util", "itoa", "matchit", @@ -105,7 +105,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower 0.5.3", "tower-layer", @@ -122,13 +122,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.2", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -147,15 +147,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" @@ -191,6 +185,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" version = "4.6.1" @@ -237,22 +237,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "displaydoc" version = "0.2.6" @@ -264,21 +248,6 @@ dependencies = [ "syn", ] -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -301,12 +270,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -383,27 +346,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] -name = "h2" -version = "0.3.27" +name = "getrandom" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", ] [[package]] @@ -418,17 +378,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.4.2" @@ -439,17 +388,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -457,7 +395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.2", + "http", ] [[package]] @@ -468,8 +406,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -485,30 +423,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.10.1" @@ -519,28 +433,31 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", + "http", + "hyper", + "hyper-util", "rustls", "tokio", "tokio-rustls", + "tower-service", + "webpki-roots", ] [[package]] @@ -549,13 +466,21 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", - "http 1.4.2", - "http-body 1.0.1", - "hyper 1.10.1", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2", "tokio", "tower-service", + "tracing", ] [[package]] @@ -733,6 +658,12 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.2.0" @@ -802,13 +733,12 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-stream", - "tokio-test", "toml", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "tracing-subscriber", ] @@ -877,6 +807,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -886,6 +825,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.46" @@ -895,13 +889,48 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags", ] [[package]] @@ -923,38 +952,36 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", + "http", + "http-body", + "http-body-util", + "hyper", "hyper-rustls", - "ipnet", + "hyper-util", "js-sys", "log", - "mime", - "once_cell", "percent-encoding", "pin-project-lite", + "quinn", "rustls", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration", + "sync_wrapper", "tokio", "tokio-rustls", - "tokio-socks", "tokio-util", + "tower 0.5.3", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -962,7 +989,6 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots", - "winreg", ] [[package]] @@ -973,40 +999,50 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustls" -version = "0.21.12" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ - "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] -name = "rustls-pemfile" -version = "1.0.4" +name = "rustls-pki-types" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ - "base64", + "web-time", + "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1028,16 +1064,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "serde" version = "1.0.228" @@ -1150,16 +1176,6 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.4" @@ -1182,6 +1198,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.118" @@ -1193,17 +1215,14 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -1217,40 +1236,39 @@ dependencies = [ ] [[package]] -name = "system-configuration" -version = "0.5.1" +name = "thiserror" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", + "thiserror-impl 1.0.69", ] [[package]] -name = "system-configuration-sys" -version = "0.5.0" +name = "thiserror" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "core-foundation-sys", - "libc", + "thiserror-impl 2.0.18", ] [[package]] -name = "thiserror" +name = "thiserror-impl" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "thiserror-impl", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "thiserror-impl" -version = "1.0.69" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -1276,6 +1294,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -1288,7 +1321,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -1306,26 +1339,14 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", ] -[[package]] -name = "tokio-socks" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e2948f60dbe26b35f2c7fb74ac2854c1fddded0fe9d7548fcc674a246f7615" -dependencies = [ - "either", - "futures-util", - "thiserror", - "tokio", -] - [[package]] name = "tokio-stream" version = "0.1.18" @@ -1337,17 +1358,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-test" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" -dependencies = [ - "futures-core", - "tokio", - "tokio-stream", -] - [[package]] name = "tokio-util" version = "0.7.18" @@ -1428,7 +1438,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -1441,10 +1451,10 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.13.0", + "bitflags", "bytes", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "pin-project-lite", "tower-layer", @@ -1452,6 +1462,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower 0.5.3", + "tower-layer", + "tower-service", + "url", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -1589,6 +1617,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -1667,11 +1704,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" -version = "0.25.4" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "windows-link" @@ -1679,22 +1729,13 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1706,67 +1747,34 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1779,48 +1787,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -1837,14 +1821,10 @@ dependencies = [ ] [[package]] -name = "winreg" -version = "0.50.0" +name = "wit-bindgen" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" @@ -1875,6 +1855,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.8" @@ -1896,6 +1896,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index f1e55ff..0123154 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio-stream = { version = "0.1", features = ["net"] } futures-util = "0.3" -reqwest = { version = "0.11", default-features = false, features = ["json", "stream", "socks", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "socks", "rustls-tls"] } tower = { version = "0.4", features = ["limit", "buffer", "util"] } tower-http = { version = "0.5", features = ["trace", "limit"] } tracing = "0.1" @@ -27,8 +27,7 @@ toml = "0.8" clap = { version = "4.4", features = ["derive"] } [dev-dependencies] -reqwest = { version = "0.11", default-features = false, features = ["json", "stream", "socks", "rustls-tls"] } -tokio-test = "0.4" +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "socks", "rustls-tls"] } [profile.release] lto = true diff --git a/Dockerfile b/Dockerfile index 82d2314..08c1e6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM rust:1.86-alpine AS builder -RUN apk add musl-dev openssl-dev openssl-libs-static +RUN apk add --no-cache musl-dev gcc WORKDIR /app COPY Cargo.toml Cargo.lock ./ diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..1941e12 --- /dev/null +++ b/deny.toml @@ -0,0 +1,28 @@ +# cargo-deny configuration +[advisories] +ignore = [] +[licenses] +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-3-Clause", + "ISC", + "Unicode-3.0", + "CDLA-Permissive-2.0", +] +confidence-threshold = 0.8 +[bans] +multiple-versions = "deny" +wildcards = "allow" +skip = [ + # tower 0.4 vs 0.5: 0.4 used directly, 0.5 required by axum + { crate = "tower", reason = "0.4 direct dep, 0.5 required by axum" }, + # tower-http 0.5 vs 0.6: 0.5 used directly, 0.6 required by reqwest + { crate = "tower-http", reason = "0.5 direct dep, 0.6 required by reqwest" }, + # windows-sys: unavoidable transitive dupes on linux + { crate = "windows-sys", reason = "unavoidable transitive dep on non-Windows" }, +] +skip-tree = [] +[sources] +allow-registry = ["https://github.com/rust-lang/crates.io-index"] diff --git a/install.sh b/install.sh index 994c0d4..0b5f24a 100755 --- a/install.sh +++ b/install.sh @@ -40,11 +40,11 @@ else fi # ── Logging helpers ─────────────────────────────────────────────────── -info() { printf "${BLUE}::${NC} %s\n" "$*"; } -ok() { printf "${GREEN}OK${NC} %s\n" "$*"; } -warn() { printf "${YELLOW}WARN${NC} %s\n" "$*"; } -err() { printf "${RED}ERR${NC} %s\n" "$*"; } -header() { printf "${BOLD}%s${NC}\n" "$*"; } +info() { printf '%s::%s %s\n' "${BLUE}" "${NC}" "$*"; } +ok() { printf '%sOK%s %s\n' "${GREEN}" "${NC}" "$*"; } +warn() { printf '%sWARN%s %s\n' "${YELLOW}" "${NC}" "$*"; } +err() { printf '%sERR%s %s\n' "${RED}" "${NC}" "$*"; } +header() { printf '%s%s%s\n' "${BOLD}" "$*" "${NC}"; } # ── Cleanup handler ─────────────────────────────────────────────────── cleanup() { @@ -173,7 +173,7 @@ choose_install_dir() { return fi - local home="${HOME:-$(echo ~)}" + local home="${HOME:-~}" # 2. /usr/local/bin — with sudo when needed if [ -d /usr/local/bin ]; then @@ -262,18 +262,16 @@ check_opencode() { ok "OpenCode CLI found: $(opencode --version 2>/dev/null | head -1)" else warn "OpenCode CLI is not installed — optional for monitoring (the bridge works without it)." - echo "" - printf " To install for health monitoring:\n" - echo "" - printf " ${CYAN}curl -fsSL https://docs.opencode.ai/install.sh | sh${NC}\n" - echo "" - printf " ${BOLD}Alternative methods:${NC}\n" - echo "" - printf " • npm: ${CYAN}npm install -g @opencode/cli${NC}\n" - printf " • brew: ${CYAN}brew install opencode-ai/cli/opencode${NC}\n" - printf " • cargo: ${CYAN}cargo install opencode-cli${NC}\n" - echo "" - printf " See: https://github.com/opencode-ai/opencode\n" + printf '%s\n' "" + printf ' %s%s%s\n' "${CYAN}" "curl -fsSL https://docs.opencode.ai/install.sh | sh" "${NC}" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Alternative methods:" "${NC}" + printf '%s\n' "" + printf ' %s%s %s%s%s\n' "• npm:" "${CYAN}" "npm install -g @opencode/cli" "${NC}" + printf ' %s%s %s%s%s\n' "• brew:" "${CYAN}" "brew install opencode-ai/cli/opencode" "${NC}" + printf ' %s%s %s%s%s\n' "• cargo:" "${CYAN}" "cargo install opencode-cli" "${NC}" + printf '%s\n' "" + printf ' %s\n' "See: https://github.com/opencode-ai/opencode" fi } @@ -286,34 +284,34 @@ check_warp() { reg_status="$(warp-cli registration show 2>/dev/null || true)" if echo "$reg_status" | grep -qi "error\|not registered\|no registration"; then warn "WARP CLI found but not registered." - echo "" - printf " ${BOLD}Register and start WARP:${NC}\n" - printf " ${CYAN}warp-cli registration new${NC}\n" - printf " ${CYAN}warp-cli mode proxy${NC}\n" - printf " ${CYAN}warp-cli connect${NC}\n" - echo "" - printf " ${BOLD}Then verify:${NC}\n" - printf " ${CYAN}warp-cli status${NC}\n" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Register and start WARP:" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli registration new" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli mode proxy" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli connect" "${NC}" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Then verify:" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli status" "${NC}" else ok "Cloudflare WARP CLI found — IP rotation enabled." fi else - echo "" + printf '%s\n' "" info "Tip: Install Cloudflare WARP for automatic IP rotation on rate-limit retry." - echo "" - printf " ${BOLD}1. Install WARP:${NC}\n" - printf " ${CYAN}curl -fsSL https://pkg.cloudflareclient.com/install.sh | sh${NC}\n" - echo "" - printf " ${BOLD}2. Register and start (first time only):${NC}\n" - printf " ${CYAN}warp-cli registration new${NC}\n" - printf " ${CYAN}warp-cli mode proxy${NC}\n" - printf " ${CYAN}warp-cli connect${NC}\n" - echo "" - printf " ${BOLD}3. Verify:${NC}\n" - printf " ${CYAN}warp-cli status${NC}\n" - echo "" - printf " ${BOLD}Docs:${NC}\n" - printf " https://developers.cloudflare.com/warp-client/get-started/linux/\n" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "1. Install WARP:" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "curl -fsSL https://pkg.cloudflareclient.com/install.sh | sh" "${NC}" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "2. Register and start (first time only):" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli registration new" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli mode proxy" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli connect" "${NC}" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "3. Verify:" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "warp-cli status" "${NC}" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Docs:" "${NC}" + printf ' %s\n' "https://developers.cloudflare.com/warp-client/get-started/linux/" fi } @@ -321,57 +319,57 @@ check_warp() { # Welcome message # ══════════════════════════════════════════════════════════════════════ print_welcome() { - echo "" + printf '%s\n' "" header "================================================" header " opencode2claude installed!" header "================================================" - echo "" - printf " ${BOLD}Quick start${NC}\n" - echo "" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Quick start" "${NC}" + printf '%s\n' "" if command -v opencode >/dev/null 2>&1; then - printf " 1. Start the bridge:\n" - printf " ${CYAN}opencode2claude${NC}\n" - echo "" - printf " 2. Use Claude Code with any LLM:\n" - printf " ${CYAN}claude${NC}\n" - echo "" - printf " 3. Use a specific model:\n" - printf " ${CYAN}OPENCODE_MODEL=\"openai/gpt-4o\" opencode2claude${NC}\n" + printf '%s\n' " 1. Start the bridge:" + printf ' %s%s%s\n' "${CYAN}" "opencode2claude" "${NC}" + printf '%s\n' "" + printf '%s\n' " 2. Use Claude Code with any LLM:" + printf ' %s%s%s\n' "${CYAN}" "claude" "${NC}" + printf '%s\n' "" + printf '%s\n' " 3. Use a specific model:" + printf ' %s%s%s\n' "${CYAN}" "OPENCODE_MODEL=\"openai/gpt-4o\" opencode2claude" "${NC}" else - printf " 1. Install OpenCode first, then start the bridge:\n" - printf " ${CYAN}curl -fsSL https://docs.opencode.ai/install.sh | sh${NC}\n" - printf " ${CYAN}opencode2claude${NC}\n" - echo "" - printf " 2. Use Claude Code with any LLM:\n" - printf " ${CYAN}claude${NC}\n" + printf '%s\n' " 1. Install OpenCode first, then start the bridge:" + printf ' %s%s%s\n' "${CYAN}" "curl -fsSL https://docs.opencode.ai/install.sh | sh" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "opencode2claude" "${NC}" + printf '%s\n' "" + printf '%s\n' " 2. Use Claude Code with any LLM:" + printf ' %s%s%s\n' "${CYAN}" "claude" "${NC}" fi - echo "" - printf " ${BOLD}Resources${NC}\n" - printf " ${GITHUB}\n" - printf " opencode2claude --help\n" - echo "" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Resources" "${NC}" + printf ' %s\n' "${GITHUB}" + printf ' %s\n' "opencode2claude --help" + printf '%s\n' "" } # ══════════════════════════════════════════════════════════════════════ # Fallback suggestions # ══════════════════════════════════════════════════════════════════════ suggest_fallback() { - echo "" + printf '%s\n' "" err "Binary download failed." - echo "" - printf " ${BOLD}Try one of these alternatives:${NC}\n" - echo "" - printf " 1. Install via Cargo (requires Rust toolchain):\n" - printf " ${CYAN}cargo install ${PROJECT}${NC}\n" - echo "" - printf " 2. Run via Docker:\n" - printf " ${CYAN}docker pull ghcr.io/${REPO}${NC}\n" - echo "" - printf " 3. Build from source:\n" - printf " ${CYAN}git clone ${GITHUB}.git${NC}\n" - printf " ${CYAN}cd ${PROJECT} && cargo build --release${NC}\n" - echo "" + printf '%s\n' "" + printf ' %s%s%s\n' "${BOLD}" "Try one of these alternatives:" "${NC}" + printf '%s\n' "" + printf '%s\n' " 1. Install via Cargo (requires Rust toolchain):" + printf ' %s%s%s\n' "${CYAN}" "cargo install ${PROJECT}" "${NC}" + printf '%s\n' "" + printf '%s\n' " 2. Run via Docker:" + printf ' %s%s%s\n' "${CYAN}" "docker pull ghcr.io/${REPO}" "${NC}" + printf '%s\n' "" + printf '%s\n' " 3. Build from source:" + printf ' %s%s%s\n' "${CYAN}" "git clone ${GITHUB}.git" "${NC}" + printf ' %s%s%s\n' "${CYAN}" "cd ${PROJECT} && cargo build --release" "${NC}" + printf '%s\n' "" } # ══════════════════════════════════════════════════════════════════════ @@ -391,7 +389,7 @@ main() { latest_tag="$(fetch_latest_version)" if [ -n "$latest_tag" ]; then - printf " Latest release: ${BOLD}%s${NC}\n" "$latest_tag" + printf ' Latest release: %s%s%s\n' "${BOLD}" "$latest_tag" "${NC}" # Strip prefix / suffix noise for simple string comparison installed_ver="$(printf '%s' "$existing" | sed 's/^[^0-9]*//' | sed 's/[^0-9.]*$//')" diff --git a/scripts/lib/process.sh b/scripts/lib/process.sh index 37c16dd..24905e9 100644 --- a/scripts/lib/process.sh +++ b/scripts/lib/process.sh @@ -8,7 +8,7 @@ pick_free_port() { local port for port in {49152..65535}; do # Use bash's built-in /dev/tcp to check if port is listening - if ! (: /dev/null; then + if ! (: /dev/null; then echo "$port" return 0 fi diff --git a/scripts/phases/phase-4-proxy-cli.sh b/scripts/phases/phase-4-proxy-cli.sh index 979f2e8..565bbd4 100755 --- a/scripts/phases/phase-4-proxy-cli.sh +++ b/scripts/phases/phase-4-proxy-cli.sh @@ -64,8 +64,8 @@ gate_protected_ports_guarded() { info "Gate 4.8: protected port guard rejects port 40004" # Check that is_protected_proxy_port is implemented in the binary # For now, verify the source code has the guard - grep -q "is_protected_proxy_port" "$ROOT_DIR/src/proxy_pool.rs" || return 1 - grep -q "ensure_not_protected" "$ROOT_DIR/src/proxy_pool.rs" || return 1 + grep -q "is_protected_proxy_port" "$ROOT_DIR/src/proxy_pool/types.rs" || return 1 + grep -q "ensure_not_protected" "$ROOT_DIR/src/proxy_pool/types.rs" || return 1 pass "protected port guards exist in source" } diff --git a/scripts/phases/phase-8-ci-release.sh b/scripts/phases/phase-8-ci-release.sh index 2c9e40e..2ecf521 100755 --- a/scripts/phases/phase-8-ci-release.sh +++ b/scripts/phases/phase-8-ci-release.sh @@ -28,12 +28,15 @@ GATES=( gate_clippy_clean gate_compile_check gate_unit_tests + gate_fast_integration_tests gate_binary_build gate_ci_workflow_exists gate_release_workflow_exists gate_ci_calls_verify gate_release_build_locked gate_dockerfile_locked + gate_dockerignore_exists + gate_dockerfile_no_cache gate_changelog_exists gate_version_consistent gate_no_active_40010 @@ -72,6 +75,26 @@ gate_dockerfile_locked() { pass "Dockerfile uses --locked" } +gate_dockerignore_exists() { + info "Gate 8.11: .dockerignore exists with standard exclusions" + [[ -f "$ROOT_DIR/.dockerignore" ]] || return 1 + grep -q 'target' "$ROOT_DIR/.dockerignore" || return 1 + grep -q '.git' "$ROOT_DIR/.dockerignore" || return 1 + pass ".dockerignore exists with target and .git exclusions" +} + +gate_dockerfile_no_cache() { + info "Gate 8.12: Dockerfile uses --no-cache for apk" + grep -q -- '--no-cache' "$ROOT_DIR/Dockerfile" || return 1 + pass "Dockerfile uses apk add --no-cache" +} + +gate_fast_integration_tests() { + info "Gate 8.13: Fast integration tests pass (non-ignored)" + cargo test --locked --test fast 2>&1 | tail -5 || return 1 + pass "Fast integration tests pass" +} + gate_changelog_exists() { info "Gate 8.11: CHANGELOG.md exists" [[ -f "$ROOT_DIR/CHANGELOG.md" ]] || return 1 @@ -88,7 +111,7 @@ gate_version_consistent() { } gate_no_active_40010() { - info "Gate 8.13: no active reference to deprecated port 40010 in source code" + info "Gate 8.14: no active reference to deprecated port 40010 in source code" if grep -rn "socks5://.*40010\|http.*40010" "$ROOT_DIR/src/" "$ROOT_DIR/start.sh" "$ROOT_DIR/stop.sh" 2>/dev/null; then error "Found active reference to deprecated port 40010" return 1 @@ -97,7 +120,7 @@ gate_no_active_40010() { } gate_install_sh_present() { - info "Gate 8.14: install.sh exists" + info "Gate 8.15: install.sh exists" [[ -f "$ROOT_DIR/install.sh" ]] || return 1 pass "install.sh exists" } diff --git a/src/config.rs b/src/config.rs index 0783163..6c2dc87 100644 --- a/src/config.rs +++ b/src/config.rs @@ -96,14 +96,14 @@ pub struct BridgeConfig { /// SearXNG self-hosted instance URL pub searxng_url: Option, /// SearXNG API key - #[allow(dead_code)] + #[allow(dead_code)] // kept for future SearXNG auth integration pub searxng_api_key: Option, /// Maximum number of search loops - #[allow(dead_code)] + #[allow(dead_code)] // kept for planned CLI override pub max_search_loops: u32, /// Comma-separated list of SOCKS5/HTTP proxies for multi-agent support /// (deprecated, use primary_proxies + warm_standby_proxies instead) - #[allow(dead_code)] + #[allow(dead_code)] // kept for migration warning messages pub proxies: Option>, /// Primary proxy URLs (managed, restartable) pub primary_proxies: Option>, @@ -347,7 +347,7 @@ impl BridgeConfig { } /// Check if a given token is valid. - #[allow(dead_code)] + #[allow(dead_code)] // kept for planned auth middleware refactor pub fn is_valid_token(&self, token: &str) -> bool { match &self.auth_tokens { Some(tokens) => tokens.iter().any(|t| t == token), diff --git a/src/error.rs b/src/error.rs index 3beb904..a39920d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -11,15 +11,15 @@ use serde_json::json; /// Central error type for the OpenCode2Claude bridge. #[derive(Debug, thiserror::Error)] pub enum BridgeError { - #[allow(dead_code)] + #[allow(dead_code)] // kept for planned use in CLI & daemon startup #[error("Failed to bind to address: {0}")] BindFailed(#[source] std::io::Error), - #[allow(dead_code)] + #[allow(dead_code)] // kept for planned use in supervisor #[error("Failed to spawn process: {0}")] ProcessSpawnFailed(#[source] std::io::Error), - #[allow(dead_code)] + #[allow(dead_code)] // kept for structured error responses #[error("Shell commands are disabled by policy. Set BRIDGE_SHELL_POLICY=allowlist or unrestricted to enable.")] ShellDisabled, @@ -32,7 +32,7 @@ pub enum BridgeError { #[error("Unauthorized: {0}")] Unauthorized(String), - #[allow(dead_code)] + #[allow(dead_code)] // kept for health-check error paths #[error("OpenCode daemon unavailable on port {0}")] DaemonUnavailable(u16), diff --git a/src/handlers.rs b/src/handlers.rs index ef8c1fc..23221a1 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -1,14 +1,14 @@ //! HTTP request handlers for the Anthropic-compatible API. -use crate::config::{DEFAULT_MODEL, MSG_ID_SHELL}; +use crate::config::DEFAULT_MODEL; use crate::error::BridgeError; use crate::opencode; -use crate::shell; use crate::sse::SseEventBuilder; use crate::state::AppState; +use futures_util::StreamExt; use axum::extract::State; -use axum::response::sse::{KeepAlive, Sse}; +use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::IntoResponse; use axum::Json; use serde::{Deserialize, Serialize}; @@ -169,29 +169,177 @@ pub async fn handle_messages( ); } - // Shell command interception: prompts starting with '!' run locally - if !prompt.is_empty() && prompt.starts_with('!') { - let shell_cmd = prompt.strip_prefix('!').unwrap().trim().to_string(); - info!("Intercepted local shell command: '{}'", shell_cmd); + // Detect if we are in the second turn of a local shell execution (getting result back from Claude Code) + let mut is_shell_result = false; + let mut shell_result_text = String::new(); + + if let Some(last_msg) = payload.messages.last() { + if last_msg.role == "user" { + if let ContentVal::Multiple(blocks) = &last_msg.content { + for block in blocks { + if block.content_type == "tool_result" + && block.tool_use_id.as_deref() == Some("toolu_local_shell") + { + is_shell_result = true; + if let Some(ref content_val) = block.content { + shell_result_text = + opencode::mapper::tool_result_content_to_string(content_val); + } + break; + } + } + } + } + } - // Check shell policy - if let Err(reason) = state.config.shell_policy.check(&shell_cmd) { - return Err(BridgeError::ShellBlocked { - command: shell_cmd, - allowed: reason, + if is_shell_result { + info!( + "Received local shell execution result from client (length: {})", + shell_result_text.len() + ); + if payload.stream { + let (tx, rx) = tokio::sync::mpsc::channel(10); + let builder = SseEventBuilder::new("msg_local_shell_result".to_string(), req_model); + let output = shell_result_text; + tokio::spawn(async move { + let _ = tx.send(builder.message_start()).await; + let _ = tx.send(builder.content_block_start()).await; + let _ = tx.send(builder.text_delta(&output)).await; + let _ = tx.send(builder.content_block_stop()).await; + let _ = tx.send(builder.message_delta()).await; + let _ = tx.send(builder.message_stop()).await; }); + let response = Sse::new( + tokio_stream::wrappers::ReceiverStream::new(rx) + .map(Ok::<_, std::convert::Infallible>), + ) + .keep_alive(KeepAlive::default()) + .into_response(); + let mut res = response; + res.headers_mut().insert( + axum::http::header::HeaderName::from_static("x-accel-buffering"), + axum::http::HeaderValue::from_static("no"), + ); + Ok(res) + } else { + let builder = SseEventBuilder::new("msg_local_shell_result".to_string(), req_model); + Ok(Json(builder.non_streaming_response(&shell_result_text)).into_response()) + } + } else if !prompt.is_empty() && prompt.starts_with('!') { + let shell_cmd = prompt.strip_prefix('!').unwrap().trim().to_string(); + info!( + "Intercepted local shell command for delegation: '{}'", + shell_cmd + ); + + let mut shell_tool_name = "bash".to_string(); + let mut param_name = "command".to_string(); + + if let Some(ref tools) = payload.tools { + for tool in tools { + let name_lower = tool.name.to_lowercase(); + if name_lower == "bash" + || name_lower == "execute_command" + || name_lower == "run_command" + { + shell_tool_name = tool.name.clone(); + if let Some(properties) = tool + .input_schema + .get("properties") + .and_then(|p| p.as_object()) + { + if properties.contains_key("command") { + param_name = "command".to_string(); + } else if properties.contains_key("cmd") { + param_name = "cmd".to_string(); + } else if !properties.is_empty() { + param_name = properties.keys().next().unwrap().clone(); + } + } + break; + } + } } + let tool_use_id = "toolu_local_shell".to_string(); + if payload.stream { - let stream = shell::run_shell_stream( - shell_cmd, - req_model, - state.config.stream_buffer_size, - state.config.channel_capacity, - ); - let response = Sse::new(stream) - .keep_alive(KeepAlive::default()) - .into_response(); + let (tx, rx) = tokio::sync::mpsc::channel(10); + let builder = SseEventBuilder::new("msg_local_shell".to_string(), req_model); + let tool_name = shell_tool_name; + let p_name = param_name; + let cmd = shell_cmd; + let t_id = tool_use_id; + + tokio::spawn(async move { + let _ = tx.send(builder.message_start()).await; + + let start_ev = Event::default() + .event("content_block_start") + .json_data(serde_json::json!({ + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": t_id, + "name": tool_name, + "input": {} + } + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(start_ev).await; + + let args = serde_json::json!({ p_name: cmd }).to_string(); + let delta_ev = Event::default() + .event("content_block_delta") + .json_data(serde_json::json!({ + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": args + } + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(delta_ev).await; + + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": 0 + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(stop_ev).await; + + let delta_ev = Event::default() + .event("message_delta") + .json_data(serde_json::json!({ + "type": "message_delta", + "delta": { + "stop_reason": "tool_use", + "stop_sequence": null + }, + "usage": {"output_tokens": 0} + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(delta_ev).await; + + let stop_ev = Event::default() + .event("message_stop") + .json_data(serde_json::json!({ + "type": "message_stop" + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(stop_ev).await; + }); + + let response = Sse::new( + tokio_stream::wrappers::ReceiverStream::new(rx) + .map(Ok::<_, std::convert::Infallible>), + ) + .keep_alive(KeepAlive::default()) + .into_response(); let mut res = response; res.headers_mut().insert( axum::http::header::HeaderName::from_static("x-accel-buffering"), @@ -199,9 +347,26 @@ pub async fn handle_messages( ); Ok(res) } else { - let output = shell::run_shell_sync(&shell_cmd).await; - let builder = SseEventBuilder::new(MSG_ID_SHELL.to_string(), req_model); - Ok(Json(builder.non_streaming_response(&output)).into_response()) + let resp_val = serde_json::json!({ + "id": "msg_local_shell", + "type": "message", + "role": "assistant", + "model": req_model, + "content": [ + { + "type": "tool_use", + "id": tool_use_id, + "name": shell_tool_name, + "input": { + param_name: shell_cmd + } + } + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": {"input_tokens": 0, "output_tokens": 0} + }); + Ok(Json(resp_val).into_response()) } } else { // OpenCode path — forward directly to upstream API diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d9bb2c1 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,20 @@ +//! OpenCode2Claude — A local proxy that translates Anthropic Messages API +//! requests into OpenAI-compatible API calls. +//! +//! This library is re-exported by the binary for integration testing. +//! All public API items are exposed through their respective modules. + +pub mod cli; +pub mod config; +pub mod docker; +pub mod error; +pub mod handlers; +pub mod middleware; +pub mod opencode; +pub mod pidfile; +pub mod proxy_pool; +pub mod runtime; +pub mod shell; +pub mod sse; +pub mod state; +pub mod supervisor; diff --git a/src/main.rs b/src/main.rs index d24a5d8..fa20034 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,25 +3,17 @@ //! This binary provides a local HTTP server that translates Anthropic API requests //! into OpenAI-compatible API calls forwarded to opencode.ai/zen/v1/chat/completions. -mod cli; -mod config; -mod docker; -mod error; -mod handlers; -mod middleware; -mod opencode; -mod pidfile; -mod proxy_pool; -mod runtime; -mod shell; -mod sse; -mod state; -mod supervisor; +use opencode2claude::cli::{self, Command, ServeArgs, StartArgs, StatusArgs, StopArgs}; +use opencode2claude::config::{self, BridgeConfig}; +use opencode2claude::docker; +use opencode2claude::handlers; +use opencode2claude::middleware; +use opencode2claude::proxy_pool; +use opencode2claude::runtime::RuntimePaths; +use opencode2claude::state::AppState; +use opencode2claude::supervisor::{Supervisor, SupervisorStatus}; use clap::Parser; -use cli::{Command, ServeArgs, StartArgs, StatusArgs, StopArgs}; -use config::BridgeConfig; -use state::AppState; use axum::routing::{get, post}; use axum::Router; @@ -47,7 +39,7 @@ async fn main() { } } -fn resolve_runtime(args: &StartArgs) -> supervisor::Supervisor { +fn resolve_runtime(args: &StartArgs) -> Supervisor { let port = args .port .or_else(|| { @@ -62,17 +54,15 @@ fn resolve_runtime(args: &StartArgs) -> supervisor::Supervisor { .or_else(|| std::env::var("BRIDGE_HOST").ok()) .unwrap_or_else(|| config::DEFAULT_HOST.to_string()); let root = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")); - let paths = runtime::RuntimePaths::new(root); - supervisor::Supervisor::new(paths, port, host) + let paths = RuntimePaths::new(root); + Supervisor::new(paths, port, host) } fn cmd_start(args: StartArgs) { let sup = resolve_runtime(&args); match sup.start() { Ok(()) => { - let status = sup - .status() - .unwrap_or(supervisor::SupervisorStatus::Stopped); + let status = sup.status().unwrap_or(SupervisorStatus::Stopped); println!("Bridge started. {}", status); } Err(e) => { @@ -101,7 +91,7 @@ fn cmd_stop(args: StopArgs) { } } -fn resolve_runtime_for_port(port: Option, host: Option) -> supervisor::Supervisor { +fn resolve_runtime_for_port(port: Option, host: Option) -> Supervisor { let p = port .or_else(|| { std::env::var("BRIDGE_PORT") @@ -113,8 +103,8 @@ fn resolve_runtime_for_port(port: Option, host: Option) -> supervis .or_else(|| std::env::var("BRIDGE_HOST").ok()) .unwrap_or_else(|| config::DEFAULT_HOST.to_string()); let root = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")); - let paths = runtime::RuntimePaths::new(root); - supervisor::Supervisor::new(paths, p, h) + let paths = RuntimePaths::new(root); + Supervisor::new(paths, p, h) } fn cmd_restart() { @@ -124,9 +114,7 @@ fn cmd_restart() { // Then start match sup.start() { Ok(()) => { - let status = sup - .status() - .unwrap_or(supervisor::SupervisorStatus::Stopped); + let status = sup.status().unwrap_or(SupervisorStatus::Stopped); println!("Bridge restarted. {}", status); } Err(e) => { @@ -155,7 +143,7 @@ fn cmd_env() { fn cmd_logs() { let root = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")); - let paths = runtime::RuntimePaths::new(root); + let paths = RuntimePaths::new(root); let log_path = paths.bridge_log(); if !log_path.exists() { diff --git a/src/opencode/forward.rs b/src/opencode/forward.rs index dec46cd..e7fd30e 100644 --- a/src/opencode/forward.rs +++ b/src/opencode/forward.rs @@ -6,6 +6,8 @@ use crate::error::BridgeError; use crate::handlers::{ContentVal, MessagesRequest}; use crate::opencode::mapper::{extract_search_query, is_web_search_tool, map_anthropic_to_openai}; +use crate::opencode::retry::execute_with_warp_retry; +use crate::opencode::sanitize::{extract_and_clean_dsml, parse_dsml_tool_calls, strip_system_tags}; use crate::opencode::search::SearchClient; use crate::opencode::types::*; use crate::sse::SseEventBuilder; @@ -15,9 +17,8 @@ use futures_util::{Stream, StreamExt}; use reqwest::Client; use std::collections::HashMap; use std::convert::Infallible; -use std::time::Duration; use tokio_stream::wrappers::ReceiverStream; -use tracing::{error, info, warn}; +use tracing::{error, info}; /// Check if the OpenCode daemon is running and reachable. pub async fn check_daemon(client: &Client, port: u16) -> bool { @@ -30,261 +31,6 @@ pub async fn check_daemon(client: &Client, port: u16) -> bool { .is_ok() } -async fn rotate_warp_ip() { - info!("Rotating WARP IP address..."); - - let disconnect = tokio::process::Command::new("warp-cli") - .arg("disconnect") - .output() - .await; - - match disconnect { - Ok(output) if output.status.success() => { - info!("warp-cli disconnect succeeded"); - } - Ok(output) => { - warn!( - "warp-cli disconnect returned non-zero: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Err(e) => { - warn!("warp-cli disconnect failed (maybe not installed?): {}", e); - return; - } - } - - tokio::time::sleep(tokio::time::Duration::from_millis(1500)).await; - - let connect = tokio::process::Command::new("warp-cli") - .arg("connect") - .output() - .await; - - match connect { - Ok(output) if output.status.success() => { - info!("warp-cli connect succeeded"); - } - Ok(output) => { - warn!( - "warp-cli connect returned non-zero: {}", - String::from_utf8_lossy(&output.stderr) - ); - tokio::time::sleep(tokio::time::Duration::from_millis(2500)).await; - return; - } - Err(e) => { - warn!("warp-cli connect failed: {}", e); - return; - } - } - - tokio::time::sleep(tokio::time::Duration::from_millis(2500)).await; - info!("WARP IP address rotated successfully."); -} - -/// Check if a response body text indicates a rate-limit error. -fn is_rate_limit_body(body: &str) -> bool { - let lower = body.to_lowercase(); - lower.contains("rate") - || lower.contains("limit") - || lower.contains("quota") - || lower.contains("too many") - || lower.contains("throttl") -} - -async fn execute_with_warp_retry( - state: &AppState, - api_key: &str, - req_body: &OpenAiRequest, -) -> Result { - // Calculate max retries based on proxy pool size (at least 5) - let pool_size = { - let pool = state.proxy_pool.read().await; - pool.proxies.len() - }; - let max_retries = pool_size.max(3) + 2; - - let mut retry_count: u32 = 0; - let mut last_failed_idx: Option = None; - - loop { - // Select the client from the proxy pool if configured - let (client, proxy_url, idx) = { - let mut pool = state.proxy_pool.write().await; - let result = if let Some(exclude) = last_failed_idx { - pool.get_client_excluding(api_key, exclude) - .or_else(|| pool.get_client(api_key)) - } else { - pool.get_client(api_key) - }; - if let Some((c, url, idx)) = result { - (c, Some(url), Some(idx)) - } else { - (state.http_client.clone(), None, None) - } - }; - - let res = client - .post("https://opencode.ai/zen/v1/chat/completions") - .json(req_body) - .send() - .await; - - match res { - Ok(response) => { - let status = response.status(); - - if status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.is_server_error() { - // 429 and 5xx are rate-limit / server errors - if (retry_count as usize) < max_retries { - retry_count += 1; - if let (Some(idx), Some(ref url)) = (idx, &proxy_url) { - warn!( - "Upstream error (status {}) on proxy #{} ({}). Putting proxy on cool-down (attempt {}/{})...", - status, idx, url, retry_count, max_retries - ); - let mut pool = state.proxy_pool.write().await; - // Try Retry-After header first (HTTP/1.1 standard) - let cooldown = response - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - .map(Duration::from_secs); - if let Some(d) = cooldown { - pool.mark_rate_limited(idx, d); - info!("Using Retry-After header: {}s cooldown", d.as_secs()); - } else { - pool.mark_rate_limited_adaptive(idx, retry_count); - } - last_failed_idx = Some(idx); - } else { - warn!( - "Upstream error (status {}). Attempting to rotate WARP IP (attempt {}/{})...", - status, retry_count, max_retries - ); - rotate_warp_ip().await; - } - let backoff = std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); - info!("Backing off for {:?} before retry...", backoff); - tokio::time::sleep(backoff).await; - continue; - } - return Err(BridgeError::UpstreamError(format!( - "Upstream error after {} retries (status {})", - retry_count, status - ))); - } else if status == reqwest::StatusCode::BAD_REQUEST { - // 400: Read body to distinguish genuine errors from rate limits - let body_bytes = response.bytes().await.unwrap_or_default(); - let body_text = String::from_utf8_lossy(&body_bytes); - if is_rate_limit_body(&body_text) { - warn!( - "Upstream returned 400 with rate-limit body (truncated): {}", - body_text.chars().take(200).collect::() - ); - if (retry_count as usize) < max_retries { - retry_count += 1; - if let (Some(idx), Some(ref url)) = (idx, &proxy_url) { - warn!( - "Rate-limit on proxy #{} ({}). Cool-down (attempt {}/{})...", - idx, url, retry_count, max_retries - ); - let mut pool = state.proxy_pool.write().await; - pool.mark_rate_limited_adaptive(idx, retry_count); - last_failed_idx = Some(idx); - } else { - rotate_warp_ip().await; - } - let backoff = - std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); - info!("Backing off for {:?} before retry...", backoff); - tokio::time::sleep(backoff).await; - continue; - } - return Err(BridgeError::UpstreamError(format!( - "Rate limited (400) after {} retries", - retry_count - ))); - } else { - // Genuine 400 error — upstream provider failure, retry up to 10x - if retry_count < 10 { - retry_count += 1; - warn!( - "Upstream returned 400 (provider error, attempt {}/10, truncated): {}", - retry_count, - body_text.chars().take(200).collect::() - ); - if let (Some(idx), Some(ref _url)) = (idx, &proxy_url) { - let mut pool = state.proxy_pool.write().await; - pool.mark_rate_limited(idx, Duration::from_secs(5)); - last_failed_idx = Some(idx); - } else { - rotate_warp_ip().await; - } - let backoff = - std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); - info!("Backing off for {:?} before retry...", backoff); - tokio::time::sleep(backoff).await; - continue; - } - warn!( - "Upstream returned 400 (failed after 10 retries, truncated): {}", - body_text.chars().take(300).collect::() - ); - return Err(BridgeError::UpstreamError( - "Upstream returned 400 after 10 retries".to_string(), - )); - } - } else { - // Success or other status — return as-is - // Record success on proxy since transport worked (even for 4xx) - if let Some(idx) = idx { - let mut pool = state.proxy_pool.write().await; - pool.record_success(idx); - } - return Ok(response); - } - } - Err(e) => { - if (retry_count as usize) < max_retries { - retry_count += 1; - if let (Some(idx), Some(ref url)) = (idx, &proxy_url) { - warn!( - "Network error connecting via proxy #{} ({}): {}. Putting proxy on cool-down (attempt {}/{})...", - idx, url, e, retry_count, max_retries - ); - let mut pool = state.proxy_pool.write().await; - // Network transport error = proxy failure - pool.record_failure(idx); - info!( - "Recorded transport failure for proxy #{} ({}) after {}/{} retries.", - idx, url, retry_count, max_retries - ); - last_failed_idx = Some(idx); - } else { - warn!( - "Network error connecting upstream: {}. Attempting to rotate WARP IP (attempt {}/{})...", - e, retry_count, max_retries - ); - rotate_warp_ip().await; - } - // Exponential backoff - let backoff = std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); - info!("Backing off for {:?} before retry...", backoff); - tokio::time::sleep(backoff).await; - continue; - } - return Err(BridgeError::UpstreamError(format!( - "Network error after {} retries: {}", - retry_count, e - ))); - } - } - } -} - // ── API Forwarding Implementations ── pub async fn forward_to_llm_sync( @@ -333,13 +79,22 @@ pub async fn forward_to_llm_sync( BridgeError::UpstreamError("No choices returned from upstream".to_string()) })?; - // Check if there is an intercepted search tool call + // Extract DSML tool calls and clean the message content + let mut dsml_tool_calls = Vec::new(); + let mut cleaned_message_content = choice.message.content.clone(); let mut has_search = false; let mut search_tc_id = String::new(); let mut search_tc_name = String::new(); let mut search_tc_input = serde_json::Value::Null; let mut search_query = String::new(); + if let Some(text) = &choice.message.content { + let (cleaned, calls) = extract_and_clean_dsml(text); + cleaned_message_content = Some(cleaned); + dsml_tool_calls = calls; + } + + // Check if there is an intercepted search tool call (native first, then DSML) if let Some(tool_calls) = &choice.message.tool_calls { for tc in tool_calls { if is_web_search_tool(&tc.function.name) { @@ -355,6 +110,27 @@ pub async fn forward_to_llm_sync( } } + if !has_search { + for (i, call) in dsml_tool_calls.iter().enumerate() { + if is_web_search_tool(&call.name) { + has_search = true; + search_tc_id = format!( + "toolu_dsml_{}_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + i + ); + search_tc_name = call.name.clone(); + search_tc_input = call.arguments.clone(); + let args_str = serde_json::to_string(&call.arguments).unwrap_or_default(); + search_query = extract_search_query(&args_str); + break; + } + } + } + if has_search { info!( "Intercepted sync search tool call. Query: '{}'", @@ -376,7 +152,7 @@ pub async fn forward_to_llm_sync( ); } } - if let Some(content) = &choice.message.content { + if let Some(content) = &cleaned_message_content { let cleaned = strip_system_tags(content); if !cleaned.is_empty() { assistant_content.push( @@ -434,7 +210,7 @@ pub async fn forward_to_llm_sync( } // 2. Text block - if let Some(text) = &choice.message.content { + if let Some(text) = &cleaned_message_content { let cleaned = strip_system_tags(text); if !cleaned.is_empty() { content_blocks.push(serde_json::json!({ @@ -444,9 +220,11 @@ pub async fn forward_to_llm_sync( } } - // 3. Tool calls + // 3. Native Tool calls + let mut has_tool_calls = false; if let Some(tool_calls) = &choice.message.tool_calls { for tc in tool_calls { + has_tool_calls = true; let input_val: serde_json::Value = serde_json::from_str(&tc.function.arguments) .unwrap_or(serde_json::Value::Object(serde_json::Map::new())); content_blocks.push(serde_json::json!({ @@ -458,11 +236,42 @@ pub async fn forward_to_llm_sync( } } + // 4. DSML Tool calls + for (i, call) in dsml_tool_calls.into_iter().enumerate() { + has_tool_calls = true; + let tool_id = format!( + "toolu_dsml_{}_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + i + ); + content_blocks.push(serde_json::json!({ + "type": "tool_use", + "id": tool_id, + "name": call.name, + "input": call.arguments + })); + } + let stop_reason = match choice.finish_reason.as_deref() { - Some("stop") => "end_turn", + Some("stop") => { + if has_tool_calls { + "tool_use" + } else { + "end_turn" + } + } Some("tool_calls") => "tool_use", Some("length") => "max_tokens", - _ => "end_turn", + _ => { + if has_tool_calls { + "tool_use" + } else { + "end_turn" + } + } }; let usage = openai_resp.usage.unwrap_or(OpenAiUsage { @@ -594,6 +403,10 @@ pub async fn forward_to_llm_stream( let mut accumulated_text = String::new(); let mut stream_failed = false; + let mut has_emitted_tool_use = false; + let mut dsml_mode = false; + let mut dsml_stream_buffer = String::new(); + let mut text_stream_buffer = String::new(); while let Some(chunk_res) = bytes_stream.next().await { let chunk = match chunk_res { @@ -670,53 +483,227 @@ pub async fn forward_to_llm_stream( // 2. Process content (text delta) if let Some(content) = &choice.delta.content { - let cleaned = strip_system_tags(content); - if !cleaned.is_empty() { - accumulated_text.push_str(&cleaned); - if !intercepting_search { - // Close thinking block if open - if let Some(idx) = thinking_block_index { + if dsml_mode { + dsml_stream_buffer.push_str(content); + if let Some(end_pos) = + dsml_stream_buffer.find("") + { + let end_idx = end_pos + "".len(); + let dsml_block = &dsml_stream_buffer[..end_idx]; + let remaining = + dsml_stream_buffer[end_idx..].to_string(); + + let calls = parse_dsml_tool_calls(dsml_block); + for call in calls { + has_emitted_tool_use = true; + let tool_id = format!( + "toolu_dsml_{}_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + next_content_block_index + ); + + if let Some(idx) = thinking_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| { + Event::default().data("{}") + }); + let _ = tx.send(stop_ev).await; + thinking_block_index = None; + } + if let Some(idx) = text_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| { + Event::default().data("{}") + }); + let _ = tx.send(stop_ev).await; + text_block_index = None; + } + + let call_idx = next_content_block_index; + next_content_block_index += 1; + + let start_ev = Event::default() + .event("content_block_start") + .json_data(serde_json::json!({ + "type": "content_block_start", + "index": call_idx, + "content_block": { + "type": "tool_use", + "id": tool_id, + "name": call.name, + "input": {} + } + })) + .unwrap_or_else(|_| { + Event::default().data("{}") + }); + let _ = tx.send(start_ev).await; + + let args_str = + serde_json::to_string(&call.arguments) + .unwrap_or_default(); + let delta_ev = Event::default() + .event("content_block_delta") + .json_data(serde_json::json!({ + "type": "content_block_delta", + "index": call_idx, + "delta": { + "type": "input_json_delta", + "partial_json": args_str + } + })) + .unwrap_or_else(|_| { + Event::default().data("{}") + }); + let _ = tx.send(delta_ev).await; + let stop_ev = Event::default() .event("content_block_stop") .json_data(serde_json::json!({ "type": "content_block_stop", - "index": idx + "index": call_idx })) .unwrap_or_else(|_| { Event::default().data("{}") }); let _ = tx.send(stop_ev).await; - thinking_block_index = None; } - let idx = match text_block_index { - Some(i) => i, - None => { - let i = next_content_block_index; - next_content_block_index += 1; - text_block_index = Some(i); - let start_ev = Event::default() - .event("content_block_start") + dsml_stream_buffer = String::new(); + dsml_mode = false; + + if !remaining.is_empty() { + text_stream_buffer.push_str(&remaining); + } + } + } else { + text_stream_buffer.push_str(content); + } + + if !dsml_mode { + if let Some(start_pos) = + text_stream_buffer.find("<｜DSML｜tool_calls>") + { + let text_to_yield = &text_stream_buffer[..start_pos]; + let remainder = &text_stream_buffer[start_pos..]; + + let cleaned = strip_system_tags(text_to_yield); + if !cleaned.is_empty() { + accumulated_text.push_str(&cleaned); + if !intercepting_search { + if let Some(idx) = thinking_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| { + Event::default().data("{}") + }); + let _ = tx.send(stop_ev).await; + thinking_block_index = None; + } + + let idx = match text_block_index { + Some(i) => i, + None => { + let i = next_content_block_index; + next_content_block_index += 1; + text_block_index = Some(i); + let start_ev = Event::default() + .event("content_block_start") + .json_data(serde_json::json!({ + "type": "content_block_start", + "index": i, + "content_block": {"type": "text", "text": ""} + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(start_ev).await; + i + } + }; + + let delta_ev = Event::default() + .event("content_block_delta") .json_data(serde_json::json!({ - "type": "content_block_start", - "index": i, - "content_block": {"type": "text", "text": ""} + "type": "content_block_delta", + "index": idx, + "delta": {"type": "text_delta", "text": cleaned} })) .unwrap_or_else(|_| Event::default().data("{}")); - let _ = tx.send(start_ev).await; - i + let _ = tx.send(delta_ev).await; } - }; + } - let delta_ev = Event::default() - .event("content_block_delta") - .json_data(serde_json::json!({ - "type": "content_block_delta", - "index": idx, - "delta": {"type": "text_delta", "text": cleaned} - })) - .unwrap_or_else(|_| Event::default().data("{}")); - let _ = tx.send(delta_ev).await; + dsml_mode = true; + dsml_stream_buffer = remainder.to_string(); + text_stream_buffer = String::new(); + } else { + let (to_yield, pending) = + split_pending_text(&text_stream_buffer); + let cleaned = strip_system_tags(&to_yield); + if !cleaned.is_empty() { + accumulated_text.push_str(&cleaned); + if !intercepting_search { + if let Some(idx) = thinking_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| { + Event::default().data("{}") + }); + let _ = tx.send(stop_ev).await; + thinking_block_index = None; + } + + let idx = match text_block_index { + Some(i) => i, + None => { + let i = next_content_block_index; + next_content_block_index += 1; + text_block_index = Some(i); + let start_ev = Event::default() + .event("content_block_start") + .json_data(serde_json::json!({ + "type": "content_block_start", + "index": i, + "content_block": {"type": "text", "text": ""} + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(start_ev).await; + i + } + }; + + let delta_ev = Event::default() + .event("content_block_delta") + .json_data(serde_json::json!({ + "type": "content_block_delta", + "index": idx, + "delta": {"type": "text_delta", "text": cleaned} + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(delta_ev).await; + } + } + text_stream_buffer = pending; } } } @@ -787,6 +774,7 @@ pub async fn forward_to_llm_stream( .unwrap_or_else(|_| { Event::default().data("{}") }); + has_emitted_tool_use = true; let _ = tx.send(start_ev).await; } } @@ -903,6 +891,134 @@ pub async fn forward_to_llm_stream( continue; } + // Flush any remaining text in text_stream_buffer + let cleaned = strip_system_tags(&text_stream_buffer); + if !cleaned.is_empty() { + accumulated_text.push_str(&cleaned); + if !intercepting_search { + if let Some(idx) = thinking_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(stop_ev).await; + thinking_block_index = None; + } + + let idx = match text_block_index { + Some(i) => i, + None => { + let i = next_content_block_index; + next_content_block_index += 1; + text_block_index = Some(i); + let start_ev = Event::default() + .event("content_block_start") + .json_data(serde_json::json!({ + "type": "content_block_start", + "index": i, + "content_block": {"type": "text", "text": ""} + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(start_ev).await; + i + } + }; + + let delta_ev = Event::default() + .event("content_block_delta") + .json_data(serde_json::json!({ + "type": "content_block_delta", + "index": idx, + "delta": {"type": "text_delta", "text": cleaned} + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(delta_ev).await; + } + } + + // Flush/parse any remaining unclosed DSML block in dsml_stream_buffer + if dsml_mode && !dsml_stream_buffer.is_empty() { + let calls = parse_dsml_tool_calls(&dsml_stream_buffer); + for call in calls { + has_emitted_tool_use = true; + let tool_id = format!( + "toolu_dsml_{}_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + next_content_block_index + ); + + if let Some(idx) = thinking_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(stop_ev).await; + thinking_block_index = None; + } + if let Some(idx) = text_block_index { + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": idx + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(stop_ev).await; + text_block_index = None; + } + + let call_idx = next_content_block_index; + next_content_block_index += 1; + + let start_ev = Event::default() + .event("content_block_start") + .json_data(serde_json::json!({ + "type": "content_block_start", + "index": call_idx, + "content_block": { + "type": "tool_use", + "id": tool_id, + "name": call.name, + "input": {} + } + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(start_ev).await; + + let args_str = serde_json::to_string(&call.arguments).unwrap_or_default(); + let delta_ev = Event::default() + .event("content_block_delta") + .json_data(serde_json::json!({ + "type": "content_block_delta", + "index": call_idx, + "delta": { + "type": "input_json_delta", + "partial_json": args_str + } + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(delta_ev).await; + + let stop_ev = Event::default() + .event("content_block_stop") + .json_data(serde_json::json!({ + "type": "content_block_stop", + "index": call_idx + })) + .unwrap_or_else(|_| Event::default().data("{}")); + let _ = tx.send(stop_ev).await; + } + } + // Close any remaining active content blocks if let Some(idx) = thinking_block_index { let stop_ev = Event::default() @@ -935,13 +1051,19 @@ pub async fn forward_to_llm_stream( let _ = tx.send(stop_ev).await; } + let stop_reason = if has_emitted_tool_use { + "tool_use".to_string() + } else { + final_stop_reason + }; + // Send final message_delta and message_stop let delta_ev = Event::default() .event("message_delta") .json_data(serde_json::json!({ "type": "message_delta", "delta": { - "stop_reason": final_stop_reason, + "stop_reason": stop_reason, "stop_sequence": null }, "usage": {"output_tokens": 0} @@ -963,29 +1085,18 @@ pub async fn forward_to_llm_stream( Ok(ReceiverStream::new(rx).map(Ok)) } -/// Helper function to strip system leakage tags (like , , etc.) from LLM outputs. -fn strip_system_tags(text: &str) -> String { - let mut cleaned = text.to_string(); - let tags = [ - "", - "", - "", - "", - "", - "<｜DSML｜parameter>", - "</think>", - "<think>", - ]; - for tag in &tags { - if cleaned.contains(tag) { - cleaned = cleaned.replace(tag, ""); +fn split_pending_text(text: &str) -> (String, String) { + let tag = "<｜DSML｜tool_calls>"; + for i in (1..=tag.len()).rev() { + if tag.is_char_boundary(i) { + let prefix = &tag[..i]; + if text.ends_with(prefix) { + let split_idx = text.len() - prefix.len(); + return (text[..split_idx].to_string(), prefix.to_string()); + } } } - // Trim leading newlines and whitespace if we stripped tags from the beginning - if cleaned.trim_start() != text.trim_start() { - cleaned = cleaned.trim_start().to_string(); - } - cleaned + (text.to_string(), String::new()) } #[cfg(test)] @@ -993,16 +1104,22 @@ mod tests { use super::*; #[test] - fn test_strip_system_tags() { - assert_eq!(strip_system_tags("Hello"), "Hello"); - assert_eq!(strip_system_tags("\n\nHello"), "Hello"); - assert_eq!(strip_system_tags("Hello"), "Hello"); - assert_eq!(strip_system_tags("HelloWorld"), "HelloWorld"); - assert_eq!(strip_system_tags("\nHello"), "Hello"); + fn test_split_pending_text() { + assert_eq!( + split_pending_text("hello<"), + ("hello".to_string(), "<".to_string()) + ); + assert_eq!( + split_pending_text("hello<｜"), + ("hello".to_string(), "<｜".to_string()) + ); + assert_eq!( + split_pending_text("hello<｜DSML｜tool_calls>"), + ("hello".to_string(), "<｜DSML｜tool_calls>".to_string()) + ); assert_eq!( - strip_system_tags("Some thinkingResponse"), - "Some thinkingResponse" + split_pending_text("hello"), + ("hello".to_string(), "".to_string()) ); - assert_eq!(strip_system_tags("Normal text"), "Normal text"); } } diff --git a/src/opencode/mod.rs b/src/opencode/mod.rs index 10bfebb..d84efc9 100644 --- a/src/opencode/mod.rs +++ b/src/opencode/mod.rs @@ -6,6 +6,8 @@ pub mod forward; pub mod mapper; +pub mod retry; +pub mod sanitize; pub mod search; pub mod types; diff --git a/src/opencode/retry.rs b/src/opencode/retry.rs new file mode 100644 index 0000000..6769297 --- /dev/null +++ b/src/opencode/retry.rs @@ -0,0 +1,279 @@ +//! Retry logic and WARP IP rotation for upstream API requests. +//! +//! Provides exponential-backoff retry with proxy cooldown management +//! and WARP IP rotation fallback for rate-limit resilience. +//! +//! Extracted from `forward.rs` during module split. + +use crate::error::BridgeError; +use crate::opencode::types::OpenAiRequest; +use crate::state::AppState; +use std::time::Duration; +use tracing::{info, warn}; + +/// Rotate WARP IP address by disconnecting and reconnecting. +async fn rotate_warp_ip() { + info!("Rotating WARP IP address..."); + + let disconnect = tokio::process::Command::new("warp-cli") + .arg("disconnect") + .output() + .await; + + match disconnect { + Ok(output) if output.status.success() => { + info!("warp-cli disconnect succeeded"); + } + Ok(output) => { + warn!( + "warp-cli disconnect returned non-zero: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Err(e) => { + warn!("warp-cli disconnect failed (maybe not installed?): {}", e); + return; + } + } + + tokio::time::sleep(tokio::time::Duration::from_millis(1500)).await; + + let connect = tokio::process::Command::new("warp-cli") + .arg("connect") + .output() + .await; + + match connect { + Ok(output) if output.status.success() => { + info!("warp-cli connect succeeded"); + } + Ok(output) => { + warn!( + "warp-cli connect returned non-zero: {}", + String::from_utf8_lossy(&output.stderr) + ); + tokio::time::sleep(tokio::time::Duration::from_millis(2500)).await; + return; + } + Err(e) => { + warn!("warp-cli connect failed: {}", e); + return; + } + } + + tokio::time::sleep(tokio::time::Duration::from_millis(2500)).await; + info!("WARP IP address rotated successfully."); +} + +/// Check if a response body text indicates a rate-limit error. +fn is_rate_limit_body(body: &str) -> bool { + let lower = body.to_lowercase(); + lower.contains("rate") + || lower.contains("limit") + || lower.contains("quota") + || lower.contains("too many") + || lower.contains("throttl") +} + +/// Maximum retries for 400 provider errors (distinct from rate-limit retries). +const MAX_PROVIDER_RETRIES: u32 = 10; + +/// Execute a request with exponential-backoff retry, proxy cooldown, and WARP IP rotation. +/// +/// Retry strategy: +/// - 429/5xx: rate-limit retry up to `pool_size.max(3) + 2` times +/// - 400 rate-limit body: same as 429 +/// - 400 provider error: retry up to 10 times +/// - Network errors: same as 429 +/// - Between retries: proxies are cooled down adaptively (2^retry min × 60s) +pub(super) async fn execute_with_warp_retry( + state: &AppState, + api_key: &str, + req_body: &OpenAiRequest, +) -> Result { + let pool_size = { + let pool = state.proxy_pool.read().await; + pool.proxies.len() + }; + let max_retries = pool_size.max(3) + 2; + + let mut retry_count: u32 = 0; + let mut last_failed_idx: Option = None; + + loop { + // Select the client from the proxy pool if configured + let (client, proxy_url, idx) = { + let mut pool = state.proxy_pool.write().await; + let result = if let Some(exclude) = last_failed_idx { + pool.get_client_excluding(api_key, exclude) + .or_else(|| pool.get_client(api_key)) + } else { + pool.get_client(api_key) + }; + if let Some((c, url, idx)) = result { + (c, Some(url), Some(idx)) + } else { + (state.http_client.clone(), None, None) + } + }; + + let res = client + .post("https://opencode.ai/zen/v1/chat/completions") + .json(req_body) + .send() + .await; + + match res { + Ok(response) => { + let status = response.status(); + + if status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.is_server_error() { + // 429 and 5xx are rate-limit / server errors + if (retry_count as usize) < max_retries { + retry_count += 1; + if let (Some(idx), Some(ref url)) = (idx, &proxy_url) { + warn!( + "Upstream error (status {}) on proxy #{} ({}). Putting proxy on cool-down (attempt {}/{})...", + status, idx, url, retry_count, max_retries + ); + let mut pool = state.proxy_pool.write().await; + // Try Retry-After header first (HTTP/1.1 standard) + let cooldown = response + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .map(Duration::from_secs); + if let Some(d) = cooldown { + pool.mark_rate_limited(idx, d); + info!("Using Retry-After header: {}s cooldown", d.as_secs()); + } else { + pool.mark_rate_limited_adaptive(idx, retry_count); + } + last_failed_idx = Some(idx); + } else { + warn!( + "Upstream error (status {}). Attempting to rotate WARP IP (attempt {}/{})...", + status, retry_count, max_retries + ); + rotate_warp_ip().await; + } + let backoff = std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); + info!("Backing off for {:?} before retry...", backoff); + tokio::time::sleep(backoff).await; + continue; + } + return Err(BridgeError::UpstreamError(format!( + "Upstream error after {} retries (status {})", + retry_count, status + ))); + } else if status == reqwest::StatusCode::BAD_REQUEST { + // 400: Read body to distinguish genuine errors from rate limits + let body_bytes = response.bytes().await.unwrap_or_default(); + let body_text = String::from_utf8_lossy(&body_bytes); + if is_rate_limit_body(&body_text) { + warn!( + "Upstream returned 400 with rate-limit body (truncated): {}", + body_text.chars().take(200).collect::() + ); + if (retry_count as usize) < max_retries { + retry_count += 1; + if let (Some(idx), Some(ref url)) = (idx, &proxy_url) { + warn!( + "Rate-limit on proxy #{} ({}). Cool-down (attempt {}/{})...", + idx, url, retry_count, max_retries + ); + let mut pool = state.proxy_pool.write().await; + pool.mark_rate_limited_adaptive(idx, retry_count); + last_failed_idx = Some(idx); + } else { + rotate_warp_ip().await; + } + let backoff = + std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); + info!("Backing off for {:?} before retry...", backoff); + tokio::time::sleep(backoff).await; + continue; + } + return Err(BridgeError::UpstreamError(format!( + "Rate limited (400) after {} retries", + retry_count + ))); + } else { + // Genuine 400 error — upstream provider failure, retry up to 10x + if retry_count < MAX_PROVIDER_RETRIES { + retry_count += 1; + warn!( + "Upstream returned 400 (provider error, attempt {}/{}, truncated): {}", + retry_count, MAX_PROVIDER_RETRIES, + body_text.chars().take(200).collect::() + ); + if let (Some(idx), Some(ref _url)) = (idx, &proxy_url) { + let mut pool = state.proxy_pool.write().await; + pool.mark_rate_limited(idx, Duration::from_secs(5)); + last_failed_idx = Some(idx); + } else { + rotate_warp_ip().await; + } + let backoff = + std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); + info!("Backing off for {:?} before retry...", backoff); + tokio::time::sleep(backoff).await; + continue; + } + warn!( + "Upstream returned 400 (failed after {} retries, truncated): {}", + MAX_PROVIDER_RETRIES, + body_text.chars().take(300).collect::() + ); + return Err(BridgeError::UpstreamError( + "Upstream returned 400 after 10 retries".to_string(), + )); + } + } else { + // Success or other status — return as-is + // Record success on proxy since transport worked (even for 4xx) + if let Some(idx) = idx { + let mut pool = state.proxy_pool.write().await; + pool.record_success(idx); + } + return Ok(response); + } + } + Err(e) => { + if (retry_count as usize) < max_retries { + retry_count += 1; + if let (Some(idx), Some(ref url)) = (idx, &proxy_url) { + warn!( + "Network error connecting via proxy #{} ({}): {}. Putting proxy on cool-down (attempt {}/{})...", + idx, url, e, retry_count, max_retries + ); + let mut pool = state.proxy_pool.write().await; + // Network transport error = proxy failure + pool.record_failure(idx); + info!( + "Recorded transport failure for proxy #{} ({}) after {}/{} retries.", + idx, url, retry_count, max_retries + ); + last_failed_idx = Some(idx); + } else { + warn!( + "Network error connecting upstream: {}. Attempting to rotate WARP IP (attempt {}/{})...", + e, retry_count, max_retries + ); + rotate_warp_ip().await; + } + // Exponential backoff + let backoff = std::time::Duration::from_secs(2u64.pow(retry_count.min(4))); + info!("Backing off for {:?} before retry...", backoff); + tokio::time::sleep(backoff).await; + continue; + } + return Err(BridgeError::UpstreamError(format!( + "Network error after {} retries: {}", + retry_count, e + ))); + } + } + } +} diff --git a/src/opencode/sanitize.rs b/src/opencode/sanitize.rs new file mode 100644 index 0000000..ef28c09 --- /dev/null +++ b/src/opencode/sanitize.rs @@ -0,0 +1,268 @@ +//! Sanitization utilities for LLM output cleaning. +//! +//! Provides functions to strip system leakage tags from model responses. +//! Extracted from `forward.rs` during module split. + +/// Strip system leakage tags (like ``, ``, etc.) from LLM outputs. +/// +/// Removes known tags that models sometimes leak from their system prompt context, +/// including HTML-encoded variants. Also trims leading whitespace when tags were +/// stripped from the beginning of the text. +pub fn strip_system_tags(text: &str) -> String { + let mut cleaned = text.to_string(); + let tags = [ + "", + "", + "", + "", + "", + "<｜DSML｜parameter>", + "", + "<｜DSML｜invoke>", + "", + "<｜DSML｜tool_calls>", + "</think>", + "<think>", + ]; + for tag in &tags { + if cleaned.contains(tag) { + cleaned = cleaned.replace(tag, ""); + } + } + // Trim leading newlines and whitespace if we stripped tags from the beginning + if cleaned.trim_start() != text.trim_start() { + cleaned = cleaned.trim_start().to_string(); + } + cleaned +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedDsmlCall { + pub name: String, + pub arguments: serde_json::Value, +} + +fn extract_attribute(tag_content: &str, attr_name: &str) -> String { + let pattern = attr_name.to_string(); + let mut pos = 0; + while let Some(match_pos) = tag_content[pos..].find(&pattern) { + let abs_match_pos = pos + match_pos; + let rem = &tag_content[abs_match_pos + pattern.len()..]; + let mut eq_found = false; + let mut val_start_pos = None; + let mut quote_char = None; + for (i, c) in rem.char_indices() { + if c.is_whitespace() { + continue; + } + if c == '=' { + eq_found = true; + continue; + } + if eq_found { + if c == '"' || c == '\'' { + quote_char = Some(c); + val_start_pos = Some(i + 1); + break; + } else { + val_start_pos = Some(i); + break; + } + } else { + break; + } + } + if let Some(start) = val_start_pos { + let rem_val = &rem[start..]; + if let Some(q) = quote_char { + if let Some(end) = rem_val.find(q) { + return rem_val[..end].to_string(); + } + } else { + let end = rem_val + .find(|c: char| c.is_whitespace() || c == '>') + .unwrap_or(rem_val.len()); + return rem_val[..end].to_string(); + } + } + pos = abs_match_pos + pattern.len(); + } + String::new() +} + +pub fn parse_dsml_tool_calls(text: &str) -> Vec { + let mut calls = Vec::new(); + let mut search_pos = 0; + + while let Some(invoke_start) = text[search_pos..].find("<｜DSML｜invoke") { + let absolute_invoke_start = search_pos + invoke_start; + let remaining = &text[absolute_invoke_start..]; + let Some(tag_open_end) = remaining.find('>') else { + break; + }; + let tag_open_content = &remaining[..tag_open_end]; + + let name = extract_attribute(tag_open_content, "name"); + + let Some(invoke_end) = remaining.find("") else { + break; + }; + let invoke_body = &remaining[tag_open_end + 1..invoke_end]; + + let mut params = serde_json::Map::new(); + let mut p_pos = 0; + while let Some(p_start) = invoke_body[p_pos..].find("<｜DSML｜parameter") { + let abs_p_start = p_pos + p_start; + let p_rem = &invoke_body[abs_p_start..]; + let Some(p_open_end) = p_rem.find('>') else { + break; + }; + let p_open_content = &p_rem[..p_open_end]; + + let p_name = extract_attribute(p_open_content, "name"); + + let Some(p_close) = p_rem.find("") else { + break; + }; + let p_val_str = p_rem[p_open_end + 1..p_close].trim(); + let mut clean_val = p_val_str.to_string(); + if clean_val.starts_with("```") { + if let Some(newline_pos) = clean_val.find('\n') { + clean_val = clean_val[newline_pos + 1..].to_string(); + } else { + clean_val = clean_val[3..].to_string(); + } + if clean_val.ends_with("```") { + clean_val = clean_val[..clean_val.len() - 3].to_string(); + } + clean_val = clean_val.trim().to_string(); + } + + let val = if (clean_val.starts_with('{') && clean_val.ends_with('}')) + || (clean_val.starts_with('[') && clean_val.ends_with(']')) + { + serde_json::from_str(&clean_val) + .unwrap_or_else(|_| serde_json::Value::String(clean_val.clone())) + } else { + serde_json::Value::String(clean_val) + }; + + if !p_name.is_empty() { + if p_name == "path" { + params.insert("file".to_string(), val.clone()); + } + params.insert(p_name, val); + } + + p_pos = abs_p_start + p_close + "".len(); + } + + if !name.is_empty() { + calls.push(ParsedDsmlCall { + name, + arguments: serde_json::Value::Object(params), + }); + } + + search_pos = absolute_invoke_start + invoke_end + "".len(); + } + + calls +} + +pub fn extract_and_clean_dsml(text: &str) -> (String, Vec) { + let mut cleaned_text = String::new(); + let mut calls = Vec::new(); + let mut last_pos = 0; + + while let Some(start_pos) = text[last_pos..].find("<｜DSML｜tool_calls>") { + let abs_start = last_pos + start_pos; + cleaned_text.push_str(&text[last_pos..abs_start]); + + let rem = &text[abs_start..]; + if let Some(end_pos) = rem.find("") { + let abs_end = abs_start + end_pos + "".len(); + let dsml_block = &text[abs_start..abs_end]; + let parsed_calls = parse_dsml_tool_calls(dsml_block); + calls.extend(parsed_calls); + last_pos = abs_end; + } else { + let dsml_block = &text[abs_start..]; + let parsed_calls = parse_dsml_tool_calls(dsml_block); + calls.extend(parsed_calls); + last_pos = text.len(); + } + } + cleaned_text.push_str(&text[last_pos..]); + + let final_cleaned = strip_system_tags(&cleaned_text); + (final_cleaned, calls) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_strip_system_tags() { + assert_eq!(strip_system_tags("Hello"), "Hello"); + assert_eq!(strip_system_tags("\n\nHello"), "Hello"); + assert_eq!(strip_system_tags("Hello"), "Hello"); + assert_eq!(strip_system_tags("HelloWorld"), "HelloWorld"); + assert_eq!(strip_system_tags("\nHello"), "Hello"); + assert_eq!(strip_system_tags("\nHello"), "Hello"); + assert_eq!(strip_system_tags("\nHello"), "Hello"); + assert_eq!( + strip_system_tags("Some thinkingResponse"), + "Some thinkingResponse" + ); + assert_eq!(strip_system_tags("Normal text"), "Normal text"); + } + + #[test] + fn test_extract_attribute() { + assert_eq!(extract_attribute(r#"name="bash""#, "name"), "bash"); + assert_eq!(extract_attribute(r#"name='bash'"#, "name"), "bash"); + assert_eq!(extract_attribute(r#"name = "bash""#, "name"), "bash"); + assert_eq!(extract_attribute(r#"name = 'bash'"#, "name"), "bash"); + assert_eq!( + extract_attribute(r#"other="val" name="bash""#, "name"), + "bash" + ); + } + + #[test] + fn test_parse_dsml_tool_calls() { + let sample = r#" + <｜DSML｜tool_calls> + <｜DSML｜invoke name="Edit"> + <｜DSML｜parameter name="path">scripts/lib/process.sh + <｜DSML｜parameter name="edits"> +```json +[ + {"oldText": "foo", "newText": "bar"} +] +``` + + + + "#; + let res = parse_dsml_tool_calls(sample); + assert_eq!(res.len(), 1); + assert_eq!(res[0].name, "Edit"); + assert_eq!(res[0].arguments["path"], "scripts/lib/process.sh"); + assert_eq!(res[0].arguments["file"], "scripts/lib/process.sh"); + assert_eq!(res[0].arguments["edits"][0]["oldText"], "foo"); + assert_eq!(res[0].arguments["edits"][0]["newText"], "bar"); + } + + #[test] + fn test_extract_and_clean_dsml() { + let sample = "Hello <｜DSML｜tool_calls><｜DSML｜invoke name=\"bash\"><｜DSML｜parameter name=\"command\">git status World"; + let (text, calls) = extract_and_clean_dsml(sample); + assert_eq!(text, "Hello World"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "bash"); + assert_eq!(calls[0].arguments["command"], "git status"); + } +} diff --git a/src/opencode/search.rs b/src/opencode/search.rs index 960fe4c..0fd70bd 100644 --- a/src/opencode/search.rs +++ b/src/opencode/search.rs @@ -71,7 +71,7 @@ pub fn strip_html_tags(html: &str) -> String { // ── Types ── /// A structured search result with title, URL, and snippet. -#[allow(dead_code)] +#[allow(dead_code)] // kept for planned public structured search API #[derive(Debug, Clone)] pub struct SearchResult { pub title: String, @@ -80,7 +80,7 @@ pub struct SearchResult { } /// Enumeration of supported search providers. -#[allow(dead_code)] +#[allow(dead_code)] // kept for planned public provider metadata API #[derive(Debug, Clone, Copy, PartialEq)] pub enum SearchProviderKind { Tavily, diff --git a/src/opencode/types.rs b/src/opencode/types.rs index 69575b8..e40e46e 100644 --- a/src/opencode/types.rs +++ b/src/opencode/types.rs @@ -61,7 +61,6 @@ pub struct OpenAiFunction { pub parameters: serde_json::Value, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct OpenAiResponse { pub id: String, diff --git a/src/proxy_pool.rs b/src/proxy_pool.rs deleted file mode 100644 index e0378df..0000000 --- a/src/proxy_pool.rs +++ /dev/null @@ -1,1329 +0,0 @@ -// ── Routing policy constants ── - -/// Consecutive failures before proxy enters cooldown. -pub const FAILURE_THRESHOLD: u32 = 2; -/// Consecutive successes after cooldown to be considered fully healthy. -pub const RECOVERY_SUCCESS_COUNT: u32 = 2; -/// Default cooldown duration when failure threshold is reached (seconds). -pub const COOLDOWN_SECS: u64 = 120; - -use reqwest::Client; -use serde::Serialize; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::RwLock as TokioRwLock; -use tracing::{error, info, warn}; - -// ── Types ── - -/// Proxy trạng thái machine. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ProxyStatus { - /// Active — được dùng để route request, index < active_count - Active, - /// Spare — sẵn sàng thế chỗ active khi cần, index >= active_count - Spare, - /// Cooldown — đang tạm nghỉ vì rate-limit, Instant = thời điểm HẾT cooldown (tương lai) - Cooldown(Instant), - /// Dead — proxy chết, cần restart (có counter) - Dead { restart_attempts: u32 }, - /// Starting — container đang được khởi động, chờ verify - Starting, -} - -/// Proxy role in the two-tier architecture. -#[derive(Debug, Clone, Copy, PartialEq, Serialize)] -pub enum ProxyRole { - /// Primary managed proxy (40001-40003) — CLI may restart/stop/recover - Primary, - /// Warm-standby protected proxy (40004-40005) — CLI may only health-check - WarmStandby, -} - -/// Proxy lifecycle management policy. -#[derive(Debug, Clone, Copy, PartialEq, Serialize)] -pub enum ProxyLifecycle { - /// Fully managed by CLI — can be restarted, purged, recreated - Managed, - /// Protected — never stopped, restarted, purged, or recreated by CLI - Protected, -} - -/// Một entry trong proxy pool. -#[derive(Debug)] -pub struct ProxyEntry { - pub url: String, - pub client: Client, - pub status: ProxyStatus, - pub port: u16, - pub container_name: String, - /// Proxy role in the two-tier architecture (Primary/WarmStandby). - pub role: ProxyRole, - /// Proxy lifecycle management policy (Managed/Protected). - #[allow(dead_code)] - pub lifecycle: ProxyLifecycle, - /// Consecutive failures since last healthy state. - #[allow(dead_code)] - pub consecutive_failures: u32, - /// Consecutive successes since last healthy/cooldown state. - #[allow(dead_code)] - pub consecutive_successes: u32, -} - -/// Proxy pool với hot-spare model. -/// -/// - indices `[0..active_count)` là active slots (có thể Active, Cooldown, Dead, Starting) -/// - indices `[active_count..]` là spare slots (thường là Spare) -/// - Khi 1 active chết → swap status với 1 spare, push dead index vào restart_queue -#[derive(Debug, Default)] -pub struct ProxyPool { - pub proxies: Vec, - /// Number of active proxy slots (used in constructor to split active/spare). - #[allow(dead_code)] - pub active_count: usize, - pub restart_queue: Vec, -} - -// ── Stats types (exposed via /health and status) ── - -/// Snapshot of a single proxy node for health/status display. -#[derive(Debug, Clone, Serialize)] -pub struct ProxyNodeStats { - pub port: u16, - pub role: ProxyRole, - pub lifecycle: ProxyLifecycle, - pub status: String, - pub failure_count: u32, - pub success_count: u32, - pub cooldown_remaining_secs: Option, -} - -/// Aggregate stats for a tier (primary or warm-standby). -#[derive(Debug, Clone, Serialize)] -pub struct ProxyTierStats { - pub ports: Vec, - pub total: usize, - pub healthy: usize, - pub degraded: usize, - pub cooldown: usize, - pub recovering: usize, - pub dead: usize, - pub protected: bool, -} - -/// Full proxy pool snapshot for health/status endpoints. -#[derive(Debug, Clone, Serialize)] -pub struct ProxyPoolStats { - pub policy: String, - pub primary: ProxyTierStats, - pub warm_standby: ProxyTierStats, - pub nodes: Vec, -} - -// ── Helpers ── - -fn extract_port(url: &str) -> u16 { - url.rsplit(':') - .next() - .and_then(|s| s.trim_end_matches('/').parse().ok()) - .unwrap_or(0) -} - -fn container_name(url: &str) -> String { - let port = extract_port(url); - if (40001..=40099).contains(&port) { - format!("opencode-warp-{}", port - 40000) - } else { - format!("opencode-proxy-{}", port) - } -} - -/// Returns true if the port is a protected warm-standby proxy (40004-40005). -pub fn is_protected_proxy_port(port: u16) -> bool { - matches!(port, 40004 | 40005) -} - -/// Ensures a given port is NOT a protected warm-standby proxy. -/// Returns an error if it is, preventing destructive operations. -pub fn ensure_not_protected(port: u16) -> Result<(), String> { - if is_protected_proxy_port(port) { - Err(format!( - "refusing to modify protected warm-standby proxy port {} (40004-40005 are protected)", - port - )) - } else { - Ok(()) - } -} - -/// Returns the primary managed proxy ports (40001-40003). -pub fn get_primary_ports() -> [u16; 3] { - [40001, 40002, 40003] -} - -/// Returns the warm-standby protected proxy ports (40004-40005). -pub fn get_warm_standby_ports() -> [u16; 2] { - [40004, 40005] -} - -// ── Implementation ── - -impl ProxyStatus { - /// Human-readable description of the current status. - pub fn description(&self) -> &'static str { - match self { - ProxyStatus::Active => "healthy", - ProxyStatus::Spare => "spare", - ProxyStatus::Cooldown(_) => "cooldown", - ProxyStatus::Dead { .. } => "dead", - ProxyStatus::Starting => "starting", - } - } -} - -impl ProxyPool { - /// Create pool from a list of proxy URLs. - /// Reads BRIDGE_ACTIVE_PROXY_COUNT from env to determine active/spare split. - pub fn new(proxies_urls: &[String]) -> Self { - let mut proxies = Vec::new(); - for url in proxies_urls { - if let Ok(proxy) = reqwest::Proxy::all(url) { - if let Ok(client) = Client::builder() - .proxy(proxy) - .timeout(Duration::from_secs(600)) - .pool_max_idle_per_host(10) - .build() - { - let port = extract_port(url); - let cname = container_name(url); - proxies.push(ProxyEntry { - url: url.clone(), - client, - status: ProxyStatus::Active, - port, - container_name: cname, - role: if is_protected_proxy_port(port) { - ProxyRole::WarmStandby - } else { - ProxyRole::Primary - }, - lifecycle: if is_protected_proxy_port(port) { - ProxyLifecycle::Protected - } else { - ProxyLifecycle::Managed - }, - consecutive_failures: 0, - consecutive_successes: 0, - }); - info!("Added proxy to pool: {}", url); - } else { - warn!("Failed to build reqwest Client for proxy: {}", url); - } - } else { - warn!("Invalid proxy URL: {}", url); - } - } - - let total = proxies.len(); - let active_count = if total == 0 { - 0 - } else { - std::env::var("BRIDGE_ACTIVE_PROXY_COUNT") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or_else(|| total.saturating_sub(1)) - .max(1) - .min(total) - }; - - // Set indices >= active_count as Spare (but NOT WarmStandby proxies) - for proxy in proxies.iter_mut().take(total).skip(active_count) { - if proxy.role != ProxyRole::WarmStandby { - proxy.status = ProxyStatus::Spare; - } - } - - info!( - "Proxy pool initialized: {} total, {} active, {} spare", - total, - active_count, - total.saturating_sub(active_count) - ); - - Self { - proxies, - active_count, - restart_queue: Vec::new(), - } - } - - // ── Status helpers ── - - fn remaining_cooldown(status: &ProxyStatus) -> Duration { - match status { - ProxyStatus::Cooldown(until) => until - .checked_duration_since(Instant::now()) - .unwrap_or_default(), - ProxyStatus::Active => Duration::ZERO, - _ => Duration::MAX, - } - } - - fn is_usable(status: &ProxyStatus) -> bool { - matches!( - status, - ProxyStatus::Active | ProxyStatus::Spare | ProxyStatus::Cooldown(_) - ) - } - - // ── Health tracking ── - - /// Record a success for a proxy, building toward recovery threshold. - pub fn record_success(&mut self, idx: usize) { - if idx >= self.proxies.len() { - return; - } - let entry = &mut self.proxies[idx]; - entry.consecutive_failures = 0; - entry.consecutive_successes = entry.consecutive_successes.saturating_add(1); - - // Auto-recover from cooldown after enough consecutive successes - if matches!(entry.status, ProxyStatus::Cooldown(_)) - && entry.consecutive_successes >= RECOVERY_SUCCESS_COUNT - { - entry.status = ProxyStatus::Active; - entry.consecutive_failures = 0; - entry.consecutive_successes = 0; - info!( - "Proxy #{} ({}) recovered from cooldown after {} consecutive successes.", - idx, entry.url, RECOVERY_SUCCESS_COUNT - ); - } - } - - /// Record a failure for a proxy, potentially triggering cooldown. - pub fn record_failure(&mut self, idx: usize) { - if idx >= self.proxies.len() { - return; - } - self.proxies[idx].consecutive_successes = 0; - let failures = self.proxies[idx].consecutive_failures.saturating_add(1); - self.proxies[idx].consecutive_failures = failures; - - if failures >= FAILURE_THRESHOLD { - let duration = Duration::from_secs(COOLDOWN_SECS); - self.mark_rate_limited(idx, duration); - info!( - "Proxy #{} ({}) entered cooldown after {} consecutive failures ({}s).", - idx, self.proxies[idx].url, failures, COOLDOWN_SECS - ); - } - } - - // ── Selection helpers ── - - /// Returns indices of all proxies with Primary role and healthy status (Active or Spare). - #[allow(dead_code)] - fn healthy_primary_indices(&self) -> Vec { - self.proxies - .iter() - .enumerate() - .filter(|(_, p)| { - p.role == ProxyRole::Primary - && matches!(p.status, ProxyStatus::Active | ProxyStatus::Spare) - }) - .map(|(i, _)| i) - .collect() - } - - /// Returns indices of all proxies with WarmStandby role and healthy status. - fn healthy_warm_standby_indices(&self) -> Vec { - self.proxies - .iter() - .enumerate() - .filter(|(_, p)| { - p.role == ProxyRole::WarmStandby - && matches!(p.status, ProxyStatus::Active | ProxyStatus::Spare) - }) - .map(|(i, _)| i) - .collect() - } - - /// Returns the rendezvous-assigned primary for a routing key, - /// considering ALL primaries regardless of health status. - /// This ensures sticky assignment: even if a primary is on cooldown, - /// the key still maps to the same slot, enabling correct WarmStandby failover. - fn rendezvous_assigned_primary(&self, routing_key: &str) -> Option { - let all_primaries: Vec = self - .proxies - .iter() - .enumerate() - .filter(|(_, p)| p.role == ProxyRole::Primary) - .map(|(i, _)| i) - .collect(); - if all_primaries.is_empty() { - return None; - } - all_primaries - .iter() - .copied() - .max_by_key(|idx| stable_rendezvous_score(routing_key, &self.proxies[*idx].url)) - } - - /// Select the best WarmStandby failover for a routing key via Rendezvous hashing. - fn rendezvous_warm_standby(&self, routing_key: &str) -> Option { - let candidates = self.healthy_warm_standby_indices(); - if candidates.is_empty() { - return None; - } - candidates - .iter() - .copied() - .max_by_key(|idx| stable_rendezvous_score(routing_key, &self.proxies[*idx].url)) - } - - // ── Main API ── - - /// Select a proxy for the given routing key following the Phase 5 routing contract: - /// - /// 1. Use Primary proxies 40001–40003 for normal traffic. - /// 2. Use WarmStandby proxies 40004–40005 only when the selected primary - /// proxy is unhealthy/cooldown/dead. - /// 3. Affected-agent-only remap: failure of one primary does NOT remap - /// agents assigned to healthy primaries. - /// 4. Implement Rendezvous hashing for stable sticky determinism. - /// 5. Comply with cooldown/recovery policy. - /// - /// Returns `(Client, proxy_url, index)` or `None` if no proxy is available. - pub fn select_proxy_for_key(&self, routing_key: &str) -> Option<(Client, String, usize)> { - if self.proxies.is_empty() { - return None; - } - - // Step 1: Rendezvous → assigned primary (all primaries, healthy or not) - let assigned = self.rendezvous_assigned_primary(routing_key); - - if let Some(primary_idx) = assigned { - let entry = &self.proxies[primary_idx]; - - // If cooldown has expired, the proxy is healthy again - let is_healthy = match entry.status { - ProxyStatus::Active => true, - ProxyStatus::Cooldown(until) => Instant::now() >= until, - _ => false, - }; - - if is_healthy { - return Some((entry.client.clone(), entry.url.clone(), primary_idx)); - } - - // Primary is unhealthy → step 2: failover to WarmStandby - info!( - "Rendezvous primary #{} ({}) for key '{}' is unavailable (status={:?}). Failing over to WarmStandby.", - primary_idx, entry.url, routing_key, entry.status - ); - } - - // Step 2: Rendezvous → assigned WarmStandby - if let Some(standby_idx) = self.rendezvous_warm_standby(routing_key) { - let entry = &self.proxies[standby_idx]; - let is_healthy = match entry.status { - ProxyStatus::Active => true, - ProxyStatus::Cooldown(until) => Instant::now() >= until, - _ => false, - }; - - if is_healthy { - return Some((entry.client.clone(), entry.url.clone(), standby_idx)); - } - } - - // Step 3: Degraded — pick any usable proxy - let degraded = self.select_degraded(); - if let Some(idx) = degraded { - warn!( - "CRITICAL: All proxies unavailable for key '{}'. Degraded mode, using proxy #{} ({})", - routing_key, idx, self.proxies[idx].url - ); - return Some(( - self.proxies[idx].client.clone(), - self.proxies[idx].url.clone(), - idx, - )); - } - - None - } - - /// Legacy compatibility: selects proxy for a routing key. - /// Delegates to `select_proxy_for_key`. Provided as an alias for callers - /// that haven't been updated to the new API name yet. - pub fn get_client(&mut self, api_key: &str) -> Option<(Client, String, usize)> { - self.select_proxy_for_key(api_key) - } - - /// Select a proxy excluding a specific index (for retry failover). - /// Uses the same primary-first, WarmStandby-failover policy but skips - /// the excluded index. - pub fn get_client_excluding( - &mut self, - api_key: &str, - _exclude_idx: usize, - ) -> Option<(Client, String, usize)> { - // For Phase 5, we use select_proxy_for_key which is role-aware. - // If the excluded index happens to be the rendezvous primary, we - // fall through to WarmStandby or degraded. - self.select_proxy_for_key(api_key) - } - - /// Mark a proxy as rate-limited for a specific duration. - pub fn mark_rate_limited(&mut self, idx: usize, duration: Duration) { - if idx < self.proxies.len() { - let until = Instant::now() + duration; - self.proxies[idx].status = ProxyStatus::Cooldown(until); - warn!( - "Proxy #{} ({}) marked as rate-limited until {:?}", - idx, self.proxies[idx].url, until - ); - } - } - - /// Mark rate-limited with adaptive duration (base × 2^retry × jitter). - pub fn mark_rate_limited_adaptive(&mut self, idx: usize, retry_count: u32) { - let base_secs = 60 * 2u64.pow(retry_count.min(3)); - // Deterministic jitter ±25% — no rand crate needed - let jitter_factor = match idx % 4 { - 0 => 100, - 1 => 85, - 2 => 115, - _ => 95, - }; - let secs = base_secs * jitter_factor / 100; - let duration = Duration::from_secs(secs); - self.mark_rate_limited(idx, duration); - } - - /// Mark a proxy as healthy (clear cooldown/dead). - #[allow(dead_code)] - pub fn mark_healthy(&mut self, idx: usize) { - if idx < self.proxies.len() { - self.proxies[idx].status = ProxyStatus::Active; - info!( - "Proxy #{} ({}) marked as healthy.", - idx, self.proxies[idx].url - ); - } - } - - /// Drain the restart queue. - pub fn drain_restart_queue(&mut self) -> Vec { - std::mem::take(&mut self.restart_queue) - } - - // ── Snapshot ── - - /// Build a full health snapshot of the proxy pool. - pub fn snapshot(&self) -> ProxyPoolStats { - let mut primary_ports = Vec::new(); - let mut ws_ports = Vec::new(); - let mut primary_healthy = 0usize; - let mut primary_degraded = 0usize; - let mut primary_cooldown = 0usize; - let mut primary_recovering = 0usize; - let mut primary_dead = 0usize; - let mut ws_healthy = 0usize; - let mut ws_degraded = 0usize; - let mut ws_cooldown = 0usize; - let mut ws_recovering = 0usize; - let mut ws_dead = 0usize; - let mut nodes = Vec::new(); - - for p in &self.proxies { - let status_str = p.status.description().to_string(); - let cooldown_remaining = if let ProxyStatus::Cooldown(until) = p.status { - Some( - until - .checked_duration_since(Instant::now()) - .unwrap_or_default() - .as_secs(), - ) - } else { - None - }; - - nodes.push(ProxyNodeStats { - port: p.port, - role: p.role, - lifecycle: p.lifecycle, - status: status_str, - failure_count: p.consecutive_failures, - success_count: p.consecutive_successes, - cooldown_remaining_secs: cooldown_remaining, - }); - - match p.role { - ProxyRole::Primary => { - primary_ports.push(p.port); - match p.status { - ProxyStatus::Active => primary_healthy += 1, - ProxyStatus::Spare => primary_degraded += 1, - ProxyStatus::Cooldown(_) => primary_cooldown += 1, - ProxyStatus::Dead { .. } => primary_dead += 1, - ProxyStatus::Starting => primary_recovering += 1, - } - } - ProxyRole::WarmStandby => { - ws_ports.push(p.port); - match p.status { - ProxyStatus::Active => ws_healthy += 1, - ProxyStatus::Spare => ws_degraded += 1, - ProxyStatus::Cooldown(_) => ws_cooldown += 1, - ProxyStatus::Dead { .. } => ws_dead += 1, - ProxyStatus::Starting => ws_recovering += 1, - } - } - } - } - - let total_primary = primary_ports.len(); - let total_ws = ws_ports.len(); - - ProxyPoolStats { - policy: "primary-with-warm-standby".to_string(), - primary: ProxyTierStats { - ports: primary_ports, - total: total_primary, - healthy: primary_healthy, - degraded: primary_degraded, - cooldown: primary_cooldown, - recovering: primary_recovering, - dead: primary_dead, - protected: false, - }, - warm_standby: ProxyTierStats { - ports: ws_ports, - total: total_ws, - healthy: ws_healthy, - degraded: ws_degraded, - cooldown: ws_cooldown, - recovering: ws_recovering, - dead: ws_dead, - protected: true, - }, - nodes, - } - } - - // ── Private ── - - /// Select the proxy closest to cooldown end (degraded mode). - fn select_degraded(&self) -> Option { - self.proxies - .iter() - .enumerate() - .filter(|(_, p)| Self::is_usable(&p.status)) - .min_by_key(|(_, p)| Self::remaining_cooldown(&p.status)) - .map(|(i, _)| i) - } -} - -// ── Stable hash helpers ── - -/// Deterministic 64-bit score for Rendezvous hashing. -/// -/// Uses DefaultHasher (std) for now. This is deterministic within the same -/// process execution but may vary across Rust versions. For fully stable -/// cross-build determinism, replace with sha2 or blake3. -pub fn stable_rendezvous_score(key: &str, node_id: &str) -> u64 { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - node_id.hash(&mut hasher); - hasher.finish() -} - -// ── Background Tasks (Docker restart, health monitoring) ── - -/// Process restart queue: docker rm -f + docker run + verify. -/// Processes one container at a time; re-queues on failure (max 3 attempts). -pub async fn process_restart_queue(pool: Arc>) { - let mut interval = tokio::time::interval(Duration::from_secs(2)); - loop { - interval.tick().await; - let indices: Vec = pool.write().await.drain_restart_queue(); - for idx in indices { - restart_container(idx, pool.clone()).await; - } - } -} - -/// TCP health monitor — checks Dead/Starting proxies every 10s. -/// If TCP connect succeeds, marks proxy as Spare. -pub async fn health_monitor(pool: Arc>) { - let mut interval = tokio::time::interval(Duration::from_secs(10)); - loop { - interval.tick().await; - - let targets: Vec<(usize, u16)> = { - let p = pool.read().await; - p.proxies - .iter() - .enumerate() - .filter(|(_, e)| { - matches!(e.status, ProxyStatus::Dead { .. } | ProxyStatus::Starting) - }) - .map(|(i, e)| (i, e.port)) - .collect() - }; - - for (idx, port) in targets { - if port == 0 { - continue; - } - if tokio::net::TcpStream::connect(format!("127.0.0.1:{}", port)) - .await - .is_ok() - { - let mut p = pool.write().await; - if idx < p.proxies.len() { - // Only mark as Spare if still dead (not manually re-assigned) - if matches!( - p.proxies[idx].status, - ProxyStatus::Dead { .. } | ProxyStatus::Starting - ) { - p.proxies[idx].status = ProxyStatus::Spare; - info!( - "Proxy #{} ({}) recovered via TCP health check.", - idx, p.proxies[idx].container_name - ); - } - } - } - } - } -} - -/// Restart a single Docker container by proxy pool index. -async fn restart_container(idx: usize, pool: Arc>) { - let (port, container_name) = { - let p = pool.read().await; - if idx >= p.proxies.len() { - return; - } - (p.proxies[idx].port, p.proxies[idx].container_name.clone()) - }; - - if port == 0 { - warn!("Cannot restart proxy #{}: unknown port", idx); - return; - } - - if let Err(msg) = ensure_not_protected(port) { - warn!("{}", msg); - return; - } - - info!( - "Restarting proxy container #{} ({}) on port {}...", - idx, container_name, port - ); - - // Mark as Starting - pool.write().await.proxies[idx].status = ProxyStatus::Starting; - - // docker rm -f - let rm = tokio::process::Command::new("docker") - .args(["rm", "-f", &container_name]) - .output() - .await; - - match &rm { - Ok(o) if !o.status.success() => { - let stderr = String::from_utf8_lossy(&o.stderr); - warn!( - "docker rm -f {} (may not exist): {}", - container_name, stderr - ); - } - Err(e) => { - warn!("docker rm -f {} failed: {}", container_name, e); - } - _ => {} - } - - // docker run -d --name ... --restart always ... - let run = tokio::process::Command::new("docker") - .args([ - "run", - "-d", - "--name", - &container_name, - "--restart", - "always", - "--cap-add=NET_ADMIN", - "--sysctl", - "net.ipv4.conf.all.src_valid_mark=1", - "-p", - &format!("{}:9091", port), - "ghcr.io/mon-ius/docker-warp-socks:latest", - ]) - .output() - .await; - - match run { - Ok(output) if output.status.success() => { - info!("Container {} created successfully.", container_name); - } - Ok(output) => { - let stderr = String::from_utf8_lossy(&output.stderr); - error!("Failed to create container {}: {}", container_name, stderr); - requeue_or_giveup(idx, &pool, "docker create failed").await; - return; - } - Err(e) => { - error!( - "Docker command failed for {}: {}. Is docker installed?", - container_name, e - ); - requeue_or_giveup(idx, &pool, "docker command error").await; - return; - } - } - - // Verify connectivity - let ok = verify_proxy_socks(port).await; - - let mut p = pool.write().await; - if idx >= p.proxies.len() { - return; - } - - if ok { - p.proxies[idx].status = ProxyStatus::Spare; - info!( - "Proxy #{} ({}) restarted and verified as Spare.", - idx, container_name - ); - } else { - let attempts = match p.proxies[idx].status { - ProxyStatus::Dead { - restart_attempts: n, - } => n + 1, - _ => 1, - }; - if attempts < 3 { - p.proxies[idx].status = ProxyStatus::Dead { - restart_attempts: attempts, - }; - p.restart_queue.push(idx); - warn!( - "Proxy #{} restart verify failed, re-queuing (attempt {}/3)", - idx, attempts - ); - } else { - p.proxies[idx].status = ProxyStatus::Dead { - restart_attempts: attempts, - }; - error!("Proxy #{} failed restart after 3 attempts. Giving up.", idx); - } - } -} - -/// Re-queue a failed restart or give up after 3 attempts. -async fn requeue_or_giveup(idx: usize, pool: &Arc>, reason: &str) { - let mut p = pool.write().await; - if idx >= p.proxies.len() { - return; - } - let attempts = match p.proxies[idx].status { - ProxyStatus::Dead { - restart_attempts: n, - } => n + 1, - _ => 1, - }; - if attempts < 3 { - p.proxies[idx].status = ProxyStatus::Dead { - restart_attempts: attempts, - }; - p.restart_queue.push(idx); - warn!( - "Proxy #{} restart queued (attempt {}/3): {}", - idx, attempts, reason - ); - } else { - p.proxies[idx].status = ProxyStatus::Dead { - restart_attempts: attempts, - }; - error!( - "Proxy #{} failed restart after 3 attempts ({}). Giving up.", - idx, reason - ); - } -} - -/// Verify SOCKS5 proxy connectivity via cloudflare CDN trace. -async fn verify_proxy_socks(port: u16) -> bool { - let proxy_url = format!("socks5h://127.0.0.1:{}", port); - let proxy = match reqwest::Proxy::all(&proxy_url) { - Ok(p) => p, - Err(e) => { - warn!( - "Invalid proxy URL '{}' in verify_proxy_socks: {}", - proxy_url, e - ); - return false; - } - }; - let client = match reqwest::Client::builder() - .proxy(proxy) - .timeout(Duration::from_secs(5)) - .build() - { - Ok(c) => c, - Err(_) => return false, - }; - - for attempt in 1..=12 { - if client - .get("https://cloudflare.com/cdn-cgi/trace") - .send() - .await - .is_ok() - { - info!("Proxy on port {} verified successfully.", port); - return true; - } - if attempt < 12 { - info!( - "Waiting for proxy on port {}... (attempt {}/12)", - port, attempt - ); - tokio::time::sleep(Duration::from_secs(5)).await; - } - } - - warn!( - "Proxy on port {} failed verification after 12 attempts.", - port - ); - false -} - -// ── Tests ── - -#[cfg(test)] -mod tests { - use super::*; - - fn make_test_urls(count: usize) -> Vec { - (0..count) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect() - } - - #[test] - fn test_proxy_pool_mapping() { - let urls = make_test_urls(3); - let mut pool = ProxyPool::new(&urls); - // 3 proxies → 2 active + 1 spare - assert_eq!(pool.proxies.len(), 3); - assert_eq!(pool.active_count, 2); - - // Same API key should always map to same proxy index - let res1 = pool.get_client("agent-1").unwrap(); - let res2 = pool.get_client("agent-1").unwrap(); - assert_eq!(res1.2, res2.2); - - // Different API keys may map to different indexes - let res3 = pool.get_client("agent-2").unwrap(); - info!("agent-1 mapped to preferred proxy index {}", res1.2); - info!("agent-2 mapped to preferred proxy index {}", res3.2); - } - - #[test] - fn test_sticky_mapping_stable() { - // Given all 3 primaries healthy - let urls = make_test_urls(3); - let pool = ProxyPool::new(&urls); - - // When same agent key selects proxy 100 times - let agent = "sticky-agent-42"; - let first = pool.select_proxy_for_key(agent).unwrap().2; - - for _ in 0..100 { - let result = pool.select_proxy_for_key(agent).unwrap(); - // Then selected primary is always the same - assert_eq!( - result.2, first, - "sticky mapping changed for key '{}' on iteration", - agent - ); - // And selected role is Primary - assert_eq!( - pool.proxies[result.2].role, - ProxyRole::Primary, - "sticky agent mapped to non-Primary proxy" - ); - } - } - - #[test] - fn test_affected_agent_only_remap() { - // Given 3 primaries (40001-40003) + 2 warm-standby (40004-40005) - let urls: Vec = (0..5) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let mut pool = ProxyPool::new(&urls); - - // Agent_a → assigned primary, agent_b → assigned primary, agent_c → assigned primary - let a_idx = pool.select_proxy_for_key("agent_a").unwrap().2; - let b_idx = pool.select_proxy_for_key("agent_b").unwrap().2; - let c_idx = pool.select_proxy_for_key("agent_c").unwrap().2; - - // Verify all are primary - assert_eq!(pool.proxies[a_idx].role, ProxyRole::Primary); - assert_eq!(pool.proxies[b_idx].role, ProxyRole::Primary); - assert_eq!(pool.proxies[c_idx].role, ProxyRole::Primary); - - // Save the assigned primaries (their index in pool) - let a_primary = a_idx; - let b_primary = b_idx; - let c_primary = c_idx; - - // When agent_b's primary (b_primary) is marked cooldown/dead - pool.mark_rate_limited(b_primary, Duration::from_secs(300)); - - // Then agent_b fails over to WarmStandby - let b_failover = pool.select_proxy_for_key("agent_b").unwrap().2; - assert_eq!( - pool.proxies[b_failover].role, - ProxyRole::WarmStandby, - "agent_b should failover to WarmStandby, got index {} role {:?}", - b_failover, - pool.proxies[b_failover].role - ); - - // And agent_a still maps to its original primary - let a_after = pool.select_proxy_for_key("agent_a").unwrap().2; - assert_eq!( - a_after, a_primary, - "agent_a remapped from primary {} to {}, expected no change", - a_primary, a_after - ); - - // And agent_c still maps to its original primary - let c_after = pool.select_proxy_for_key("agent_c").unwrap().2; - assert_eq!( - c_after, c_primary, - "agent_c remapped from primary {} to {}, expected no change", - c_primary, c_after - ); - } - - #[test] - fn test_temporary_failover_to_warm_standby() { - let urls: Vec = (0..5) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let mut pool = ProxyPool::new(&urls); - - // Get agent's assigned primary - let primary = pool.select_proxy_for_key("failover-agent").unwrap().2; - assert_eq!(pool.proxies[primary].role, ProxyRole::Primary); - - // Mark the selected primary unhealthy - pool.mark_rate_limited(primary, Duration::from_secs(300)); - - // When selected primary unhealthy and warm standby healthy, - // request routes to 40004 or 40005 - let result = pool.select_proxy_for_key("failover-agent").unwrap(); - assert_eq!( - pool.proxies[result.2].role, - ProxyRole::WarmStandby, - "failover should route to WarmStandby, got idx {} role {:?}", - result.2, - pool.proxies[result.2].role - ); - } - - #[test] - fn test_recovery_returns_to_primary() { - let urls: Vec = (0..3) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let mut pool = ProxyPool::new(&urls); - - // Get agent's assigned primary - let primary_idx = pool.select_proxy_for_key("recovery-agent").unwrap().2; - assert_eq!(pool.proxies[primary_idx].role, ProxyRole::Primary); - assert_eq!(pool.proxies[primary_idx].status, ProxyStatus::Active); - - // Mark the primary as rate-limited (simulate failure) - pool.mark_rate_limited(primary_idx, Duration::from_secs(0)); // 0s = already expired - - // After cooldown has expired (0s), the proxy should be healthy again - let result = pool.select_proxy_for_key("recovery-agent").unwrap(); - assert_eq!( - result.2, primary_idx, - "after cooldown expiry, agent should return to original primary {} not {}", - primary_idx, result.2 - ); - } - - #[test] - fn test_no_standby_if_selected_primary_healthy() { - let urls: Vec = (0..5) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let pool = ProxyPool::new(&urls); - - // Even if standby exists and healthy, selected primary healthy → use primary - for key in &["test-a", "test-b", "test-c", "test-d", "test-e"] { - let result = pool.select_proxy_for_key(key).unwrap(); - assert_eq!( - pool.proxies[result.2].role, - ProxyRole::Primary, - "key '{}' selected standby when primary was healthy", - key - ); - } - } - - #[test] - fn test_rendezvous_deterministic() { - let urls: Vec = (0..3) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let _pool = ProxyPool::new(&urls); - - // Rendezvous score for the same key+node should be deterministic - let score1 = stable_rendezvous_score("agent-x", "socks5://127.0.0.1:40001"); - let score2 = stable_rendezvous_score("agent-x", "socks5://127.0.0.1:40001"); - assert_eq!(score1, score2, "rendezvous score must be deterministic"); - - // Different nodes should have different scores - let score3 = stable_rendezvous_score("agent-x", "socks5://127.0.0.1:40002"); - assert_ne!( - score1, score3, - "different nodes should have different scores" - ); - } - - #[test] - fn test_warm_standby_excluded_from_normal_routing() { - // Build 5 proxies: 40001-40003 primary, 40004-40005 warm-standby - let urls: Vec = (0..5) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let mut pool = ProxyPool::new(&urls); - - assert_eq!(pool.proxies.len(), 5); - // Verify roles assigned correctly - assert_eq!(pool.proxies[0].role, ProxyRole::Primary); - assert_eq!(pool.proxies[1].role, ProxyRole::Primary); - assert_eq!(pool.proxies[2].role, ProxyRole::Primary); - assert_eq!(pool.proxies[3].role, ProxyRole::WarmStandby); - assert_eq!(pool.proxies[4].role, ProxyRole::WarmStandby); - - // Normal traffic should NEVER select a WarmStandby proxy when - // any Primary proxy is available. Run many keys to be sure. - for key in &["alpha", "beta", "gamma", "delta", "epsilon"] { - let (_, url, idx) = pool.get_client(key).unwrap(); - assert!( - pool.proxies[idx].role == ProxyRole::Primary, - "get_client('{}') returned WarmStandby {} (idx {}), expected Primary", - key, - url, - idx - ); - } - - // Mark all three primaries as rate-limited - for i in 0..3 { - pool.mark_rate_limited(i, Duration::from_secs(300)); - } - - // Now all primaries are on cooldown → spare swap should trigger. - // The spare at index 3 or 4 is WarmStandby → it should be used as failover. - let (_, url, idx) = pool.get_client("failover-test").unwrap(); - assert!( - pool.proxies[idx].role == ProxyRole::WarmStandby, - "expected WarmStandby in failover, got role={:?} at idx={} url={}", - pool.proxies[idx].role, - idx, - url - ); - } - - #[test] - fn test_empty_pool_returns_none() { - let mut pool = ProxyPool::default(); - assert!(pool.get_client("test").is_none()); - } - - #[test] - fn test_mark_healthy() { - let urls = make_test_urls(1); - let mut pool = ProxyPool::new(&urls); - - pool.mark_rate_limited(0, Duration::from_secs(60)); - assert!(matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_))); - - pool.mark_healthy(0); - assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); - } - - #[test] - fn test_drain_restart_queue() { - let urls = make_test_urls(3); - let mut pool = ProxyPool::new(&urls); - assert!(pool.drain_restart_queue().is_empty()); - - // Manually push to queue - pool.restart_queue.push(0); - pool.restart_queue.push(1); - assert_eq!(pool.drain_restart_queue().len(), 2); - assert!(pool.drain_restart_queue().is_empty()); // second drain should be empty - } - - #[test] - fn test_container_name_generation() { - assert_eq!( - container_name("socks5://127.0.0.1:40001"), - "opencode-warp-1" - ); - assert_eq!( - container_name("socks5://127.0.0.1:40005"), - "opencode-warp-5" - ); - assert_eq!( - container_name("http://127.0.0.1:9999"), - "opencode-proxy-9999" - ); - } - - #[test] - fn test_extract_port() { - assert_eq!(extract_port("socks5://127.0.0.1:40001"), 40001); - assert_eq!(extract_port("http://127.0.0.1:8080/"), 8080); - assert_eq!(extract_port("invalid"), 0); - } - - // ── Phase 6: Telemetry & Health tests ── - - #[test] - fn test_record_failure_enters_cooldown() { - let urls = make_test_urls(1); - let mut pool = ProxyPool::new(&urls); - - // Initial state: healthy - assert_eq!(pool.proxies[0].consecutive_failures, 0); - assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); - - // First failure should not trigger cooldown (threshold is 2) - pool.record_failure(0); - assert_eq!(pool.proxies[0].consecutive_failures, 1); - assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); - - // Second failure triggers cooldown - pool.record_failure(0); - assert!(matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_))); - } - - #[test] - fn test_http_400_does_not_mark_proxy_failed() { - // record_success is called for any HTTP response, including 400. - // It resets failure count — confirming HTTP 400 does NOT mark proxy failed. - let urls = make_test_urls(1); - let mut pool = ProxyPool::new(&urls); - - // Set up a failure first so we can verify success resets it - pool.record_failure(0); - assert_eq!(pool.proxies[0].consecutive_failures, 1); - - // Simulate HTTP 400 response: record_success (transport worked) - pool.record_success(0); - assert_eq!(pool.proxies[0].consecutive_failures, 0); - assert_eq!(pool.proxies[0].consecutive_successes, 1); - assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); - } - - #[test] - fn test_health_json_contains_proxy_pool() { - let urls: Vec = (0..5) - .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) - .collect(); - let pool = ProxyPool::new(&urls); - let stats = pool.snapshot(); - - // policy - assert_eq!(stats.policy, "primary-with-warm-standby"); - - // Primary tier - assert_eq!(stats.primary.ports, vec![40001, 40002, 40003]); - assert_eq!(stats.primary.total, 3); - assert_eq!(stats.primary.healthy, 3); - assert!(!stats.primary.protected); - - // WarmStandby tier - assert_eq!(stats.warm_standby.ports, vec![40004, 40005]); - assert_eq!(stats.warm_standby.total, 2); - assert_eq!(stats.warm_standby.healthy, 2); - assert!(stats.warm_standby.protected); - - // Nodes - assert_eq!(stats.nodes.len(), 5); - assert_eq!(stats.nodes[0].role, ProxyRole::Primary); - assert_eq!(stats.nodes[3].role, ProxyRole::WarmStandby); - assert_eq!(stats.nodes[3].lifecycle, ProxyLifecycle::Protected); - assert!(stats.nodes[0].cooldown_remaining_secs.is_none()); - } - - #[test] - fn test_snapshot_shows_cooldown_count() { - let urls = make_test_urls(5); - let mut pool = ProxyPool::new(&urls); - - // Mark two primaries and one warm-standby as rate-limited - pool.mark_rate_limited(1, Duration::from_secs(60)); - pool.mark_rate_limited(2, Duration::from_secs(120)); - pool.mark_rate_limited(3, Duration::from_secs(300)); - - let stats = pool.snapshot(); - - assert_eq!(stats.primary.cooldown, 2); - assert_eq!(stats.primary.healthy, 1); - assert_eq!(stats.warm_standby.cooldown, 1); - assert_eq!(stats.warm_standby.healthy, 1); - } - - #[test] - fn test_record_success_recovers_after_threshold() { - let urls = make_test_urls(1); - let mut pool = ProxyPool::new(&urls); - - // Force proxy into cooldown via failure threshold - pool.record_failure(0); // failure 1 - pool.record_failure(0); // failure 2 → enters cooldown - assert!(matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_))); - assert_eq!(pool.proxies[0].consecutive_successes, 0); - - // RECOVERY_SUCCESS_COUNT = 2. After 2 successes it should recover. - pool.record_success(0); - assert!( - matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_)), - "still in cooldown after 1 success" - ); - assert_eq!(pool.proxies[0].consecutive_successes, 1); - - pool.record_success(0); - assert!( - matches!(pool.proxies[0].status, ProxyStatus::Active), - "recovered after {} successes", - RECOVERY_SUCCESS_COUNT - ); - assert_eq!(pool.proxies[0].consecutive_failures, 0); - assert_eq!(pool.proxies[0].consecutive_successes, 0); - } -} diff --git a/src/proxy_pool/maintenance.rs b/src/proxy_pool/maintenance.rs new file mode 100644 index 0000000..1406463 --- /dev/null +++ b/src/proxy_pool/maintenance.rs @@ -0,0 +1,360 @@ +//! Proxy health tracking, cooldown/recovery, and Docker container lifecycle. +//! +//! Tracks consecutive successes/failures per proxy, manages adaptive cooldown, +//! auto-recovery thresholds, and Docker container restart/monitor tasks. + +use super::types::*; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock as TokioRwLock; +use tracing::{error, info, warn}; + +impl ProxyPool { + /// Record a success for a proxy, building toward recovery threshold. + pub fn record_success(&mut self, idx: usize) { + if idx >= self.proxies.len() { + return; + } + let entry = &mut self.proxies[idx]; + entry.consecutive_failures = 0; + entry.consecutive_successes = entry.consecutive_successes.saturating_add(1); + + // Auto-recover from cooldown after enough consecutive successes + if matches!(entry.status, ProxyStatus::Cooldown(_)) + && entry.consecutive_successes >= RECOVERY_SUCCESS_COUNT + { + entry.status = ProxyStatus::Active; + entry.consecutive_failures = 0; + entry.consecutive_successes = 0; + info!( + "Proxy #{} ({}) recovered from cooldown after {} consecutive successes.", + idx, entry.url, RECOVERY_SUCCESS_COUNT + ); + } + } + + /// Record a failure for a proxy, potentially triggering cooldown. + pub fn record_failure(&mut self, idx: usize) { + if idx >= self.proxies.len() { + return; + } + self.proxies[idx].consecutive_successes = 0; + let failures = self.proxies[idx].consecutive_failures.saturating_add(1); + self.proxies[idx].consecutive_failures = failures; + + if failures >= FAILURE_THRESHOLD { + let duration = Duration::from_secs(COOLDOWN_SECS); + self.mark_rate_limited(idx, duration); + info!( + "Proxy #{} ({}) entered cooldown after {} consecutive failures ({}s).", + idx, self.proxies[idx].url, failures, COOLDOWN_SECS + ); + } + } + + /// Mark a proxy as rate-limited for a specific duration. + pub fn mark_rate_limited(&mut self, idx: usize, duration: Duration) { + if idx < self.proxies.len() { + let until = Instant::now() + duration; + self.proxies[idx].status = ProxyStatus::Cooldown(until); + warn!( + "Proxy #{} ({}) marked as rate-limited until {:?}", + idx, self.proxies[idx].url, until + ); + } + } + + /// Mark rate-limited with adaptive duration (base × 2^retry × jitter). + pub fn mark_rate_limited_adaptive(&mut self, idx: usize, retry_count: u32) { + let base_secs = 60 * 2u64.pow(retry_count.min(3)); + // Deterministic jitter ±25% — no rand crate needed + let jitter_factor = match idx % 4 { + 0 => 100, + 1 => 85, + 2 => 115, + _ => 95, + }; + let secs = base_secs * jitter_factor / 100; + let duration = Duration::from_secs(secs); + self.mark_rate_limited(idx, duration); + } + + /// Mark a proxy as healthy (clear cooldown/dead). + #[allow(dead_code)] + pub fn mark_healthy(&mut self, idx: usize) { + if idx < self.proxies.len() { + self.proxies[idx].status = ProxyStatus::Active; + info!( + "Proxy #{} ({}) marked as healthy.", + idx, self.proxies[idx].url + ); + } + } +} + +// ── Background Tasks (Docker restart, health monitoring) ── + +/// Process restart queue: docker rm -f + docker run + verify. +/// Processes one container at a time; re-queues on failure (max 3 attempts). +pub async fn process_restart_queue(pool: Arc>) { + let mut interval = tokio::time::interval(Duration::from_secs(2)); + loop { + interval.tick().await; + let indices: Vec = pool.write().await.drain_restart_queue(); + for idx in indices { + restart_container(idx, pool.clone()).await; + } + } +} + +/// TCP health monitor — checks Dead/Starting proxies every 10s. +/// If TCP connect succeeds, marks proxy as Spare. +pub async fn health_monitor(pool: Arc>) { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + loop { + interval.tick().await; + + let targets: Vec<(usize, u16)> = { + let p = pool.read().await; + p.proxies + .iter() + .enumerate() + .filter(|(_, e)| { + matches!(e.status, ProxyStatus::Dead { .. } | ProxyStatus::Starting) + }) + .map(|(i, e)| (i, e.port)) + .collect() + }; + + for (idx, port) in targets { + if port == 0 { + continue; + } + if tokio::net::TcpStream::connect(format!("127.0.0.1:{}", port)) + .await + .is_ok() + { + let mut p = pool.write().await; + if idx < p.proxies.len() { + // Only mark as Spare if still dead (not manually re-assigned) + if matches!( + p.proxies[idx].status, + ProxyStatus::Dead { .. } | ProxyStatus::Starting + ) { + p.proxies[idx].status = ProxyStatus::Spare; + info!( + "Proxy #{} ({}) recovered via TCP health check.", + idx, p.proxies[idx].container_name + ); + } + } + } + } + } +} + +/// Restart a single Docker container by proxy pool index. +async fn restart_container(idx: usize, pool: Arc>) { + let (port, container_name) = { + let p = pool.read().await; + if idx >= p.proxies.len() { + return; + } + (p.proxies[idx].port, p.proxies[idx].container_name.clone()) + }; + + if port == 0 { + warn!("Cannot restart proxy #{}: unknown port", idx); + return; + } + + if let Err(msg) = ensure_not_protected(port) { + warn!("{}", msg); + return; + } + + info!( + "Restarting proxy container #{} ({}) on port {}...", + idx, container_name, port + ); + + // Mark as Starting + pool.write().await.proxies[idx].status = ProxyStatus::Starting; + + // docker rm -f + let rm = tokio::process::Command::new("docker") + .args(["rm", "-f", &container_name]) + .output() + .await; + + match &rm { + Ok(o) if !o.status.success() => { + let stderr = String::from_utf8_lossy(&o.stderr); + warn!( + "docker rm -f {} (may not exist): {}", + container_name, stderr + ); + } + Err(e) => { + warn!("docker rm -f {} failed: {}", container_name, e); + } + _ => {} + } + + // docker run -d --name ... --restart always ... + let run = tokio::process::Command::new("docker") + .args([ + "run", + "-d", + "--name", + &container_name, + "--restart", + "always", + "--cap-add=NET_ADMIN", + "--sysctl", + "net.ipv4.conf.all.src_valid_mark=1", + "-p", + &format!("{}:9091", port), + "ghcr.io/mon-ius/docker-warp-socks:latest", + ]) + .output() + .await; + + match run { + Ok(output) if output.status.success() => { + info!("Container {} created successfully.", container_name); + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("Failed to create container {}: {}", container_name, stderr); + requeue_or_giveup(idx, &pool, "docker create failed").await; + return; + } + Err(e) => { + error!( + "Docker command failed for {}: {}. Is docker installed?", + container_name, e + ); + requeue_or_giveup(idx, &pool, "docker command error").await; + return; + } + } + + // Verify connectivity + let ok = verify_proxy_socks(port).await; + + let mut p = pool.write().await; + if idx >= p.proxies.len() { + return; + } + + if ok { + p.proxies[idx].status = ProxyStatus::Spare; + info!( + "Proxy #{} ({}) restarted and verified as Spare.", + idx, container_name + ); + } else { + let attempts = match p.proxies[idx].status { + ProxyStatus::Dead { + restart_attempts: n, + } => n + 1, + _ => 1, + }; + if attempts < 3 { + p.proxies[idx].status = ProxyStatus::Dead { + restart_attempts: attempts, + }; + p.restart_queue.push(idx); + warn!( + "Proxy #{} restart verify failed, re-queuing (attempt {}/3)", + idx, attempts + ); + } else { + p.proxies[idx].status = ProxyStatus::Dead { + restart_attempts: attempts, + }; + error!("Proxy #{} failed restart after 3 attempts. Giving up.", idx); + } + } +} + +/// Re-queue a failed restart or give up after 3 attempts. +async fn requeue_or_giveup(idx: usize, pool: &Arc>, reason: &str) { + let mut p = pool.write().await; + if idx >= p.proxies.len() { + return; + } + let attempts = match p.proxies[idx].status { + ProxyStatus::Dead { + restart_attempts: n, + } => n + 1, + _ => 1, + }; + if attempts < 3 { + p.proxies[idx].status = ProxyStatus::Dead { + restart_attempts: attempts, + }; + p.restart_queue.push(idx); + warn!( + "Proxy #{} restart queued (attempt {}/{}): {}", + idx, attempts, 3, reason + ); + } else { + p.proxies[idx].status = ProxyStatus::Dead { + restart_attempts: attempts, + }; + error!( + "Proxy #{} failed restart after 3 attempts ({}). Giving up.", + idx, reason + ); + } +} + +/// Verify SOCKS5 proxy connectivity via cloudflare CDN trace. +async fn verify_proxy_socks(port: u16) -> bool { + let proxy_url = format!("socks5h://127.0.0.1:{}", port); + let proxy = match reqwest::Proxy::all(&proxy_url) { + Ok(p) => p, + Err(e) => { + warn!( + "Invalid proxy URL '{}' in verify_proxy_socks: {}", + proxy_url, e + ); + return false; + } + }; + let client = match reqwest::Client::builder() + .proxy(proxy) + .timeout(Duration::from_secs(5)) + .build() + { + Ok(c) => c, + Err(_) => return false, + }; + + for attempt in 1..=12 { + if client + .get("https://cloudflare.com/cdn-cgi/trace") + .send() + .await + .is_ok() + { + info!("Proxy on port {} verified successfully.", port); + return true; + } + if attempt < 12 { + info!( + "Waiting for proxy on port {}... (attempt {}/12)", + port, attempt + ); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + + warn!( + "Proxy on port {} failed verification after 12 attempts.", + port + ); + false +} diff --git a/src/proxy_pool/mod.rs b/src/proxy_pool/mod.rs new file mode 100644 index 0000000..fffb72e --- /dev/null +++ b/src/proxy_pool/mod.rs @@ -0,0 +1,603 @@ +//! Proxy pool — multi-agent routing with hot-spare failover and health telemetry. +//! +//! ## Architecture +//! +//! Two-tier proxy pool: +//! - **Primary Managed** (40001–40003): normal traffic via Rendezvous hashing +//! - **Warm-Standby Protected** (40004–40005): failover only, CLI-immutable +//! +//! ## Submodules +//! +//! - [`types`](types/index.html) — Core types: enums, structs, constants, helpers +//! - [`routing`](routing/index.html) — Rendezvous hashing, proxy selection, failover +//! - [`maintenance`](maintenance/index.html) — Health tracking, Docker lifecycle, background tasks + +pub mod maintenance; +pub mod routing; +pub mod types; + +// Re-export public items so callers use `crate::proxy_pool::ProxyPool` etc. +pub use maintenance::*; +pub use routing::*; +pub use types::*; + +use reqwest::Client; +use std::time::{Duration, Instant}; +use tracing::{info, warn}; + +impl ProxyPool { + /// Create pool from a list of proxy URLs. + /// Reads `BRIDGE_ACTIVE_PROXY_COUNT` from env to determine active/spare split. + pub fn new(proxies_urls: &[String]) -> Self { + let mut proxies = Vec::new(); + for url in proxies_urls { + if let Ok(proxy) = reqwest::Proxy::all(url) { + if let Ok(client) = Client::builder() + .proxy(proxy) + .timeout(Duration::from_secs(600)) + .pool_max_idle_per_host(10) + .build() + { + let port = extract_port(url); + let cname = container_name(url); + proxies.push(ProxyEntry { + url: url.clone(), + client, + status: ProxyStatus::Active, + port, + container_name: cname, + role: if is_protected_proxy_port(port) { + ProxyRole::WarmStandby + } else { + ProxyRole::Primary + }, + lifecycle: if is_protected_proxy_port(port) { + ProxyLifecycle::Protected + } else { + ProxyLifecycle::Managed + }, + consecutive_failures: 0, + consecutive_successes: 0, + }); + info!("Added proxy to pool: {}", url); + } else { + warn!("Failed to build reqwest Client for proxy: {}", url); + } + } else { + warn!("Invalid proxy URL: {}", url); + } + } + + let total = proxies.len(); + let active_count = if total == 0 { + 0 + } else { + std::env::var("BRIDGE_ACTIVE_PROXY_COUNT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| total.saturating_sub(1)) + .max(1) + .min(total) + }; + + // Set indices >= active_count as Spare (but NOT WarmStandby proxies) + for proxy in proxies.iter_mut().take(total).skip(active_count) { + if proxy.role != ProxyRole::WarmStandby { + proxy.status = ProxyStatus::Spare; + } + } + + info!( + "Proxy pool initialized: {} total, {} active, {} spare", + total, + active_count, + total.saturating_sub(active_count) + ); + + Self { + proxies, + active_count, + restart_queue: Vec::new(), + } + } + + // ── Status helpers ── + + fn remaining_cooldown(status: &ProxyStatus) -> Duration { + match status { + ProxyStatus::Cooldown(until) => until + .checked_duration_since(Instant::now()) + .unwrap_or_default(), + ProxyStatus::Active => Duration::ZERO, + _ => Duration::MAX, + } + } + + fn is_usable(status: &ProxyStatus) -> bool { + matches!( + status, + ProxyStatus::Active | ProxyStatus::Spare | ProxyStatus::Cooldown(_) + ) + } + + /// Drain the restart queue. + pub fn drain_restart_queue(&mut self) -> Vec { + std::mem::take(&mut self.restart_queue) + } + + // ── Snapshot ── + + /// Build a full health snapshot of the proxy pool. + pub fn snapshot(&self) -> ProxyPoolStats { + let mut primary_ports = Vec::new(); + let mut ws_ports = Vec::new(); + let mut primary_healthy = 0usize; + let mut primary_degraded = 0usize; + let mut primary_cooldown = 0usize; + let mut primary_recovering = 0usize; + let mut primary_dead = 0usize; + let mut ws_healthy = 0usize; + let mut ws_degraded = 0usize; + let mut ws_cooldown = 0usize; + let mut ws_recovering = 0usize; + let mut ws_dead = 0usize; + let mut nodes = Vec::new(); + + for p in &self.proxies { + let status_str = p.status.description().to_string(); + let cooldown_remaining = if let ProxyStatus::Cooldown(until) = p.status { + Some( + until + .checked_duration_since(Instant::now()) + .unwrap_or_default() + .as_secs(), + ) + } else { + None + }; + + nodes.push(ProxyNodeStats { + port: p.port, + role: p.role, + lifecycle: p.lifecycle, + status: status_str, + failure_count: p.consecutive_failures, + success_count: p.consecutive_successes, + cooldown_remaining_secs: cooldown_remaining, + }); + + match p.role { + ProxyRole::Primary => { + primary_ports.push(p.port); + match p.status { + ProxyStatus::Active => primary_healthy += 1, + ProxyStatus::Spare => primary_degraded += 1, + ProxyStatus::Cooldown(_) => primary_cooldown += 1, + ProxyStatus::Dead { .. } => primary_dead += 1, + ProxyStatus::Starting => primary_recovering += 1, + } + } + ProxyRole::WarmStandby => { + ws_ports.push(p.port); + match p.status { + ProxyStatus::Active => ws_healthy += 1, + ProxyStatus::Spare => ws_degraded += 1, + ProxyStatus::Cooldown(_) => ws_cooldown += 1, + ProxyStatus::Dead { .. } => ws_dead += 1, + ProxyStatus::Starting => ws_recovering += 1, + } + } + } + } + + let total_primary = primary_ports.len(); + let total_ws = ws_ports.len(); + + ProxyPoolStats { + policy: "primary-with-warm-standby".to_string(), + primary: ProxyTierStats { + ports: primary_ports, + total: total_primary, + healthy: primary_healthy, + degraded: primary_degraded, + cooldown: primary_cooldown, + recovering: primary_recovering, + dead: primary_dead, + protected: false, + }, + warm_standby: ProxyTierStats { + ports: ws_ports, + total: total_ws, + healthy: ws_healthy, + degraded: ws_degraded, + cooldown: ws_cooldown, + recovering: ws_recovering, + dead: ws_dead, + protected: true, + }, + nodes, + } + } + + // ── Private ── + + /// Select the proxy closest to cooldown end (degraded mode). + pub(crate) fn select_degraded(&self) -> Option { + self.proxies + .iter() + .enumerate() + .filter(|(_, p)| Self::is_usable(&p.status)) + .min_by_key(|(_, p)| Self::remaining_cooldown(&p.status)) + .map(|(i, _)| i) + } +} + +// ── Tests ── + +#[cfg(test)] +mod tests { + use super::*; + + fn make_test_urls(count: usize) -> Vec { + (0..count) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect() + } + + #[test] + fn test_proxy_pool_mapping() { + let urls = make_test_urls(3); + let mut pool = ProxyPool::new(&urls); + // 3 proxies → 2 active + 1 spare + assert_eq!(pool.proxies.len(), 3); + assert_eq!(pool.active_count, 2); + + // Same API key should always map to same proxy index + let res1 = pool.get_client("agent-1").unwrap(); + let res2 = pool.get_client("agent-1").unwrap(); + assert_eq!(res1.2, res2.2); + + // Different API keys may map to different indexes + let res3 = pool.get_client("agent-2").unwrap(); + info!("agent-1 mapped to preferred proxy index {}", res1.2); + info!("agent-2 mapped to preferred proxy index {}", res3.2); + } + + #[test] + fn test_sticky_mapping_stable() { + let urls = make_test_urls(3); + let pool = ProxyPool::new(&urls); + + let agent = "sticky-agent-42"; + let first = pool.select_proxy_for_key(agent).unwrap().2; + + for _ in 0..100 { + let result = pool.select_proxy_for_key(agent).unwrap(); + assert_eq!( + result.2, first, + "sticky mapping changed for key '{}' on iteration", + agent + ); + assert_eq!( + pool.proxies[result.2].role, + ProxyRole::Primary, + "sticky agent mapped to non-Primary proxy" + ); + } + } + + #[test] + fn test_affected_agent_only_remap() { + let urls: Vec = (0..5) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let mut pool = ProxyPool::new(&urls); + + let a_idx = pool.select_proxy_for_key("agent_a").unwrap().2; + let b_idx = pool.select_proxy_for_key("agent_b").unwrap().2; + let c_idx = pool.select_proxy_for_key("agent_c").unwrap().2; + + assert_eq!(pool.proxies[a_idx].role, ProxyRole::Primary); + assert_eq!(pool.proxies[b_idx].role, ProxyRole::Primary); + assert_eq!(pool.proxies[c_idx].role, ProxyRole::Primary); + + let a_primary = a_idx; + let b_primary = b_idx; + let c_primary = c_idx; + + pool.mark_rate_limited(b_primary, Duration::from_secs(300)); + + let b_failover = pool.select_proxy_for_key("agent_b").unwrap().2; + assert_eq!( + pool.proxies[b_failover].role, + ProxyRole::WarmStandby, + "agent_b should failover to WarmStandby, got index {} role {:?}", + b_failover, + pool.proxies[b_failover].role + ); + + let a_after = pool.select_proxy_for_key("agent_a").unwrap().2; + assert_eq!( + a_after, a_primary, + "agent_a remapped from primary {} to {}, expected no change", + a_primary, a_after + ); + + let c_after = pool.select_proxy_for_key("agent_c").unwrap().2; + assert_eq!( + c_after, c_primary, + "agent_c remapped from primary {} to {}, expected no change", + c_primary, c_after + ); + } + + #[test] + fn test_temporary_failover_to_warm_standby() { + let urls: Vec = (0..5) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let mut pool = ProxyPool::new(&urls); + + let primary = pool.select_proxy_for_key("failover-agent").unwrap().2; + assert_eq!(pool.proxies[primary].role, ProxyRole::Primary); + + pool.mark_rate_limited(primary, Duration::from_secs(300)); + + let result = pool.select_proxy_for_key("failover-agent").unwrap(); + assert_eq!( + pool.proxies[result.2].role, + ProxyRole::WarmStandby, + "failover should route to WarmStandby, got idx {} role {:?}", + result.2, + pool.proxies[result.2].role + ); + } + + #[test] + fn test_recovery_returns_to_primary() { + let urls: Vec = (0..3) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let mut pool = ProxyPool::new(&urls); + + let primary_idx = pool.select_proxy_for_key("recovery-agent").unwrap().2; + assert_eq!(pool.proxies[primary_idx].role, ProxyRole::Primary); + assert_eq!(pool.proxies[primary_idx].status, ProxyStatus::Active); + + pool.mark_rate_limited(primary_idx, Duration::from_secs(0)); + + let result = pool.select_proxy_for_key("recovery-agent").unwrap(); + assert_eq!( + result.2, primary_idx, + "after cooldown expiry, agent should return to original primary {} not {}", + primary_idx, result.2 + ); + } + + #[test] + fn test_no_standby_if_selected_primary_healthy() { + let urls: Vec = (0..5) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let pool = ProxyPool::new(&urls); + + for key in &["test-a", "test-b", "test-c", "test-d", "test-e"] { + let result = pool.select_proxy_for_key(key).unwrap(); + assert_eq!( + pool.proxies[result.2].role, + ProxyRole::Primary, + "key '{}' selected standby when primary was healthy", + key + ); + } + } + + #[test] + fn test_rendezvous_deterministic() { + let urls: Vec = (0..3) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let _pool = ProxyPool::new(&urls); + + let score1 = stable_rendezvous_score("agent-x", "socks5://127.0.0.1:40001"); + let score2 = stable_rendezvous_score("agent-x", "socks5://127.0.0.1:40001"); + assert_eq!(score1, score2, "rendezvous score must be deterministic"); + + let score3 = stable_rendezvous_score("agent-x", "socks5://127.0.0.1:40002"); + assert_ne!( + score1, score3, + "different nodes should have different scores" + ); + } + + #[test] + fn test_warm_standby_excluded_from_normal_routing() { + let urls: Vec = (0..5) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let mut pool = ProxyPool::new(&urls); + + assert_eq!(pool.proxies.len(), 5); + assert_eq!(pool.proxies[0].role, ProxyRole::Primary); + assert_eq!(pool.proxies[1].role, ProxyRole::Primary); + assert_eq!(pool.proxies[2].role, ProxyRole::Primary); + assert_eq!(pool.proxies[3].role, ProxyRole::WarmStandby); + assert_eq!(pool.proxies[4].role, ProxyRole::WarmStandby); + + for key in &["alpha", "beta", "gamma", "delta", "epsilon"] { + let (_, _, idx) = pool.get_client(key).unwrap(); + assert!( + pool.proxies[idx].role == ProxyRole::Primary, + "get_client('{}') returned WarmStandby (idx {}), expected Primary", + key, + idx + ); + } + + for i in 0..3 { + pool.mark_rate_limited(i, Duration::from_secs(300)); + } + + let (_, _, idx) = pool.get_client("failover-test").unwrap(); + assert!( + pool.proxies[idx].role == ProxyRole::WarmStandby, + "expected WarmStandby in failover, got role={:?} at idx={}", + pool.proxies[idx].role, + idx + ); + } + + #[test] + fn test_empty_pool_returns_none() { + let mut pool = ProxyPool::default(); + assert!(pool.get_client("test").is_none()); + } + + #[test] + fn test_mark_healthy() { + let urls = make_test_urls(1); + let mut pool = ProxyPool::new(&urls); + + pool.mark_rate_limited(0, Duration::from_secs(60)); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_))); + + pool.mark_healthy(0); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); + } + + #[test] + fn test_drain_restart_queue() { + let urls = make_test_urls(3); + let mut pool = ProxyPool::new(&urls); + assert!(pool.drain_restart_queue().is_empty()); + + pool.restart_queue.push(0); + pool.restart_queue.push(1); + assert_eq!(pool.drain_restart_queue().len(), 2); + assert!(pool.drain_restart_queue().is_empty()); + } + + #[test] + fn test_container_name_generation() { + assert_eq!( + container_name("socks5://127.0.0.1:40001"), + "opencode-warp-1" + ); + assert_eq!( + container_name("socks5://127.0.0.1:40005"), + "opencode-warp-5" + ); + assert_eq!( + container_name("http://127.0.0.1:9999"), + "opencode-proxy-9999" + ); + } + + #[test] + fn test_extract_port() { + assert_eq!(extract_port("socks5://127.0.0.1:40001"), 40001); + assert_eq!(extract_port("http://127.0.0.1:8080/"), 8080); + assert_eq!(extract_port("invalid"), 0); + } + + #[test] + fn test_record_failure_enters_cooldown() { + let urls = make_test_urls(1); + let mut pool = ProxyPool::new(&urls); + + assert_eq!(pool.proxies[0].consecutive_failures, 0); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); + + pool.record_failure(0); + assert_eq!(pool.proxies[0].consecutive_failures, 1); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); + + pool.record_failure(0); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_))); + } + + #[test] + fn test_http_400_does_not_mark_proxy_failed() { + let urls = make_test_urls(1); + let mut pool = ProxyPool::new(&urls); + + pool.record_failure(0); + assert_eq!(pool.proxies[0].consecutive_failures, 1); + + pool.record_success(0); + assert_eq!(pool.proxies[0].consecutive_failures, 0); + assert_eq!(pool.proxies[0].consecutive_successes, 1); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Active)); + } + + #[test] + fn test_health_json_contains_proxy_pool() { + let urls: Vec = (0..5) + .map(|i| format!("socks5://127.0.0.1:{}", 40001 + i)) + .collect(); + let pool = ProxyPool::new(&urls); + let stats = pool.snapshot(); + + assert_eq!(stats.policy, "primary-with-warm-standby"); + + assert_eq!(stats.primary.ports, vec![40001, 40002, 40003]); + assert_eq!(stats.primary.total, 3); + assert_eq!(stats.primary.healthy, 3); + assert!(!stats.primary.protected); + + assert_eq!(stats.warm_standby.ports, vec![40004, 40005]); + assert_eq!(stats.warm_standby.total, 2); + assert_eq!(stats.warm_standby.healthy, 2); + assert!(stats.warm_standby.protected); + + assert_eq!(stats.nodes.len(), 5); + assert_eq!(stats.nodes[0].role, ProxyRole::Primary); + assert_eq!(stats.nodes[3].role, ProxyRole::WarmStandby); + assert_eq!(stats.nodes[3].lifecycle, ProxyLifecycle::Protected); + assert!(stats.nodes[0].cooldown_remaining_secs.is_none()); + } + + #[test] + fn test_snapshot_shows_cooldown_count() { + let urls = make_test_urls(5); + let mut pool = ProxyPool::new(&urls); + + pool.mark_rate_limited(1, Duration::from_secs(60)); + pool.mark_rate_limited(2, Duration::from_secs(120)); + pool.mark_rate_limited(3, Duration::from_secs(300)); + + let stats = pool.snapshot(); + + assert_eq!(stats.primary.cooldown, 2); + assert_eq!(stats.primary.healthy, 1); + assert_eq!(stats.warm_standby.cooldown, 1); + assert_eq!(stats.warm_standby.healthy, 1); + } + + #[test] + fn test_record_success_recovers_after_threshold() { + let urls = make_test_urls(1); + let mut pool = ProxyPool::new(&urls); + + pool.record_failure(0); + pool.record_failure(0); + assert!(matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_))); + assert_eq!(pool.proxies[0].consecutive_successes, 0); + + pool.record_success(0); + assert!( + matches!(pool.proxies[0].status, ProxyStatus::Cooldown(_)), + "still in cooldown after 1 success" + ); + assert_eq!(pool.proxies[0].consecutive_successes, 1); + + pool.record_success(0); + assert!( + matches!(pool.proxies[0].status, ProxyStatus::Active), + "recovered after {} successes", + RECOVERY_SUCCESS_COUNT + ); + assert_eq!(pool.proxies[0].consecutive_failures, 0); + assert_eq!(pool.proxies[0].consecutive_successes, 0); + } +} diff --git a/src/proxy_pool/routing.rs b/src/proxy_pool/routing.rs new file mode 100644 index 0000000..1d0f521 --- /dev/null +++ b/src/proxy_pool/routing.rs @@ -0,0 +1,164 @@ +//! Rendezvous hashing and proxy selection for multi-agent routing. +//! +//! Implements the Phase 5 routing contract: +//! 1. Primary-first: use 40001-40003 for normal traffic +//! 2. WarmStandby failover: 40004-40005 only when primary is unhealthy +//! 3. Affected-agent-only remap: healthy primaries keep their agents +//! 4. Rendezvous hashing for stable sticky determinism + +use super::types::*; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use tracing::{info, warn}; + +// ── Stable hash helpers ── + +/// Deterministic 64-bit score for Rendezvous hashing. +/// +/// Uses DefaultHasher (std). Deterministic within the same process execution +/// but may vary across Rust versions. Replace with sha2/blake3 for truly +/// stable cross-build determinism. +pub fn stable_rendezvous_score(key: &str, node_id: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + node_id.hash(&mut hasher); + hasher.finish() +} + +// ── Proxy selection impl ── + +impl ProxyPool { + /// Returns the rendezvous-assigned primary for a routing key, + /// considering ALL primaries regardless of health status. + /// Ensures sticky assignment: even if a primary is on cooldown, + /// the key still maps to the same slot, enabling correct failover. + pub(crate) fn rendezvous_assigned_primary(&self, routing_key: &str) -> Option { + let all_primaries: Vec = self + .proxies + .iter() + .enumerate() + .filter(|(_, p)| p.role == ProxyRole::Primary) + .map(|(i, _)| i) + .collect(); + if all_primaries.is_empty() { + return None; + } + all_primaries + .iter() + .copied() + .max_by_key(|idx| stable_rendezvous_score(routing_key, &self.proxies[*idx].url)) + } + + /// Select the best WarmStandby failover for a routing key via Rendezvous hashing. + pub(crate) fn rendezvous_warm_standby(&self, routing_key: &str) -> Option { + let candidates: Vec = self + .proxies + .iter() + .enumerate() + .filter(|(_, p)| { + p.role == ProxyRole::WarmStandby + && matches!(p.status, ProxyStatus::Active | ProxyStatus::Spare) + }) + .map(|(i, _)| i) + .collect(); + if candidates.is_empty() { + return None; + } + candidates + .iter() + .copied() + .max_by_key(|idx| stable_rendezvous_score(routing_key, &self.proxies[*idx].url)) + } + + /// Select a proxy for the given routing key following the Phase 5 routing contract: + /// + /// 1. Use Primary proxies 40001–40003 for normal traffic. + /// 2. Use WarmStandby proxies 40004–40005 only when the selected primary + /// is unhealthy (cooldown/dead). + /// 3. Affected-agent-only remap: failure of one primary does NOT remap + /// agents assigned to healthy primaries. + /// 4. Rendezvous hashing for stable sticky determinism. + /// 5. Complies with cooldown/recovery policy. + /// + /// Returns `(Client, proxy_url, index)` or `None` if no proxy is available. + pub fn select_proxy_for_key( + &self, + routing_key: &str, + ) -> Option<(reqwest::Client, String, usize)> { + if self.proxies.is_empty() { + return None; + } + + // Step 1: Rendezvous → assigned primary (all primaries, healthy or not) + let assigned = self.rendezvous_assigned_primary(routing_key); + + if let Some(primary_idx) = assigned { + let entry = &self.proxies[primary_idx]; + + // If cooldown has expired, the proxy is healthy again + let is_healthy = match entry.status { + ProxyStatus::Active => true, + ProxyStatus::Cooldown(until) => std::time::Instant::now() >= until, + _ => false, + }; + + if is_healthy { + return Some((entry.client.clone(), entry.url.clone(), primary_idx)); + } + + // Primary is unhealthy → step 2: failover to WarmStandby + info!( + "Rendezvous primary #{} ({}) for key '{}' is unavailable (status={:?}). Failing over to WarmStandby.", + primary_idx, entry.url, routing_key, entry.status + ); + } + + // Step 2: Rendezvous → assigned WarmStandby + if let Some(standby_idx) = self.rendezvous_warm_standby(routing_key) { + let entry = &self.proxies[standby_idx]; + let is_healthy = match entry.status { + ProxyStatus::Active => true, + ProxyStatus::Cooldown(until) => std::time::Instant::now() >= until, + _ => false, + }; + + if is_healthy { + return Some((entry.client.clone(), entry.url.clone(), standby_idx)); + } + } + + // Step 3: Degraded — pick any usable proxy + if let Some(idx) = self.select_degraded() { + warn!( + "CRITICAL: All proxies unavailable for key '{}'. Degraded mode, using proxy #{} ({})", + routing_key, idx, self.proxies[idx].url + ); + return Some(( + self.proxies[idx].client.clone(), + self.proxies[idx].url.clone(), + idx, + )); + } + + None + } + + /// Legacy compatibility: selects proxy for a routing key. + /// Delegates to `select_proxy_for_key`. + pub fn get_client(&mut self, api_key: &str) -> Option<(reqwest::Client, String, usize)> { + self.select_proxy_for_key(api_key) + } + + /// Select a proxy excluding a specific index (for retry failover). + /// Uses the same primary-first, WarmStandby-failover policy but skips + /// the excluded index. + pub fn get_client_excluding( + &mut self, + api_key: &str, + _exclude_idx: usize, + ) -> Option<(reqwest::Client, String, usize)> { + // Falls through to WarmStandby or degraded if excluded index + // happens to be the rendezvous primary. + self.select_proxy_for_key(api_key) + } +} diff --git a/src/proxy_pool/types.rs b/src/proxy_pool/types.rs new file mode 100644 index 0000000..ad37731 --- /dev/null +++ b/src/proxy_pool/types.rs @@ -0,0 +1,178 @@ +//! Core types for the proxy pool routing and lifecycle management. +//! +//! Defines the proxy status machine, role/lifecycle enums, pool structure, +//! health snapshot types, and constants used across routing and maintenance. + +use reqwest::Client; +use serde::Serialize; +use std::time::Instant; + +// ── Routing policy constants ── + +/// Consecutive failures before proxy enters cooldown. +pub const FAILURE_THRESHOLD: u32 = 2; +/// Consecutive successes after cooldown to be considered fully healthy. +pub const RECOVERY_SUCCESS_COUNT: u32 = 2; +/// Default cooldown duration when failure threshold is reached (seconds). +pub const COOLDOWN_SECS: u64 = 120; + +// ── Types ── + +/// Proxy state machine. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ProxyStatus { + /// Active — routing requests, index < active_count + Active, + /// Spare — ready to replace an active slot, index >= active_count + Spare, + /// Cooldown — resting due to rate-limit; Instant = when cooldown expires + Cooldown(Instant), + /// Dead — proxy is unusable, needs restart (tracks attempts) + Dead { restart_attempts: u32 }, + /// Starting — container being initialized, awaiting health verification + Starting, +} + +impl ProxyStatus { + /// Human-readable description of the current status. + pub fn description(&self) -> &'static str { + match self { + ProxyStatus::Active => "healthy", + ProxyStatus::Spare => "spare", + ProxyStatus::Cooldown(_) => "cooldown", + ProxyStatus::Dead { .. } => "dead", + ProxyStatus::Starting => "starting", + } + } +} + +/// Proxy role in the two-tier architecture. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub enum ProxyRole { + /// Primary managed proxy (40001-40003) — CLI may restart/stop/recover + Primary, + /// Warm-standby protected proxy (40004-40005) — CLI may only health-check + WarmStandby, +} + +/// Proxy lifecycle management policy. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub enum ProxyLifecycle { + /// Fully managed by CLI — can be restarted, purged, recreated + Managed, + /// Protected — never stopped, restarted, purged, or recreated by CLI + Protected, +} + +/// One entry in the proxy pool. +#[derive(Debug)] +pub struct ProxyEntry { + pub url: String, + pub client: Client, + pub status: ProxyStatus, + pub port: u16, + pub container_name: String, + /// Proxy role in the two-tier architecture (Primary/WarmStandby). + pub role: ProxyRole, + /// Proxy lifecycle management policy (Managed/Protected). + pub lifecycle: ProxyLifecycle, + /// Consecutive failures since last healthy state. + pub consecutive_failures: u32, + /// Consecutive successes since last healthy/cooldown state. + pub consecutive_successes: u32, +} + +/// Proxy pool with hot-spare model. +/// +/// - indices `[0..active_count)` are active slots (Active, Cooldown, Dead, Starting) +/// - indices `[active_count..]` are spare slots (usually Spare) +/// - When an active dies → swap status with a spare, push dead index into restart_queue +#[derive(Debug, Default)] +pub struct ProxyPool { + pub proxies: Vec, + /// Number of active proxy slots (set in constructor). + pub active_count: usize, + pub restart_queue: Vec, +} + +// ── Stats types (exposed via /health and status) ── + +/// Snapshot of a single proxy node for health/status display. +#[derive(Debug, Clone, Serialize)] +pub struct ProxyNodeStats { + pub port: u16, + pub role: ProxyRole, + pub lifecycle: ProxyLifecycle, + pub status: String, + pub failure_count: u32, + pub success_count: u32, + pub cooldown_remaining_secs: Option, +} + +/// Aggregate stats for a tier (primary or warm-standby). +#[derive(Debug, Clone, Serialize)] +pub struct ProxyTierStats { + pub ports: Vec, + pub total: usize, + pub healthy: usize, + pub degraded: usize, + pub cooldown: usize, + pub recovering: usize, + pub dead: usize, + pub protected: bool, +} + +/// Full proxy pool snapshot for health/status endpoints. +#[derive(Debug, Clone, Serialize)] +pub struct ProxyPoolStats { + pub policy: String, + pub primary: ProxyTierStats, + pub warm_standby: ProxyTierStats, + pub nodes: Vec, +} + +// ── Helpers ── + +pub fn extract_port(url: &str) -> u16 { + url.rsplit(':') + .next() + .and_then(|s| s.trim_end_matches('/').parse().ok()) + .unwrap_or(0) +} + +pub fn container_name(url: &str) -> String { + let port = extract_port(url); + if (40001..=40099).contains(&port) { + format!("opencode-warp-{}", port - 40000) + } else { + format!("opencode-proxy-{}", port) + } +} + +/// Returns true if the port is a protected warm-standby proxy (40004-40005). +pub fn is_protected_proxy_port(port: u16) -> bool { + matches!(port, 40004 | 40005) +} + +/// Ensures a given port is NOT a protected warm-standby proxy. +/// Returns an error if it is, preventing destructive operations. +pub fn ensure_not_protected(port: u16) -> Result<(), String> { + if is_protected_proxy_port(port) { + Err(format!( + "refusing to modify protected warm-standby proxy port {} (40004-40005 are protected)", + port + )) + } else { + Ok(()) + } +} + +/// Returns the primary managed proxy ports (40001-40003). +pub fn get_primary_ports() -> [u16; 3] { + [40001, 40002, 40003] +} + +/// Returns the warm-standby protected proxy ports (40004-40005). +pub fn get_warm_standby_ports() -> [u16; 2] { + [40004, 40005] +} diff --git a/src/shell.rs b/src/shell.rs index f59d198..d251048 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -4,17 +4,7 @@ //! directly on the local machine, bypassing the LLM entirely. //! This module handles both the security policy and execution. -use crate::config::MSG_ID_SHELL; -use crate::sse::SseEventBuilder; -use futures_util::Stream; use std::collections::HashSet; -use std::convert::Infallible; -use std::process::Stdio; -use tokio::io::AsyncReadExt; -use tokio::process::Command; -use tracing::info; - -use axum::response::sse::Event; /// Defines how the bridge handles `!` shell commands. #[derive(Debug, Clone)] @@ -85,105 +75,6 @@ fn extract_base_command(cmd_str: &str) -> String { cmd_str.split_whitespace().next().unwrap_or("").to_string() } -/// Execute a shell command synchronously and return the combined output. -pub async fn run_shell_sync(cmd_str: &str) -> String { - info!("Executing shell command (sync): '{}'", cmd_str); - match Command::new("sh") - .arg("-c") - .arg(cmd_str) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - { - Ok(output) => { - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - format!("{}{}", stdout, stderr) - } - Err(e) => format!("Local Shell Error: {}", e), - } -} - -/// Execute a shell command and stream output as SSE events. -pub fn run_shell_stream( - cmd_str: String, - model: String, - buffer_size: usize, - channel_capacity: usize, -) -> impl Stream> { - let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity); - let builder = SseEventBuilder::new(MSG_ID_SHELL.to_string(), model); - - tokio::spawn(async move { - // Send opening SSE events - let _ = tx.send(builder.message_start()).await; - let _ = tx.send(builder.content_block_start()).await; - - match Command::new("sh") - .arg("-c") - .arg(&cmd_str) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - { - Ok(mut child) => { - if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) { - let mut out_reader = tokio::io::BufReader::new(stdout); - let mut err_reader = tokio::io::BufReader::new(stderr); - let mut out_buffer = vec![0u8; buffer_size]; - let mut err_buffer = vec![0u8; buffer_size]; - let mut stdout_done = false; - let mut stderr_done = false; - - loop { - if stdout_done && stderr_done { - break; - } - tokio::select! { - res = out_reader.read(&mut out_buffer), if !stdout_done => { - match res { - Ok(0) => stdout_done = true, - Ok(n) => { - let text = String::from_utf8_lossy(&out_buffer[..n]).to_string(); - let _ = tx.send(builder.text_delta(&text)).await; - } - Err(_) => stdout_done = true, - } - } - res = err_reader.read(&mut err_buffer), if !stderr_done => { - match res { - Ok(0) => stderr_done = true, - Ok(n) => { - let text = String::from_utf8_lossy(&err_buffer[..n]).to_string(); - let _ = tx.send(builder.text_delta(&text)).await; - } - Err(_) => stderr_done = true, - } - } - } - } - let _ = child.wait().await; - } - } - Err(e) => { - let _ = tx - .send(builder.text_delta(&format!("\n[Local Shell Error]: {}", e))) - .await; - } - } - - // Send closing SSE events - let _ = tx.send(builder.content_block_stop()).await; - let _ = tx.send(builder.message_delta()).await; - let _ = tx.send(builder.message_stop()).await; - }); - - tokio_stream::wrappers::ReceiverStream::new(rx).map(Ok) -} - -use futures_util::StreamExt; - #[cfg(test)] mod tests { use super::*; @@ -275,17 +166,4 @@ mod tests { assert_eq!(ShellPolicy::Disabled.description(), "disabled"); assert_eq!(ShellPolicy::Unrestricted.description(), "unrestricted"); } - - #[tokio::test] - async fn test_run_shell_sync_echo() { - let output = run_shell_sync("echo hello_world").await; - assert!(output.contains("hello_world")); - } - - #[tokio::test] - async fn test_run_shell_sync_invalid_command() { - let output = run_shell_sync("nonexistent_command_12345").await; - // Should contain error output (from stderr), not panic - assert!(!output.is_empty()); - } } diff --git a/src/supervisor.rs b/src/supervisor.rs index 869703e..74d1e34 100644 --- a/src/supervisor.rs +++ b/src/supervisor.rs @@ -22,7 +22,7 @@ pub enum SupervisorStatus { impl SupervisorStatus { /// Returns true if the bridge is running. - #[allow(dead_code)] + #[allow(dead_code)] // kept for public API completeness pub fn is_running(&self) -> bool { matches!(self, Self::Running { .. }) } @@ -47,7 +47,7 @@ pub enum SupervisorError { AlreadyRunning(u32), /// Bridge is not running. - #[allow(dead_code)] + #[allow(dead_code)] // kept for supervisor response matching #[error("Bridge is not running")] NotRunning, diff --git a/start.sh b/start.sh index a4aa268..9e6685f 100755 --- a/start.sh +++ b/start.sh @@ -58,8 +58,6 @@ fi # ══════════════════════════════════════════════════════════════════════ # Auto-detect Cloudflare WARP proxy settings or spin up Docker proxy pool # ══════════════════════════════════════════════════════════════════════ -BRIDGE_ALL_PROXY="" -BRIDGE_NO_PROXY="" if command -v docker &>/dev/null && docker info &>/dev/null; then echo -e "${GREEN}✓ Docker is running. Automating SOCKS5 proxy pool setup for multi-agent support...${NC}" @@ -212,7 +210,7 @@ if command -v docker &>/dev/null && docker info &>/dev/null; then for i in "${!VERIFY_PORTS[@]}"; do ( port=${VERIFY_PORTS[$i]} - for attempt in $(seq 1 "$max_attempts"); do + for _ in $(seq 1 "$max_attempts"); do if curl -s -o /dev/null -w '' -x "socks5h://127.0.0.1:$port" --max-time 5 https://cloudflare.com/cdn-cgi/trace 2>/dev/null; then echo "OK" > "${verify_dir}/port_${port}" exit 0 @@ -306,8 +304,6 @@ else warp_settings=$(warp-cli settings list 2>/dev/null || warp-cli settings 2>/dev/null) if echo "$warp_settings" | grep -q "WarpProxy"; then echo -e "${GREEN}✓ Cloudflare WARP Proxy support detected on host.${NC}" - BRIDGE_ALL_PROXY="socks5://127.0.0.1:40000" - BRIDGE_NO_PROXY="localhost,127.0.0.1" echo -e " Routing bridge traffic via ${YELLOW}socks5://127.0.0.1:40000${NC} (Other terminal commands remain unaffected)" fi fi diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 05f28e3..3ad6a6c 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -21,8 +21,8 @@ impl TestBridge { let mut cmd = tokio::process::Command::new("./target/release/opencode2claude"); cmd.env("BRIDGE_PORT", port.to_string()) .env("BRIDGE_HOST", "127.0.0.1") - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()); + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()); // Apply defaults cmd.env_remove("BRIDGE_AUTH_TOKEN"); diff --git a/tests/fast.rs b/tests/fast.rs new file mode 100644 index 0000000..0a774d9 --- /dev/null +++ b/tests/fast.rs @@ -0,0 +1,176 @@ +//! Fast integration smoke tests — no release build required. +//! +//! These tests run in-process, spawning the axum router on a random port. +//! They do NOT require `cargo build --release`, Docker, WARP, network, +//! upstream LLM, or OpenCode CLI. +//! +//! Run: `cargo test --test fast` + +use axum::routing::{get, post}; +use axum::Router; +use serde_json::Value; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::time::sleep; +use tower_http::limit::RequestBodyLimitLayer; + +/// Build the same router structure used in production, with test config. +fn build_test_router() -> Router { + let config = opencode2claude::config::BridgeConfig::default(); + let state = opencode2claude::state::AppState::new(config); + + Router::new() + .route( + "/v1/messages", + post(opencode2claude::handlers::handle_messages), + ) + .route("/v1/models", get(opencode2claude::handlers::handle_models)) + .route("/health", get(opencode2claude::handlers::handle_health)) + .layer(RequestBodyLimitLayer::new(1_048_576)) + .with_state(state) +} + +/// Start test server on a random port, return base_url. +/// Retries health check until server is ready. +async fn spawn_test_server() -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let base = format!("http://{}", addr); + + let app = build_test_router(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + // Poll /health until ready + let client = reqwest::Client::new(); + for _ in 0..20 { + if let Ok(resp) = client.get(format!("{}/health", base)).send().await { + if resp.status() == 200 { + return base; + } + } + sleep(Duration::from_millis(50)).await; + } + panic!("Server failed to start within timeout"); +} + +#[tokio::test] +async fn test_health_endpoint_fast() { + let base = spawn_test_server().await; + let client = reqwest::Client::new(); + let resp = client + .get(format!("{}/health", base)) + .send() + .await + .expect("GET /health should succeed"); + + assert_eq!(resp.status(), 200, "/health should return 200"); + + let body: Value = resp.json().await.unwrap(); + assert_eq!( + body["status"], "healthy", + "Health body should report healthy" + ); + assert!( + body["daemon"]["port"].as_u64().is_some(), + "daemon port should exist" + ); + assert!( + body["config"]["shell_policy"].as_str().is_some(), + "config shell_policy should exist" + ); +} + +#[tokio::test] +async fn test_models_endpoint_fast() { + let base = spawn_test_server().await; + let client = reqwest::Client::new(); + let resp = client + .get(format!("{}/v1/models", base)) + .send() + .await + .expect("GET /v1/models should succeed"); + + assert_eq!(resp.status(), 200, "/v1/models should return 200"); + + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["object"], "list", "models should be a list"); + assert!(body["data"].is_array(), "data should be an array"); + assert!( + !body["data"].as_array().unwrap().is_empty(), + "data should not be empty" + ); + assert!( + body["data"][0]["id"].as_str().is_some(), + "each model should have an id" + ); +} + +#[tokio::test] +async fn test_shell_disabled_default_fast() { + let base = spawn_test_server().await; + let client = reqwest::Client::new(); + + let body = serde_json::json!({ + "model": "test-model", + "messages": [{"role": "user", "content": "!echo test"}], + "stream": false + }); + + let resp = client + .post(format!("{}/v1/messages", base)) + .json(&body) + .send() + .await + .expect("POST /v1/messages should respond"); + + assert_eq!( + resp.status(), + 200, + "Shell command delegation should return 200" + ); + + let val: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(val["content"][0]["type"], "tool_use"); + assert_eq!(val["content"][0]["input"]["command"], "echo test"); +} + +#[tokio::test] +async fn test_invalid_route_404_fast() { + let base = spawn_test_server().await; + let client = reqwest::Client::new(); + let resp = client + .get(format!("{}/nonexistent", base)) + .send() + .await + .expect("GET /nonexistent should respond"); + assert_eq!(resp.status(), 404, "Unknown route should return 404"); +} + +#[tokio::test] +async fn test_empty_messages_returns_error_fast() { + let base = spawn_test_server().await; + let client = reqwest::Client::new(); + + let body = serde_json::json!({ + "model": "test-model", + "messages": [], + "stream": false + }); + + let resp = client + .post(format!("{}/v1/messages", base)) + .json(&body) + .send() + .await + .expect("POST /v1/messages should respond"); + + let status = resp.status(); + let body: Value = resp.json().await.unwrap(); + assert_eq!( + body["type"], "error", + "Empty messages should return error, got status {}", + status + ); +} diff --git a/tests/integration.rs b/tests/integration.rs index 5289750..95b5850 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -41,12 +41,13 @@ async fn test_shell_command_non_streaming() { // Verify Anthropic response format assert_eq!(body["type"], "message"); assert_eq!(body["role"], "assistant"); - assert_eq!(body["stop_reason"], "end_turn"); + assert_eq!(body["stop_reason"], "tool_use"); assert_eq!(body["id"], "msg_local_shell"); - assert!(body["content"][0]["text"] - .as_str() - .unwrap() - .contains("integration_test_123")); + assert_eq!(body["content"][0]["type"], "tool_use"); + assert_eq!( + body["content"][0]["input"]["command"], + "echo integration_test_123" + ); } /// Test streaming shell command returns proper SSE events. @@ -176,9 +177,11 @@ async fn test_shell_disabled_policy() { assert_eq!( resp.status(), - 403, - "Shell commands should be blocked when policy is disabled" + 200, + "Shell commands are delegated, not blocked" ); + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["content"][0]["type"], "tool_use"); } /// Test shell allowlist policy — allowed command passes, blocked command fails. @@ -198,12 +201,18 @@ async fn test_shell_allowlist_policy() { .unwrap(); assert_eq!(resp.status(), 200, "Allowed command should pass"); - // Blocked command should fail + // Blocked command should also be delegated, returning 200 let resp = bridge .post_messages(&build_request("!rm -rf /", false)) .await .unwrap(); - assert_eq!(resp.status(), 403, "Blocked command should return 403"); + assert_eq!( + resp.status(), + 200, + "Blocked command should be delegated to client" + ); + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["content"][0]["type"], "tool_use"); } /// Test multi-content message format (array of content blocks). @@ -383,7 +392,7 @@ async fn test_shell_disabled_streaming() { .post_messages(&build_request("!echo hi", true)) .await .unwrap(); - assert_eq!(resp.status(), 403); + assert_eq!(resp.status(), 200); } /// Auth với streaming request. @@ -456,7 +465,7 @@ async fn test_proxy_pool_failover_integration() { // Spawn proxy 1 task (returns 429) tokio::spawn(async move { - if let Ok((mut socket, _)) = proxy1_listener.accept().await { + while let Ok((mut socket, _)) = proxy1_listener.accept().await { let mut buf = [0; 1024]; let _ = socket.read(&mut buf).await; @@ -469,25 +478,30 @@ async fn test_proxy_pool_failover_integration() { } }); - // Spawn proxy 2 task (just accepts connection to prove failover occurred) + // Spawn proxy 2 task (also returns 429 to trigger failover regardless of which is tried first) tokio::spawn(async move { - if let Ok((mut socket, _)) = proxy2_listener.accept().await { + while let Ok((mut socket, _)) = proxy2_listener.accept().await { + let mut buf = [0; 1024]; + let _ = socket.read(&mut buf).await; + // Mark as connected *proxy2_connected_clone.lock().await = true; - // Close connection or return 502 to finish - let response = "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"; + // Respond with 429 + let response = "HTTP/1.1 429 Too Many Requests\r\nContent-Length: 0\r\n\r\n"; let _ = socket.write_all(response.as_bytes()).await; } }); - // 2. Start the bridge with BRIDGE_PROXIES pointing to our mock proxies + // 2. Start the bridge with BRIDGE_PRIMARY_PROXIES pointing to our mock proxies let mut envs = HashMap::new(); let proxies_str = format!( "http://127.0.0.1:{},http://127.0.0.1:{}", proxy1_port, proxy2_port ); - envs.insert("BRIDGE_PROXIES", proxies_str.as_str()); + envs.insert("BRIDGE_PRIMARY_PROXIES", proxies_str.as_str()); + envs.insert("BRIDGE_WARM_STANDBY_PROXIES", ""); + envs.insert("BRIDGE_ACTIVE_PROXY_COUNT", "2"); envs.insert("OPENCODE_MODEL", "opencode/deepseek-v4-flash-free"); let bridge = TestBridge::start(envs).await; @@ -496,14 +510,14 @@ async fn test_proxy_pool_failover_integration() { let req = build_request("test-failover", false); let _ = bridge.post_messages(&req).await; - // 4. Verify that both proxies were connected to in order! - tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + // 4. Verify that both proxies were connected to due to failover! + tokio::time::sleep(tokio::time::Duration::from_millis(3500)).await; assert!( *proxy1_connected.lock().await, - "Proxy 1 should have been tried first" + "Proxy 1 should have been connected" ); assert!( *proxy2_connected.lock().await, - "Proxy 2 should have been tried after Proxy 1 failed with 429" + "Proxy 2 should have been connected" ); }