diff --git a/CHANGELOG.md b/CHANGELOG.md index 3161ccca3..291021f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + - Webhooks: event creation in GraphQL when works are changed, event handling via Loco ## [[0.13.13]](https://github.com/thoth-pub/thoth/releases/tag/v0.13.13) - 2025-06-05 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 437983ad9..c16047fd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -196,7 +196,7 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "cookie", + "cookie 0.16.2", "derive_more 2.0.1", "encoding_rs", "foldhash", @@ -283,6 +283,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom 0.2.15", + "once_cell", + "version_check", + "zerocopy 0.7.35", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -307,6 +320,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -396,6 +415,18 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] + [[package]] name = "argon2rs" version = "0.2.5" @@ -421,6 +452,32 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0cf008e5e1a9e9e22a7d3c9a4992e21a350290069e36d8fb72304ed17e8f2d2" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-trait" version = "0.1.88" @@ -438,6 +495,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto-future" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1e7e457ea78e524f48639f551fd79703ac3f2237f5ecccdf4708f8a75ad373" + [[package]] name = "auto_enums" version = "0.8.7" @@ -456,6 +519,135 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +dependencies = [ + "axum-core", + "axum-macros", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.3.1", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +dependencies = [ + "bytes", + "futures-util", + "http 1.3.1", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-extra" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" +dependencies = [ + "axum", + "axum-core", + "bytes", + "cookie 0.18.1", + "futures-util", + "http 1.3.1", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "serde", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "axum-test" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317c1f4ecc1e68e0ad5decb78478421055c963ce215e736ed97463fa609cd196" +dependencies = [ + "anyhow", + "assert-json-diff", + "auto-future", + "axum", + "bytes", + "bytesize", + "cookie 0.18.1", + "http 1.3.1", + "http-body-util", + "hyper", + "hyper-util", + "mime", + "pretty_assertions", + "reserve-port", + "rust-multipart-rfc7578_2", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "tokio", + "tower 0.5.2", + "url", +] + +[[package]] +name = "backon" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970d91570c01a8a5959b36ad7dd1c30642df24b6b3068710066f6809f7033bb7" +dependencies = [ + "fastrand", + "gloo-timers 0.3.0", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -471,6 +663,18 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "backtrace_printer" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d28de81c708c843640982b66573df0f0168d87e42854b563971f326745aab7" +dependencies = [ + "btparse-stable", + "colored", + "regex", + "thiserror 1.0.69", +] + [[package]] name = "base64" version = "0.20.0" @@ -483,6 +687,24 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" + +[[package]] +name = "bb8" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" +dependencies = [ + "async-trait", + "futures-util", + "parking_lot 0.12.3", + "tokio", +] + [[package]] name = "bincode" version = "1.3.3" @@ -504,6 +726,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "blake2-rfc" version = "0.2.18" @@ -550,12 +781,38 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "btparse-stable" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d75b8252ed252f881d1dc4482ae3c3854df6ee8183c1906bac50ff358f4f89f" + [[package]] name = "bumpalo" version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "byte-unit" +version = "4.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c" +dependencies = [ + "serde", + "utf8-width", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -568,6 +825,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bytesize" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e" + [[package]] name = "bytestring" version = "1.4.0" @@ -579,9 +842,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.16" +version = "1.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" dependencies = [ "jobserver", "libc", @@ -609,6 +872,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8100e46ff92eb85bf6dc2930c73f2a4f7176393c84a9446b3d501e1b354e7b34" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.40" @@ -624,6 +893,38 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "chumsky" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" +dependencies = [ + "hashbrown 0.14.5", + "stacker", +] + [[package]] name = "cipher" version = "0.4.4" @@ -641,6 +942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -655,6 +957,18 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.5.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "clap_lex" version = "0.7.4" @@ -676,6 +990,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -699,7 +1023,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width", + "unicode-width 0.2.0", "windows-sys 0.59.0", ] @@ -719,6 +1043,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -737,6 +1070,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -771,6 +1115,82 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cron" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +dependencies = [ + "chrono", + "nom 7.1.3", + "once_cell", +] + +[[package]] +name = "cron_clock" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8699d8ed16e3db689f8ae04d8dc3c6666a4ba7e724e5a157884b7cc385d16b" +dependencies = [ + "chrono", + "nom 7.1.3", + "once_cell", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "cruet" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "113a9e83d8f614be76de8df1f25bf9d0ea6e85ea573710a3d3f3abe1438ae49c" +dependencies = [ + "once_cell", + "regex", +] + +[[package]] +name = "cruet" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6132609543972496bc97b1e01f1ce6586768870aeb4cabeb3385f4e05b5caead" +dependencies = [ + "once_cell", + "regex", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -782,6 +1202,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.100", +] + [[package]] name = "csv" version = "1.3.1" @@ -865,7 +1308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c136f185b3ca9d1f4e4e19c11570e1002f4bfdd592d589053e225716d613851f" dependencies = [ "deadpool", - "redis", + "redis 0.29.2", ] [[package]] @@ -879,13 +1322,24 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "0.99.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -939,6 +1393,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "deunicode" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc55fe0d1f6c107595572ec8b107c0999bb1a2e0b75e37429a4fb0d6474a0e7d" + [[package]] name = "dialoguer" version = "0.11.0" @@ -1025,6 +1485,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1036,6 +1502,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1067,12 +1554,70 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "dtoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "duct_sh" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6633cadba557545fbbe0299a2f9adc4bb2fc5fb238773f5e841e0c23d62146" +dependencies = [ + "duct", +] + +[[package]] +name = "ego-tree" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c6ba7d4eec39eaa9ab24d44a0e73a7949a1095a8b3f3abb11eddf27dbb56a53" + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "email-encoding" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20b9cde6a71f9f758440470f3de16db6c09a02c443ce66850d87f5410548fb8e" +dependencies = [ + "base64 0.22.1", + "memchr", +] + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -1088,6 +1633,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "english-to-cron" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a13a7d5e0ab3872c3ee478366eae624d89ab953d30276b0eee08169774ceb73" +dependencies = [ + "regex", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -1133,6 +1687,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "flagset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec" + [[package]] name = "flate2" version = "1.1.0" @@ -1179,6 +1739,25 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.31" @@ -1250,6 +1829,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -1268,6 +1853,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1278,6 +1872,25 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width 0.1.14", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -1298,9 +1911,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1319,6 +1934,36 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "globset" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags 2.9.0", + "ignore", + "walkdir", +] + [[package]] name = "gloo" version = "0.2.1" @@ -1472,6 +2117,8 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" dependencies = [ + "futures-channel", + "futures-core", "js-sys", "wasm-bindgen", ] @@ -1604,6 +2251,16 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1628,6 +2285,18 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hkdf" version = "0.12.4" @@ -1646,6 +2315,29 @@ dependencies = [ "digest", ] +[[package]] +name = "hostname" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" +dependencies = [ + "cfg-if", + "libc", + "windows", +] + +[[package]] +name = "html5ever" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" +dependencies = [ + "log", + "mac", + "markup5ever", + "match_token", +] + [[package]] name = "http" version = "0.2.12" @@ -1691,6 +2383,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -1703,6 +2401,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "hyper" version = "1.6.0" @@ -1716,6 +2423,7 @@ dependencies = [ "http 1.3.1", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1738,6 +2446,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -1777,14 +2486,15 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", "windows-core", ] @@ -1943,12 +2653,47 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata 0.4.9", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "impl-more" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1979,6 +2724,23 @@ dependencies = [ "generic-array", ] +[[package]] +name = "insta" +version = "1.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50259abbaa67d11d2bcafc7ba1d094ed7a0c70e3ce893f0d0997f73558cb3084" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "pest", + "pest_derive", + "pin-project", + "regex", + "serde", + "similar", +] + [[package]] name = "instant" version = "0.1.13" @@ -1997,6 +2759,26 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "ipnetwork" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" +dependencies = [ + "serde", +] + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2031,9 +2813,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" +checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" dependencies = [ "jiff-static", "log", @@ -2044,9 +2826,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" +checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" dependencies = [ "proc-macro2", "quote", @@ -2132,12 +2914,63 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lettre" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759bc2b8eabb6a30b235d6f716f7f36479f4b38cbe65b8747aefee51f89e8437" +dependencies = [ + "async-trait", + "base64 0.22.1", + "chumsky", + "email-encoding", + "email_address", + "fastrand", + "futures-io", + "futures-util", + "hostname", + "httpdate", + "idna", + "mime", + "nom 8.0.0", + "percent-encoding", + "quoted_printable", + "rustls", + "socket2", + "tokio", + "tokio-rustls", + "url", + "webpki-roots", +] + [[package]] name = "libc" version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -2177,19 +3010,154 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "loco-gen" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec8762a66ade27c5157288ca46db51a77bbd95cb2d7ad87bcaada9621e3a4d9" +dependencies = [ + "chrono", + "clap", + "colored", + "cruet 0.14.0", + "duct", + "include_dir", + "regex", + "rrgen", + "serde", + "serde_json", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "loco-rs" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19decee5529e36ffd3e22f83cf601c1fcb35a8932886fc33c19e8482bc1baf88" +dependencies = [ + "argon2", + "async-trait", + "axum", + "axum-extra", + "axum-test", + "backtrace_printer", + "bb8", + "byte-unit", + "bytes", + "cfg-if", + "chrono", + "clap", + "colored", + "cruet 0.13.3", + "duct", + "duct_sh", + "english-to-cron", + "fs-err", + "futures-util", + "heck 0.4.1", + "hyper", + "include_dir", + "ipnetwork", + "lettre", + "loco-gen", + "mime", + "opendal", + "rand 0.8.5", + "regex", + "reqwest", + "rusty-sidekiq", + "scraper", + "semver", + "serde", + "serde_json", + "serde_variant", + "serde_yaml", + "tera", + "thiserror 1.0.69", + "thousands", + "tokio", + "tokio-cron-scheduler", + "tokio-util", + "toml", + "tower 0.4.13", + "tower-http", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", + "validator", +] + [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "mac" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] name = "marc" version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7158861fa6320edf04e974b504ad3c3a69e1aa9500d2e4899e6dc41265c0a56" +checksum = "c7158861fa6320edf04e974b504ad3c3a69e1aa9500d2e4899e6dc41265c0a56" +dependencies = [ + "xml-rs", +] + +[[package]] +name = "markup5ever" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "xml-rs", + "cfg-if", + "digest", ] [[package]] @@ -2235,6 +3203,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -2273,12 +3247,47 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nodrop" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2295,6 +3304,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2319,7 +3339,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -2355,6 +3375,34 @@ dependencies = [ "serde_json", ] +[[package]] +name = "opendal" +version = "0.50.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb28bb6c64e116ceaf8dd4e87099d3cfea4a58e85e62b104fef74c91afba0f44" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "flagset", + "futures", + "getrandom 0.2.15", + "http 1.3.1", + "log", + "md-5", + "once_cell", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + [[package]] name = "openssl" version = "0.10.72" @@ -2399,6 +3447,22 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_pipe" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "paperclip" version = "0.9.5" @@ -2524,6 +3588,26 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "pem" version = "3.0.5" @@ -2540,6 +3624,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror 2.0.12", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "pest_meta" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "phf" version = "0.11.3" @@ -2550,6 +3679,16 @@ dependencies = [ "phf_shared", ] +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + [[package]] name = "phf_generator" version = "0.11.3" @@ -2582,6 +3721,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -2639,7 +3798,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.24", ] [[package]] @@ -2652,6 +3811,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2707,6 +3891,79 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" +dependencies = [ + "cc", +] + +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +dependencies = [ + "bytes", + "getrandom 0.3.2", + "rand 0.9.0", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -2716,6 +3973,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quoted_printable" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c9bd8497b02465aeef5375144c26062e0dcd5939dfcbb0f5db76cb8c17c73" + [[package]] name = "r-efi" version = "5.2.0" @@ -2752,7 +4015,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy", + "zerocopy 0.8.24", ] [[package]] @@ -2795,9 +4058,29 @@ dependencies = [ [[package]] name = "redis" -version = "0.29.1" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8455fa3621f6b41c514946de66ea0531f57ca017b2e6c7cc368035ea5b46df" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "redis" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8034fb926579ff49d3fe58d288d5dcb580bf11e9bccd33224b45adebf0fd0c23" +checksum = "b110459d6e323b7cda23980c46c77157601199c9da6241552b284cd565a7a133" dependencies = [ "arc-swap", "bytes", @@ -2832,6 +4115,17 @@ dependencies = [ "bitflags 2.9.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -2840,8 +4134,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2852,7 +4155,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] [[package]] @@ -2861,17 +4164,29 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e327e510263980e231de548a33e63d34962d29ae61b467389a1a09627a254" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", @@ -2894,7 +4209,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2902,12 +4220,16 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tower", + "tokio-rustls", + "tokio-util", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", + "webpki-roots", "windows-registry", ] @@ -2948,6 +4270,16 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "reserve-port" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "359fc315ed556eb0e42ce74e76f4b1cd807b50fa6307f3de4e51f92dbe86e2d5" +dependencies = [ + "lazy_static", + "thiserror 2.0.12", +] + [[package]] name = "retry-policies" version = "0.4.0" @@ -2986,17 +4318,97 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rrgen" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e27f5f254d89b0b5b76445442e5c935b63a566ee5a735c9d877ca2029b4ce9" +dependencies = [ + "cruet 0.13.3", + "fs-err", + "glob", + "heck 0.4.1", + "regex", + "serde", + "serde_json", + "serde_regex", + "serde_yaml", + "tera", + "thiserror 1.0.69", +] + +[[package]] +name = "rstest" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afd55a67069d6e434a95161415f5beeada95a01c7b815508a82dcb0e1593682" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.100", + "unicode-ident", +] + +[[package]] +name = "rust-multipart-rfc7578_2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc4bb9e7c9abe5fa5f30c2d8f8fefb9e0080a2c1e3c2e567318d2907054b35d3" +dependencies = [ + "bytes", + "futures-core", + "futures-util", + "http 1.3.1", + "mime", + "mime_guess", + "rand 0.9.0", + "thiserror 2.0.12", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" dependencies = [ "bitflags 2.9.0", "errno", @@ -3011,7 +4423,9 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3032,6 +4446,9 @@ name = "rustls-pki-types" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -3050,12 +4467,58 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +[[package]] +name = "rusty-sidekiq" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15544f047600b602c7b11ff7ee0882f9034f9cbe2c205693edd5615e2a6c03ee" +dependencies = [ + "async-trait", + "bb8", + "chrono", + "convert_case", + "cron_clock", + "gethostname", + "hex", + "num_cpus", + "rand 0.8.5", + "redis 0.22.3", + "serde", + "serde_json", + "serial_test", + "sha2", + "slog-term", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea091f6cac2595aa38993f04f4ee692ed43757035c36e67c180b6828356385b1" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.27" @@ -3092,6 +4555,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e749d29b2064585327af5038a5a8eb73aeebad4a3472e83531a436563f7208" +dependencies = [ + "ahash", + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "indexmap 2.8.0", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "sdd" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "584e070911c7017da6cb2eb0788d09f43d789029b5877d3e5ecc8acf86ceee21" + [[package]] name = "security-framework" version = "2.11.1" @@ -3115,6 +4601,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags 2.9.0", + "cssparser", + "derive_more 0.99.19", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.26" @@ -3165,6 +4670,26 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -3186,6 +4711,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_variant" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a0068df419f9d9b6488fdded3f1c818522cdea328e02ce9d9f147380265a432" +dependencies = [ + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -3199,6 +4733,40 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +dependencies = [ + "futures", + "log", + "once_cell", + "parking_lot 0.12.3", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "servo_arc" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae65c4249478a2647db249fb43e23cec56a2c8974a427e7bd8cb5a1d0964921a" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.10.6" @@ -3210,6 +4778,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.8" @@ -3221,6 +4795,25 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shared_child" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "shell-words" version = "1.1.0" @@ -3242,6 +4835,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "simple_asn1" version = "0.6.3" @@ -3269,6 +4868,35 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-term" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +dependencies = [ + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + +[[package]] +name = "slug" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" +dependencies = [ + "deunicode", + "wasm-bindgen", +] + [[package]] name = "smallvec" version = "1.14.0" @@ -3302,12 +4930,50 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601f9201feb9b09c00266478bf459952b9ef9a6b94edb2f21eba14ab681a60a9" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string_cache" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938d512196766101d333398efde81bc1f37b00cb42c2f8350e5df639f040bbbe" +dependencies = [ + "new_debug_unreachable", + "parking_lot 0.12.3", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3426,9 +5092,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.19.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" dependencies = [ "fastrand", "getrandom 0.3.2", @@ -3437,6 +5103,50 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "tera" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9d851b45e865f178319da0abdbfe6acbc4328759ff18dafc3a41c16b4cd2ee" +dependencies = [ + "chrono", + "chrono-tz", + "globwalk", + "humansize", + "lazy_static", + "percent-encoding", + "pest", + "pest_derive", + "rand 0.8.5", + "regex", + "serde", + "serde_json", + "slug", + "unic-segment", +] + +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -3490,6 +5200,7 @@ dependencies = [ "thoth-app-server", "thoth-errors", "thoth-export-server", + "thoth-processor", "tokio", ] @@ -3634,17 +5345,54 @@ dependencies = [ "serde", "serde_json", "thoth-api", - "thoth-client", - "thoth-errors", + "thoth-client", + "thoth-errors", + "uuid", + "xml-rs", +] + +[[package]] +name = "thoth-processor" +version = "0.13.13" +dependencies = [ + "async-trait", + "axum", + "insta", + "loco-rs", + "regex", + "reqwest", + "rstest", + "serde", + "serde_json", + "serial_test", + "thoth-api", + "tokio", + "tracing", + "tracing-subscriber", "uuid", - "xml-rs", +] + +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", ] [[package]] name = "time" -version = "0.3.40" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -3663,9 +5411,9 @@ checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -3681,6 +5429,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.44.2" @@ -3699,6 +5462,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-cron-scheduler" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2594dd7c2abbbafbb1c78d167fd10860dc7bd75f814cb051a1e0d3e796b9702" +dependencies = [ + "chrono", + "cron", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "tokio-macros" version = "2.5.0" @@ -3777,6 +5555,17 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.2" @@ -3790,6 +5579,34 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +dependencies = [ + "async-compression", + "bitflags 2.9.0", + "bytes", + "futures-core", + "futures-util", + "http 1.3.1", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -3816,6 +5633,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror 1.0.69", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.28" @@ -3834,6 +5663,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -3848,6 +5720,62 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + [[package]] name = "unicase" version = "2.8.1" @@ -3860,6 +5788,18 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.0" @@ -3905,12 +5845,24 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf16_iter" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3931,10 +5883,47 @@ checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "getrandom 0.3.2", "js-sys", + "rand 0.9.0", "serde", "wasm-bindgen", ] +[[package]] +name = "validator" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b4a29d8709210980a09379f27ee31549b73292c87ab9899beee1c0d3be6303" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac855a2ce6f843beb229757e6e570a42e837bcb15e5f449dd48d5747d41bf77" +dependencies = [ + "darling", + "once_cell", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3953,6 +5942,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4061,6 +6060,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-timer" version = "0.2.5" @@ -4086,6 +6098,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4102,12 +6133,31 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -4119,9 +6169,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" [[package]] name = "windows-registry" @@ -4136,9 +6186,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ "windows-link", ] @@ -4340,6 +6390,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yew" version = "0.18.0" @@ -4505,18 +6561,38 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.23" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ - "zerocopy-derive", + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] name = "zerocopy-derive" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", @@ -4583,18 +6659,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.3" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.14+zstd.1.5.7" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index e1d02626e..c523fc172 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ readme = "README.md" maintenance = { status = "actively-developed" } [workspace] -members = ["thoth-api", "thoth-api-server", "thoth-app", "thoth-app-server", "thoth-client", "thoth-errors", "thoth-export-server"] +members = ["thoth-api", "thoth-api-server", "thoth-app", "thoth-app-server", "thoth-client", "thoth-errors", "thoth-export-server", "thoth-processor"] [dependencies] thoth-api = { version = "=0.13.13", path = "thoth-api", features = ["backend"] } @@ -20,6 +20,7 @@ thoth-api-server = { version = "=0.13.13", path = "thoth-api-server" } thoth-app-server = { version = "=0.13.13", path = "thoth-app-server" } thoth-errors = { version = "=0.13.13", path = "thoth-errors" } thoth-export-server = { version = "=0.13.13", path = "thoth-export-server" } +thoth-processor = { version = "=0.13.13", path = "thoth-processor" } clap = { version = "4.5.32", features = ["cargo", "env"] } dialoguer = { version = "0.11.0", features = ["password"] } dotenv = "0.15.0" diff --git a/Dockerfile.dev b/Dockerfile.dev index 8107eed29..461e7492f 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -32,18 +32,19 @@ COPY thoth-app-server/Cargo.toml thoth-app-server/Cargo.toml COPY thoth-client/Cargo.toml thoth-client/Cargo.toml COPY thoth-errors/Cargo.toml thoth-errors/Cargo.toml COPY thoth-export-server/Cargo.toml thoth-export-server/Cargo.toml +COPY thoth-processor/Cargo.toml thoth-processor/Cargo.toml RUN mkdir thoth-api/src thoth-api-server/src thoth-app/src \ thoth-app-server/src thoth-client/src thoth-errors/src \ - thoth-export-server/src + thoth-export-server/src thoth-processor/src RUN touch thoth-api/src/lib.rs thoth-api-server/src/lib.rs \ thoth-app/src/lib.rs thoth-app-server/src/lib.rs thoth-client/src/lib.rs \ - thoth-errors/src/lib.rs thoth-export-server/src/lib.rs + thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-processor/src/lib.rs RUN echo "fn main() {}" > thoth-client/build.rs RUN echo "fn main() {}" > thoth-app-server/build.rs RUN echo "fn main() {}" > thoth-export-server/build.rs RUN cargo build RUN rm -rf src thoth-api thoth-api-server thoth-app thoth-app-server thoth-client \ - thoth-errors thoth-export-server Cargo.toml Cargo.lock + thoth-errors thoth-export-server thoth-processor Cargo.toml Cargo.lock # Get the actual source COPY . . @@ -52,8 +53,8 @@ COPY . . # it needs to (re)compile these modules RUN touch -a -m thoth-api/src/lib.rs thoth-api-server/src/lib.rs \ thoth-app/src/lib.rs thoth-app-server/src/lib.rs thoth-client/src/lib.rs \ - thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-app-server/build.rs \ - thoth-export-server/build.rs + thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-processor/src/lib.rs \ + thoth-app-server/build.rs thoth-export-server/build.rs # Build Thoth for debug RUN cargo build diff --git a/Makefile b/Makefile index 6b0bf3bc7..ea4c76b6d 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ build-graphql-api \ build-export-api \ build-app \ + build-processor run-app \ run-graphql-api \ run-export-api \ + run-processor watch-app \ docker-dev \ docker-dev-build \ @@ -19,9 +21,12 @@ check \ check-all \ -all: build-graphql-api build-export-api build-app +all: build-graphql-api build-export-api build-app build-processor check-all: test check clippy check-format +run-processor: build-processor + RUST_BACKTRACE=1 cargo run start processor + run-app: build-app RUST_BACKTRACE=1 cargo run start app @@ -57,6 +62,8 @@ build-export-api: build build-app: build +build-processor: build + test: cargo test --workspace diff --git a/diesel.toml b/diesel.toml index 752265c18..880a0107b 100644 --- a/diesel.toml +++ b/diesel.toml @@ -17,4 +17,5 @@ custom_type_derives = [ "crate::model::subject::Subject_type" "crate::model::institution::Country_code" "crate::model::work_relation::Relation_type" + "crate::event::model::Event_type" ] diff --git a/docker-compose.yml b/docker-compose.yml index 1fba394cb..fef5a6968 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,5 +44,15 @@ services: - graphql-api - export-api + processor: + image: ghcr.io/thoth-pub/thoth + container_name: "thoth_processor" + restart: unless-stopped + command: ["start", "processor"] + env_file: + - .env + depends_on: + - redis + volumes: db: diff --git a/src/bin/commands/mod.rs b/src/bin/commands/mod.rs index f6a585c9e..3128cd626 100644 --- a/src/bin/commands/mod.rs +++ b/src/bin/commands/mod.rs @@ -20,6 +20,7 @@ lazy_static! { pub(super) static ref INIT: Command = Command::new("init") .about("Run the database migrations and start the thoth API server") .arg(arguments::database()) + .arg(arguments::redis()) .arg(arguments::host("GRAPHQL_API_HOST")) .arg(arguments::port("8000", "GRAPHQL_API_PORT")) .arg(arguments::threads("GRAPHQL_API_THREADS")) diff --git a/src/bin/commands/start.rs b/src/bin/commands/start.rs index 9ef2f3c8d..bf03ed98a 100644 --- a/src/bin/commands/start.rs +++ b/src/bin/commands/start.rs @@ -1,7 +1,7 @@ use crate::arguments; use clap::{ArgMatches, Command}; use lazy_static::lazy_static; -use thoth::{api_server, app_server, errors::ThothResult, export_server}; +use thoth::{api_server, app_server, errors::ThothResult, export_server, processor_server}; lazy_static! { pub(crate) static ref COMMAND: Command = Command::new("start") @@ -12,6 +12,7 @@ lazy_static! { Command::new("graphql-api") .about("Start the thoth GraphQL API server") .arg(arguments::database()) + .arg(arguments::redis()) .arg(arguments::host("GRAPHQL_API_HOST")) .arg(arguments::port("8000", "GRAPHQL_API_PORT")) .arg(arguments::threads("GRAPHQL_API_THREADS")) @@ -39,11 +40,13 @@ lazy_static! { .arg(arguments::keep_alive("EXPORT_API_KEEP_ALIVE")) .arg(arguments::export_url()) .arg(arguments::gql_endpoint()), - ); + ) + .subcommand(Command::new("processor").about("Start the thoth event processor")); } pub fn graphql_api(arguments: &ArgMatches) -> ThothResult<()> { let database_url = arguments.get_one::("db").unwrap().to_owned(); + let redis_url = arguments.get_one::("redis").unwrap().to_owned(); let host = arguments.get_one::("host").unwrap().to_owned(); let port = arguments.get_one::("port").unwrap().to_owned(); let threads = *arguments.get_one::("threads").unwrap(); @@ -54,6 +57,7 @@ pub fn graphql_api(arguments: &ArgMatches) -> ThothResult<()> { let session_duration = *arguments.get_one::("duration").unwrap(); api_server( database_url, + redis_url, host, port, threads, @@ -99,3 +103,7 @@ pub fn export_api(arguments: &ArgMatches) -> ThothResult<()> { ) .map_err(|e| e.into()) } + +pub fn processor() -> ThothResult<()> { + processor_server().map_err(|e| thoth_errors::ThothError::InternalError(e.to_string())) +} diff --git a/src/bin/thoth.rs b/src/bin/thoth.rs index 42597884b..57de3ba81 100644 --- a/src/bin/thoth.rs +++ b/src/bin/thoth.rs @@ -24,6 +24,7 @@ fn main() -> thoth::errors::ThothResult<()> { Some(("graphql-api", arguments)) => commands::start::graphql_api(arguments), Some(("app", arguments)) => commands::start::app(arguments), Some(("export-api", arguments)) => commands::start::export_api(arguments), + Some(("processor", _)) => commands::start::processor(), _ => unreachable!(), }, Some(("migrate", arguments)) => commands::migrate(arguments), diff --git a/src/lib.rs b/src/lib.rs index b0e60dc41..d1dafe6f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,3 +3,4 @@ pub use thoth_api_server::start_server as api_server; pub use thoth_app_server::start_server as app_server; pub use thoth_errors as errors; pub use thoth_export_server::{start_server as export_server, ALL_SPECIFICATIONS}; +pub use thoth_processor::start_server as processor_server; diff --git a/thoth-api-server/src/lib.rs b/thoth-api-server/src/lib.rs index 318faaf0d..0336960b0 100644 --- a/thoth-api-server/src/lib.rs +++ b/thoth-api-server/src/lib.rs @@ -19,11 +19,12 @@ use serde::Serialize; use thoth_api::{ account::model::{AccountDetails, DecodedToken, LoginCredentials}, account::service::{get_account, get_account_details, login}, - db::{init_pool, PgPool}, + db::{init_pool as init_pg_pool, PgPool}, graphql::{ model::{create_schema, Context, Schema}, GraphQLRequest, }, + redis::{init_pool as init_redis_pool, RedisPool}, }; use thoth_errors::ThothError; @@ -91,10 +92,11 @@ async fn graphql_schema(st: Data>) -> HttpResponse { async fn graphql( st: Data>, pool: Data, + redis_pool: Data, token: DecodedToken, data: Json, ) -> Result { - let ctx = Context::new(pool.into_inner(), token); + let ctx = Context::new(pool.into_inner(), redis_pool.into_inner(), token); let result = data.execute(&st, &ctx).await; match result.is_ok() { true => Ok(HttpResponse::Ok().json(result)), @@ -186,6 +188,7 @@ async fn account_details( #[actix_web::main] pub async fn start_server( database_url: String, + redis_url: String, host: String, port: String, threads: usize, @@ -227,7 +230,8 @@ pub async fn start_server( .supports_credentials(), ) .app_data(Data::new(ApiConfig::new(public_url.clone()))) - .app_data(Data::new(init_pool(&database_url))) + .app_data(Data::new(init_pg_pool(&database_url))) + .app_data(Data::new(init_redis_pool(&redis_url))) .app_data(Data::new(Arc::new(create_schema()))) .service(index) .service(graphql_index) diff --git a/thoth-api/migrations/v0.13.14/down.sql b/thoth-api/migrations/v0.13.14/down.sql new file mode 100644 index 000000000..cce46d74a --- /dev/null +++ b/thoth-api/migrations/v0.13.14/down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS webhook_history; +DROP TRIGGER set_updated_at ON webhook; +DROP TABLE IF EXISTS webhook; +DROP TYPE IF EXISTS event_type; diff --git a/thoth-api/migrations/v0.13.14/up.sql b/thoth-api/migrations/v0.13.14/up.sql new file mode 100644 index 000000000..1e5571a78 --- /dev/null +++ b/thoth-api/migrations/v0.13.14/up.sql @@ -0,0 +1,29 @@ +CREATE TYPE event_type AS ENUM ( + 'WorkCreated', + 'WorkUpdated', + 'WorkPublished' +); + +CREATE TABLE webhook ( + webhook_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + publisher_id UUID NOT NULL REFERENCES publisher(publisher_id) ON DELETE CASCADE, + endpoint TEXT NOT NULL CHECK (endpoint ~* '^[^:]*:\/\/(?:[^\/:]*:[^\/@]*@)?(?:[^\/:.]*\.)+([^:\/]+)'), + token TEXT CHECK (OCTET_LENGTH(token) >= 1), + is_published BOOLEAN NOT NULL, + event_type event_type NOT NULL, + payload TEXT CHECK (OCTET_LENGTH(payload) >= 1), + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +SELECT diesel_manage_updated_at('webhook'); + +CREATE INDEX idx_webhook_endpoint ON webhook (endpoint); +CREATE INDEX idx_webhook_publisher_id ON webhook (publisher_id); + +CREATE TABLE webhook_history ( + webhook_history_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + webhook_id UUID NOT NULL REFERENCES webhook(webhook_id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES account(account_id), + data JSONB NOT NULL, + timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs new file mode 100644 index 000000000..7d6361a7c --- /dev/null +++ b/thoth-api/src/event/handler.rs @@ -0,0 +1,21 @@ +use super::model::{Event, EventType}; +use crate::model::work::{Work, WorkProperties}; +use crate::redis::{rpush, RedisPool}; +use thoth_errors::ThothResult; + +pub const QUEUE_KEY: &str = "events:graphql"; + +pub async fn send_event( + redis: &RedisPool, + event_type: EventType, + work: &Work, +) -> ThothResult { + let event = Event { + event_type, + work_id: *work.work_id(), + is_published: work.is_published(), + event_timestamp: work.updated_at, + thoth_version: env!("CARGO_PKG_VERSION").parse().unwrap(), + }; + rpush(redis, QUEUE_KEY, &serde_json::to_string(&event)?).await +} diff --git a/thoth-api/src/event/mod.rs b/thoth-api/src/event/mod.rs new file mode 100644 index 000000000..0386dfea0 --- /dev/null +++ b/thoth-api/src/event/mod.rs @@ -0,0 +1,3 @@ +#[cfg(feature = "backend")] +pub mod handler; +pub mod model; diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs new file mode 100644 index 000000000..76ce9b8a0 --- /dev/null +++ b/thoth-api/src/event/model.rs @@ -0,0 +1,59 @@ +use crate::model::Timestamp; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumString}; +use uuid::Uuid; + +#[cfg_attr( + feature = "backend", + derive(DbEnum, juniper::GraphQLEnum), + graphql(description = "Nature of an event"), + ExistingTypePath = "crate::schema::sql_types::EventType" +)] +#[derive( + Debug, Clone, Default, Copy, PartialEq, Eq, Deserialize, Serialize, EnumString, Display, +)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +#[strum(serialize_all = "title_case")] +pub enum EventType { + #[cfg_attr( + feature = "backend", + db_rename = "WorkCreated", + graphql(description = "Work creation event") + )] + WorkCreated, + #[default] + #[cfg_attr( + feature = "backend", + db_rename = "WorkUpdated", + graphql(description = "Work update event") + )] + WorkUpdated, + #[cfg_attr( + feature = "backend", + db_rename = "WorkPublished", + graphql(description = "Work publication event") + )] + WorkPublished, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLObject), + graphql( + description = "Details of a change made to a record which may require follow-up processing" + ) +)] +#[derive(Debug, Deserialize, Serialize)] +pub struct Event { + pub event_type: EventType, + pub work_id: Uuid, + pub is_published: bool, + pub event_timestamp: Timestamp, + pub thoth_version: String, +} + +#[cfg_attr(feature = "backend", derive(juniper::GraphQLObject))] +#[derive(Debug, Deserialize, Serialize)] +pub struct EventWrapper { + pub event: Event, +} diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 8a157c526..41c8a05ef 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -7,6 +7,10 @@ use uuid::Uuid; use crate::account::model::AccountAccess; use crate::account::model::DecodedToken; use crate::db::PgPool; +use crate::event::handler::send_event; +use crate::event::model::{EventType, EventWrapper}; +use crate::job::handler::get_jobs; +use crate::job::model::Job; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; @@ -22,6 +26,7 @@ use crate::model::publisher::*; use crate::model::reference::*; use crate::model::series::*; use crate::model::subject::*; +use crate::model::webhook::*; use crate::model::work::*; use crate::model::work_relation::*; use crate::model::Convert; @@ -33,6 +38,7 @@ use crate::model::Orcid; use crate::model::Ror; use crate::model::Timestamp; use crate::model::WeightUnit; +use crate::redis::RedisPool; use thoth_errors::{ThothError, ThothResult}; use super::utils::{Direction, Expression}; @@ -42,14 +48,16 @@ impl juniper::Context for Context {} #[derive(Clone)] pub struct Context { pub db: Arc, + pub redis: Arc, pub account_access: AccountAccess, pub token: DecodedToken, } impl Context { - pub fn new(pool: Arc, token: DecodedToken) -> Self { + pub fn new(pool: Arc, redis_pool: Arc, token: DecodedToken) -> Self { Self { db: pool, + redis: redis_pool, account_access: token.get_user_permissions(), token, } @@ -1461,6 +1469,56 @@ impl QueryRoot { fn reference_count(context: &Context) -> FieldResult { Reference::count(&context.db, None, vec![], vec![], vec![], None).map_err(|e| e.into()) } + + #[graphql(description = "Query the full list of webhooks")] + fn webhooks( + context: &Context, + #[graphql(default = 100, description = "The number of items to return")] limit: Option, + #[graphql(default = 0, description = "The number of items to skip")] offset: Option, + #[graphql( + default = WebhookOrderBy::default(), + description = "The order in which to sort the results" + )] + order: Option, + #[graphql( + default = vec![], + description = "If set, only shows results connected to publishers with these IDs" + )] + publishers: Option>, + ) -> FieldResult> { + Webhook::all( + &context.db, + limit.unwrap_or_default(), + offset.unwrap_or_default(), + None, + order.unwrap_or_default(), + publishers.unwrap_or_default(), + None, + None, + vec![], + vec![], + None, + ) + .map_err(|e| e.into()) + } + + #[graphql(description = "Query a single webhook using its ID")] + fn webhook( + context: &Context, + #[graphql(description = "Thoth webhook ID to search on")] webhook_id: Uuid, + ) -> FieldResult { + Webhook::from_id(&context.db, &webhook_id).map_err(|e| e.into()) + } + + #[graphql(description = "Get the total number of webhooks")] + fn webhook_count(context: &Context) -> FieldResult { + Webhook::count(&context.db, None, vec![], vec![], vec![], None).map_err(|e| e.into()) + } + + #[graphql(description = "Get information about retried jobs")] + async fn retried_jobs(context: &Context) -> FieldResult> { + get_jobs(&context.redis).await.map_err(|e| e.into()) + } } pub struct MutationRoot; @@ -1468,7 +1526,7 @@ pub struct MutationRoot; #[juniper::graphql_object(Context = Context)] impl MutationRoot { #[graphql(description = "Create a new work with the specified values")] - fn create_work( + async fn create_work( context: &Context, #[graphql(description = "Values for work to be created")] data: NewWork, ) -> FieldResult { @@ -1479,7 +1537,17 @@ impl MutationRoot { data.validate()?; - Work::create(&context.db, &data).map_err(|e| e.into()) + let result = Work::create(&context.db, &data).map_err(|e| e.into()); + + if let Ok(ref created_work) = result { + // TODO handle results throughout + let _ = send_event(&context.redis, EventType::WorkCreated, created_work).await; + if created_work.work_status == WorkStatus::Active { + let _ = send_event(&context.redis, EventType::WorkPublished, created_work).await; + } + } + + result } #[graphql(description = "Create a new publisher with the specified values")] @@ -1719,8 +1787,19 @@ impl MutationRoot { Reference::create(&context.db, &data).map_err(|e| e.into()) } + #[graphql(description = "Create a new webhook with the specified values")] + fn create_webhook( + context: &Context, + #[graphql(description = "Values for webhook to be created")] data: NewWebhook, + ) -> FieldResult { + context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; + context.account_access.can_edit(data.publisher_id)?; + + Webhook::create(&context.db, &data).map_err(|e| e.into()) + } + #[graphql(description = "Update an existing work with the specified values")] - fn update_work( + async fn update_work( context: &Context, #[graphql(description = "Values to apply to existing work")] data: PatchWork, ) -> FieldResult { @@ -1751,6 +1830,11 @@ impl MutationRoot { // update the work and, if it succeeds, synchronise its children statuses and pub. date match work.update(&context.db, &data, &account_id) { Ok(w) => { + if w.work_status == WorkStatus::Active && work.work_status != WorkStatus::Active { + let _ = send_event(&context.redis, EventType::WorkPublished, &w).await; + } else { + let _ = send_event(&context.redis, EventType::WorkUpdated, &w).await; + } // update chapters if their pub. data, withdrawn_date or work_status doesn't match the parent's for child in work.children(&context.db)? { if child.publication_date != w.publication_date @@ -2166,6 +2250,25 @@ impl MutationRoot { .map_err(|e| e.into()) } + #[graphql(description = "Update an existing webhook with the specified values")] + fn update_webhook( + context: &Context, + #[graphql(description = "Values to apply to existing webhook")] data: PatchWebhook, + ) -> FieldResult { + context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; + let webhook = Webhook::from_id(&context.db, &data.webhook_id).unwrap(); + context.account_access.can_edit(webhook.publisher_id())?; + + if data.publisher_id != webhook.publisher_id { + context.account_access.can_edit(data.publisher_id)?; + } + + let account_id = context.token.jwt.as_ref().unwrap().account_id(&context.db); + webhook + .update(&context.db, &data, &account_id) + .map_err(|e| e.into()) + } + #[graphql(description = "Delete a single work using its ID")] fn delete_work( context: &Context, @@ -2416,6 +2519,18 @@ impl MutationRoot { reference.delete(&context.db).map_err(|e| e.into()) } + + #[graphql(description = "Delete a single webhook using its ID")] + fn delete_webhook( + context: &Context, + #[graphql(description = "Thoth ID of webhook to be deleted")] webhook_id: Uuid, + ) -> FieldResult { + context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; + let webhook = Webhook::from_id(&context.db, &webhook_id).unwrap(); + context.account_access.can_edit(webhook.publisher_id())?; + + webhook.delete(&context.db).map_err(|e| e.into()) + } } #[juniper::graphql_object(Context = Context, description = "A written text that can be published")] @@ -3165,6 +3280,41 @@ impl Publisher { ) .map_err(|e| e.into()) } + + #[graphql(description = "Get webhooks linked to this publisher")] + pub fn webhooks( + &self, + context: &Context, + #[graphql(default = 100, description = "The number of items to return")] limit: Option, + #[graphql(default = 0, description = "The number of items to skip")] offset: Option, + #[graphql( + default = WebhookOrderBy::default(), + description = "The order in which to sort the results" + )] + order: Option, + #[graphql( + default = vec![], + description = "Specific types to filter by", + )] + event_types: Option>, + #[graphql(description = "Only show results where IsPublished is True")] + is_published: Option, + ) -> FieldResult> { + Webhook::all( + &context.db, + limit.unwrap_or_default(), + offset.unwrap_or_default(), + None, + order.unwrap_or_default(), + vec![], + Some(self.publisher_id), + None, + event_types.unwrap_or_default(), + vec![], + is_published, + ) + .map_err(|e| e.into()) + } } #[juniper::graphql_object(Context = Context, description = "The brand under which a publisher issues works.")] @@ -4189,6 +4339,126 @@ impl Reference { } } +#[juniper::graphql_object(Context = Context, description = "A web request to be made when a specified data event occurs")] +impl Webhook { + #[graphql(description = "Thoth ID of the webhook")] + pub fn webhook_id(&self) -> Uuid { + self.webhook_id + } + + #[graphql(description = "Thoth ID of the publisher to which this webhook belongs")] + pub fn publisher_id(&self) -> Uuid { + self.publisher_id + } + + #[graphql(description = "URL which is called by the webhook")] + pub fn endpoint(&self) -> &String { + &self.endpoint + } + + #[graphql(description = "Authentication token required for the webhook")] + pub fn token(&self) -> Option<&String> { + self.token.as_ref() + } + + #[graphql(description = "Type of event which triggers the webhook")] + pub fn event_type(&self) -> &EventType { + &self.event_type + } + + #[graphql(description = "Whether the activation of the webhook depends on publication status")] + pub fn is_published(&self) -> bool { + self.is_published + } + + #[graphql(description = "Payload to be sent in the webhook request body (usually JSON)")] + pub fn payload(&self) -> Option<&String> { + self.payload.as_ref() + } + + #[graphql(description = "Date and time at which the webhook record was created")] + pub fn created_at(&self) -> Timestamp { + self.created_at + } + + #[graphql(description = "Date and time at which the webhook record was last updated")] + pub fn updated_at(&self) -> Timestamp { + self.updated_at + } + + #[graphql(description = "Get the publisher to which this webhook belongs")] + pub fn publisher(&self, context: &Context) -> FieldResult { + Publisher::from_id(&context.db, &self.publisher_id).map_err(|e| e.into()) + } +} + +#[juniper::graphql_object(Context = Context, description = "A task generated when a webhook is triggered")] +impl Job { + #[graphql(description = "Name of the processing queue where this job was initially created")] + pub fn queue(&self) -> &String { + &self.queue + } + + #[graphql(description = "Arguments which were supplied when generating the job")] + pub fn args(&self) -> &Vec { + &self.args + } + + #[graphql(description = "Whether or not the job should be retried on failure")] + pub fn retry(&self) -> bool { + self.retry + } + + #[graphql(description = "Type of the job (defined by the name of the worker processing it)")] + pub fn class(&self) -> &String { + &self.class + } + + #[graphql(description = "Unique identifier of the job")] + pub fn jid(&self) -> &String { + &self.jid + } + + #[graphql(description = "Date and time at which the job was created (in unix epoch format)")] + pub fn created_at(&self) -> f64 { + self.created_at + } + + #[graphql( + description = "Date and time at which the job was added to the queue (in unix epoch format)" + )] + pub fn enqueued_at(&self) -> f64 { + self.enqueued_at + } + + #[graphql(description = "Date and time at which the job failed (in unix epoch format)")] + pub fn failed_at(&self) -> f64 { + self.failed_at + } + + #[graphql(description = "Error message returned on failure of the job")] + pub fn error_message(&self) -> &String { + &self.error_message + } + + #[graphql(description = "Type of error with which the job failed")] + pub fn error_class(&self) -> Option<&String> { + self.error_class.as_ref() + } + + #[graphql(description = "Number of times the job has been retried")] + pub fn retry_count(&self) -> i32 { + self.retry_count + } + + #[graphql( + description = "Date and time at which the job was last retried (in unix epoch format)" + )] + pub fn retried_at(&self) -> f64 { + self.retried_at + } +} + pub type Schema = RootNode<'static, QueryRoot, MutationRoot, EmptySubscription>; pub fn create_schema() -> Schema { diff --git a/thoth-api/src/job/handler.rs b/thoth-api/src/job/handler.rs new file mode 100644 index 000000000..42449f53e --- /dev/null +++ b/thoth-api/src/job/handler.rs @@ -0,0 +1,13 @@ +use super::model::Job; +use crate::redis::{zrange, RedisPool}; +use thoth_errors::ThothResult; + +pub const RETRY_QUEUE_KEY: &str = "retry"; + +pub async fn get_jobs(redis: &RedisPool) -> ThothResult> { + let jobs_string = zrange(redis, RETRY_QUEUE_KEY, 0, -1).await?; + jobs_string + .into_iter() + .map(|j| serde_json::from_str(&j).map_err(|e| e.into())) + .collect::>>() +} diff --git a/thoth-api/src/job/mod.rs b/thoth-api/src/job/mod.rs new file mode 100644 index 000000000..8583a1b23 --- /dev/null +++ b/thoth-api/src/job/mod.rs @@ -0,0 +1,2 @@ +pub mod handler; +pub mod model; diff --git a/thoth-api/src/job/model.rs b/thoth-api/src/job/model.rs new file mode 100644 index 000000000..6c038862e --- /dev/null +++ b/thoth-api/src/job/model.rs @@ -0,0 +1,18 @@ +use crate::event::model::EventWrapper; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub struct Job { + pub queue: String, + pub args: Vec, + pub retry: bool, + pub class: String, + pub jid: String, + pub created_at: f64, + pub enqueued_at: f64, + pub failed_at: f64, + pub error_message: String, + pub error_class: Option, + pub retry_count: i32, + pub retried_at: f64, +} diff --git a/thoth-api/src/lib.rs b/thoth-api/src/lib.rs index 8495057b3..030c52f5a 100644 --- a/thoth-api/src/lib.rs +++ b/thoth-api/src/lib.rs @@ -17,7 +17,10 @@ extern crate juniper; pub mod account; #[cfg(feature = "backend")] pub mod db; +pub mod event; pub mod graphql; +#[cfg(feature = "backend")] +pub mod job; #[macro_use] pub mod model; #[cfg(feature = "backend")] diff --git a/thoth-api/src/model/mod.rs b/thoth-api/src/model/mod.rs index b8a087479..7a417c0c1 100644 --- a/thoth-api/src/model/mod.rs +++ b/thoth-api/src/model/mod.rs @@ -1095,5 +1095,6 @@ pub mod publisher; pub mod reference; pub mod series; pub mod subject; +pub mod webhook; pub mod work; pub mod work_relation; diff --git a/thoth-api/src/model/webhook/crud.rs b/thoth-api/src/model/webhook/crud.rs new file mode 100644 index 000000000..d62a28946 --- /dev/null +++ b/thoth-api/src/model/webhook/crud.rs @@ -0,0 +1,179 @@ +use super::{ + NewWebhook, NewWebhookHistory, PatchWebhook, Webhook, WebhookField, WebhookHistory, + WebhookOrderBy, +}; +use crate::event::model::EventType; +use crate::graphql::utils::Direction; +use crate::model::{Crud, DbInsert, HistoryEntry}; +use crate::schema::{webhook, webhook_history}; +use crate::{crud_methods, db_insert}; +use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; +use thoth_errors::ThothResult; +use uuid::Uuid; + +impl Crud for Webhook { + type NewEntity = NewWebhook; + type PatchEntity = PatchWebhook; + type OrderByEntity = WebhookOrderBy; + type FilterParameter1 = EventType; + type FilterParameter2 = (); + type FilterParameter3 = bool; + + fn pk(&self) -> Uuid { + self.webhook_id + } + + fn all( + db: &crate::db::PgPool, + limit: i32, + offset: i32, + _: Option, + order: Self::OrderByEntity, + publishers: Vec, + parent_id_1: Option, + _: Option, + event_types: Vec, + _: Vec, + published: Option, + ) -> ThothResult> { + use crate::schema::webhook::dsl::*; + let mut connection = db.get()?; + let mut query = webhook.into_boxed(); + + query = match order.field { + WebhookField::WebhookId => match order.direction { + Direction::Asc => query.order(webhook_id.asc()), + Direction::Desc => query.order(webhook_id.desc()), + }, + WebhookField::PublisherId => match order.direction { + Direction::Asc => query.order(publisher_id.asc()), + Direction::Desc => query.order(publisher_id.desc()), + }, + WebhookField::Endpoint => match order.direction { + Direction::Asc => query.order(endpoint.asc()), + Direction::Desc => query.order(endpoint.desc()), + }, + WebhookField::Token => match order.direction { + Direction::Asc => query.order(token.asc()), + Direction::Desc => query.order(token.desc()), + }, + WebhookField::IsPublished => match order.direction { + Direction::Asc => query.order(is_published.asc()), + Direction::Desc => query.order(is_published.desc()), + }, + WebhookField::EventType => match order.direction { + Direction::Asc => query.order(event_type.asc()), + Direction::Desc => query.order(event_type.desc()), + }, + WebhookField::Payload => match order.direction { + Direction::Asc => query.order(payload.asc()), + Direction::Desc => query.order(payload.desc()), + }, + WebhookField::CreatedAt => match order.direction { + Direction::Asc => query.order(created_at.asc()), + Direction::Desc => query.order(created_at.desc()), + }, + WebhookField::UpdatedAt => match order.direction { + Direction::Asc => query.order(updated_at.asc()), + Direction::Desc => query.order(updated_at.desc()), + }, + }; + if !publishers.is_empty() { + query = query.filter(publisher_id.eq_any(publishers)); + } + if let Some(pid) = parent_id_1 { + query = query.filter(publisher_id.eq(pid)); + } + if !event_types.is_empty() { + query = query.filter(event_type.eq_any(event_types)); + } + if let Some(publ) = published { + query = query.filter(is_published.eq(publ)); + } + query + .limit(limit.into()) + .offset(offset.into()) + .load::(&mut connection) + .map_err(Into::into) + } + + fn count( + db: &crate::db::PgPool, + _: Option, + publishers: Vec, + event_types: Vec, + _: Vec, + published: Option, + ) -> ThothResult { + use crate::schema::webhook::dsl::*; + let mut connection = db.get()?; + let mut query = webhook.into_boxed(); + if !publishers.is_empty() { + query = query.filter(publisher_id.eq_any(publishers)); + } + if !event_types.is_empty() { + query = query.filter(event_type.eq_any(event_types)); + } + if let Some(publ) = published { + query = query.filter(is_published.eq(publ)); + } + + // `SELECT COUNT(*)` in postgres returns a BIGINT, which diesel parses as i64. Juniper does + // not implement i64 yet, only i32. The only sensible way, albeit shameful, to solve this + // is converting i64 to string and then parsing it as i32. This should work until we reach + // 2147483647 records - if you are fixing this bug, congratulations on book number 2147483647! + query + .count() + .get_result::(&mut connection) + .map(|t| t.to_string().parse::().unwrap()) + .map_err(Into::into) + } + + fn publisher_id(&self, _db: &crate::db::PgPool) -> ThothResult { + Ok(self.publisher_id) + } + + crud_methods!(webhook::table, webhook::dsl::webhook); +} + +impl HistoryEntry for Webhook { + type NewHistoryEntity = NewWebhookHistory; + + fn new_history_entry(&self, account_id: &Uuid) -> Self::NewHistoryEntity { + Self::NewHistoryEntity { + webhook_id: self.webhook_id, + account_id: *account_id, + data: serde_json::Value::String(serde_json::to_string(&self).unwrap()), + } + } +} + +impl DbInsert for NewWebhookHistory { + type MainEntity = WebhookHistory; + + db_insert!(webhook_history::table); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_webhook_pk() { + let webhook: Webhook = Default::default(); + assert_eq!(webhook.pk(), webhook.webhook_id); + } + + #[test] + fn test_new_webhook_history_from_webhook() { + let webhook: Webhook = Default::default(); + let account_id: Uuid = Default::default(); + let new_webhook_history = webhook.new_history_entry(&account_id); + assert_eq!(new_webhook_history.webhook_id, webhook.webhook_id); + assert_eq!(new_webhook_history.account_id, account_id); + assert_eq!( + new_webhook_history.data, + serde_json::Value::String(serde_json::to_string(&webhook).unwrap()) + ); + } +} diff --git a/thoth-api/src/model/webhook/mod.rs b/thoth-api/src/model/webhook/mod.rs new file mode 100644 index 000000000..e0d09fa35 --- /dev/null +++ b/thoth-api/src/model/webhook/mod.rs @@ -0,0 +1,117 @@ +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumString}; +use uuid::Uuid; + +use crate::event::model::EventType; +use crate::graphql::utils::Direction; +use crate::model::Timestamp; +#[cfg(feature = "backend")] +use crate::schema::webhook; +#[cfg(feature = "backend")] +use crate::schema::webhook_history; + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLEnum), + graphql(description = "Field to use when sorting webhooks list") +)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, EnumString, Display)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum WebhookField { + WebhookId, + PublisherId, + #[default] + Endpoint, + Token, + IsPublished, + EventType, + Payload, + CreatedAt, + UpdatedAt, +} + +#[cfg_attr(feature = "backend", derive(Queryable))] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub webhook_id: Uuid, + pub publisher_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, + pub payload: Option, + pub created_at: Timestamp, + pub updated_at: Timestamp, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLInputObject, Insertable), + graphql(description = "Set of values required to define a new webhook"), + diesel(table_name = webhook) +)] +pub struct NewWebhook { + pub publisher_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, + pub payload: Option, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLInputObject, AsChangeset), + graphql(description = "Set of values required to update an existing webhook"), + diesel(table_name = webhook, treat_none_as_null = true) +)] +pub struct PatchWebhook { + pub webhook_id: Uuid, + pub publisher_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, + pub payload: Option, +} + +#[cfg_attr(feature = "backend", derive(Queryable))] +pub struct WebhookHistory { + pub webhook_history_id: Uuid, + pub webhook_id: Uuid, + pub account_id: Uuid, + pub data: serde_json::Value, + pub timestamp: Timestamp, +} + +#[cfg_attr( + feature = "backend", + derive(Insertable), + diesel(table_name = webhook_history) +)] +pub struct NewWebhookHistory { + pub webhook_id: Uuid, + pub account_id: Uuid, + pub data: serde_json::Value, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLInputObject), + graphql(description = "Field and order to use when sorting webhooks list") +)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct WebhookOrderBy { + pub field: WebhookField, + pub direction: Direction, +} + +#[test] +fn test_webhookfield_default() { + let webfield: WebhookField = Default::default(); + assert_eq!(webfield, WebhookField::Endpoint); +} + +#[cfg(feature = "backend")] +pub mod crud; diff --git a/thoth-api/src/redis.rs b/thoth-api/src/redis.rs index 04ea75d81..9f73a59b5 100644 --- a/thoth-api/src/redis.rs +++ b/thoth-api/src/redis.rs @@ -38,6 +38,27 @@ pub async fn scan_match(pool: &RedisPool, pattern: &str) -> ThothResult ThothResult { + let mut con = create_connection(pool).await?; + con.rpush(key, value).await.map_err(Into::into) +} + +pub async fn blpop(pool: &RedisPool, key: &str) -> ThothResult { + let mut con = create_connection(pool).await?; + let (_, value): (_, String) = con.blpop::<_, (String, String)>(key, 0.0).await?; + Ok(value) +} + +pub async fn zrange( + pool: &RedisPool, + key: &str, + start: isize, + stop: isize, +) -> ThothResult> { + let mut con = create_connection(pool).await?; + con.zrange(key, start, stop).await.map_err(Into::into) +} + #[cfg(test)] mod tests { use super::*; @@ -72,6 +93,29 @@ mod tests { assert_eq!(get_result.unwrap(), test_value); } + #[tokio::test] + async fn test_rpush_and_blpop() { + let pool = get_pool().await; + + let test_key = "test_queue"; + let test_value_1 = "test_value_1"; + let test_value_2 = "test_value_2"; + + let rpush_result_1 = rpush(&pool, test_key, test_value_1).await; + assert!(rpush_result_1.is_ok()); + + let rpush_result_2 = rpush(&pool, test_key, test_value_2).await; + assert!(rpush_result_2.is_ok()); + + let blpop_result_1 = blpop(&pool, test_key).await; + assert!(blpop_result_1.is_ok()); + assert_eq!(blpop_result_1.unwrap(), test_value_1); + + let blpop_result_2 = blpop(&pool, test_key).await; + assert!(blpop_result_2.is_ok()); + assert_eq!(blpop_result_2.unwrap(), test_value_2); + } + #[tokio::test] async fn test_get_nonexistent_key() { let pool = get_pool().await; diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index e78c5350f..294422da0 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -46,6 +46,10 @@ pub mod sql_types { #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] #[diesel(postgres_type(name = "relation_type"))] pub struct RelationType; + + #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] + #[diesel(postgres_type(name = "event_type"))] + pub struct EventType; } table! { @@ -514,6 +518,35 @@ table! { } } +table! { + use diesel::sql_types::*; + use super::sql_types::EventType; + + webhook (webhook_id) { + webhook_id -> Uuid, + publisher_id -> Uuid, + endpoint -> Text, + token -> Nullable, + is_published -> Bool, + event_type -> EventType, + payload -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + +table! { + use diesel::sql_types::*; + + webhook_history (webhook_history_id) { + webhook_history_id -> Uuid, + webhook_id -> Uuid, + account_id -> Uuid, + data -> Jsonb, + timestamp -> Timestamptz, + } +} + table! { use diesel::sql_types::*; use super::sql_types::WorkType; @@ -647,6 +680,9 @@ joinable!(series_history -> series (series_id)); joinable!(subject -> work (work_id)); joinable!(subject_history -> account (account_id)); joinable!(subject_history -> subject (subject_id)); +joinable!(webhook -> publisher (publisher_id)); +joinable!(webhook_history -> account (account_id)); +joinable!(webhook_history -> webhook (webhook_id)); joinable!(work -> imprint (imprint_id)); joinable!(work_history -> account (account_id)); joinable!(work_history -> work (work_id)); @@ -687,6 +723,8 @@ allow_tables_to_appear_in_same_query!( series_history, subject, subject_history, + webhook, + webhook_history, work, work_history, work_relation, diff --git a/thoth-errors/src/database_errors.rs b/thoth-errors/src/database_errors.rs index 6f53b9069..4fffe8081 100644 --- a/thoth-errors/src/database_errors.rs +++ b/thoth-errors/src/database_errors.rs @@ -179,7 +179,7 @@ impl From for ThothError { ThothError::DatabaseError(info.message().to_string()) } Error::NotFound => ThothError::EntityNotFound, - _ => ThothError::InternalError("".into()), + _ => ThothError::InternalError(error.to_string()), } } } diff --git a/thoth-processor/Cargo.toml b/thoth-processor/Cargo.toml new file mode 100644 index 000000000..8a336f29e --- /dev/null +++ b/thoth-processor/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "thoth-processor" +version = "0.13.13" +authors = ["Javier Arias ", "Ross Higman ", "Brendan O'Connell "] +edition = "2021" +license = "Apache-2.0" +description = "Event processor for Thoth" +repository = "https://github.com/thoth-pub/thoth" +readme = "README.md" + +[dependencies] +loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1" } +tokio = { version = "1.33.0", default-features = false, features = [ + "rt-multi-thread", +] } +async-trait = { version = "0.1.74" } +axum = { version = "0.8.1" } +tracing = { version = "0.1.40" } +tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] } +regex = { version = "1.11.1" } +reqwest = { version = "0.12", features = ["json"] } +uuid = { version = "1.16.0", features = ["serde"] } +thoth-api = { version = "=0.13.13", path = "../thoth-api" } + +[dev-dependencies] +loco-rs = { version = "0.14.0", default-features = false, features = ["testing"] } +serial_test = { version = "3.1.1" } +rstest = { version = "0.21.0" } +insta = { version = "1.34.0", features = ["redactions", "yaml", "filters"] } diff --git a/thoth-processor/config/development.yaml b/thoth-processor/config/development.yaml new file mode 100644 index 000000000..1c93ff406 --- /dev/null +++ b/thoth-processor/config/development.yaml @@ -0,0 +1,50 @@ +# Loco (thoth-processor) configuration file documentation + +# Application logging configuration +logger: + # Enable or disable logging. + enable: true + # Enable pretty backtrace (sets RUST_BACKTRACE=1) + pretty_backtrace: true + # Log level, options: trace, debug, info, warn or error. + level: debug + # Define the logging format. options: compact, pretty or json + format: compact + # By default the logger has filtering only logs that came from your code or logs that came from `loco` framework. to see all third party libraries + # Uncomment the line below to override to see all third party libraries you can enable this config and override the logger filters. + # override_filter: trace + +# Web server configuration +server: + # Port on which the server will listen. the server binding is 0.0.0.0:{PORT} + port: 8282 + # Binding for the server (which interface to bind to) + binding: localhost + # The UI hostname or IP address that mailers will point to. + host: http://localhost + # Out of the box middleware configuration. to disable middleware you can changed the `enable` field to `false` of comment the middleware block + middlewares: + +# Initializers Configuration +# initializers: +# oauth2: +# authorization_code: # Authorization code grant type +# - client_identifier: google # Identifier for the OAuth2 provider. Replace 'google' with your provider's name if different, must be unique within the oauth2 config. +# ... other fields + +# Worker Configuration +workers: + # specifies the worker mode. Options: + # - BackgroundQueue - Workers operate asynchronously in the background, processing queued. + # - ForegroundBlocking - Workers operate in the foreground and block until tasks are completed. + # - BackgroundAsync - Workers operate asynchronously in the background, processing tasks with async capabilities. + mode: BackgroundQueue + +queue: + kind: Redis + # Redis connection URI + uri: {{ get_env(name="REDIS_URL") }} + dangerously_flush: false + +settings: + thoth_graphql_api: {{ get_env(name="THOTH_GRAPHQL_API") }} diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml new file mode 100644 index 000000000..82a82f992 --- /dev/null +++ b/thoth-processor/config/test.yaml @@ -0,0 +1,50 @@ +# Loco (thoth-processor) configuration file documentation + +# Application logging configuration +logger: + # Enable or disable logging. + enable: true + # Enable pretty backtrace (sets RUST_BACKTRACE=1) + pretty_backtrace: true + # Log level, options: trace, debug, info, warn or error. + level: debug + # Define the logging format. options: compact, pretty or json + format: compact + # By default the logger has filtering only logs that came from your code or logs that came from `loco` framework. to see all third party libraries + # Uncomment the line below to override to see all third party libraries you can enable this config and override the logger filters. + # override_filter: trace + +# Web server configuration +server: + # Port on which the server will listen. the server binding is 0.0.0.0:{PORT} + port: 8282 + # Binding for the server (which interface to bind to) + binding: localhost + # The UI hostname or IP address that mailers will point to. + host: http://localhost + # Out of the box middleware configuration. to disable middleware you can changed the `enable` field to `false` of comment the middleware block + middlewares: + +# Initializers Configuration +# initializers: +# oauth2: +# authorization_code: # Authorization code grant type +# - client_identifier: google # Identifier for the OAuth2 provider. Replace 'google' with your provider's name if different, must be unique within the oauth2 config. +# ... other fields + +# Worker Configuration +workers: + # specifies the worker mode. Options: + # - BackgroundQueue - Workers operate asynchronously in the background, processing queued. + # - ForegroundBlocking - Workers operate in the foreground and block until tasks are completed. + # - BackgroundAsync - Workers operate asynchronously in the background, processing tasks with async capabilities. + mode: BackgroundQueue + +queue: + kind: Redis + # Redis connection URI + uri: {{ get_env(name="TEST_REDIS_URL") }} + dangerously_flush: false + +settings: + thoth_graphql_api: {{ get_env(name="THOTH_GRAPHQL_API") }} diff --git a/thoth-processor/src/app.rs b/thoth-processor/src/app.rs new file mode 100644 index 000000000..08c90db00 --- /dev/null +++ b/thoth-processor/src/app.rs @@ -0,0 +1,64 @@ +use crate::{ + initializers::handle_events::HandleEvents, + workers::{ + fire_webhook_worker::FireWebhookWorker, work_created_worker::WorkCreatedWorker, + work_published_worker::WorkPublishedWorker, work_updated_worker::WorkUpdatedWorker, + }, +}; +use async_trait::async_trait; +use loco_rs::{ + app::{AppContext, Hooks, Initializer}, + bgworker::{BackgroundWorker, Queue}, + boot::{create_app, BootResult, StartMode}, + config::Config, + controller::AppRoutes, + environment::Environment, + task::Tasks, + Result, +}; + +pub struct App; +#[async_trait] +impl Hooks for App { + fn app_name() -> &'static str { + env!("CARGO_CRATE_NAME") + } + + fn app_version() -> String { + format!( + "{} ({})", + env!("CARGO_PKG_VERSION"), + option_env!("BUILD_SHA") + .or(option_env!("GITHUB_SHA")) + .unwrap_or("dev") + ) + } + + async fn boot( + mode: StartMode, + environment: &Environment, + config: Config, + ) -> Result { + create_app::(mode, environment, config).await + } + + async fn initializers(_ctx: &AppContext) -> Result>> { + Ok(vec![Box::new(HandleEvents)]) + } + + fn routes(_ctx: &AppContext) -> AppRoutes { + AppRoutes::with_default_routes() + } + async fn connect_workers(ctx: &AppContext, queue: &Queue) -> Result<()> { + queue.register(FireWebhookWorker::build(ctx)).await?; + queue.register(WorkCreatedWorker::build(ctx)).await?; + queue.register(WorkUpdatedWorker::build(ctx)).await?; + queue.register(WorkPublishedWorker::build(ctx)).await?; + Ok(()) + } + + #[allow(unused_variables)] + fn register_tasks(tasks: &mut Tasks) { + // tasks-inject (do not remove) + } +} diff --git a/thoth-processor/src/common/mod.rs b/thoth-processor/src/common/mod.rs new file mode 100644 index 000000000..6e98cefd0 --- /dev/null +++ b/thoth-processor/src/common/mod.rs @@ -0,0 +1 @@ +pub mod settings; diff --git a/thoth-processor/src/common/settings.rs b/thoth-processor/src/common/settings.rs new file mode 100644 index 000000000..e0939d1f0 --- /dev/null +++ b/thoth-processor/src/common/settings.rs @@ -0,0 +1,13 @@ +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Default, Debug)] +pub struct Settings { + pub thoth_graphql_api: String, +} + +impl Settings { + pub fn from_json(value: &serde_json::Value) -> Result { + Ok(serde_json::from_value(value.clone())?) + } +} diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs new file mode 100644 index 000000000..5c5ee3140 --- /dev/null +++ b/thoth-processor/src/initializers/handle_events.rs @@ -0,0 +1,73 @@ +use crate::workers::{ + work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}, + work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}, + work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}, +}; +use async_trait::async_trait; +use loco_rs::{config::QueueConfig, prelude::*}; +use thoth_api::{ + event::{ + handler::QUEUE_KEY, + model::{Event, EventType}, + }, + redis::{blpop, init_pool}, +}; + +pub struct HandleEvents; + +#[async_trait] +impl Initializer for HandleEvents { + fn name(&self) -> String { + "handle-events".to_string() + } + + async fn before_run(&self, ctx: &AppContext) -> Result<()> { + let ctx = ctx.clone(); + let redis_url = match ctx.config.queue.as_ref().unwrap() { + QueueConfig::Redis(queue) => &queue.uri, + _ => unreachable!(), + }; + let redis = init_pool(redis_url); + + tokio::spawn(async move { + loop { + if let Ok(payload) = blpop(&redis, QUEUE_KEY).await { + tracing::info!("Initializer received payload: {:?}", payload); + match serde_json::from_str::(&payload) { + Ok(event) => { + tracing::info!("Received event: {:?}", event); + let _ = match event.event_type { + EventType::WorkCreated => { + WorkCreatedWorker::perform_later( + &ctx, + WorkCreatedWorkerArgs { event }, + ) + .await + } + EventType::WorkUpdated => { + WorkUpdatedWorker::perform_later( + &ctx, + WorkUpdatedWorkerArgs { event }, + ) + .await + } + EventType::WorkPublished => { + WorkPublishedWorker::perform_later( + &ctx, + WorkPublishedWorkerArgs { event }, + ) + .await + } + }; + } + Err(e) => { + tracing::error!("Invalid event payload: {}", e); + } + } + } + } + }); + + Ok(()) + } +} diff --git a/thoth-processor/src/initializers/mod.rs b/thoth-processor/src/initializers/mod.rs new file mode 100644 index 000000000..4ab300b81 --- /dev/null +++ b/thoth-processor/src/initializers/mod.rs @@ -0,0 +1 @@ +pub mod handle_events; diff --git a/thoth-processor/src/lib.rs b/thoth-processor/src/lib.rs new file mode 100644 index 000000000..60a20ded2 --- /dev/null +++ b/thoth-processor/src/lib.rs @@ -0,0 +1,36 @@ +pub mod app; +pub mod common; +pub mod initializers; +pub mod requests; +pub mod workers; + +use crate::app::App; +use loco_rs::{ + app::Hooks, + boot::{start, ServeParams, StartMode}, + environment::{resolve_from_env, Environment}, + logger, Result, +}; +use std::path::Path; + +#[tokio::main] +#[allow(clippy::result_large_err)] +pub async fn start_server() -> Result<()> { + start_app::().await +} + +pub async fn start_app() -> Result<()> { + let environment: Environment = resolve_from_env().into(); + let config = environment.load_from_folder(Path::new("thoth-processor/config/"))?; + + if !H::init_logger(&config, &environment)? { + logger::init::(&config.logger)?; + } + + let boot_result = H::boot(StartMode::ServerAndWorker, &environment, config).await?; + let serve_params = ServeParams { + port: boot_result.app_context.config.server.port, + binding: boot_result.app_context.config.server.binding.to_string(), + }; + start::(boot_result, serve_params, false).await +} diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs new file mode 100644 index 000000000..141f55abc --- /dev/null +++ b/thoth-processor/src/requests/graphql.rs @@ -0,0 +1,109 @@ +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; +use thoth_api::event::model::{Event, EventType}; +use uuid::Uuid; + +const WEBHOOKS_QUERY: &str = " + query WebhooksQuery( + $workId: Uuid!, + $eventTypes: [EventType!], + $isPublished: Boolean!, + ) { + work(workId: $workId) { + imprint { + publisher { + webhooks(eventTypes: $eventTypes, isPublished: $isPublished) { + endpoint + token + isPublished + eventType + payload + } + } + } + } + } +"; + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksVariables { + work_id: Uuid, + event_types: Vec, + is_published: bool, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksQueryBody { + pub query: String, + pub variables: WebhooksVariables, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, + pub payload: Option, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponsePublisher { + pub webhooks: Vec, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseImprint { + pub publisher: WebhooksResponsePublisher, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseWork { + pub imprint: WebhooksResponseImprint, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseData { + pub work: WebhooksResponseWork, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseBody { + pub data: WebhooksResponseData, +} + +pub async fn query_webhooks(url: String, event: Event) -> Result, Error> { + let client = reqwest::Client::new(); + let variables = WebhooksVariables { + work_id: event.work_id, + event_types: vec![event.event_type], + is_published: event.is_published, + }; + let body = WebhooksQueryBody { + query: WEBHOOKS_QUERY.to_string(), + variables, + }; + let token = "placeholder".to_string(); + + let response = client + .post(&url) + .json(&body) + .bearer_auth(token) + .send() + .await? + .error_for_status()?; + + tracing::info!("response: {:?}", response); + let response_text = response.json::().await?; + tracing::info!("response_text: {:?}", response_text); + + Ok(response_text.data.work.imprint.publisher.webhooks) +} diff --git a/thoth-processor/src/requests/mod.rs b/thoth-processor/src/requests/mod.rs new file mode 100644 index 000000000..eba80b67a --- /dev/null +++ b/thoth-processor/src/requests/mod.rs @@ -0,0 +1,2 @@ +pub mod graphql; +pub mod target; diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs new file mode 100644 index 000000000..ff56f0f5b --- /dev/null +++ b/thoth-processor/src/requests/target.rs @@ -0,0 +1,38 @@ +use loco_rs::prelude::*; +use uuid::Uuid; + +pub async fn fire_webhook( + url: String, + token: Option, + work_id: Uuid, + payload: Option, +) -> Result { + let client = reqwest::Client::new(); + + let mut request = client + .post(&url) + // GitHub Actions repository dispatch events require a User-Agent header + .header("User-Agent", "Thoth"); + + if let Some(token_value) = token { + request = request.bearer_auth(token_value); + } + + // References for constructing payloads: + // GitHub Actions: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#repository_dispatch + // Mattermost: https://developers.mattermost.com/integrate/webhooks/incoming/ + if let Some(payload_value) = payload { + let interpolated_payload = payload_value.replace("${work_id}", &work_id.to_string()); + request = request.body(interpolated_payload); + } + + let response = request + .send() + .await? + .error_for_status()?; + + tracing::info!("response: {:?}", response); + tracing::info!("response_status: {:?}", response.status()); + + Ok(response.status().to_string()) +} diff --git a/thoth-processor/src/workers/fire_webhook_worker.rs b/thoth-processor/src/workers/fire_webhook_worker.rs new file mode 100644 index 000000000..050b0723f --- /dev/null +++ b/thoth-processor/src/workers/fire_webhook_worker.rs @@ -0,0 +1,36 @@ +use crate::requests::graphql::Webhook; +use crate::requests::target::fire_webhook; +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +pub struct FireWebhookWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct FireWebhookWorkerArgs { + pub work_id: Uuid, + pub webhook: Webhook, +} + +#[async_trait] +impl BackgroundWorker for FireWebhookWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: FireWebhookWorkerArgs) -> Result<()> { + tracing::info!("Webhook: {:?}", args.webhook); + let target_rsp = fire_webhook( + args.webhook.endpoint, + args.webhook.token, + args.work_id, + args.webhook.payload, + ) + .await?; + tracing::info!("Target response: {:?}", target_rsp); + + Ok(()) + } +} diff --git a/thoth-processor/src/workers/mod.rs b/thoth-processor/src/workers/mod.rs new file mode 100644 index 000000000..0ff891e6d --- /dev/null +++ b/thoth-processor/src/workers/mod.rs @@ -0,0 +1,4 @@ +pub mod fire_webhook_worker; +pub mod work_created_worker; +pub mod work_published_worker; +pub mod work_updated_worker; diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs new file mode 100644 index 000000000..68611a5de --- /dev/null +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -0,0 +1,45 @@ +use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; +use crate::{common::settings::Settings, requests::graphql::query_webhooks}; +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; +use thoth_api::event::model::Event; + +pub struct WorkCreatedWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct WorkCreatedWorkerArgs { + pub event: Event, +} + +#[async_trait] +impl BackgroundWorker for WorkCreatedWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: WorkCreatedWorkerArgs) -> Result<()> { + tracing::info!("Event: {:?}", args.event); + let work_id = args.event.work_id; + let webhooks = query_webhooks( + format!( + "{}/graphql", + Settings::from_json(&self.ctx.config.settings.as_ref().unwrap())?.thoth_graphql_api + ), + args.event, + ) + .await?; + tracing::info!("Webhooks: {:?}", webhooks); + + for webhook in webhooks { + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { work_id, webhook }, + ) + .await; + } + + Ok(()) + } +} diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs new file mode 100644 index 000000000..da80c44a1 --- /dev/null +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -0,0 +1,45 @@ +use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; +use crate::{common::settings::Settings, requests::graphql::query_webhooks}; +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; +use thoth_api::event::model::Event; + +pub struct WorkPublishedWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct WorkPublishedWorkerArgs { + pub event: Event, +} + +#[async_trait] +impl BackgroundWorker for WorkPublishedWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: WorkPublishedWorkerArgs) -> Result<()> { + tracing::info!("Event: {:?}", args.event); + let work_id = args.event.work_id; + let webhooks = query_webhooks( + format!( + "{}/graphql", + Settings::from_json(&self.ctx.config.settings.as_ref().unwrap())?.thoth_graphql_api + ), + args.event, + ) + .await?; + tracing::info!("Webhooks: {:?}", webhooks); + + for webhook in webhooks { + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { work_id, webhook }, + ) + .await; + } + + Ok(()) + } +} diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs new file mode 100644 index 000000000..1e806553e --- /dev/null +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -0,0 +1,45 @@ +use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; +use crate::{common::settings::Settings, requests::graphql::query_webhooks}; +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; +use thoth_api::event::model::Event; + +pub struct WorkUpdatedWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct WorkUpdatedWorkerArgs { + pub event: Event, +} + +#[async_trait] +impl BackgroundWorker for WorkUpdatedWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: WorkUpdatedWorkerArgs) -> Result<()> { + tracing::info!("Event: {:?}", args.event); + let work_id = args.event.work_id; + let webhooks = query_webhooks( + format!( + "{}/graphql", + Settings::from_json(&self.ctx.config.settings.as_ref().unwrap())?.thoth_graphql_api + ), + args.event, + ) + .await?; + tracing::info!("Webhooks: {:?}", webhooks); + + for webhook in webhooks { + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { work_id, webhook }, + ) + .await; + } + + Ok(()) + } +} diff --git a/thoth-processor/tests/mod.rs b/thoth-processor/tests/mod.rs new file mode 100644 index 000000000..4d55549bf --- /dev/null +++ b/thoth-processor/tests/mod.rs @@ -0,0 +1 @@ +mod workers; diff --git a/thoth-processor/tests/workers/fire_webhook_worker.rs b/thoth-processor/tests/workers/fire_webhook_worker.rs new file mode 100644 index 000000000..848e8e3bf --- /dev/null +++ b/thoth-processor/tests/workers/fire_webhook_worker.rs @@ -0,0 +1,31 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; +use thoth_processor::{ + app::App, + requests::graphql::Webhook, + workers::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}, +}; + +#[tokio::test] +#[serial] +async fn test_run_work_created_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!(FireWebhookWorker::perform_later( + &boot.app_context, + FireWebhookWorkerArgs { + work_id: Default::default(), + webhook: Webhook { + endpoint: Default::default(), + token: Default::default(), + is_published: Default::default(), + event_type: Default::default(), + payload: Default::default(), + } + } + ) + .await + .is_ok()); + // Include additional assert validations after the execution of the worker +} diff --git a/thoth-processor/tests/workers/mod.rs b/thoth-processor/tests/workers/mod.rs new file mode 100644 index 000000000..fd6a481d6 --- /dev/null +++ b/thoth-processor/tests/workers/mod.rs @@ -0,0 +1,4 @@ +mod fire_webhook_worker; +mod work_created_worker; +mod work_published_worker; +mod work_updated_worker; diff --git a/thoth-processor/tests/workers/work_created_worker.rs b/thoth-processor/tests/workers/work_created_worker.rs new file mode 100644 index 000000000..ce7e6b8c4 --- /dev/null +++ b/thoth-processor/tests/workers/work_created_worker.rs @@ -0,0 +1,30 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; +use thoth_api::event::model::{Event, EventType}; +use thoth_processor::{ + app::App, + workers::work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}, +}; + +#[tokio::test] +#[serial] +async fn test_run_work_created_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!(WorkCreatedWorker::perform_later( + &boot.app_context, + WorkCreatedWorkerArgs { + event: Event { + event_type: EventType::WorkCreated, + work_id: Default::default(), + is_published: Default::default(), + event_timestamp: Default::default(), + thoth_version: Default::default(), + } + } + ) + .await + .is_ok()); + // Include additional assert validations after the execution of the worker +} diff --git a/thoth-processor/tests/workers/work_published_worker.rs b/thoth-processor/tests/workers/work_published_worker.rs new file mode 100644 index 000000000..fccd193e9 --- /dev/null +++ b/thoth-processor/tests/workers/work_published_worker.rs @@ -0,0 +1,30 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; +use thoth_api::event::model::{Event, EventType}; +use thoth_processor::{ + app::App, + workers::work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}, +}; + +#[tokio::test] +#[serial] +async fn test_run_work_published_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!(WorkPublishedWorker::perform_later( + &boot.app_context, + WorkPublishedWorkerArgs { + event: Event { + event_type: EventType::WorkPublished, + work_id: Default::default(), + is_published: Default::default(), + event_timestamp: Default::default(), + thoth_version: Default::default(), + } + } + ) + .await + .is_ok()); + // Include additional assert validations after the execution of the worker +} diff --git a/thoth-processor/tests/workers/work_updated_worker.rs b/thoth-processor/tests/workers/work_updated_worker.rs new file mode 100644 index 000000000..42007c3da --- /dev/null +++ b/thoth-processor/tests/workers/work_updated_worker.rs @@ -0,0 +1,30 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; +use thoth_api::event::model::{Event, EventType}; +use thoth_processor::{ + app::App, + workers::work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}, +}; + +#[tokio::test] +#[serial] +async fn test_run_work_updated_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!(WorkUpdatedWorker::perform_later( + &boot.app_context, + WorkUpdatedWorkerArgs { + event: Event { + event_type: EventType::WorkUpdated, + work_id: Default::default(), + is_published: Default::default(), + event_timestamp: Default::default(), + thoth_version: Default::default(), + } + } + ) + .await + .is_ok()); + // Include additional assert validations after the execution of the worker +}