From 99ca79e4d96fb4b84916e51d9bd0b260a9ad99b8 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:04:16 +0000 Subject: [PATCH 01/69] GraphQL Subscription backed by new Event DB table (incomplete) --- thoth-api/migrations/v0.13.2/down.sql | 2 ++ thoth-api/migrations/v0.13.2/up.sql | 13 ++++++++ thoth-api/src/graphql/model.rs | 21 +++++++++--- thoth-api/src/model/event/crud.rs | 48 +++++++++++++++++++++++++++ thoth-api/src/model/event/mod.rs | 41 +++++++++++++++++++++++ thoth-api/src/model/mod.rs | 1 + thoth-api/src/schema.rs | 19 +++++++++++ 7 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 thoth-api/migrations/v0.13.2/down.sql create mode 100644 thoth-api/migrations/v0.13.2/up.sql create mode 100644 thoth-api/src/model/event/crud.rs create mode 100644 thoth-api/src/model/event/mod.rs diff --git a/thoth-api/migrations/v0.13.2/down.sql b/thoth-api/migrations/v0.13.2/down.sql new file mode 100644 index 000000000..8d45b39bf --- /dev/null +++ b/thoth-api/migrations/v0.13.2/down.sql @@ -0,0 +1,2 @@ +DROP TYPE IF EXISTS event_type; +DROP TABLE IF EXISTS event; \ No newline at end of file diff --git a/thoth-api/migrations/v0.13.2/up.sql b/thoth-api/migrations/v0.13.2/up.sql new file mode 100644 index 000000000..c4b3ab215 --- /dev/null +++ b/thoth-api/migrations/v0.13.2/up.sql @@ -0,0 +1,13 @@ +CREATE TYPE event_type AS ENUM ( + 'work-created', + 'work-updated', + 'work-published', +); + +CREATE TABLE event ( + event_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + event_type event_type NOT NULL, + work_id UUID NOT NULL REFERENCES work(work_id) ON DELETE CASCADE, + is_published BOOLEAN NOT NULL DEFAULT False, + event_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, +) \ No newline at end of file diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 339ff4e1c..b31a90f9a 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1,6 +1,7 @@ use chrono::naive::NaiveDate; -use juniper::RootNode; -use juniper::{EmptySubscription, FieldResult}; +use futures::Stream; +use juniper::{FieldResult, RootNode}; +use std::pin::Pin; use std::sync::Arc; use uuid::Uuid; @@ -10,6 +11,7 @@ use crate::db::PgPool; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; +use crate::model::event::*; use crate::model::funding::*; use crate::model::imprint::*; use crate::model::institution::*; @@ -2411,6 +2413,17 @@ impl MutationRoot { } } +pub struct SubscriptionRoot; + +type TestStream = Pin> + Send>>; + +#[juniper::graphql_subscription(Context = Context)] +impl SubscriptionRoot { + async fn test_stream(context: &Context) -> TestStream { + Event::all(&context.db) + } +} + #[juniper::graphql_object(Context = Context, description = "A written text that can be published")] impl Work { #[graphql(description = "Thoth ID of the work")] @@ -4182,10 +4195,10 @@ impl Reference { } } -pub type Schema = RootNode<'static, QueryRoot, MutationRoot, EmptySubscription>; +pub type Schema = RootNode<'static, QueryRoot, MutationRoot, SubscriptionRoot>; pub fn create_schema() -> Schema { - Schema::new(QueryRoot {}, MutationRoot {}, EmptySubscription::new()) + Schema::new(QueryRoot {}, MutationRoot {}, SubscriptionRoot {}) } fn publisher_id_from_imprint_id(db: &crate::db::PgPool, imprint_id: Uuid) -> ThothResult { diff --git a/thoth-api/src/model/event/crud.rs b/thoth-api/src/model/event/crud.rs new file mode 100644 index 000000000..b7c0c6ea3 --- /dev/null +++ b/thoth-api/src/model/event/crud.rs @@ -0,0 +1,48 @@ +use super::{ + Event, EventType, NewEvent, +}; +use crate::model::DbInsert; +use crate::schema::event; +use crate::db_insert; +use thoth_errors::{ThothError, ThothResult}; + +impl Event { + fn pk(&self) -> Uuid { + self.event_id + } + + fn create(db: &crate::db::PgPool, data: &NewEvent) -> ThothResult { + let mut connection = db.get()?; + diesel::insert_into(event::table) + .values(data) + .get_result::(&mut connection) + .map_err(Into::into) + } + + fn all(db: &crate::db::PgPool) -> ThothResult> { + let mut connection = db.get()?; + dsl::event + .select(crate::schema::event::all_columns) + .order(dsl::event_timestamp.desc()) + .then_order_by(dsl::event_id) + .load::(&mut connection) + .map_err(Into::into) + } +} + +impl DbInsert for NewEvent { + type MainEntity = Event; + + db_insert!(event::table); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_event_pk() { + let event: Event = Default::default(); + assert_eq!(event.pk(), event.event_id); + } +} \ No newline at end of file diff --git a/thoth-api/src/model/event/mod.rs b/thoth-api/src/model/event/mod.rs new file mode 100644 index 000000000..99768d04c --- /dev/null +++ b/thoth-api/src/model/event/mod.rs @@ -0,0 +1,41 @@ +use uuid::Uuid; + +use crate::model::Timestamp; +#[cfg(feature = "backend")] +use crate::schema::event; + +#[cfg_attr( + feature = "backend", + derive(DbEnum, juniper::GraphQLEnum), + ExistingTypePath = "crate::schema::sql_types::EventType" +)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventType { + #[cfg_attr(feature = "backend", db_rename = "work-created")] + WorkCreated, + #[cfg_attr(feature = "backend", db_rename = "work-updated")] + WorkUpdated, + #[cfg_attr(feature = "backend", db_rename = "work-published")] + WorkPublished, +} + +#[cfg_attr(feature = "backend", derive(Queryable))] +pub struct Event { + pub event_id: Uuid, + pub event_type: EventType, + pub work_id: Uuid, + pub is_published: bool, + pub event_timestamp: Timestamp, +} + +#[cfg_attr( + feature = "backend", + derive(Insertable), + diesel(table_name = event) +)] +pub struct NewEvent { + pub event_type: EventType, + pub work_id: Uuid, + pub is_published: bool, + pub event_timestamp: Timestamp, +} diff --git a/thoth-api/src/model/mod.rs b/thoth-api/src/model/mod.rs index b8a087479..550ff2b90 100644 --- a/thoth-api/src/model/mod.rs +++ b/thoth-api/src/model/mod.rs @@ -1083,6 +1083,7 @@ mod tests { pub mod affiliation; pub mod contribution; pub mod contributor; +pub mod event; pub mod funding; pub mod imprint; pub mod institution; diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index e78c5350f..0699b0e8a 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -46,6 +46,10 @@ pub mod sql_types { #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] #[diesel(postgres_type(name = "relation_type"))] pub struct RelationType; + + #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] + #[diesel(postgres_type(name = "event_type"))] + pub struct EventType; } table! { @@ -152,6 +156,19 @@ table! { } } +table! { + use diesel::sql_types::*; + use super::sql_types::EventType; + + event (event_id) { + event_id -> Uuid, + event_type -> EventType, + work_id -> Uuid, + is_published -> Bool, + event_timestamp -> Timestamptz, + } +} + table! { use diesel::sql_types::*; @@ -609,6 +626,7 @@ joinable!(contribution_history -> account (account_id)); joinable!(contribution_history -> contribution (contribution_id)); joinable!(contributor_history -> account (account_id)); joinable!(contributor_history -> contributor (contributor_id)); +joinable!(event -> work (work_id)); joinable!(funding -> institution (institution_id)); joinable!(funding -> work (work_id)); joinable!(funding_history -> account (account_id)); @@ -662,6 +680,7 @@ allow_tables_to_appear_in_same_query!( contribution_history, contributor, contributor_history, + event, funding, funding_history, imprint, From 36f8e99b82df93bbfdf8a2cd7dfc712d2ee493c0 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:04:24 +0000 Subject: [PATCH 02/69] Revert "GraphQL Subscription backed by new Event DB table (incomplete)" This reverts commit 99ca79e4d96fb4b84916e51d9bd0b260a9ad99b8. DB table not required: send events to Redis directly --- thoth-api/migrations/v0.13.2/down.sql | 2 -- thoth-api/migrations/v0.13.2/up.sql | 13 -------- thoth-api/src/graphql/model.rs | 21 +++--------- thoth-api/src/model/event/crud.rs | 48 --------------------------- thoth-api/src/model/event/mod.rs | 41 ----------------------- thoth-api/src/model/mod.rs | 1 - thoth-api/src/schema.rs | 19 ----------- 7 files changed, 4 insertions(+), 141 deletions(-) delete mode 100644 thoth-api/migrations/v0.13.2/down.sql delete mode 100644 thoth-api/migrations/v0.13.2/up.sql delete mode 100644 thoth-api/src/model/event/crud.rs delete mode 100644 thoth-api/src/model/event/mod.rs diff --git a/thoth-api/migrations/v0.13.2/down.sql b/thoth-api/migrations/v0.13.2/down.sql deleted file mode 100644 index 8d45b39bf..000000000 --- a/thoth-api/migrations/v0.13.2/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TYPE IF EXISTS event_type; -DROP TABLE IF EXISTS event; \ No newline at end of file diff --git a/thoth-api/migrations/v0.13.2/up.sql b/thoth-api/migrations/v0.13.2/up.sql deleted file mode 100644 index c4b3ab215..000000000 --- a/thoth-api/migrations/v0.13.2/up.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TYPE event_type AS ENUM ( - 'work-created', - 'work-updated', - 'work-published', -); - -CREATE TABLE event ( - event_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - event_type event_type NOT NULL, - work_id UUID NOT NULL REFERENCES work(work_id) ON DELETE CASCADE, - is_published BOOLEAN NOT NULL DEFAULT False, - event_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -) \ No newline at end of file diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index b31a90f9a..339ff4e1c 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1,7 +1,6 @@ use chrono::naive::NaiveDate; -use futures::Stream; -use juniper::{FieldResult, RootNode}; -use std::pin::Pin; +use juniper::RootNode; +use juniper::{EmptySubscription, FieldResult}; use std::sync::Arc; use uuid::Uuid; @@ -11,7 +10,6 @@ use crate::db::PgPool; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; -use crate::model::event::*; use crate::model::funding::*; use crate::model::imprint::*; use crate::model::institution::*; @@ -2413,17 +2411,6 @@ impl MutationRoot { } } -pub struct SubscriptionRoot; - -type TestStream = Pin> + Send>>; - -#[juniper::graphql_subscription(Context = Context)] -impl SubscriptionRoot { - async fn test_stream(context: &Context) -> TestStream { - Event::all(&context.db) - } -} - #[juniper::graphql_object(Context = Context, description = "A written text that can be published")] impl Work { #[graphql(description = "Thoth ID of the work")] @@ -4195,10 +4182,10 @@ impl Reference { } } -pub type Schema = RootNode<'static, QueryRoot, MutationRoot, SubscriptionRoot>; +pub type Schema = RootNode<'static, QueryRoot, MutationRoot, EmptySubscription>; pub fn create_schema() -> Schema { - Schema::new(QueryRoot {}, MutationRoot {}, SubscriptionRoot {}) + Schema::new(QueryRoot {}, MutationRoot {}, EmptySubscription::new()) } fn publisher_id_from_imprint_id(db: &crate::db::PgPool, imprint_id: Uuid) -> ThothResult { diff --git a/thoth-api/src/model/event/crud.rs b/thoth-api/src/model/event/crud.rs deleted file mode 100644 index b7c0c6ea3..000000000 --- a/thoth-api/src/model/event/crud.rs +++ /dev/null @@ -1,48 +0,0 @@ -use super::{ - Event, EventType, NewEvent, -}; -use crate::model::DbInsert; -use crate::schema::event; -use crate::db_insert; -use thoth_errors::{ThothError, ThothResult}; - -impl Event { - fn pk(&self) -> Uuid { - self.event_id - } - - fn create(db: &crate::db::PgPool, data: &NewEvent) -> ThothResult { - let mut connection = db.get()?; - diesel::insert_into(event::table) - .values(data) - .get_result::(&mut connection) - .map_err(Into::into) - } - - fn all(db: &crate::db::PgPool) -> ThothResult> { - let mut connection = db.get()?; - dsl::event - .select(crate::schema::event::all_columns) - .order(dsl::event_timestamp.desc()) - .then_order_by(dsl::event_id) - .load::(&mut connection) - .map_err(Into::into) - } -} - -impl DbInsert for NewEvent { - type MainEntity = Event; - - db_insert!(event::table); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_event_pk() { - let event: Event = Default::default(); - assert_eq!(event.pk(), event.event_id); - } -} \ No newline at end of file diff --git a/thoth-api/src/model/event/mod.rs b/thoth-api/src/model/event/mod.rs deleted file mode 100644 index 99768d04c..000000000 --- a/thoth-api/src/model/event/mod.rs +++ /dev/null @@ -1,41 +0,0 @@ -use uuid::Uuid; - -use crate::model::Timestamp; -#[cfg(feature = "backend")] -use crate::schema::event; - -#[cfg_attr( - feature = "backend", - derive(DbEnum, juniper::GraphQLEnum), - ExistingTypePath = "crate::schema::sql_types::EventType" -)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EventType { - #[cfg_attr(feature = "backend", db_rename = "work-created")] - WorkCreated, - #[cfg_attr(feature = "backend", db_rename = "work-updated")] - WorkUpdated, - #[cfg_attr(feature = "backend", db_rename = "work-published")] - WorkPublished, -} - -#[cfg_attr(feature = "backend", derive(Queryable))] -pub struct Event { - pub event_id: Uuid, - pub event_type: EventType, - pub work_id: Uuid, - pub is_published: bool, - pub event_timestamp: Timestamp, -} - -#[cfg_attr( - feature = "backend", - derive(Insertable), - diesel(table_name = event) -)] -pub struct NewEvent { - pub event_type: EventType, - pub work_id: Uuid, - pub is_published: bool, - pub event_timestamp: Timestamp, -} diff --git a/thoth-api/src/model/mod.rs b/thoth-api/src/model/mod.rs index 550ff2b90..b8a087479 100644 --- a/thoth-api/src/model/mod.rs +++ b/thoth-api/src/model/mod.rs @@ -1083,7 +1083,6 @@ mod tests { pub mod affiliation; pub mod contribution; pub mod contributor; -pub mod event; pub mod funding; pub mod imprint; pub mod institution; diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index 0699b0e8a..e78c5350f 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -46,10 +46,6 @@ pub mod sql_types { #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] #[diesel(postgres_type(name = "relation_type"))] pub struct RelationType; - - #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] - #[diesel(postgres_type(name = "event_type"))] - pub struct EventType; } table! { @@ -156,19 +152,6 @@ table! { } } -table! { - use diesel::sql_types::*; - use super::sql_types::EventType; - - event (event_id) { - event_id -> Uuid, - event_type -> EventType, - work_id -> Uuid, - is_published -> Bool, - event_timestamp -> Timestamptz, - } -} - table! { use diesel::sql_types::*; @@ -626,7 +609,6 @@ joinable!(contribution_history -> account (account_id)); joinable!(contribution_history -> contribution (contribution_id)); joinable!(contributor_history -> account (account_id)); joinable!(contributor_history -> contributor (contributor_id)); -joinable!(event -> work (work_id)); joinable!(funding -> institution (institution_id)); joinable!(funding -> work (work_id)); joinable!(funding_history -> account (account_id)); @@ -680,7 +662,6 @@ allow_tables_to_appear_in_same_query!( contribution_history, contributor, contributor_history, - event, funding, funding_history, imprint, From 39b246e304d01967ef3540943726594ce9f6c3f1 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 15 Jan 2025 11:32:20 +0000 Subject: [PATCH 03/69] Attach Redis pool to GraphQL API and send a value when create_work mutation is called (working) --- src/bin/thoth.rs | 6 ++++++ thoth-api-server/src/lib.rs | 6 +++++- thoth-api/src/graphql/model.rs | 16 +++++++++++++--- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/bin/thoth.rs b/src/bin/thoth.rs index c89731b8a..f987fb42d 100644 --- a/src/bin/thoth.rs +++ b/src/bin/thoth.rs @@ -169,6 +169,7 @@ fn thoth_commands() -> Command { Command::new("graphql-api") .about("Start the thoth GraphQL API server") .arg(database_argument()) + .arg(redis_argument()) .arg(host_argument("GRAPHQL_API_HOST")) .arg(port_argument("8000", "GRAPHQL_API_PORT")) .arg(threads_argument("GRAPHQL_API_THREADS")) @@ -202,6 +203,7 @@ fn thoth_commands() -> Command { Command::new("init") .about("Run the database migrations and start the thoth API server") .arg(database_argument()) + .arg(redis_argument()) .arg(host_argument("GRAPHQL_API_HOST")) .arg(port_argument("8000", "GRAPHQL_API_PORT")) .arg(threads_argument("GRAPHQL_API_THREADS")) @@ -230,6 +232,7 @@ fn main() -> ThothResult<()> { Some(("start", start_matches)) => match start_matches.subcommand() { Some(("graphql-api", api_matches)) => { let database_url = api_matches.get_one::("db").unwrap().to_owned(); + let redis_url = api_matches.get_one::("redis").unwrap().to_owned(); let host = api_matches.get_one::("host").unwrap().to_owned(); let port = api_matches.get_one::("port").unwrap().to_owned(); let threads = *api_matches.get_one::("threads").unwrap(); @@ -240,6 +243,7 @@ fn main() -> ThothResult<()> { let session_duration = *api_matches.get_one::("duration").unwrap(); api_server( database_url, + redis_url, host, port, threads, @@ -297,6 +301,7 @@ fn main() -> ThothResult<()> { } Some(("init", init_matches)) => { let database_url = init_matches.get_one::("db").unwrap().to_owned(); + let redis_url = init_matches.get_one::("redis").unwrap().to_owned(); let host = init_matches.get_one::("host").unwrap().to_owned(); let port = init_matches.get_one::("port").unwrap().to_owned(); let threads = *init_matches.get_one::("threads").unwrap(); @@ -311,6 +316,7 @@ fn main() -> ThothResult<()> { run_migrations(&database_url)?; api_server( database_url, + redis_url, host, port, threads, diff --git a/thoth-api-server/src/lib.rs b/thoth-api-server/src/lib.rs index 318faaf0d..7f53049a5 100644 --- a/thoth-api-server/src/lib.rs +++ b/thoth-api-server/src/lib.rs @@ -24,6 +24,7 @@ use thoth_api::{ model::{create_schema, Context, Schema}, GraphQLRequest, }, + redis::{init_pool as redis_init_pool, RedisPool}, }; use thoth_errors::ThothError; @@ -91,10 +92,11 @@ async fn graphql_schema(st: Data>) -> HttpResponse { async fn graphql( st: Data>, pool: Data, + redis_pool: Data, token: DecodedToken, data: Json, ) -> Result { - let ctx = Context::new(pool.into_inner(), token); + let ctx = Context::new(pool.into_inner(), redis_pool.into_inner(), token); let result = data.execute(&st, &ctx).await; match result.is_ok() { true => Ok(HttpResponse::Ok().json(result)), @@ -186,6 +188,7 @@ async fn account_details( #[actix_web::main] pub async fn start_server( database_url: String, + redis_url: String, host: String, port: String, threads: usize, @@ -228,6 +231,7 @@ pub async fn start_server( ) .app_data(Data::new(ApiConfig::new(public_url.clone()))) .app_data(Data::new(init_pool(&database_url))) + .app_data(Data::new(redis_init_pool(&redis_url))) .app_data(Data::new(Arc::new(create_schema()))) .service(index) .service(graphql_index) diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index b79a1f119..c6e544846 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -35,6 +35,7 @@ use crate::model::Orcid; use crate::model::Ror; use crate::model::Timestamp; use crate::model::WeightUnit; +use crate::redis::{set, RedisPool}; use thoth_errors::{ThothError, ThothResult}; use super::utils::{Direction, Expression}; @@ -44,14 +45,16 @@ impl juniper::Context for Context {} #[derive(Clone)] pub struct Context { pub db: Arc, + pub redis: Arc, pub account_access: AccountAccess, pub token: DecodedToken, } impl Context { - pub fn new(pool: Arc, token: DecodedToken) -> Self { + pub fn new(pool: Arc, redis_pool: Arc, token: DecodedToken) -> Self { Self { db: pool, + redis: redis_pool, account_access: token.get_user_permissions(), token, } @@ -1470,7 +1473,7 @@ pub struct MutationRoot; #[juniper::graphql_object(Context = Context)] impl MutationRoot { #[graphql(description = "Create a new work with the specified values")] - fn create_work( + async fn create_work( context: &Context, #[graphql(description = "Values for work to be created")] data: NewWork, ) -> FieldResult { @@ -1481,7 +1484,14 @@ impl MutationRoot { data.validate()?; - Work::create(&context.db, &data).map_err(|e| e.into()) + let result = Work::create(&context.db, &data).map_err(|e| e.into()); + + if let Ok(created_work) = result.clone() { + // placeholder (TODO handle this result) + let _ = set(&context.redis, "test", &created_work.work_id().to_string()).await; + } + + result } #[graphql(description = "Create a new publisher with the specified values")] From 1d846e55a9bec85936051936cda39e4310cc06f8 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 20 Jan 2025 13:54:47 +0000 Subject: [PATCH 04/69] Rename pool imports to be consistent with new thoth.rs changes --- thoth-api-server/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/thoth-api-server/src/lib.rs b/thoth-api-server/src/lib.rs index 7f53049a5..0336960b0 100644 --- a/thoth-api-server/src/lib.rs +++ b/thoth-api-server/src/lib.rs @@ -19,12 +19,12 @@ use serde::Serialize; use thoth_api::{ account::model::{AccountDetails, DecodedToken, LoginCredentials}, account::service::{get_account, get_account_details, login}, - db::{init_pool, PgPool}, + db::{init_pool as init_pg_pool, PgPool}, graphql::{ model::{create_schema, Context, Schema}, GraphQLRequest, }, - redis::{init_pool as redis_init_pool, RedisPool}, + redis::{init_pool as init_redis_pool, RedisPool}, }; use thoth_errors::ThothError; @@ -230,8 +230,8 @@ pub async fn start_server( .supports_credentials(), ) .app_data(Data::new(ApiConfig::new(public_url.clone()))) - .app_data(Data::new(init_pool(&database_url))) - .app_data(Data::new(redis_init_pool(&redis_url))) + .app_data(Data::new(init_pg_pool(&database_url))) + .app_data(Data::new(init_redis_pool(&redis_url))) .app_data(Data::new(Arc::new(create_schema()))) .service(index) .service(graphql_index) From 6531a23d5b5d8fcc3c29c70445a04db9fd888a78 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:27:45 +0000 Subject: [PATCH 05/69] Implement Event model and Redis lpush method as basic queue logic (working) --- thoth-api/src/event/handler.rs | 20 ++++++++++++++++++++ thoth-api/src/event/mod.rs | 4 ++++ thoth-api/src/event/model.rs | 18 ++++++++++++++++++ thoth-api/src/graphql/model.rs | 17 +++++++++++++---- thoth-api/src/lib.rs | 1 + thoth-api/src/redis.rs | 20 ++++++++++++++++++++ 6 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 thoth-api/src/event/handler.rs create mode 100644 thoth-api/src/event/mod.rs create mode 100644 thoth-api/src/event/model.rs diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs new file mode 100644 index 000000000..e4d2eb841 --- /dev/null +++ b/thoth-api/src/event/handler.rs @@ -0,0 +1,20 @@ +use super::model::{Event, EventType}; +use crate::model::work::{Work, WorkProperties}; +use crate::redis::{lpush, RedisPool}; +use thoth_errors::ThothResult; + +const QUEUE_KEY: &str = "events:graphql"; + +pub async fn send_event( + redis: &RedisPool, + event_type: EventType, + work: &Work, +) -> ThothResult { + let event = Event { + event_type, + work_id: *work.work_id(), + is_published: work.is_active_withdrawn_superseded(), + event_timestamp: work.updated_at, + }; + lpush(redis, QUEUE_KEY, &serde_json::to_string(&event)?).await +} diff --git a/thoth-api/src/event/mod.rs b/thoth-api/src/event/mod.rs new file mode 100644 index 000000000..0a88473cd --- /dev/null +++ b/thoth-api/src/event/mod.rs @@ -0,0 +1,4 @@ +#[cfg(feature = "backend")] +pub mod handler; +#[cfg(feature = "backend")] +pub mod model; diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs new file mode 100644 index 000000000..cb7b58dc4 --- /dev/null +++ b/thoth-api/src/event/model.rs @@ -0,0 +1,18 @@ +use crate::model::Timestamp; +use serde::Serialize; +use uuid::Uuid; + +#[derive(Serialize, juniper::GraphQLEnum)] +pub enum EventType { + WorkCreated, + WorkUpdated, + WorkPublished, +} + +#[derive(Serialize, juniper::GraphQLInputObject)] +pub struct Event { + pub event_type: EventType, + pub work_id: Uuid, + pub is_published: bool, + pub event_timestamp: Timestamp, +} diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index c6e544846..fba1a68e3 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -7,6 +7,8 @@ use uuid::Uuid; use crate::account::model::AccountAccess; use crate::account::model::DecodedToken; use crate::db::PgPool; +use crate::event::handler::send_event; +use crate::event::model::EventType; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; @@ -35,7 +37,7 @@ use crate::model::Orcid; use crate::model::Ror; use crate::model::Timestamp; use crate::model::WeightUnit; -use crate::redis::{set, RedisPool}; +use crate::redis::RedisPool; use thoth_errors::{ThothError, ThothResult}; use super::utils::{Direction, Expression}; @@ -1487,8 +1489,11 @@ impl MutationRoot { let result = Work::create(&context.db, &data).map_err(|e| e.into()); if let Ok(created_work) = result.clone() { - // placeholder (TODO handle this result) - let _ = set(&context.redis, "test", &created_work.work_id().to_string()).await; + // TODO handle results throughout + let _ = send_event(&context.redis, EventType::WorkCreated, &created_work).await; + if created_work.work_status == WorkStatus::Active { + let _ = send_event(&context.redis, EventType::WorkPublished, &created_work).await; + } } result @@ -1732,7 +1737,7 @@ impl MutationRoot { } #[graphql(description = "Update an existing work with the specified values")] - fn update_work( + async fn update_work( context: &Context, #[graphql(description = "Values to apply to existing work")] data: PatchWork, ) -> FieldResult { @@ -1758,6 +1763,10 @@ impl MutationRoot { // update the work and, if it succeeds, synchronise its children statuses and pub. date match work.update(&context.db, &data, &account_id) { Ok(w) => { + let _ = send_event(&context.redis, EventType::WorkUpdated, &w).await; + if w.work_status == WorkStatus::Active && work.work_status != WorkStatus::Active { + let _ = send_event(&context.redis, EventType::WorkPublished, &w).await; + } // update chapters if their pub. data, withdrawn_date or work_status doesn't match the parent's for child in work.children(&context.db)? { if child.publication_date != w.publication_date diff --git a/thoth-api/src/lib.rs b/thoth-api/src/lib.rs index 8495057b3..b7e400ec9 100644 --- a/thoth-api/src/lib.rs +++ b/thoth-api/src/lib.rs @@ -17,6 +17,7 @@ extern crate juniper; pub mod account; #[cfg(feature = "backend")] pub mod db; +pub mod event; pub mod graphql; #[macro_use] pub mod model; diff --git a/thoth-api/src/redis.rs b/thoth-api/src/redis.rs index 04ea75d81..071ccfee3 100644 --- a/thoth-api/src/redis.rs +++ b/thoth-api/src/redis.rs @@ -38,6 +38,11 @@ pub async fn scan_match(pool: &RedisPool, pattern: &str) -> ThothResult ThothResult { + let mut con = create_connection(pool).await?; + con.lpush(key, value).await.map_err(Into::into) +} + #[cfg(test)] mod tests { use super::*; @@ -72,6 +77,21 @@ mod tests { assert_eq!(get_result.unwrap(), test_value); } + #[tokio::test] + async fn test_lpush() { + let pool = get_pool().await; + + let test_key = "test_queue"; + let test_value_1 = "test_value_1"; + let test_value_2 = "test_value_2"; + + let lpush_result_1 = lpush(&pool, test_key, test_value_1).await; + assert!(lpush_result_1.is_ok()); + + let lpush_result_2 = lpush(&pool, test_key, test_value_2).await; + assert!(lpush_result_2.is_ok()); + } + #[tokio::test] async fn test_get_nonexistent_key() { let pool = get_pool().await; From cc4314757da8b9e1051905d2b096b704fe712e5d Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 4 Feb 2025 17:50:18 +0000 Subject: [PATCH 06/69] Minimal implementation of Loco modulebased on starter code (compiles, runs, config missing) --- Cargo.lock | 2070 ++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 +- src/bin/commands/start.rs | 9 +- src/bin/thoth.rs | 1 + src/lib.rs | 1 + thoth-loco-test/Cargo.toml | 26 + thoth-loco-test/src/app.rs | 53 + thoth-loco-test/src/lib.rs | 30 + 8 files changed, 2119 insertions(+), 74 deletions(-) create mode 100644 thoth-loco-test/Cargo.toml create mode 100644 thoth-loco-test/src/app.rs create mode 100644 thoth-loco-test/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 2e0cc447e..870bb3166 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ dependencies = [ "ahash", "base64 0.22.1", "bitflags 2.6.0", - "brotli", + "brotli 6.0.0", "bytes", "bytestring", "derive_more 0.99.18", @@ -64,7 +64,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand", + "rand 0.8.5", "sha1", "smallvec", "tokio", @@ -163,7 +163,7 @@ dependencies = [ "actix-web", "anyhow", "derive_more 1.0.0", - "rand", + "rand 0.8.5", "serde", "serde_json", "tracing", @@ -198,7 +198,7 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "cookie", + "cookie 0.16.2", "derive_more 0.99.18", "encoding_rs", "futures-core", @@ -290,10 +290,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -320,6 +320,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -408,6 +414,18 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] + [[package]] name = "argon2rs" version = "0.2.5" @@ -439,6 +457,32 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +dependencies = [ + "brotli 7.0.0", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -456,6 +500,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto-future" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c1e7e457ea78e524f48639f551fd79703ac3f2237f5ecccdf4708f8a75ad373" + [[package]] name = "auto_enums" version = "0.8.6" @@ -474,6 +524,135 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +dependencies = [ + "axum-core", + "axum-macros", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.2.0", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +dependencies = [ + "bytes", + "futures-util", + "http 1.2.0", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-extra" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" +dependencies = [ + "axum", + "axum-core", + "bytes", + "cookie 0.18.1", + "futures-util", + "http 1.2.0", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "serde", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "axum-test" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317c1f4ecc1e68e0ad5decb78478421055c963ce215e736ed97463fa609cd196" +dependencies = [ + "anyhow", + "assert-json-diff", + "auto-future", + "axum", + "bytes", + "bytesize", + "cookie 0.18.1", + "http 1.2.0", + "http-body-util", + "hyper", + "hyper-util", + "mime", + "pretty_assertions", + "reserve-port", + "rust-multipart-rfc7578_2", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "tokio", + "tower 0.5.2", + "url", +] + +[[package]] +name = "backon" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5289ec98f68f28dd809fd601059e6aa908bb8f6108620930828283d4ee23d7" +dependencies = [ + "fastrand", + "gloo-timers 0.3.0", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -489,6 +668,18 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "backtrace_printer" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d28de81c708c843640982b66573df0f0168d87e42854b563971f326745aab7" +dependencies = [ + "btparse-stable", + "colored", + "regex", + "thiserror 1.0.69", +] + [[package]] name = "base64" version = "0.20.0" @@ -507,6 +698,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bincode" version = "1.3.3" @@ -528,6 +725,15 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "blake2-rfc" version = "0.2.18" @@ -564,6 +770,17 @@ dependencies = [ "brotli-decompressor", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + [[package]] name = "brotli-decompressor" version = "4.0.1" @@ -574,12 +791,38 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "btparse-stable" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d75b8252ed252f881d1dc4482ae3c3854df6ee8183c1906bac50ff358f4f89f" + [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byte-unit" +version = "4.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c" +dependencies = [ + "serde", + "utf8-width", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -588,9 +831,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "bytesize" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" [[package]] name = "bytestring" @@ -633,6 +882,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8100e46ff92eb85bf6dc2930c73f2a4f7176393c84a9446b3d501e1b354e7b34" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -648,6 +903,38 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "chumsky" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" +dependencies = [ + "hashbrown 0.14.5", + "stacker", +] + [[package]] name = "cipher" version = "0.4.4" @@ -665,6 +952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -679,6 +967,18 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "clap_lex" version = "0.7.3" @@ -700,6 +1000,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "combine" version = "3.8.1" @@ -773,13 +1083,24 @@ dependencies = [ "hkdf", "hmac", "percent-encoding", - "rand", + "rand 0.8.5", "sha2", "subtle", "time", "version_check", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -814,6 +1135,71 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cron" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "cruet" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "113a9e83d8f614be76de8df1f25bf9d0ea6e85ea573710a3d3f3abe1438ae49c" +dependencies = [ + "once_cell", + "regex", +] + +[[package]] +name = "cruet" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6132609543972496bc97b1e01f1ce6586768870aeb4cabeb3385f4e05b5caead" +dependencies = [ + "once_cell", + "regex", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -821,10 +1207,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.89", +] + [[package]] name = "csv" version = "1.3.1" @@ -974,6 +1383,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "deunicode" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" + [[package]] name = "dialoguer" version = "0.11.0" @@ -1060,6 +1475,12 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1102,12 +1523,70 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "dtoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "duct_sh" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6633cadba557545fbbe0299a2f9adc4bb2fc5fb238773f5e841e0c23d62146" +dependencies = [ + "duct", +] + +[[package]] +name = "ego-tree" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c6ba7d4eec39eaa9ab24d44a0e73a7949a1095a8b3f3abb11eddf27dbb56a53" + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "email-encoding" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3d894bbbab314476b265f9b2d46bf24b123a36dd0e96b06a1b49545b9d9dcc" +dependencies = [ + "base64 0.22.1", + "memchr", +] + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -1123,6 +1602,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "english-to-cron" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c16423ac933fee80f05a52b435a912d5b08edbbbfe936e0042ebb3accdf303da" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "env_filter" version = "0.1.2" @@ -1168,6 +1657,12 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +[[package]] +name = "flagset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec" + [[package]] name = "flate2" version = "1.0.35" @@ -1208,6 +1703,25 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.31" @@ -1279,6 +1793,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -1297,6 +1817,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1307,6 +1836,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -1316,10 +1854,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets", +] + [[package]] name = "ghash" version = "0.5.1" @@ -1336,6 +1886,36 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "globset" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags 2.6.0", + "ignore", + "walkdir", +] + [[package]] name = "gloo" version = "0.2.1" @@ -1489,6 +2069,8 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" dependencies = [ + "futures-channel", + "futures-core", "js-sys", "wasm-bindgen", ] @@ -1607,7 +2189,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.2.0", "indexmap 2.6.0", "slab", "tokio", @@ -1621,6 +2203,16 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1663,6 +2255,31 @@ dependencies = [ "digest", ] +[[package]] +name = "hostname" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" +dependencies = [ + "cfg-if", + "libc", + "windows", +] + +[[package]] +name = "html5ever" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e15626aaf9c351bc696217cbe29cb9b5e86c43f8a46b5e2f5c6c5cf7cb904ce" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "http" version = "0.2.12" @@ -1676,9 +2293,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1692,7 +2309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -1703,11 +2320,17 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body", "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.9.5" @@ -1720,6 +2343,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.1.0" @@ -1736,9 +2368,10 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1753,7 +2386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", + "http 1.2.0", "hyper", "hyper-util", "rustls", @@ -1761,6 +2394,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -1788,7 +2422,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body", "hyper", "pin-project-lite", @@ -1966,12 +2600,47 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata 0.4.9", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "impl-more" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2003,6 +2672,23 @@ dependencies = [ "generic-array", ] +[[package]] +name = "insta" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "pest", + "pest_derive", + "pin-project", + "regex", + "serde", + "similar", +] + [[package]] name = "instant" version = "0.1.13" @@ -2021,6 +2707,15 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "ipnetwork" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" +dependencies = [ + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2131,6 +2826,37 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lettre" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4c9a167ff73df98a5ecc07e8bf5ce90b583665da3d1762eb1f775ad4d0d6f5" +dependencies = [ + "async-trait", + "base64 0.22.1", + "chumsky", + "email-encoding", + "email_address", + "fastrand", + "futures-io", + "futures-util", + "hostname", + "httpdate", + "idna", + "mime", + "nom", + "percent-encoding", + "quoted_printable", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "socket2", + "tokio", + "tokio-rustls", + "url", + "webpki-roots", +] + [[package]] name = "libc" version = "0.2.169" @@ -2138,57 +2864,191 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] -name = "linux-raw-sys" -version = "0.4.14" +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "local-channel" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +dependencies = [ + "futures-core", + "futures-sink", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "loco-gen" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec8762a66ade27c5157288ca46db51a77bbd95cb2d7ad87bcaada9621e3a4d9" +dependencies = [ + "chrono", + "clap", + "colored", + "cruet 0.14.0", + "duct", + "include_dir", + "regex", + "rrgen", + "serde", + "serde_json", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "loco-rs" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19decee5529e36ffd3e22f83cf601c1fcb35a8932886fc33c19e8482bc1baf88" +dependencies = [ + "argon2", + "async-trait", + "axum", + "axum-extra", + "axum-test", + "backtrace_printer", + "byte-unit", + "bytes", + "cfg-if", + "chrono", + "clap", + "colored", + "cruet 0.13.3", + "duct", + "duct_sh", + "english-to-cron", + "fs-err", + "futures-util", + "heck 0.4.1", + "hyper", + "include_dir", + "ipnetwork", + "lettre", + "loco-gen", + "mime", + "opendal", + "rand 0.8.5", + "regex", + "reqwest", + "scraper", + "semver", + "serde", + "serde_json", + "serde_variant", + "serde_yaml", + "tera", + "thiserror 1.0.69", + "thousands", + "tokio", + "tokio-cron-scheduler", + "tokio-util", + "toml", + "tower 0.4.13", + "tower-http", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", + "validator", +] + +[[package]] +name = "log" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] -name = "litemap" -version = "0.7.4" +name = "mac" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" [[package]] -name = "local-channel" -version = "0.1.5" +name = "marc" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +checksum = "c7158861fa6320edf04e974b504ad3c3a69e1aa9500d2e4899e6dc41265c0a56" dependencies = [ - "futures-core", - "futures-sink", - "local-waker", + "xml-rs", ] [[package]] -name = "local-waker" -version = "0.1.4" +name = "markup5ever" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" +checksum = "82c88c6129bd24319e62a0359cb6b958fa7e8be6e19bb1663bc396b90883aca5" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] [[package]] -name = "lock_api" -version = "0.4.12" +name = "matchers" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "autocfg", - "scopeguard", + "regex-automata 0.1.10", ] [[package]] -name = "log" -version = "0.4.22" +name = "matchit" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] -name = "marc" -version = "3.1.1" +name = "md-5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7158861fa6320edf04e974b504ad3c3a69e1aa9500d2e4899e6dc41265c0a56" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "xml-rs", + "cfg-if", + "digest", ] [[package]] @@ -2234,6 +3094,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2252,7 +3118,7 @@ dependencies = [ "hermit-abi", "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2273,12 +3139,38 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nodrop" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2295,6 +3187,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2366,6 +3269,34 @@ dependencies = [ "serde_json", ] +[[package]] +name = "opendal" +version = "0.50.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb28bb6c64e116ceaf8dd4e87099d3cfea4a58e85e62b104fef74c91afba0f44" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "flagset", + "futures", + "getrandom 0.2.15", + "http 1.2.0", + "log", + "md-5", + "once_cell", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + [[package]] name = "openssl" version = "0.10.68" @@ -2410,6 +3341,22 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_pipe" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "paperclip" version = "0.9.4" @@ -2535,6 +3482,26 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.15" @@ -2557,6 +3524,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror 2.0.3", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "pest_meta" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "phf" version = "0.11.2" @@ -2564,7 +3576,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", - "phf_shared", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand 0.8.5", ] [[package]] @@ -2573,8 +3605,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_shared", - "rand", + "phf_shared 0.11.2", + "rand 0.8.5", ] [[package]] @@ -2583,13 +3615,22 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.2", + "phf_shared 0.11.2", "proc-macro2", "quote", "syn 2.0.89", ] +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.11.2" @@ -2599,6 +3640,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "pin-project-lite" version = "0.2.15" @@ -2641,7 +3702,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -2653,6 +3714,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2708,6 +3794,77 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.3", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +dependencies = [ + "bytes", + "getrandom 0.2.15", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.3", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.37" @@ -2717,6 +3874,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quoted_printable" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c9bd8497b02465aeef5375144c26062e0dcd5939dfcbb0f5db76cb8c17c73" + [[package]] name = "r2d2" version = "0.8.10" @@ -2732,11 +3895,22 @@ dependencies = [ name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ - "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.0", + "zerocopy 0.8.14", ] [[package]] @@ -2746,7 +3920,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.0", ] [[package]] @@ -2755,7 +3939,17 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +dependencies = [ + "getrandom 0.3.1", + "zerocopy 0.8.14", ] [[package]] @@ -2805,8 +3999,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2817,7 +4020,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] [[package]] @@ -2826,12 +4029,24 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.9" @@ -2844,7 +4059,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body", "http-body-util", "hyper", @@ -2859,7 +4074,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2867,11 +4085,15 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", + "webpki-roots", "windows-registry", ] @@ -2883,7 +4105,7 @@ checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" dependencies = [ "anyhow", "async-trait", - "http 1.1.0", + "http 1.2.0", "reqwest", "serde", "thiserror 1.0.69", @@ -2899,8 +4121,8 @@ dependencies = [ "anyhow", "async-trait", "futures", - "getrandom", - "http 1.1.0", + "getrandom 0.2.15", + "http 1.2.0", "hyper", "parking_lot 0.11.2", "reqwest", @@ -2912,13 +4134,23 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "reserve-port" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "359fc315ed556eb0e42ce74e76f4b1cd807b50fa6307f3de4e51f92dbe86e2d5" +dependencies = [ + "lazy_static", + "thiserror 2.0.3", +] + [[package]] name = "retry-policies" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" dependencies = [ - "rand", + "rand 0.8.5", ] [[package]] @@ -2929,7 +4161,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -2951,12 +4183,83 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rrgen" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e27f5f254d89b0b5b76445442e5c935b63a566ee5a735c9d877ca2029b4ce9" +dependencies = [ + "cruet 0.13.3", + "fs-err", + "glob", + "heck 0.4.1", + "regex", + "serde", + "serde_json", + "serde_regex", + "serde_yaml", + "tera", + "thiserror 1.0.69", +] + +[[package]] +name = "rstest" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afd55a67069d6e434a95161415f5beeada95a01c7b815508a82dcb0e1593682" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.89", + "unicode-ident", +] + +[[package]] +name = "rust-multipart-rfc7578_2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc4bb9e7c9abe5fa5f30c2d8f8fefb9e0080a2c1e3c2e567318d2907054b35d3" +dependencies = [ + "bytes", + "futures-core", + "futures-util", + "http 1.2.0", + "mime", + "mime_guess", + "rand 0.9.0", + "thiserror 2.0.3", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + [[package]] name = "rustc_version" version = "0.4.1" @@ -2985,7 +4288,9 @@ version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3006,6 +4311,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -3030,6 +4338,24 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e1c91382686d21b5ac7959341fcb9780fa7c03773646995a87c950fa7be640" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.27" @@ -3066,6 +4392,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e749d29b2064585327af5038a5a8eb73aeebad4a3472e83531a436563f7208" +dependencies = [ + "ahash", + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "indexmap 2.6.0", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "sdd" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478f121bb72bbf63c52c93011ea1791dca40140dfe13f8336c4c5ac952c33aa9" + [[package]] name = "security-framework" version = "2.11.1" @@ -3089,6 +4438,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags 2.6.0", + "cssparser", + "derive_more 0.99.18", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.23" @@ -3139,6 +4507,26 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -3160,6 +4548,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_variant" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a0068df419f9d9b6488fdded3f1c818522cdea328e02ce9d9f147380265a432" +dependencies = [ + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -3173,6 +4570,40 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +dependencies = [ + "futures", + "log", + "once_cell", + "parking_lot 0.12.3", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "servo_arc" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae65c4249478a2647db249fb43e23cec56a2c8974a427e7bd8cb5a1d0964921a" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.10.6" @@ -3195,6 +4626,25 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shared_child" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "shell-words" version = "1.1.0" @@ -3216,6 +4666,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -3243,6 +4699,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slug" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" +dependencies = [ + "deunicode", + "wasm-bindgen", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -3282,12 +4748,51 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot 0.12.3", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3417,6 +4922,39 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "tera" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9d851b45e865f178319da0abdbfe6acbc4328759ff18dafc3a41c16b4cd2ee" +dependencies = [ + "chrono", + "chrono-tz", + "globwalk", + "humansize", + "lazy_static", + "percent-encoding", + "pest", + "pest_derive", + "rand 0.8.5", + "regex", + "serde", + "serde_json", + "slug", + "unic-segment", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -3470,6 +5008,7 @@ dependencies = [ "thoth-app-server", "thoth-errors", "thoth-export-server", + "thoth-loco-test", "tokio", ] @@ -3492,7 +5031,7 @@ dependencies = [ "juniper", "lazy_static", "phf", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -3620,6 +5159,40 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "thoth-loco-test" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "insta", + "loco-rs", + "regex", + "rstest", + "serde", + "serde_json", + "serial_test", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.36" @@ -3661,6 +5234,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.43.0" @@ -3679,6 +5267,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-cron-scheduler" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2594dd7c2abbbafbb1c78d167fd10860dc7bd75f814cb051a1e0d3e796b9702" +dependencies = [ + "chrono", + "cron", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "tokio-macros" version = "2.5.0" @@ -3758,6 +5361,66 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +dependencies = [ + "async-compression", + "bitflags 2.6.0", + "bytes", + "futures-core", + "futures-util", + "http 1.2.0", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -3776,6 +5439,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror 1.0.69", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.28" @@ -3794,6 +5469,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -3808,6 +5526,62 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + [[package]] name = "unicase" version = "2.8.0" @@ -3874,12 +5648,24 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf16_iter" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3898,11 +5684,48 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ - "getrandom", + "getrandom 0.2.15", + "rand 0.8.5", "serde", "wasm-bindgen", ] +[[package]] +name = "validator" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b4a29d8709210980a09379f27ee31549b73292c87ab9899beee1c0d3be6303" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac855a2ce6f843beb229757e6e570a42e837bcb15e5f449dd48d5747d41bf77" +dependencies = [ + "darling", + "once_cell", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.89", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3921,6 +5744,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -3936,6 +5769,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.95" @@ -4016,6 +5858,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-timer" version = "0.2.5" @@ -4041,6 +5896,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4057,12 +5931,31 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -4193,6 +6086,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -4217,6 +6119,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yew" version = "0.18.0" @@ -4387,7 +6295,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a367f292d93d4eab890745e75a778da40909cab4d6ff8173693812f79c4a2468" +dependencies = [ + "zerocopy-derive 0.8.14", ] [[package]] @@ -4401,6 +6318,17 @@ dependencies = [ "syn 2.0.89", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3931cb58c62c13adec22e38686b559c86a30565e16ad6e8510a337cedc611e1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.89", +] + [[package]] name = "zerofrom" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index be855b797..2d4695d48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ readme = "README.md" maintenance = { status = "actively-developed" } [workspace] -members = ["thoth-api", "thoth-api-server", "thoth-app", "thoth-app-server", "thoth-client", "thoth-errors", "thoth-export-server"] +members = ["thoth-api", "thoth-api-server", "thoth-app", "thoth-app-server", "thoth-client", "thoth-errors", "thoth-export-server", "thoth-loco-test"] [dependencies] thoth-api = { version = "=0.13.6", path = "thoth-api", features = ["backend"] } @@ -20,6 +20,7 @@ thoth-api-server = { version = "=0.13.6", path = "thoth-api-server" } thoth-app-server = { version = "=0.13.6", path = "thoth-app-server" } thoth-errors = { version = "=0.13.6", path = "thoth-errors" } thoth-export-server = { version = "=0.13.6", path = "thoth-export-server" } +thoth-loco-test = { version = "=0.1.0", path = "thoth-loco-test" } clap = { version = "4.5.21", features = ["cargo", "env"] } dialoguer = { version = "0.11.0", features = ["password"] } dotenv = "0.15.0" diff --git a/src/bin/commands/start.rs b/src/bin/commands/start.rs index 53251acbd..b91ce484a 100644 --- a/src/bin/commands/start.rs +++ b/src/bin/commands/start.rs @@ -1,7 +1,7 @@ use crate::arguments; use clap::{ArgMatches, Command}; use lazy_static::lazy_static; -use thoth::{api_server, app_server, errors::ThothResult, export_server}; +use thoth::{api_server, app_server, errors::ThothResult, export_server, loco_server}; lazy_static! { pub(crate) static ref COMMAND: Command = Command::new("start") @@ -40,7 +40,8 @@ lazy_static! { .arg(arguments::keep_alive("EXPORT_API_KEEP_ALIVE")) .arg(arguments::export_url()) .arg(arguments::gql_endpoint()), - ); + ) + .subcommand(Command::new("loco").about("Start loco")); } pub fn graphql_api(arguments: &ArgMatches) -> ThothResult<()> { @@ -102,3 +103,7 @@ pub fn export_api(arguments: &ArgMatches) -> ThothResult<()> { ) .map_err(|e| e.into()) } + +pub fn loco() -> ThothResult<()> { + loco_server().map_err(|e| thoth_errors::ThothError::InternalError(e.to_string())) +} diff --git a/src/bin/thoth.rs b/src/bin/thoth.rs index 42597884b..5baee1812 100644 --- a/src/bin/thoth.rs +++ b/src/bin/thoth.rs @@ -24,6 +24,7 @@ fn main() -> thoth::errors::ThothResult<()> { Some(("graphql-api", arguments)) => commands::start::graphql_api(arguments), Some(("app", arguments)) => commands::start::app(arguments), Some(("export-api", arguments)) => commands::start::export_api(arguments), + Some(("loco", _)) => commands::start::loco(), _ => unreachable!(), }, Some(("migrate", arguments)) => commands::migrate(arguments), diff --git a/src/lib.rs b/src/lib.rs index b0e60dc41..93d0b61af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,3 +3,4 @@ pub use thoth_api_server::start_server as api_server; pub use thoth_app_server::start_server as app_server; pub use thoth_errors as errors; pub use thoth_export_server::{start_server as export_server, ALL_SPECIFICATIONS}; +pub use thoth_loco_test::start_server as loco_server; diff --git a/thoth-loco-test/Cargo.toml b/thoth-loco-test/Cargo.toml new file mode 100644 index 000000000..94fc8c18e --- /dev/null +++ b/thoth-loco-test/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "thoth-loco-test" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +loco-rs = { version = "0.14.0", default-features = false, features = ["cli"] } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1" } +tokio = { version = "1.33.0", default-features = false, features = [ + "rt-multi-thread", +] } +async-trait = { version = "0.1.74" } +axum = { version = "0.8.1" } +tracing = { version = "0.1.40" } +tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] } +regex = { version = "1.11.1" } + +[dev-dependencies] +loco-rs = { version = "0.14.0", default-features = false, features = ["testing"] } +serial_test = { version = "3.1.1" } +rstest = { version = "0.21.0" } +insta = { version = "1.34.0", features = ["redactions", "yaml", "filters"] } diff --git a/thoth-loco-test/src/app.rs b/thoth-loco-test/src/app.rs new file mode 100644 index 000000000..8b8b07b22 --- /dev/null +++ b/thoth-loco-test/src/app.rs @@ -0,0 +1,53 @@ +use async_trait::async_trait; +use loco_rs::{ + app::{AppContext, Hooks, Initializer}, + bgworker::Queue, + boot::{create_app, BootResult, StartMode}, + config::Config, + controller::AppRoutes, + environment::Environment, + task::Tasks, + Result, +}; + +pub struct App; +#[async_trait] +impl Hooks for App { + fn app_name() -> &'static str { + env!("CARGO_CRATE_NAME") + } + + fn app_version() -> String { + format!( + "{} ({})", + env!("CARGO_PKG_VERSION"), + option_env!("BUILD_SHA") + .or(option_env!("GITHUB_SHA")) + .unwrap_or("dev") + ) + } + + async fn boot( + mode: StartMode, + environment: &Environment, + config: Config, + ) -> Result { + create_app::(mode, environment, config).await + } + + async fn initializers(_ctx: &AppContext) -> Result>> { + Ok(vec![]) + } + + fn routes(_ctx: &AppContext) -> AppRoutes { + AppRoutes::with_default_routes() + } + async fn connect_workers(_ctx: &AppContext, _queue: &Queue) -> Result<()> { + Ok(()) + } + + #[allow(unused_variables)] + fn register_tasks(tasks: &mut Tasks) { + // tasks-inject (do not remove) + } +} diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs new file mode 100644 index 000000000..367971416 --- /dev/null +++ b/thoth-loco-test/src/lib.rs @@ -0,0 +1,30 @@ +pub mod app; + +use crate::app::App; +use loco_rs::{ + app::Hooks, + boot::{start, ServeParams, StartMode}, + environment::resolve_from_env, + logger, Result, +}; + +#[tokio::main] +pub async fn start_server() -> Result<()> { + start_app::().await +} + +pub async fn start_app() -> Result<()> { + let environment = resolve_from_env().into(); + let config = H::load_config(&environment).await?; + + if !H::init_logger(&config, &environment)? { + logger::init::(&config.logger)?; + } + + let boot_result = H::boot(StartMode::WorkerOnly, &environment, config).await?; + let serve_params = ServeParams { + port: boot_result.app_context.config.server.port, + binding: boot_result.app_context.config.server.binding.to_string(), + }; + start::(boot_result, serve_params, false).await +} From b0685b71c08c776f6fa889a8f173ef9c77ce8d4d Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:50:02 +0000 Subject: [PATCH 07/69] Add minimal Loco config based on autogen plus docs for Workers --- config/development.yaml | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 config/development.yaml diff --git a/config/development.yaml b/config/development.yaml new file mode 100644 index 000000000..4bbe37ec9 --- /dev/null +++ b/config/development.yaml @@ -0,0 +1,50 @@ +# Loco configuration file documentation +# TODO a better location would be thoth/thoth-loco-test/config, but that doesn't get picked up - +# see if the path where it searches can be changed + +# Application logging configuration +logger: + # Enable or disable logging. + enable: true + # Enable pretty backtrace (sets RUST_BACKTRACE=1) + pretty_backtrace: true + # Log level, options: trace, debug, info, warn or error. + level: debug + # Define the logging format. options: compact, pretty or json + format: compact + # By default the logger has filtering only logs that came from your code or logs that came from `loco` framework. to see all third party libraries + # Uncomment the line below to override to see all third party libraries you can enable this config and override the logger filters. + # override_filter: trace + +# Web server configuration +server: + # Port on which the server will listen. the server binding is 0.0.0.0:{PORT} + port: 5150 + # Binding for the server (which interface to bind to) + binding: localhost + # The UI hostname or IP address that mailers will point to. + host: http://localhost + # Out of the box middleware configuration. to disable middleware you can changed the `enable` field to `false` of comment the middleware block + middlewares: + +# Initializers Configuration +# initializers: +# oauth2: +# authorization_code: # Authorization code grant type +# - client_identifier: google # Identifier for the OAuth2 provider. Replace 'google' with your provider's name if different, must be unique within the oauth2 config. +# ... other fields + +# Worker Configuration +workers: + # specifies the worker mode. Options: + # - BackgroundQueue - Workers operate asynchronously in the background, processing queued. + # - ForegroundBlocking - Workers operate in the foreground and block until tasks are completed. + # - BackgroundAsync - Workers operate asynchronously in the background, processing tasks with async capabilities. + mode: BackgroundQueue + +queue: + kind: Redis + # Redis connection URI + # TODO get from environment + uri: "redis://localhost:6379" + dangerously_flush: false From 0a5d8abe0c0a4e60d0b35806d96d2d38b3577afd Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:52:06 +0000 Subject: [PATCH 08/69] Changelog placeholder --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d835b19..6ac7292b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + - Webhooks: event creation in GraphQL when works are changed, event handling via Loco ## [[0.13.6]](https://github.com/thoth-pub/thoth/releases/tag/v0.13.6) - 2025-01-28 ### Changed From 1a28ebd89bd2b00c038139879c31219adbe05983 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 6 Feb 2025 10:12:00 +0000 Subject: [PATCH 09/69] Correct config by enabling redis feature (on by default, but defaults are off) --- Cargo.lock | 207 ++++++++++++++++++++++++++++++++++++- thoth-loco-test/Cargo.toml | 2 +- 2 files changed, 204 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 870bb3166..dd1bc4206 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -704,6 +704,18 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bb8" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" +dependencies = [ + "async-trait", + "futures-util", + "parking_lot 0.12.3", + "tokio", +] + [[package]] name = "bincode" version = "1.3.3" @@ -1072,6 +1084,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1146,6 +1167,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "cron_clock" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8699d8ed16e3db689f8ae04d8dc3c6666a4ba7e724e5a157884b7cc385d16b" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + [[package]] name = "crossbeam-channel" version = "0.5.14" @@ -1317,7 +1349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfae6799b68a735270e4344ee3e834365f707c72da09c9a8bb89b45cc3351395" dependencies = [ "deadpool", - "redis", + "redis 0.27.5", ] [[package]] @@ -1344,7 +1376,7 @@ version = "0.99.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", @@ -1492,6 +1524,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1836,6 +1889,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "getopts" version = "0.2.21" @@ -2237,6 +2300,18 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hkdf" version = "0.12.4" @@ -2716,6 +2791,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2869,6 +2955,16 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2946,6 +3042,7 @@ dependencies = [ "axum-extra", "axum-test", "backtrace_printer", + "bb8", "byte-unit", "bytes", "cfg-if", @@ -2969,6 +3066,7 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest", + "rusty-sidekiq", "scraper", "semver", "serde", @@ -3115,7 +3213,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3222,7 +3320,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -3952,6 +4050,26 @@ dependencies = [ "zerocopy 0.8.14", ] +[[package]] +name = "redis" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8455fa3621f6b41c514946de66ea0531f57ca017b2e6c7cc368035ea5b46df" +dependencies = [ + "async-trait", + "bytes", + "combine 4.6.7", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "redis" version = "0.27.5" @@ -3991,6 +4109,17 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -4332,6 +4461,34 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +[[package]] +name = "rusty-sidekiq" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15544f047600b602c7b11ff7ee0882f9034f9cbe2c205693edd5615e2a6c03ee" +dependencies = [ + "async-trait", + "bb8", + "chrono", + "convert_case 0.6.0", + "cron_clock", + "gethostname", + "hex", + "num_cpus", + "rand 0.8.5", + "redis 0.22.3", + "serde", + "serde_json", + "serial_test", + "sha2", + "slog-term", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + [[package]] name = "ryu" version = "1.0.18" @@ -4615,6 +4772,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.8" @@ -4699,6 +4862,25 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-term" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +dependencies = [ + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "slug" version = "0.1.6" @@ -4955,6 +5137,17 @@ dependencies = [ "unic-segment", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -5594,6 +5787,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-width" version = "0.1.14" diff --git a/thoth-loco-test/Cargo.toml b/thoth-loco-test/Cargo.toml index 94fc8c18e..5402b2162 100644 --- a/thoth-loco-test/Cargo.toml +++ b/thoth-loco-test/Cargo.toml @@ -7,7 +7,7 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -loco-rs = { version = "0.14.0", default-features = false, features = ["cli"] } +loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } tokio = { version = "1.33.0", default-features = false, features = [ From dcff2b710bfa12a5763fa1d3c4b3b56b5bf6d03e Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:42:44 +0000 Subject: [PATCH 10/69] Add Loco module to dev Dockerfile --- Dockerfile.dev | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 8107eed29..96ff62fb8 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -32,18 +32,19 @@ COPY thoth-app-server/Cargo.toml thoth-app-server/Cargo.toml COPY thoth-client/Cargo.toml thoth-client/Cargo.toml COPY thoth-errors/Cargo.toml thoth-errors/Cargo.toml COPY thoth-export-server/Cargo.toml thoth-export-server/Cargo.toml +COPY thoth-loco-test/Cargo.toml thoth-loco-test/Cargo.toml RUN mkdir thoth-api/src thoth-api-server/src thoth-app/src \ thoth-app-server/src thoth-client/src thoth-errors/src \ - thoth-export-server/src + thoth-export-server/src thoth-loco-test/src RUN touch thoth-api/src/lib.rs thoth-api-server/src/lib.rs \ thoth-app/src/lib.rs thoth-app-server/src/lib.rs thoth-client/src/lib.rs \ - thoth-errors/src/lib.rs thoth-export-server/src/lib.rs + thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-loco-test/src/lib.rs RUN echo "fn main() {}" > thoth-client/build.rs RUN echo "fn main() {}" > thoth-app-server/build.rs RUN echo "fn main() {}" > thoth-export-server/build.rs RUN cargo build RUN rm -rf src thoth-api thoth-api-server thoth-app thoth-app-server thoth-client \ - thoth-errors thoth-export-server Cargo.toml Cargo.lock + thoth-errors thoth-export-server thoth-loco-test Cargo.toml Cargo.lock # Get the actual source COPY . . @@ -52,8 +53,8 @@ COPY . . # it needs to (re)compile these modules RUN touch -a -m thoth-api/src/lib.rs thoth-api-server/src/lib.rs \ thoth-app/src/lib.rs thoth-app-server/src/lib.rs thoth-client/src/lib.rs \ - thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-app-server/build.rs \ - thoth-export-server/build.rs + thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-loco-test/src/lib.rs \ + thoth-app-server/build.rs thoth-export-server/build.rs # Build Thoth for debug RUN cargo build From 45acb838f05bc20752106e50c4a5f5fc3fa8b7a6 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 15 Apr 2025 13:31:07 +0100 Subject: [PATCH 11/69] Add a skeleton background worker based on Loco docs/examples --- thoth-loco-test/src/app.rs | 6 ++++-- thoth-loco-test/src/lib.rs | 1 + thoth-loco-test/src/workers/mod.rs | 1 + thoth-loco-test/src/workers/test_worker.rs | 22 ++++++++++++++++++++ thoth-loco-test/tests/mod.rs | 1 + thoth-loco-test/tests/workers/mod.rs | 1 + thoth-loco-test/tests/workers/test_worker.rs | 20 ++++++++++++++++++ 7 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 thoth-loco-test/src/workers/mod.rs create mode 100644 thoth-loco-test/src/workers/test_worker.rs create mode 100644 thoth-loco-test/tests/mod.rs create mode 100644 thoth-loco-test/tests/workers/mod.rs create mode 100644 thoth-loco-test/tests/workers/test_worker.rs diff --git a/thoth-loco-test/src/app.rs b/thoth-loco-test/src/app.rs index 8b8b07b22..d0a29ad6a 100644 --- a/thoth-loco-test/src/app.rs +++ b/thoth-loco-test/src/app.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use loco_rs::{ app::{AppContext, Hooks, Initializer}, - bgworker::Queue, + bgworker::{BackgroundWorker, Queue}, boot::{create_app, BootResult, StartMode}, config::Config, controller::AppRoutes, @@ -9,6 +9,7 @@ use loco_rs::{ task::Tasks, Result, }; +use crate::workers::test_worker::TestWorker; pub struct App; #[async_trait] @@ -42,7 +43,8 @@ impl Hooks for App { fn routes(_ctx: &AppContext) -> AppRoutes { AppRoutes::with_default_routes() } - async fn connect_workers(_ctx: &AppContext, _queue: &Queue) -> Result<()> { + async fn connect_workers(ctx: &AppContext, queue: &Queue) -> Result<()> { + queue.register(TestWorker::build(ctx)).await?; Ok(()) } diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs index 367971416..f0429cb7e 100644 --- a/thoth-loco-test/src/lib.rs +++ b/thoth-loco-test/src/lib.rs @@ -1,4 +1,5 @@ pub mod app; +pub mod workers; use crate::app::App; use loco_rs::{ diff --git a/thoth-loco-test/src/workers/mod.rs b/thoth-loco-test/src/workers/mod.rs new file mode 100644 index 000000000..4bfbcfdd4 --- /dev/null +++ b/thoth-loco-test/src/workers/mod.rs @@ -0,0 +1 @@ +pub mod test_worker; diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-loco-test/src/workers/test_worker.rs new file mode 100644 index 000000000..c876b7490 --- /dev/null +++ b/thoth-loco-test/src/workers/test_worker.rs @@ -0,0 +1,22 @@ +use serde::{Deserialize, Serialize}; +use loco_rs::prelude::*; + +pub struct TestWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct TestWorkerArgs { +} + +#[async_trait] +impl BackgroundWorker for TestWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + async fn perform(&self, _args: TestWorkerArgs) -> Result<()> { + println!("=================TestWorker======================="); + // TODO: Some actual work goes here... + Ok(()) + } +} diff --git a/thoth-loco-test/tests/mod.rs b/thoth-loco-test/tests/mod.rs new file mode 100644 index 000000000..4d55549bf --- /dev/null +++ b/thoth-loco-test/tests/mod.rs @@ -0,0 +1 @@ +mod workers; diff --git a/thoth-loco-test/tests/workers/mod.rs b/thoth-loco-test/tests/workers/mod.rs new file mode 100644 index 000000000..1116aea79 --- /dev/null +++ b/thoth-loco-test/tests/workers/mod.rs @@ -0,0 +1 @@ +mod test_worker; diff --git a/thoth-loco-test/tests/workers/test_worker.rs b/thoth-loco-test/tests/workers/test_worker.rs new file mode 100644 index 000000000..5e2615eea --- /dev/null +++ b/thoth-loco-test/tests/workers/test_worker.rs @@ -0,0 +1,20 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use thoth_loco_test::{ + app::App, + workers::test_worker::{TestWorker, TestWorkerArgs}, +}; +use serial_test::serial; + +#[tokio::test] +#[serial] +async fn test_run_test_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!( + TestWorker::perform_later(&boot.app_context, TestWorkerArgs {}) + .await + .is_ok() + ); + // Include additional assert validations after the execution of the worker +} From 3939b0d27f1482fea513c82d3d68ab38e6fa54f4 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 15 Apr 2025 15:03:12 +0100 Subject: [PATCH 12/69] Add Task which fires Worker: works, but doesn't connect to Redis --- thoth-loco-test/src/app.rs | 10 ++++-- thoth-loco-test/src/initializers/mod.rs | 1 + .../src/initializers/test_initializer.rs | 36 +++++++++++++++++++ thoth-loco-test/src/lib.rs | 2 ++ thoth-loco-test/src/tasks/mod.rs | 1 + thoth-loco-test/src/tasks/test_task.rs | 24 +++++++++++++ 6 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 thoth-loco-test/src/initializers/mod.rs create mode 100644 thoth-loco-test/src/initializers/test_initializer.rs create mode 100644 thoth-loco-test/src/tasks/mod.rs create mode 100644 thoth-loco-test/src/tasks/test_task.rs diff --git a/thoth-loco-test/src/app.rs b/thoth-loco-test/src/app.rs index d0a29ad6a..fc30e181f 100644 --- a/thoth-loco-test/src/app.rs +++ b/thoth-loco-test/src/app.rs @@ -9,7 +9,10 @@ use loco_rs::{ task::Tasks, Result, }; -use crate::workers::test_worker::TestWorker; +use crate::{ + initializers::test_initializer::TestInitializer, + tasks::test_task::TestTask, + workers::test_worker::TestWorker}; pub struct App; #[async_trait] @@ -37,7 +40,9 @@ impl Hooks for App { } async fn initializers(_ctx: &AppContext) -> Result>> { - Ok(vec![]) + Ok(vec![ + Box::new(TestInitializer), + ]) } fn routes(_ctx: &AppContext) -> AppRoutes { @@ -50,6 +55,7 @@ impl Hooks for App { #[allow(unused_variables)] fn register_tasks(tasks: &mut Tasks) { + tasks.register(TestTask); // tasks-inject (do not remove) } } diff --git a/thoth-loco-test/src/initializers/mod.rs b/thoth-loco-test/src/initializers/mod.rs new file mode 100644 index 000000000..34a478bea --- /dev/null +++ b/thoth-loco-test/src/initializers/mod.rs @@ -0,0 +1 @@ +pub mod test_initializer; diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs new file mode 100644 index 000000000..73d7722ce --- /dev/null +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -0,0 +1,36 @@ +use async_trait::async_trait; +use loco_rs::{ + app::{AppContext, Initializer}, + task::Task, + Result, +}; +use tracing::error; + +use crate::tasks::test_task::TestTask; + +pub struct TestInitializer; + +#[async_trait] +impl Initializer for TestInitializer { + fn name(&self) -> String { + "test-initializer".to_string() + } + + async fn before_run(&self, ctx: &AppContext) -> Result<()> { + let ctx = ctx.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + + if let Err(e) = (TestTask) + .run(&ctx, &loco_rs::task::Vars::default()) + .await + { + error!("TestTask error: {:?}", e); + } + } + }); + + Ok(()) + } +} \ No newline at end of file diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs index f0429cb7e..188c7d094 100644 --- a/thoth-loco-test/src/lib.rs +++ b/thoth-loco-test/src/lib.rs @@ -1,5 +1,7 @@ pub mod app; pub mod workers; +pub mod initializers; +pub mod tasks; use crate::app::App; use loco_rs::{ diff --git a/thoth-loco-test/src/tasks/mod.rs b/thoth-loco-test/src/tasks/mod.rs new file mode 100644 index 000000000..fa62f7efe --- /dev/null +++ b/thoth-loco-test/src/tasks/mod.rs @@ -0,0 +1 @@ +pub mod test_task; diff --git a/thoth-loco-test/src/tasks/test_task.rs b/thoth-loco-test/src/tasks/test_task.rs new file mode 100644 index 000000000..23c354d42 --- /dev/null +++ b/thoth-loco-test/src/tasks/test_task.rs @@ -0,0 +1,24 @@ +use loco_rs::prelude::*; +use crate::workers::test_worker::{TestWorker, TestWorkerArgs}; + +pub struct TestTask; + +#[async_trait] +impl Task for TestTask { + fn task(&self) -> TaskInfo { + TaskInfo { + name: "test_task".to_string(), + detail: "This is a test task".to_string(), + } + } + + async fn run(&self, ctx: &AppContext, _vars: &task::Vars) -> Result<()> { + TestWorker::perform_later( + &ctx, + TestWorkerArgs { + // user_guid: "foo".to_string(), + }, + ) + .await + } +} From b708cd4701d78c0be93d346594c71af2a7ba80bb Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 23 Apr 2025 15:36:32 +0100 Subject: [PATCH 13/69] Main Loco instance should be both server and worker --- thoth-loco-test/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs index 188c7d094..e6c21ed86 100644 --- a/thoth-loco-test/src/lib.rs +++ b/thoth-loco-test/src/lib.rs @@ -24,7 +24,7 @@ pub async fn start_app() -> Result<()> { logger::init::(&config.logger)?; } - let boot_result = H::boot(StartMode::WorkerOnly, &environment, config).await?; + let boot_result = H::boot(StartMode::ServerAndWorker, &environment, config).await?; let serve_params = ServeParams { port: boot_result.app_context.config.server.port, binding: boot_result.app_context.config.server.binding.to_string(), From 608ff74ba15a87bcf89c3a54b52164be2cdf0a34 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:11:24 +0100 Subject: [PATCH 14/69] Connect Loco worker to correct Redis queue --- config/development.yaml | 1 + thoth-loco-test/src/workers/test_worker.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/config/development.yaml b/config/development.yaml index 4bbe37ec9..a5666310b 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -48,3 +48,4 @@ queue: # TODO get from environment uri: "redis://localhost:6379" dangerously_flush: false + queues: ["events:graphql"] diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-loco-test/src/workers/test_worker.rs index c876b7490..6f6e95511 100644 --- a/thoth-loco-test/src/workers/test_worker.rs +++ b/thoth-loco-test/src/workers/test_worker.rs @@ -14,6 +14,11 @@ impl BackgroundWorker for TestWorker { fn build(ctx: &AppContext) -> Self { Self { ctx: ctx.clone() } } + + fn queue() -> Option { + Some("events:graphql".to_string()) + } + async fn perform(&self, _args: TestWorkerArgs) -> Result<()> { println!("=================TestWorker======================="); // TODO: Some actual work goes here... From 8e18364c42d90a55178f384db5eaf985796cfd8d Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:35:51 +0100 Subject: [PATCH 15/69] Loco seems to expect Redis queue names to be prepended with 'queue:' --- thoth-api/src/event/handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs index e4d2eb841..36a7ccb49 100644 --- a/thoth-api/src/event/handler.rs +++ b/thoth-api/src/event/handler.rs @@ -3,7 +3,7 @@ use crate::model::work::{Work, WorkProperties}; use crate::redis::{lpush, RedisPool}; use thoth_errors::ThothResult; -const QUEUE_KEY: &str = "events:graphql"; +const QUEUE_KEY: &str = "queue:events:graphql"; pub async fn send_event( redis: &RedisPool, From b4442a683efec6f3acab4431abb96d381f02a5f7 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 24 Apr 2025 17:06:35 +0100 Subject: [PATCH 16/69] Simple output of tracing to console --- thoth-loco-test/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs index e6c21ed86..3757b403e 100644 --- a/thoth-loco-test/src/lib.rs +++ b/thoth-loco-test/src/lib.rs @@ -13,6 +13,7 @@ use loco_rs::{ #[tokio::main] pub async fn start_server() -> Result<()> { + tracing_subscriber::fmt().with_max_level(tracing::Level::DEBUG).init(); start_app::().await } From 4b0056b17b2fde87537cf26baad2caf35855e80a Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 13 May 2025 10:09:44 +0100 Subject: [PATCH 17/69] Revert queue name prefix as this is for Loco-created queues (i.e. for workers) --- config/development.yaml | 1 - thoth-api/src/event/handler.rs | 2 +- thoth-loco-test/src/workers/test_worker.rs | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/development.yaml b/config/development.yaml index a5666310b..4bbe37ec9 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -48,4 +48,3 @@ queue: # TODO get from environment uri: "redis://localhost:6379" dangerously_flush: false - queues: ["events:graphql"] diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs index 36a7ccb49..e4d2eb841 100644 --- a/thoth-api/src/event/handler.rs +++ b/thoth-api/src/event/handler.rs @@ -3,7 +3,7 @@ use crate::model::work::{Work, WorkProperties}; use crate::redis::{lpush, RedisPool}; use thoth_errors::ThothResult; -const QUEUE_KEY: &str = "queue:events:graphql"; +const QUEUE_KEY: &str = "events:graphql"; pub async fn send_event( redis: &RedisPool, diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-loco-test/src/workers/test_worker.rs index 6f6e95511..80344c191 100644 --- a/thoth-loco-test/src/workers/test_worker.rs +++ b/thoth-loco-test/src/workers/test_worker.rs @@ -15,10 +15,6 @@ impl BackgroundWorker for TestWorker { Self { ctx: ctx.clone() } } - fn queue() -> Option { - Some("events:graphql".to_string()) - } - async fn perform(&self, _args: TestWorkerArgs) -> Result<()> { println!("=================TestWorker======================="); // TODO: Some actual work goes here... From d9774eb1e738121685694330baddaf3fe3c62724 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 13 May 2025 10:17:11 +0100 Subject: [PATCH 18/69] Hardcode Redis connection (successfully retrieves queue items) --- Cargo.lock | 1 + thoth-loco-test/Cargo.toml | 1 + thoth-loco-test/src/workers/test_worker.rs | 12 +++++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b215486e4..0c4a44c20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5357,6 +5357,7 @@ version = "0.1.0" dependencies = [ "async-trait", "axum", + "deadpool-redis", "insta", "loco-rs", "regex", diff --git a/thoth-loco-test/Cargo.toml b/thoth-loco-test/Cargo.toml index 5402b2162..66bd7ce2b 100644 --- a/thoth-loco-test/Cargo.toml +++ b/thoth-loco-test/Cargo.toml @@ -7,6 +7,7 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +deadpool-redis = "0.20.0" loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-loco-test/src/workers/test_worker.rs index 80344c191..8bf0f0393 100644 --- a/thoth-loco-test/src/workers/test_worker.rs +++ b/thoth-loco-test/src/workers/test_worker.rs @@ -18,6 +18,16 @@ impl BackgroundWorker for TestWorker { async fn perform(&self, _args: TestWorkerArgs) -> Result<()> { println!("=================TestWorker======================="); // TODO: Some actual work goes here... - Ok(()) + let redis: &deadpool_redis::Pool = &deadpool_redis::Config::from_url("redis://localhost:6379") + .builder() + .expect("Failed to create redis pool.") + .build() + .expect("Failed to build redis pool."); + let mut conn = redis.get().await.expect("Failed to connect to redis pool."); + loop { + if let Ok((_, payload)) = deadpool_redis::redis::AsyncCommands::blpop::<_,(String, String)>(&mut conn, "events:graphql", 0.0).await { + tracing::info!("Received payload: {:?}", payload); + } + } } } From caafab55386c4e5452ca1f0da23946083e151a21 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 13 May 2025 10:35:51 +0100 Subject: [PATCH 19/69] Revert "Simple output of tracing to console" This reverts commit b4442a683efec6f3acab4431abb96d381f02a5f7. Clash with local untracked config --- thoth-loco-test/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs index 3757b403e..e6c21ed86 100644 --- a/thoth-loco-test/src/lib.rs +++ b/thoth-loco-test/src/lib.rs @@ -13,7 +13,6 @@ use loco_rs::{ #[tokio::main] pub async fn start_server() -> Result<()> { - tracing_subscriber::fmt().with_max_level(tracing::Level::DEBUG).init(); start_app::().await } From 9043811ca54c59ee7929a4783fcb03cfdbc5ac37 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 13 May 2025 16:41:06 +0100 Subject: [PATCH 20/69] Move Redis connection to Initializer --- .../src/initializers/test_initializer.rs | 23 ++++++++----------- thoth-loco-test/src/workers/test_worker.rs | 12 +--------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index 73d7722ce..d628aa477 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -1,12 +1,8 @@ use async_trait::async_trait; use loco_rs::{ app::{AppContext, Initializer}, - task::Task, Result, }; -use tracing::error; - -use crate::tasks::test_task::TestTask; pub struct TestInitializer; @@ -16,17 +12,18 @@ impl Initializer for TestInitializer { "test-initializer".to_string() } - async fn before_run(&self, ctx: &AppContext) -> Result<()> { - let ctx = ctx.clone(); + async fn before_run(&self, _ctx: &AppContext) -> Result<()> { + let redis: &deadpool_redis::Pool = &deadpool_redis::Config::from_url("redis://localhost:6379") + .builder() + .expect("Failed to create redis pool.") + .build() + .expect("Failed to build redis pool."); + let mut conn = redis.get().await.expect("Failed to connect to redis pool."); + tokio::spawn(async move { loop { - tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; - - if let Err(e) = (TestTask) - .run(&ctx, &loco_rs::task::Vars::default()) - .await - { - error!("TestTask error: {:?}", e); + if let Ok((_, payload)) = deadpool_redis::redis::AsyncCommands::blpop::<_,(String, String)>(&mut conn, "events:graphql", 0.0).await { + tracing::info!("Initializer received payload: {:?}", payload); } } }); diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-loco-test/src/workers/test_worker.rs index 8bf0f0393..80344c191 100644 --- a/thoth-loco-test/src/workers/test_worker.rs +++ b/thoth-loco-test/src/workers/test_worker.rs @@ -18,16 +18,6 @@ impl BackgroundWorker for TestWorker { async fn perform(&self, _args: TestWorkerArgs) -> Result<()> { println!("=================TestWorker======================="); // TODO: Some actual work goes here... - let redis: &deadpool_redis::Pool = &deadpool_redis::Config::from_url("redis://localhost:6379") - .builder() - .expect("Failed to create redis pool.") - .build() - .expect("Failed to build redis pool."); - let mut conn = redis.get().await.expect("Failed to connect to redis pool."); - loop { - if let Ok((_, payload)) = deadpool_redis::redis::AsyncCommands::blpop::<_,(String, String)>(&mut conn, "events:graphql", 0.0).await { - tracing::info!("Received payload: {:?}", payload); - } - } + Ok(()) } } From 497cf5befb00ccbd7ce4c84cd5636af550fcbbe3 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 13 May 2025 16:45:25 +0100 Subject: [PATCH 21/69] Deserialize queue items into thoth-api Event structure --- Cargo.lock | 1 + thoth-api/src/event/model.rs | 6 +++--- thoth-loco-test/Cargo.toml | 1 + thoth-loco-test/src/initializers/test_initializer.rs | 8 ++++++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c4a44c20..a0cf24957 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5365,6 +5365,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "thoth-api", "tokio", "tracing", "tracing-subscriber", diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index cb7b58dc4..5b9f23f12 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -1,15 +1,15 @@ use crate::model::Timestamp; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Serialize, juniper::GraphQLEnum)] +#[derive(Debug, Deserialize, Serialize, juniper::GraphQLEnum)] pub enum EventType { WorkCreated, WorkUpdated, WorkPublished, } -#[derive(Serialize, juniper::GraphQLInputObject)] +#[derive(Debug, Deserialize, Serialize, juniper::GraphQLInputObject)] pub struct Event { pub event_type: EventType, pub work_id: Uuid, diff --git a/thoth-loco-test/Cargo.toml b/thoth-loco-test/Cargo.toml index 66bd7ce2b..09443d12c 100644 --- a/thoth-loco-test/Cargo.toml +++ b/thoth-loco-test/Cargo.toml @@ -19,6 +19,7 @@ axum = { version = "0.8.1" } tracing = { version = "0.1.40" } tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] } regex = { version = "1.11.1" } +thoth-api = { version = "=0.13.9", path = "../thoth-api" } [dev-dependencies] loco-rs = { version = "0.14.0", default-features = false, features = ["testing"] } diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index d628aa477..55f0e84a8 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -24,6 +24,14 @@ impl Initializer for TestInitializer { loop { if let Ok((_, payload)) = deadpool_redis::redis::AsyncCommands::blpop::<_,(String, String)>(&mut conn, "events:graphql", 0.0).await { tracing::info!("Initializer received payload: {:?}", payload); + match serde_json::from_str::(&payload) { + Ok(event) => { + tracing::info!("Received event: {:?}", event); + } + Err(e) => { + tracing::error!("Invalid event payload: {}", e); + } + } } } }); From 559551b4d09df33bf615cb1941740eadcd3f8e4f Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 13 May 2025 17:08:16 +0100 Subject: [PATCH 22/69] Tidy syntax --- thoth-api/src/event/handler.rs | 2 +- thoth-loco-test/src/initializers/test_initializer.rs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs index e4d2eb841..7e176ba00 100644 --- a/thoth-api/src/event/handler.rs +++ b/thoth-api/src/event/handler.rs @@ -3,7 +3,7 @@ use crate::model::work::{Work, WorkProperties}; use crate::redis::{lpush, RedisPool}; use thoth_errors::ThothResult; -const QUEUE_KEY: &str = "events:graphql"; +pub const QUEUE_KEY: &str = "events:graphql"; pub async fn send_event( redis: &RedisPool, diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index 55f0e84a8..2f8b1060a 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -1,8 +1,10 @@ use async_trait::async_trait; +use deadpool_redis::{Config, Pool, redis::AsyncCommands}; use loco_rs::{ app::{AppContext, Initializer}, Result, }; +use thoth_api::event::{model::Event, handler::QUEUE_KEY}; pub struct TestInitializer; @@ -13,7 +15,7 @@ impl Initializer for TestInitializer { } async fn before_run(&self, _ctx: &AppContext) -> Result<()> { - let redis: &deadpool_redis::Pool = &deadpool_redis::Config::from_url("redis://localhost:6379") + let redis: &Pool = &Config::from_url("redis://localhost:6379") .builder() .expect("Failed to create redis pool.") .build() @@ -22,9 +24,9 @@ impl Initializer for TestInitializer { tokio::spawn(async move { loop { - if let Ok((_, payload)) = deadpool_redis::redis::AsyncCommands::blpop::<_,(String, String)>(&mut conn, "events:graphql", 0.0).await { + if let Ok((_, payload)) = conn.blpop::<_,(String, String)>(QUEUE_KEY, 0.0).await { tracing::info!("Initializer received payload: {:?}", payload); - match serde_json::from_str::(&payload) { + match serde_json::from_str::(&payload) { Ok(event) => { tracing::info!("Received event: {:?}", event); } From 4e943ea45cc376f0667ed7bf8c779ec9c09e2b05 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 14 May 2025 09:52:24 +0100 Subject: [PATCH 23/69] Review markups --- thoth-api/src/event/handler.rs | 4 ++-- thoth-api/src/event/model.rs | 4 ++-- thoth-api/src/graphql/model.rs | 6 ++++-- thoth-api/src/redis.rs | 15 ++++++++------- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs index 7e176ba00..46ddcf459 100644 --- a/thoth-api/src/event/handler.rs +++ b/thoth-api/src/event/handler.rs @@ -1,6 +1,6 @@ use super::model::{Event, EventType}; use crate::model::work::{Work, WorkProperties}; -use crate::redis::{lpush, RedisPool}; +use crate::redis::{rpush, RedisPool}; use thoth_errors::ThothResult; pub const QUEUE_KEY: &str = "events:graphql"; @@ -16,5 +16,5 @@ pub async fn send_event( is_published: work.is_active_withdrawn_superseded(), event_timestamp: work.updated_at, }; - lpush(redis, QUEUE_KEY, &serde_json::to_string(&event)?).await + rpush(redis, QUEUE_KEY, &serde_json::to_string(&event)?).await } diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 5b9f23f12..588b53c91 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -2,14 +2,14 @@ use crate::model::Timestamp; use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Debug, Deserialize, Serialize, juniper::GraphQLEnum)] +#[derive(Debug, Deserialize, Serialize)] pub enum EventType { WorkCreated, WorkUpdated, WorkPublished, } -#[derive(Debug, Deserialize, Serialize, juniper::GraphQLInputObject)] +#[derive(Debug, Deserialize, Serialize)] pub struct Event { pub event_type: EventType, pub work_id: Uuid, diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index fba1a68e3..95a6eb835 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1488,7 +1488,7 @@ impl MutationRoot { let result = Work::create(&context.db, &data).map_err(|e| e.into()); - if let Ok(created_work) = result.clone() { + if let Ok(ref created_work) = result { // TODO handle results throughout let _ = send_event(&context.redis, EventType::WorkCreated, &created_work).await; if created_work.work_status == WorkStatus::Active { @@ -1763,10 +1763,12 @@ impl MutationRoot { // update the work and, if it succeeds, synchronise its children statuses and pub. date match work.update(&context.db, &data, &account_id) { Ok(w) => { - let _ = send_event(&context.redis, EventType::WorkUpdated, &w).await; if w.work_status == WorkStatus::Active && work.work_status != WorkStatus::Active { let _ = send_event(&context.redis, EventType::WorkPublished, &w).await; } + else { + let _ = send_event(&context.redis, EventType::WorkUpdated, &w).await; + } // update chapters if their pub. data, withdrawn_date or work_status doesn't match the parent's for child in work.children(&context.db)? { if child.publication_date != w.publication_date diff --git a/thoth-api/src/redis.rs b/thoth-api/src/redis.rs index 071ccfee3..5fd512ff4 100644 --- a/thoth-api/src/redis.rs +++ b/thoth-api/src/redis.rs @@ -38,9 +38,9 @@ pub async fn scan_match(pool: &RedisPool, pattern: &str) -> ThothResult ThothResult { +pub async fn rpush(pool: &RedisPool, key: &str, value: &str) -> ThothResult { let mut con = create_connection(pool).await?; - con.lpush(key, value).await.map_err(Into::into) + con.rpush(key, value).await.map_err(Into::into) } #[cfg(test)] @@ -78,18 +78,19 @@ mod tests { } #[tokio::test] - async fn test_lpush() { + async fn test_rpush() { let pool = get_pool().await; let test_key = "test_queue"; let test_value_1 = "test_value_1"; let test_value_2 = "test_value_2"; - let lpush_result_1 = lpush(&pool, test_key, test_value_1).await; - assert!(lpush_result_1.is_ok()); + let rpush_result_1 = rpush(&pool, test_key, test_value_1).await; + assert!(rpush_result_1.is_ok()); + + let rpush_result_2 = rpush(&pool, test_key, test_value_2).await; + assert!(rpush_result_2.is_ok()); - let lpush_result_2 = lpush(&pool, test_key, test_value_2).await; - assert!(lpush_result_2.is_ok()); } #[tokio::test] From c3101a2588815b73542d722f0828dcdecda0d3eb Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 14 May 2025 10:40:57 +0100 Subject: [PATCH 24/69] Add Thoth version to Event --- thoth-api/src/event/handler.rs | 1 + thoth-api/src/event/model.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/thoth-api/src/event/handler.rs b/thoth-api/src/event/handler.rs index 46ddcf459..69faec6ba 100644 --- a/thoth-api/src/event/handler.rs +++ b/thoth-api/src/event/handler.rs @@ -15,6 +15,7 @@ pub async fn send_event( work_id: *work.work_id(), is_published: work.is_active_withdrawn_superseded(), event_timestamp: work.updated_at, + thoth_version: env!("CARGO_PKG_VERSION").parse().unwrap(), }; rpush(redis, QUEUE_KEY, &serde_json::to_string(&event)?).await } diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 588b53c91..bbcfada1d 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -15,4 +15,5 @@ pub struct Event { pub work_id: Uuid, pub is_published: bool, pub event_timestamp: Timestamp, + pub thoth_version: String, } From bafc926ae1d6cf6f4b06bd773220891b10be20f8 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 14 May 2025 10:47:03 +0100 Subject: [PATCH 25/69] Tidy syntax --- .../src/initializers/test_initializer.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index 2f8b1060a..bb2479192 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -1,10 +1,13 @@ use async_trait::async_trait; -use deadpool_redis::{Config, Pool, redis::AsyncCommands}; +use deadpool_redis::redis::AsyncCommands; use loco_rs::{ app::{AppContext, Initializer}, Result, }; -use thoth_api::event::{model::Event, handler::QUEUE_KEY}; +use thoth_api::{ + event::{model::Event, handler::QUEUE_KEY}, + redis::{init_pool}, +}; pub struct TestInitializer; @@ -15,11 +18,7 @@ impl Initializer for TestInitializer { } async fn before_run(&self, _ctx: &AppContext) -> Result<()> { - let redis: &Pool = &Config::from_url("redis://localhost:6379") - .builder() - .expect("Failed to create redis pool.") - .build() - .expect("Failed to build redis pool."); + let redis = init_pool("redis://localhost:6379"); let mut conn = redis.get().await.expect("Failed to connect to redis pool."); tokio::spawn(async move { From 9ef64f045805927bfd71e10397a1c940d1da6f8d Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 14 May 2025 15:57:41 +0100 Subject: [PATCH 26/69] Call Worker from Initializer loop --- .../src/initializers/test_initializer.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index bb2479192..77d35b525 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -1,13 +1,11 @@ use async_trait::async_trait; use deadpool_redis::redis::AsyncCommands; -use loco_rs::{ - app::{AppContext, Initializer}, - Result, -}; +use loco_rs::prelude::*; use thoth_api::{ event::{model::Event, handler::QUEUE_KEY}, redis::{init_pool}, }; +use crate::workers::test_worker::{TestWorker, TestWorkerArgs}; pub struct TestInitializer; @@ -17,10 +15,12 @@ impl Initializer for TestInitializer { "test-initializer".to_string() } - async fn before_run(&self, _ctx: &AppContext) -> Result<()> { + async fn before_run(&self, ctx: &AppContext) -> Result<()> { let redis = init_pool("redis://localhost:6379"); let mut conn = redis.get().await.expect("Failed to connect to redis pool."); + let ctx = ctx.clone(); + tokio::spawn(async move { loop { if let Ok((_, payload)) = conn.blpop::<_,(String, String)>(QUEUE_KEY, 0.0).await { @@ -28,6 +28,11 @@ impl Initializer for TestInitializer { match serde_json::from_str::(&payload) { Ok(event) => { tracing::info!("Received event: {:?}", event); + let _ = TestWorker::perform_later( + &ctx, + TestWorkerArgs {}, + ) + .await; } Err(e) => { tracing::error!("Invalid event payload: {}", e); From 97cee9046afca2c9460c49b805c61d7e7a45ef19 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 14 May 2025 16:54:04 +0100 Subject: [PATCH 27/69] Add BLPOP to Thoth Redis methods and replace deadpool dependency --- Cargo.lock | 1 - thoth-api/src/redis.rs | 15 ++++++++++++++- thoth-loco-test/Cargo.toml | 1 - .../src/initializers/test_initializer.rs | 7 ++----- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0cf24957..76fcadd6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5357,7 +5357,6 @@ version = "0.1.0" dependencies = [ "async-trait", "axum", - "deadpool-redis", "insta", "loco-rs", "regex", diff --git a/thoth-api/src/redis.rs b/thoth-api/src/redis.rs index 5fd512ff4..89ea4aa44 100644 --- a/thoth-api/src/redis.rs +++ b/thoth-api/src/redis.rs @@ -43,6 +43,12 @@ pub async fn rpush(pool: &RedisPool, key: &str, value: &str) -> ThothResult ThothResult { + let mut con = create_connection(pool).await?; + let (_, value): (_, String) = con.blpop::<_, (String, String)>(key, 0.0).await?; + Ok(value) +} + #[cfg(test)] mod tests { use super::*; @@ -78,7 +84,7 @@ mod tests { } #[tokio::test] - async fn test_rpush() { + async fn test_rpush_and_blpop() { let pool = get_pool().await; let test_key = "test_queue"; @@ -91,6 +97,13 @@ mod tests { let rpush_result_2 = rpush(&pool, test_key, test_value_2).await; assert!(rpush_result_2.is_ok()); + let blpop_result_1 = blpop(&pool, test_key).await; + assert!(blpop_result_1.is_ok()); + assert_eq!(blpop_result_1.unwrap(), test_value_1); + + let blpop_result_2 = blpop(&pool, test_key).await; + assert!(blpop_result_2.is_ok()); + assert_eq!(blpop_result_2.unwrap(), test_value_2); } #[tokio::test] diff --git a/thoth-loco-test/Cargo.toml b/thoth-loco-test/Cargo.toml index 09443d12c..a48981a40 100644 --- a/thoth-loco-test/Cargo.toml +++ b/thoth-loco-test/Cargo.toml @@ -7,7 +7,6 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -deadpool-redis = "0.20.0" loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index 77d35b525..715d8ceb2 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -1,9 +1,8 @@ use async_trait::async_trait; -use deadpool_redis::redis::AsyncCommands; use loco_rs::prelude::*; use thoth_api::{ event::{model::Event, handler::QUEUE_KEY}, - redis::{init_pool}, + redis::{blpop, init_pool}, }; use crate::workers::test_worker::{TestWorker, TestWorkerArgs}; @@ -17,13 +16,11 @@ impl Initializer for TestInitializer { async fn before_run(&self, ctx: &AppContext) -> Result<()> { let redis = init_pool("redis://localhost:6379"); - let mut conn = redis.get().await.expect("Failed to connect to redis pool."); - let ctx = ctx.clone(); tokio::spawn(async move { loop { - if let Ok((_, payload)) = conn.blpop::<_,(String, String)>(QUEUE_KEY, 0.0).await { + if let Ok(payload) = blpop(&redis, QUEUE_KEY).await { tracing::info!("Initializer received payload: {:?}", payload); match serde_json::from_str::(&payload) { Ok(event) => { From ddf086e68fc5ddddcf6d144e6a6d3c1d58ed2e58 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 14 May 2025 17:16:08 +0100 Subject: [PATCH 28/69] Remove Task which is now unused --- thoth-loco-test/src/app.rs | 2 -- thoth-loco-test/src/lib.rs | 1 - thoth-loco-test/src/tasks/mod.rs | 1 - thoth-loco-test/src/tasks/test_task.rs | 24 ------------------------ 4 files changed, 28 deletions(-) delete mode 100644 thoth-loco-test/src/tasks/mod.rs delete mode 100644 thoth-loco-test/src/tasks/test_task.rs diff --git a/thoth-loco-test/src/app.rs b/thoth-loco-test/src/app.rs index fc30e181f..45a908833 100644 --- a/thoth-loco-test/src/app.rs +++ b/thoth-loco-test/src/app.rs @@ -11,7 +11,6 @@ use loco_rs::{ }; use crate::{ initializers::test_initializer::TestInitializer, - tasks::test_task::TestTask, workers::test_worker::TestWorker}; pub struct App; @@ -55,7 +54,6 @@ impl Hooks for App { #[allow(unused_variables)] fn register_tasks(tasks: &mut Tasks) { - tasks.register(TestTask); // tasks-inject (do not remove) } } diff --git a/thoth-loco-test/src/lib.rs b/thoth-loco-test/src/lib.rs index e6c21ed86..ecd9e3640 100644 --- a/thoth-loco-test/src/lib.rs +++ b/thoth-loco-test/src/lib.rs @@ -1,7 +1,6 @@ pub mod app; pub mod workers; pub mod initializers; -pub mod tasks; use crate::app::App; use loco_rs::{ diff --git a/thoth-loco-test/src/tasks/mod.rs b/thoth-loco-test/src/tasks/mod.rs deleted file mode 100644 index fa62f7efe..000000000 --- a/thoth-loco-test/src/tasks/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod test_task; diff --git a/thoth-loco-test/src/tasks/test_task.rs b/thoth-loco-test/src/tasks/test_task.rs deleted file mode 100644 index 23c354d42..000000000 --- a/thoth-loco-test/src/tasks/test_task.rs +++ /dev/null @@ -1,24 +0,0 @@ -use loco_rs::prelude::*; -use crate::workers::test_worker::{TestWorker, TestWorkerArgs}; - -pub struct TestTask; - -#[async_trait] -impl Task for TestTask { - fn task(&self) -> TaskInfo { - TaskInfo { - name: "test_task".to_string(), - detail: "This is a test task".to_string(), - } - } - - async fn run(&self, ctx: &AppContext, _vars: &task::Vars) -> Result<()> { - TestWorker::perform_later( - &ctx, - TestWorkerArgs { - // user_guid: "foo".to_string(), - }, - ) - .await - } -} From 45effe57d968a394d00dbc464ece543bb3a31bff Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 15 May 2025 16:00:42 +0100 Subject: [PATCH 29/69] Pass Event from Initializer to Worker to be processed further --- thoth-loco-test/src/initializers/test_initializer.rs | 6 +++++- thoth-loco-test/src/workers/test_worker.rs | 11 ++++++++--- thoth-loco-test/tests/workers/test_worker.rs | 11 ++++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-loco-test/src/initializers/test_initializer.rs index 715d8ceb2..7dd2acd86 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-loco-test/src/initializers/test_initializer.rs @@ -15,6 +15,7 @@ impl Initializer for TestInitializer { } async fn before_run(&self, ctx: &AppContext) -> Result<()> { + //TODO remove hardcoding let redis = init_pool("redis://localhost:6379"); let ctx = ctx.clone(); @@ -25,9 +26,12 @@ impl Initializer for TestInitializer { match serde_json::from_str::(&payload) { Ok(event) => { tracing::info!("Received event: {:?}", event); + //TODO match on the type of event & pass to different workers let _ = TestWorker::perform_later( &ctx, - TestWorkerArgs {}, + TestWorkerArgs { + event: event, + }, ) .await; } diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-loco-test/src/workers/test_worker.rs index 80344c191..f237e013c 100644 --- a/thoth-loco-test/src/workers/test_worker.rs +++ b/thoth-loco-test/src/workers/test_worker.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use loco_rs::prelude::*; +use thoth_api::event::model::Event; pub struct TestWorker { pub ctx: AppContext, @@ -7,6 +8,7 @@ pub struct TestWorker { #[derive(Deserialize, Debug, Serialize)] pub struct TestWorkerArgs { + pub event: Event, } #[async_trait] @@ -15,9 +17,12 @@ impl BackgroundWorker for TestWorker { Self { ctx: ctx.clone() } } - async fn perform(&self, _args: TestWorkerArgs) -> Result<()> { - println!("=================TestWorker======================="); - // TODO: Some actual work goes here... + async fn perform(&self, args: TestWorkerArgs) -> Result<()> { + tracing::info!("TestWorker start"); + tracing::info!("Event: {:?}", args.event); + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + tracing::info!("TestWorker end"); + Ok(()) } } diff --git a/thoth-loco-test/tests/workers/test_worker.rs b/thoth-loco-test/tests/workers/test_worker.rs index 5e2615eea..7105b1339 100644 --- a/thoth-loco-test/tests/workers/test_worker.rs +++ b/thoth-loco-test/tests/workers/test_worker.rs @@ -1,4 +1,5 @@ use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use thoth_api::event::model::{Event, EventType}; use thoth_loco_test::{ app::App, workers::test_worker::{TestWorker, TestWorkerArgs}, @@ -12,7 +13,15 @@ async fn test_run_test_worker_worker() { // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background assert!( - TestWorker::perform_later(&boot.app_context, TestWorkerArgs {}) + TestWorker::perform_later(&boot.app_context, TestWorkerArgs { + event: Event { + event_type: EventType::WorkUpdated, + work_id: Default::default(), + is_published: Default::default(), + event_timestamp: Default::default(), + thoth_version: Default::default(), + } + }) .await .is_ok() ); From fbb76f6de12456b4674c58fcc9cce617569ce93a Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 11:24:52 +0100 Subject: [PATCH 30/69] Proper naming --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- Dockerfile.dev | 10 +++++----- config/development.yaml | 4 ++-- src/bin/commands/start.rs | 8 ++++---- src/bin/thoth.rs | 2 +- src/lib.rs | 2 +- thoth-loco-test/src/initializers/mod.rs | 1 - thoth-loco-test/src/workers/mod.rs | 1 - thoth-loco-test/tests/workers/mod.rs | 1 - {thoth-loco-test => thoth-processor}/Cargo.toml | 2 +- {thoth-loco-test => thoth-processor}/src/app.rs | 8 ++++---- .../src/initializers/handle_events.rs | 12 ++++++------ thoth-processor/src/initializers/mod.rs | 1 + {thoth-loco-test => thoth-processor}/src/lib.rs | 0 thoth-processor/src/workers/mod.rs | 1 + .../src/workers/work_updated_worker.rs | 12 ++++++------ {thoth-loco-test => thoth-processor}/tests/mod.rs | 0 thoth-processor/tests/workers/mod.rs | 1 + .../tests/workers/work_updated_worker.rs | 8 ++++---- 20 files changed, 41 insertions(+), 41 deletions(-) delete mode 100644 thoth-loco-test/src/initializers/mod.rs delete mode 100644 thoth-loco-test/src/workers/mod.rs delete mode 100644 thoth-loco-test/tests/workers/mod.rs rename {thoth-loco-test => thoth-processor}/Cargo.toml (97%) rename {thoth-loco-test => thoth-processor}/src/app.rs (86%) rename thoth-loco-test/src/initializers/test_initializer.rs => thoth-processor/src/initializers/handle_events.rs (81%) create mode 100644 thoth-processor/src/initializers/mod.rs rename {thoth-loco-test => thoth-processor}/src/lib.rs (100%) create mode 100644 thoth-processor/src/workers/mod.rs rename thoth-loco-test/src/workers/test_worker.rs => thoth-processor/src/workers/work_updated_worker.rs (58%) rename {thoth-loco-test => thoth-processor}/tests/mod.rs (100%) create mode 100644 thoth-processor/tests/workers/mod.rs rename thoth-loco-test/tests/workers/test_worker.rs => thoth-processor/tests/workers/work_updated_worker.rs (77%) diff --git a/Cargo.lock b/Cargo.lock index 277eee0ba..bf20c8c89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5200,7 +5200,7 @@ dependencies = [ "thoth-app-server", "thoth-errors", "thoth-export-server", - "thoth-loco-test", + "thoth-processor", "tokio", ] @@ -5352,7 +5352,7 @@ dependencies = [ ] [[package]] -name = "thoth-loco-test" +name = "thoth-processor" version = "0.1.0" dependencies = [ "async-trait", diff --git a/Cargo.toml b/Cargo.toml index ff97258af..8e3a0ec08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ readme = "README.md" maintenance = { status = "actively-developed" } [workspace] -members = ["thoth-api", "thoth-api-server", "thoth-app", "thoth-app-server", "thoth-client", "thoth-errors", "thoth-export-server", "thoth-loco-test"] +members = ["thoth-api", "thoth-api-server", "thoth-app", "thoth-app-server", "thoth-client", "thoth-errors", "thoth-export-server", "thoth-processor"] [dependencies] thoth-api = { version = "=0.13.11", path = "thoth-api", features = ["backend"] } @@ -20,7 +20,7 @@ thoth-api-server = { version = "=0.13.11", path = "thoth-api-server" } thoth-app-server = { version = "=0.13.11", path = "thoth-app-server" } thoth-errors = { version = "=0.13.11", path = "thoth-errors" } thoth-export-server = { version = "=0.13.11", path = "thoth-export-server" } -thoth-loco-test = { version = "=0.1.0", path = "thoth-loco-test" } +thoth-processor = { version = "=0.1.0", path = "thoth-processor" } clap = { version = "4.5.32", features = ["cargo", "env"] } dialoguer = { version = "0.11.0", features = ["password"] } dotenv = "0.15.0" diff --git a/Dockerfile.dev b/Dockerfile.dev index 96ff62fb8..461e7492f 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -32,19 +32,19 @@ COPY thoth-app-server/Cargo.toml thoth-app-server/Cargo.toml COPY thoth-client/Cargo.toml thoth-client/Cargo.toml COPY thoth-errors/Cargo.toml thoth-errors/Cargo.toml COPY thoth-export-server/Cargo.toml thoth-export-server/Cargo.toml -COPY thoth-loco-test/Cargo.toml thoth-loco-test/Cargo.toml +COPY thoth-processor/Cargo.toml thoth-processor/Cargo.toml RUN mkdir thoth-api/src thoth-api-server/src thoth-app/src \ thoth-app-server/src thoth-client/src thoth-errors/src \ - thoth-export-server/src thoth-loco-test/src + thoth-export-server/src thoth-processor/src RUN touch thoth-api/src/lib.rs thoth-api-server/src/lib.rs \ thoth-app/src/lib.rs thoth-app-server/src/lib.rs thoth-client/src/lib.rs \ - thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-loco-test/src/lib.rs + thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-processor/src/lib.rs RUN echo "fn main() {}" > thoth-client/build.rs RUN echo "fn main() {}" > thoth-app-server/build.rs RUN echo "fn main() {}" > thoth-export-server/build.rs RUN cargo build RUN rm -rf src thoth-api thoth-api-server thoth-app thoth-app-server thoth-client \ - thoth-errors thoth-export-server thoth-loco-test Cargo.toml Cargo.lock + thoth-errors thoth-export-server thoth-processor Cargo.toml Cargo.lock # Get the actual source COPY . . @@ -53,7 +53,7 @@ COPY . . # it needs to (re)compile these modules RUN touch -a -m thoth-api/src/lib.rs thoth-api-server/src/lib.rs \ thoth-app/src/lib.rs thoth-app-server/src/lib.rs thoth-client/src/lib.rs \ - thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-loco-test/src/lib.rs \ + thoth-errors/src/lib.rs thoth-export-server/src/lib.rs thoth-processor/src/lib.rs \ thoth-app-server/build.rs thoth-export-server/build.rs # Build Thoth for debug diff --git a/config/development.yaml b/config/development.yaml index 4bbe37ec9..a7ccdc6e6 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -1,5 +1,5 @@ -# Loco configuration file documentation -# TODO a better location would be thoth/thoth-loco-test/config, but that doesn't get picked up - +# Loco (thoth-processor) configuration file documentation +# TODO a better location would be thoth/thoth-processor/config, but that doesn't get picked up - # see if the path where it searches can be changed # Application logging configuration diff --git a/src/bin/commands/start.rs b/src/bin/commands/start.rs index b91ce484a..bf03ed98a 100644 --- a/src/bin/commands/start.rs +++ b/src/bin/commands/start.rs @@ -1,7 +1,7 @@ use crate::arguments; use clap::{ArgMatches, Command}; use lazy_static::lazy_static; -use thoth::{api_server, app_server, errors::ThothResult, export_server, loco_server}; +use thoth::{api_server, app_server, errors::ThothResult, export_server, processor_server}; lazy_static! { pub(crate) static ref COMMAND: Command = Command::new("start") @@ -41,7 +41,7 @@ lazy_static! { .arg(arguments::export_url()) .arg(arguments::gql_endpoint()), ) - .subcommand(Command::new("loco").about("Start loco")); + .subcommand(Command::new("processor").about("Start the thoth event processor")); } pub fn graphql_api(arguments: &ArgMatches) -> ThothResult<()> { @@ -104,6 +104,6 @@ pub fn export_api(arguments: &ArgMatches) -> ThothResult<()> { .map_err(|e| e.into()) } -pub fn loco() -> ThothResult<()> { - loco_server().map_err(|e| thoth_errors::ThothError::InternalError(e.to_string())) +pub fn processor() -> ThothResult<()> { + processor_server().map_err(|e| thoth_errors::ThothError::InternalError(e.to_string())) } diff --git a/src/bin/thoth.rs b/src/bin/thoth.rs index 5baee1812..57de3ba81 100644 --- a/src/bin/thoth.rs +++ b/src/bin/thoth.rs @@ -24,7 +24,7 @@ fn main() -> thoth::errors::ThothResult<()> { Some(("graphql-api", arguments)) => commands::start::graphql_api(arguments), Some(("app", arguments)) => commands::start::app(arguments), Some(("export-api", arguments)) => commands::start::export_api(arguments), - Some(("loco", _)) => commands::start::loco(), + Some(("processor", _)) => commands::start::processor(), _ => unreachable!(), }, Some(("migrate", arguments)) => commands::migrate(arguments), diff --git a/src/lib.rs b/src/lib.rs index 93d0b61af..d1dafe6f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,4 +3,4 @@ pub use thoth_api_server::start_server as api_server; pub use thoth_app_server::start_server as app_server; pub use thoth_errors as errors; pub use thoth_export_server::{start_server as export_server, ALL_SPECIFICATIONS}; -pub use thoth_loco_test::start_server as loco_server; +pub use thoth_processor::start_server as processor_server; diff --git a/thoth-loco-test/src/initializers/mod.rs b/thoth-loco-test/src/initializers/mod.rs deleted file mode 100644 index 34a478bea..000000000 --- a/thoth-loco-test/src/initializers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod test_initializer; diff --git a/thoth-loco-test/src/workers/mod.rs b/thoth-loco-test/src/workers/mod.rs deleted file mode 100644 index 4bfbcfdd4..000000000 --- a/thoth-loco-test/src/workers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod test_worker; diff --git a/thoth-loco-test/tests/workers/mod.rs b/thoth-loco-test/tests/workers/mod.rs deleted file mode 100644 index 1116aea79..000000000 --- a/thoth-loco-test/tests/workers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod test_worker; diff --git a/thoth-loco-test/Cargo.toml b/thoth-processor/Cargo.toml similarity index 97% rename from thoth-loco-test/Cargo.toml rename to thoth-processor/Cargo.toml index 05c33522b..fa26d5efc 100644 --- a/thoth-loco-test/Cargo.toml +++ b/thoth-processor/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "thoth-loco-test" +name = "thoth-processor" version = "0.1.0" edition = "2021" publish = false diff --git a/thoth-loco-test/src/app.rs b/thoth-processor/src/app.rs similarity index 86% rename from thoth-loco-test/src/app.rs rename to thoth-processor/src/app.rs index 45a908833..ad3f599fc 100644 --- a/thoth-loco-test/src/app.rs +++ b/thoth-processor/src/app.rs @@ -10,8 +10,8 @@ use loco_rs::{ Result, }; use crate::{ - initializers::test_initializer::TestInitializer, - workers::test_worker::TestWorker}; + initializers::handle_events::HandleEvents, + workers::work_updated_worker::WorkUpdatedWorker}; pub struct App; #[async_trait] @@ -40,7 +40,7 @@ impl Hooks for App { async fn initializers(_ctx: &AppContext) -> Result>> { Ok(vec![ - Box::new(TestInitializer), + Box::new(HandleEvents), ]) } @@ -48,7 +48,7 @@ impl Hooks for App { AppRoutes::with_default_routes() } async fn connect_workers(ctx: &AppContext, queue: &Queue) -> Result<()> { - queue.register(TestWorker::build(ctx)).await?; + queue.register(WorkUpdatedWorker::build(ctx)).await?; Ok(()) } diff --git a/thoth-loco-test/src/initializers/test_initializer.rs b/thoth-processor/src/initializers/handle_events.rs similarity index 81% rename from thoth-loco-test/src/initializers/test_initializer.rs rename to thoth-processor/src/initializers/handle_events.rs index 7dd2acd86..471c48bc7 100644 --- a/thoth-loco-test/src/initializers/test_initializer.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -4,14 +4,14 @@ use thoth_api::{ event::{model::Event, handler::QUEUE_KEY}, redis::{blpop, init_pool}, }; -use crate::workers::test_worker::{TestWorker, TestWorkerArgs}; +use crate::workers::work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}; -pub struct TestInitializer; +pub struct HandleEvents; #[async_trait] -impl Initializer for TestInitializer { +impl Initializer for HandleEvents { fn name(&self) -> String { - "test-initializer".to_string() + "handle-events".to_string() } async fn before_run(&self, ctx: &AppContext) -> Result<()> { @@ -27,9 +27,9 @@ impl Initializer for TestInitializer { Ok(event) => { tracing::info!("Received event: {:?}", event); //TODO match on the type of event & pass to different workers - let _ = TestWorker::perform_later( + let _ = WorkUpdatedWorker::perform_later( &ctx, - TestWorkerArgs { + WorkUpdatedWorkerArgs { event: event, }, ) diff --git a/thoth-processor/src/initializers/mod.rs b/thoth-processor/src/initializers/mod.rs new file mode 100644 index 000000000..4ab300b81 --- /dev/null +++ b/thoth-processor/src/initializers/mod.rs @@ -0,0 +1 @@ +pub mod handle_events; diff --git a/thoth-loco-test/src/lib.rs b/thoth-processor/src/lib.rs similarity index 100% rename from thoth-loco-test/src/lib.rs rename to thoth-processor/src/lib.rs diff --git a/thoth-processor/src/workers/mod.rs b/thoth-processor/src/workers/mod.rs new file mode 100644 index 000000000..04befbe05 --- /dev/null +++ b/thoth-processor/src/workers/mod.rs @@ -0,0 +1 @@ +pub mod work_updated_worker; diff --git a/thoth-loco-test/src/workers/test_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs similarity index 58% rename from thoth-loco-test/src/workers/test_worker.rs rename to thoth-processor/src/workers/work_updated_worker.rs index f237e013c..7679593fb 100644 --- a/thoth-loco-test/src/workers/test_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -2,26 +2,26 @@ use serde::{Deserialize, Serialize}; use loco_rs::prelude::*; use thoth_api::event::model::Event; -pub struct TestWorker { +pub struct WorkUpdatedWorker { pub ctx: AppContext, } #[derive(Deserialize, Debug, Serialize)] -pub struct TestWorkerArgs { +pub struct WorkUpdatedWorkerArgs { pub event: Event, } #[async_trait] -impl BackgroundWorker for TestWorker { +impl BackgroundWorker for WorkUpdatedWorker { fn build(ctx: &AppContext) -> Self { Self { ctx: ctx.clone() } } - async fn perform(&self, args: TestWorkerArgs) -> Result<()> { - tracing::info!("TestWorker start"); + async fn perform(&self, args: WorkUpdatedWorkerArgs) -> Result<()> { + tracing::info!("WorkUpdatedWorker start"); tracing::info!("Event: {:?}", args.event); tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; - tracing::info!("TestWorker end"); + tracing::info!("WorkUpdatedWorker end"); Ok(()) } diff --git a/thoth-loco-test/tests/mod.rs b/thoth-processor/tests/mod.rs similarity index 100% rename from thoth-loco-test/tests/mod.rs rename to thoth-processor/tests/mod.rs diff --git a/thoth-processor/tests/workers/mod.rs b/thoth-processor/tests/workers/mod.rs new file mode 100644 index 000000000..03ebe2880 --- /dev/null +++ b/thoth-processor/tests/workers/mod.rs @@ -0,0 +1 @@ +mod work_updated_worker; diff --git a/thoth-loco-test/tests/workers/test_worker.rs b/thoth-processor/tests/workers/work_updated_worker.rs similarity index 77% rename from thoth-loco-test/tests/workers/test_worker.rs rename to thoth-processor/tests/workers/work_updated_worker.rs index 7105b1339..ffd06ad27 100644 --- a/thoth-loco-test/tests/workers/test_worker.rs +++ b/thoth-processor/tests/workers/work_updated_worker.rs @@ -1,19 +1,19 @@ use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; use thoth_api::event::model::{Event, EventType}; -use thoth_loco_test::{ +use thoth_processor::{ app::App, - workers::test_worker::{TestWorker, TestWorkerArgs}, + workers::work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}, }; use serial_test::serial; #[tokio::test] #[serial] -async fn test_run_test_worker_worker() { +async fn test_run_work_updated_worker_worker() { let boot = boot_test::().await.unwrap(); // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background assert!( - TestWorker::perform_later(&boot.app_context, TestWorkerArgs { + WorkUpdatedWorker::perform_later(&boot.app_context, WorkUpdatedWorkerArgs { event: Event { event_type: EventType::WorkUpdated, work_id: Default::default(), From e75c06fea9add86789a760ef249441b8191da2d7 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 11:51:11 +0100 Subject: [PATCH 31/69] Send Event to different Worker based on event type --- thoth-processor/src/app.rs | 9 ++++- .../src/initializers/handle_events.rs | 38 ++++++++++++++----- thoth-processor/src/workers/mod.rs | 2 + .../src/workers/work_created_worker.rs | 28 ++++++++++++++ .../src/workers/work_published_worker.rs | 28 ++++++++++++++ thoth-processor/tests/workers/mod.rs | 2 + .../tests/workers/work_created_worker.rs | 29 ++++++++++++++ .../tests/workers/work_published_worker.rs | 29 ++++++++++++++ 8 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 thoth-processor/src/workers/work_created_worker.rs create mode 100644 thoth-processor/src/workers/work_published_worker.rs create mode 100644 thoth-processor/tests/workers/work_created_worker.rs create mode 100644 thoth-processor/tests/workers/work_published_worker.rs diff --git a/thoth-processor/src/app.rs b/thoth-processor/src/app.rs index ad3f599fc..2633d1c41 100644 --- a/thoth-processor/src/app.rs +++ b/thoth-processor/src/app.rs @@ -11,7 +11,12 @@ use loco_rs::{ }; use crate::{ initializers::handle_events::HandleEvents, - workers::work_updated_worker::WorkUpdatedWorker}; + workers::{ + work_created_worker::WorkCreatedWorker, + work_updated_worker::WorkUpdatedWorker, + work_published_worker::WorkPublishedWorker, + }, +}; pub struct App; #[async_trait] @@ -48,7 +53,9 @@ impl Hooks for App { AppRoutes::with_default_routes() } async fn connect_workers(ctx: &AppContext, queue: &Queue) -> Result<()> { + queue.register(WorkCreatedWorker::build(ctx)).await?; queue.register(WorkUpdatedWorker::build(ctx)).await?; + queue.register(WorkPublishedWorker::build(ctx)).await?; Ok(()) } diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs index 471c48bc7..1b77a852b 100644 --- a/thoth-processor/src/initializers/handle_events.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -1,10 +1,12 @@ use async_trait::async_trait; use loco_rs::prelude::*; use thoth_api::{ - event::{model::Event, handler::QUEUE_KEY}, + event::{model::{Event, EventType}, handler::QUEUE_KEY}, redis::{blpop, init_pool}, }; +use crate::workers::work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}; use crate::workers::work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}; +use crate::workers::work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}; pub struct HandleEvents; @@ -26,14 +28,32 @@ impl Initializer for HandleEvents { match serde_json::from_str::(&payload) { Ok(event) => { tracing::info!("Received event: {:?}", event); - //TODO match on the type of event & pass to different workers - let _ = WorkUpdatedWorker::perform_later( - &ctx, - WorkUpdatedWorkerArgs { - event: event, - }, - ) - .await; + let _ = match event.event_type { + EventType::WorkCreated => WorkCreatedWorker::perform_later + ( + &ctx, + WorkCreatedWorkerArgs { + event: event, + }, + ) + .await, + EventType::WorkUpdated => WorkUpdatedWorker::perform_later + ( + &ctx, + WorkUpdatedWorkerArgs { + event: event, + }, + ) + .await, + EventType::WorkPublished => WorkPublishedWorker::perform_later + ( + &ctx, + WorkPublishedWorkerArgs { + event: event, + }, + ) + .await, + }; } Err(e) => { tracing::error!("Invalid event payload: {}", e); diff --git a/thoth-processor/src/workers/mod.rs b/thoth-processor/src/workers/mod.rs index 04befbe05..8e525d4ac 100644 --- a/thoth-processor/src/workers/mod.rs +++ b/thoth-processor/src/workers/mod.rs @@ -1 +1,3 @@ +pub mod work_created_worker; pub mod work_updated_worker; +pub mod work_published_worker; diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs new file mode 100644 index 000000000..a0091c3f7 --- /dev/null +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; +use loco_rs::prelude::*; +use thoth_api::event::model::Event; + +pub struct WorkCreatedWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct WorkCreatedWorkerArgs { + pub event: Event, +} + +#[async_trait] +impl BackgroundWorker for WorkCreatedWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: WorkCreatedWorkerArgs) -> Result<()> { + tracing::info!("WorkCreatedWorker start"); + tracing::info!("Event: {:?}", args.event); + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + tracing::info!("WorkCreatedWorker end"); + + Ok(()) + } +} diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs new file mode 100644 index 000000000..b52eca9db --- /dev/null +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; +use loco_rs::prelude::*; +use thoth_api::event::model::Event; + +pub struct WorkPublishedWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct WorkPublishedWorkerArgs { + pub event: Event, +} + +#[async_trait] +impl BackgroundWorker for WorkPublishedWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: WorkPublishedWorkerArgs) -> Result<()> { + tracing::info!("WorkPublishedWorker start"); + tracing::info!("Event: {:?}", args.event); + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + tracing::info!("WorkPublishedWorker end"); + + Ok(()) + } +} diff --git a/thoth-processor/tests/workers/mod.rs b/thoth-processor/tests/workers/mod.rs index 03ebe2880..c250d0a07 100644 --- a/thoth-processor/tests/workers/mod.rs +++ b/thoth-processor/tests/workers/mod.rs @@ -1 +1,3 @@ +mod work_created_worker; mod work_updated_worker; +mod work_published_worker; diff --git a/thoth-processor/tests/workers/work_created_worker.rs b/thoth-processor/tests/workers/work_created_worker.rs new file mode 100644 index 000000000..f4c93c370 --- /dev/null +++ b/thoth-processor/tests/workers/work_created_worker.rs @@ -0,0 +1,29 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use thoth_api::event::model::{Event, EventType}; +use thoth_processor::{ + app::App, + workers::work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}, +}; +use serial_test::serial; + +#[tokio::test] +#[serial] +async fn test_run_work_created_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!( + WorkCreatedWorker::perform_later(&boot.app_context, WorkCreatedWorkerArgs { + event: Event { + event_type: EventType::WorkCreated, + work_id: Default::default(), + is_published: Default::default(), + event_timestamp: Default::default(), + thoth_version: Default::default(), + } + }) + .await + .is_ok() + ); + // Include additional assert validations after the execution of the worker +} diff --git a/thoth-processor/tests/workers/work_published_worker.rs b/thoth-processor/tests/workers/work_published_worker.rs new file mode 100644 index 000000000..261f97f82 --- /dev/null +++ b/thoth-processor/tests/workers/work_published_worker.rs @@ -0,0 +1,29 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use thoth_api::event::model::{Event, EventType}; +use thoth_processor::{ + app::App, + workers::work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}, +}; +use serial_test::serial; + +#[tokio::test] +#[serial] +async fn test_run_work_published_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!( + WorkPublishedWorker::perform_later(&boot.app_context, WorkPublishedWorkerArgs { + event: Event { + event_type: EventType::WorkPublished, + work_id: Default::default(), + is_published: Default::default(), + event_timestamp: Default::default(), + thoth_version: Default::default(), + } + }) + .await + .is_ok() + ); + // Include additional assert validations after the execution of the worker +} From d869a829c279517f7f81aba63ec409886c9a302c Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 11:53:34 +0100 Subject: [PATCH 32/69] Fix formatting --- thoth-api/src/graphql/model.rs | 3 +- thoth-processor/src/app.rs | 19 ++++---- .../src/initializers/handle_events.rs | 48 ++++++++++--------- thoth-processor/src/lib.rs | 2 +- thoth-processor/src/workers/mod.rs | 2 +- .../src/workers/work_created_worker.rs | 2 +- .../src/workers/work_published_worker.rs | 2 +- .../src/workers/work_updated_worker.rs | 2 +- thoth-processor/tests/workers/mod.rs | 2 +- .../tests/workers/work_created_worker.rs | 15 +++--- .../tests/workers/work_published_worker.rs | 15 +++--- .../tests/workers/work_updated_worker.rs | 15 +++--- 12 files changed, 64 insertions(+), 63 deletions(-) diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index bdb36abeb..d20670d88 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1768,8 +1768,7 @@ impl MutationRoot { Ok(w) => { if w.work_status == WorkStatus::Active && work.work_status != WorkStatus::Active { let _ = send_event(&context.redis, EventType::WorkPublished, &w).await; - } - else { + } else { let _ = send_event(&context.redis, EventType::WorkUpdated, &w).await; } // update chapters if their pub. data, withdrawn_date or work_status doesn't match the parent's diff --git a/thoth-processor/src/app.rs b/thoth-processor/src/app.rs index 2633d1c41..07c240e7b 100644 --- a/thoth-processor/src/app.rs +++ b/thoth-processor/src/app.rs @@ -1,3 +1,10 @@ +use crate::{ + initializers::handle_events::HandleEvents, + workers::{ + work_created_worker::WorkCreatedWorker, work_published_worker::WorkPublishedWorker, + work_updated_worker::WorkUpdatedWorker, + }, +}; use async_trait::async_trait; use loco_rs::{ app::{AppContext, Hooks, Initializer}, @@ -9,14 +16,6 @@ use loco_rs::{ task::Tasks, Result, }; -use crate::{ - initializers::handle_events::HandleEvents, - workers::{ - work_created_worker::WorkCreatedWorker, - work_updated_worker::WorkUpdatedWorker, - work_published_worker::WorkPublishedWorker, - }, -}; pub struct App; #[async_trait] @@ -44,9 +43,7 @@ impl Hooks for App { } async fn initializers(_ctx: &AppContext) -> Result>> { - Ok(vec![ - Box::new(HandleEvents), - ]) + Ok(vec![Box::new(HandleEvents)]) } fn routes(_ctx: &AppContext) -> AppRoutes { diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs index 1b77a852b..25f10b1b8 100644 --- a/thoth-processor/src/initializers/handle_events.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -1,12 +1,17 @@ +use crate::workers::{ + work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}, + work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}, + work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}, +}; use async_trait::async_trait; use loco_rs::prelude::*; use thoth_api::{ - event::{model::{Event, EventType}, handler::QUEUE_KEY}, + event::{ + handler::QUEUE_KEY, + model::{Event, EventType}, + }, redis::{blpop, init_pool}, }; -use crate::workers::work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}; -use crate::workers::work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}; -use crate::workers::work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}; pub struct HandleEvents; @@ -29,30 +34,27 @@ impl Initializer for HandleEvents { Ok(event) => { tracing::info!("Received event: {:?}", event); let _ = match event.event_type { - EventType::WorkCreated => WorkCreatedWorker::perform_later - ( + EventType::WorkCreated => { + WorkCreatedWorker::perform_later( &ctx, - WorkCreatedWorkerArgs { - event: event, - }, + WorkCreatedWorkerArgs { event: event }, ) - .await, - EventType::WorkUpdated => WorkUpdatedWorker::perform_later - ( + .await + } + EventType::WorkUpdated => { + WorkUpdatedWorker::perform_later( &ctx, - WorkUpdatedWorkerArgs { - event: event, - }, + WorkUpdatedWorkerArgs { event: event }, ) - .await, - EventType::WorkPublished => WorkPublishedWorker::perform_later - ( + .await + } + EventType::WorkPublished => { + WorkPublishedWorker::perform_later( &ctx, - WorkPublishedWorkerArgs { - event: event, - }, + WorkPublishedWorkerArgs { event: event }, ) - .await, + .await + } }; } Err(e) => { @@ -65,4 +67,4 @@ impl Initializer for HandleEvents { Ok(()) } -} \ No newline at end of file +} diff --git a/thoth-processor/src/lib.rs b/thoth-processor/src/lib.rs index ecd9e3640..16ad4df7b 100644 --- a/thoth-processor/src/lib.rs +++ b/thoth-processor/src/lib.rs @@ -1,6 +1,6 @@ pub mod app; -pub mod workers; pub mod initializers; +pub mod workers; use crate::app::App; use loco_rs::{ diff --git a/thoth-processor/src/workers/mod.rs b/thoth-processor/src/workers/mod.rs index 8e525d4ac..72682cae2 100644 --- a/thoth-processor/src/workers/mod.rs +++ b/thoth-processor/src/workers/mod.rs @@ -1,3 +1,3 @@ pub mod work_created_worker; -pub mod work_updated_worker; pub mod work_published_worker; +pub mod work_updated_worker; diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs index a0091c3f7..83ecd1740 100644 --- a/thoth-processor/src/workers/work_created_worker.rs +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -1,5 +1,5 @@ -use serde::{Deserialize, Serialize}; use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; pub struct WorkCreatedWorker { diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index b52eca9db..057e9eb02 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -1,5 +1,5 @@ -use serde::{Deserialize, Serialize}; use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; pub struct WorkPublishedWorker { diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs index 7679593fb..845266dcd 100644 --- a/thoth-processor/src/workers/work_updated_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -1,5 +1,5 @@ -use serde::{Deserialize, Serialize}; use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; pub struct WorkUpdatedWorker { diff --git a/thoth-processor/tests/workers/mod.rs b/thoth-processor/tests/workers/mod.rs index c250d0a07..cc74d4c06 100644 --- a/thoth-processor/tests/workers/mod.rs +++ b/thoth-processor/tests/workers/mod.rs @@ -1,3 +1,3 @@ mod work_created_worker; -mod work_updated_worker; mod work_published_worker; +mod work_updated_worker; diff --git a/thoth-processor/tests/workers/work_created_worker.rs b/thoth-processor/tests/workers/work_created_worker.rs index f4c93c370..ce7e6b8c4 100644 --- a/thoth-processor/tests/workers/work_created_worker.rs +++ b/thoth-processor/tests/workers/work_created_worker.rs @@ -1,10 +1,10 @@ use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; use thoth_api::event::model::{Event, EventType}; use thoth_processor::{ app::App, workers::work_created_worker::{WorkCreatedWorker, WorkCreatedWorkerArgs}, }; -use serial_test::serial; #[tokio::test] #[serial] @@ -12,8 +12,9 @@ async fn test_run_work_created_worker_worker() { let boot = boot_test::().await.unwrap(); // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background - assert!( - WorkCreatedWorker::perform_later(&boot.app_context, WorkCreatedWorkerArgs { + assert!(WorkCreatedWorker::perform_later( + &boot.app_context, + WorkCreatedWorkerArgs { event: Event { event_type: EventType::WorkCreated, work_id: Default::default(), @@ -21,9 +22,9 @@ async fn test_run_work_created_worker_worker() { event_timestamp: Default::default(), thoth_version: Default::default(), } - }) - .await - .is_ok() - ); + } + ) + .await + .is_ok()); // Include additional assert validations after the execution of the worker } diff --git a/thoth-processor/tests/workers/work_published_worker.rs b/thoth-processor/tests/workers/work_published_worker.rs index 261f97f82..fccd193e9 100644 --- a/thoth-processor/tests/workers/work_published_worker.rs +++ b/thoth-processor/tests/workers/work_published_worker.rs @@ -1,10 +1,10 @@ use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; use thoth_api::event::model::{Event, EventType}; use thoth_processor::{ app::App, workers::work_published_worker::{WorkPublishedWorker, WorkPublishedWorkerArgs}, }; -use serial_test::serial; #[tokio::test] #[serial] @@ -12,8 +12,9 @@ async fn test_run_work_published_worker_worker() { let boot = boot_test::().await.unwrap(); // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background - assert!( - WorkPublishedWorker::perform_later(&boot.app_context, WorkPublishedWorkerArgs { + assert!(WorkPublishedWorker::perform_later( + &boot.app_context, + WorkPublishedWorkerArgs { event: Event { event_type: EventType::WorkPublished, work_id: Default::default(), @@ -21,9 +22,9 @@ async fn test_run_work_published_worker_worker() { event_timestamp: Default::default(), thoth_version: Default::default(), } - }) - .await - .is_ok() - ); + } + ) + .await + .is_ok()); // Include additional assert validations after the execution of the worker } diff --git a/thoth-processor/tests/workers/work_updated_worker.rs b/thoth-processor/tests/workers/work_updated_worker.rs index ffd06ad27..42007c3da 100644 --- a/thoth-processor/tests/workers/work_updated_worker.rs +++ b/thoth-processor/tests/workers/work_updated_worker.rs @@ -1,10 +1,10 @@ use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; use thoth_api::event::model::{Event, EventType}; use thoth_processor::{ app::App, workers::work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}, }; -use serial_test::serial; #[tokio::test] #[serial] @@ -12,8 +12,9 @@ async fn test_run_work_updated_worker_worker() { let boot = boot_test::().await.unwrap(); // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background - assert!( - WorkUpdatedWorker::perform_later(&boot.app_context, WorkUpdatedWorkerArgs { + assert!(WorkUpdatedWorker::perform_later( + &boot.app_context, + WorkUpdatedWorkerArgs { event: Event { event_type: EventType::WorkUpdated, work_id: Default::default(), @@ -21,9 +22,9 @@ async fn test_run_work_updated_worker_worker() { event_timestamp: Default::default(), thoth_version: Default::default(), } - }) - .await - .is_ok() - ); + } + ) + .await + .is_ok()); // Include additional assert validations after the execution of the worker } From 1550400e8c930a7551304fcad64db16ad3251bbe Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 11:56:59 +0100 Subject: [PATCH 33/69] Fix linting --- thoth-api/src/graphql/model.rs | 4 ++-- thoth-processor/src/initializers/handle_events.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index d20670d88..a934472f7 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1488,9 +1488,9 @@ impl MutationRoot { if let Ok(ref created_work) = result { // TODO handle results throughout - let _ = send_event(&context.redis, EventType::WorkCreated, &created_work).await; + let _ = send_event(&context.redis, EventType::WorkCreated, created_work).await; if created_work.work_status == WorkStatus::Active { - let _ = send_event(&context.redis, EventType::WorkPublished, &created_work).await; + let _ = send_event(&context.redis, EventType::WorkPublished, created_work).await; } } diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs index 25f10b1b8..918e9f9c6 100644 --- a/thoth-processor/src/initializers/handle_events.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -37,21 +37,21 @@ impl Initializer for HandleEvents { EventType::WorkCreated => { WorkCreatedWorker::perform_later( &ctx, - WorkCreatedWorkerArgs { event: event }, + WorkCreatedWorkerArgs { event }, ) .await } EventType::WorkUpdated => { WorkUpdatedWorker::perform_later( &ctx, - WorkUpdatedWorkerArgs { event: event }, + WorkUpdatedWorkerArgs { event }, ) .await } EventType::WorkPublished => { WorkPublishedWorker::perform_later( &ctx, - WorkPublishedWorkerArgs { event: event }, + WorkPublishedWorkerArgs { event }, ) .await } From f94ef3c67ddb40bbe75a7df8c2f16887881d9573 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 12:11:43 +0100 Subject: [PATCH 34/69] Loco tests require thoth-processor/config/test.yaml file to pass (copied from config/development.yaml) --- thoth-processor/config/test.yaml | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 thoth-processor/config/test.yaml diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml new file mode 100644 index 000000000..ebf808376 --- /dev/null +++ b/thoth-processor/config/test.yaml @@ -0,0 +1,48 @@ +# Loco (thoth-processor) configuration file documentation + +# Application logging configuration +logger: + # Enable or disable logging. + enable: true + # Enable pretty backtrace (sets RUST_BACKTRACE=1) + pretty_backtrace: true + # Log level, options: trace, debug, info, warn or error. + level: debug + # Define the logging format. options: compact, pretty or json + format: compact + # By default the logger has filtering only logs that came from your code or logs that came from `loco` framework. to see all third party libraries + # Uncomment the line below to override to see all third party libraries you can enable this config and override the logger filters. + # override_filter: trace + +# Web server configuration +server: + # Port on which the server will listen. the server binding is 0.0.0.0:{PORT} + port: 5150 + # Binding for the server (which interface to bind to) + binding: localhost + # The UI hostname or IP address that mailers will point to. + host: http://localhost + # Out of the box middleware configuration. to disable middleware you can changed the `enable` field to `false` of comment the middleware block + middlewares: + +# Initializers Configuration +# initializers: +# oauth2: +# authorization_code: # Authorization code grant type +# - client_identifier: google # Identifier for the OAuth2 provider. Replace 'google' with your provider's name if different, must be unique within the oauth2 config. +# ... other fields + +# Worker Configuration +workers: + # specifies the worker mode. Options: + # - BackgroundQueue - Workers operate asynchronously in the background, processing queued. + # - ForegroundBlocking - Workers operate in the foreground and block until tasks are completed. + # - BackgroundAsync - Workers operate asynchronously in the background, processing tasks with async capabilities. + mode: BackgroundQueue + +queue: + kind: Redis + # Redis connection URI + # TODO get from environment + uri: "redis://localhost:6379" + dangerously_flush: false From 62973f9439ffe81b734587089dc357c7395de8ce Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 14:38:34 +0100 Subject: [PATCH 35/69] Nest Loco config under thoth-processor --- .env.example | 2 ++ {config => thoth-processor/config}/development.yaml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) rename {config => thoth-processor/config}/development.yaml (93%) diff --git a/.env.example b/.env.example index 311a47a38..edab17732 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,8 @@ REDIS_URL=redis://localhost:6379 SECRET_KEY=an_up_to_255_bytes_random_key # Logging level RUST_LOG=info +# Location of Loco (thoth-processor) config folder +LOCO_CONFIG_FOLDER=thoth-processor/config/ # Uncomment the following if running with docker # DATABASE_URL=postgres://thoth:thoth@db/thoth diff --git a/config/development.yaml b/thoth-processor/config/development.yaml similarity index 93% rename from config/development.yaml rename to thoth-processor/config/development.yaml index a7ccdc6e6..ebf808376 100644 --- a/config/development.yaml +++ b/thoth-processor/config/development.yaml @@ -1,6 +1,4 @@ # Loco (thoth-processor) configuration file documentation -# TODO a better location would be thoth/thoth-processor/config, but that doesn't get picked up - -# see if the path where it searches can be changed # Application logging configuration logger: From b6e3c88906b53e62d29777a403acac56122adb62 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 14:47:00 +0100 Subject: [PATCH 36/69] Add processor to infrastructure files --- Makefile | 9 ++++++++- docker-compose.yml | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6b0bf3bc7..ea4c76b6d 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ build-graphql-api \ build-export-api \ build-app \ + build-processor run-app \ run-graphql-api \ run-export-api \ + run-processor watch-app \ docker-dev \ docker-dev-build \ @@ -19,9 +21,12 @@ check \ check-all \ -all: build-graphql-api build-export-api build-app +all: build-graphql-api build-export-api build-app build-processor check-all: test check clippy check-format +run-processor: build-processor + RUST_BACKTRACE=1 cargo run start processor + run-app: build-app RUST_BACKTRACE=1 cargo run start app @@ -57,6 +62,8 @@ build-export-api: build build-app: build +build-processor: build + test: cargo test --workspace diff --git a/docker-compose.yml b/docker-compose.yml index 1fba394cb..fef5a6968 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,5 +44,15 @@ services: - graphql-api - export-api + processor: + image: ghcr.io/thoth-pub/thoth + container_name: "thoth_processor" + restart: unless-stopped + command: ["start", "processor"] + env_file: + - .env + depends_on: + - redis + volumes: db: From 94d1b4a661a053e1b3c307743b87d0928ac85c2f Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 15:33:47 +0100 Subject: [PATCH 37/69] Get Redis URI from environment --- thoth-processor/config/development.yaml | 3 +-- thoth-processor/config/test.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/thoth-processor/config/development.yaml b/thoth-processor/config/development.yaml index ebf808376..05a86bb54 100644 --- a/thoth-processor/config/development.yaml +++ b/thoth-processor/config/development.yaml @@ -43,6 +43,5 @@ workers: queue: kind: Redis # Redis connection URI - # TODO get from environment - uri: "redis://localhost:6379" + uri: {{ get_env(name="REDIS_URL") }} dangerously_flush: false diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml index ebf808376..a4fbaa956 100644 --- a/thoth-processor/config/test.yaml +++ b/thoth-processor/config/test.yaml @@ -43,6 +43,5 @@ workers: queue: kind: Redis # Redis connection URI - # TODO get from environment - uri: "redis://localhost:6379" + uri: {{ get_env(name="TEST_REDIS_URL") }} dangerously_flush: false From fa53d25e41dba5326d7948f2b4fe60399edf4769 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 19 May 2025 17:08:02 +0100 Subject: [PATCH 38/69] Retrive Redis URL from config instead of hardcoding --- thoth-processor/src/initializers/handle_events.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs index 918e9f9c6..5c5ee3140 100644 --- a/thoth-processor/src/initializers/handle_events.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -4,7 +4,7 @@ use crate::workers::{ work_updated_worker::{WorkUpdatedWorker, WorkUpdatedWorkerArgs}, }; use async_trait::async_trait; -use loco_rs::prelude::*; +use loco_rs::{config::QueueConfig, prelude::*}; use thoth_api::{ event::{ handler::QUEUE_KEY, @@ -22,9 +22,12 @@ impl Initializer for HandleEvents { } async fn before_run(&self, ctx: &AppContext) -> Result<()> { - //TODO remove hardcoding - let redis = init_pool("redis://localhost:6379"); let ctx = ctx.clone(); + let redis_url = match ctx.config.queue.as_ref().unwrap() { + QueueConfig::Redis(queue) => &queue.uri, + _ => unreachable!(), + }; + let redis = init_pool(redis_url); tokio::spawn(async move { loop { From e56c42a33ca7d514b4eae9d91433ffff453a8198 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 20 May 2025 14:17:24 +0100 Subject: [PATCH 39/69] Ignore Clippy warning (can't change size of loco_rs::Error) --- thoth-processor/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/thoth-processor/src/lib.rs b/thoth-processor/src/lib.rs index 16ad4df7b..b69d00408 100644 --- a/thoth-processor/src/lib.rs +++ b/thoth-processor/src/lib.rs @@ -11,6 +11,7 @@ use loco_rs::{ }; #[tokio::main] +#[allow(clippy::result_large_err)] pub async fn start_server() -> Result<()> { start_app::().await } From 3c402f80c1c976c9ebd26efc2b2696f099d1e36b Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 21 May 2025 17:29:38 +0100 Subject: [PATCH 40/69] Add Webhooks to database model and implement minimal API query --- thoth-api/migrations/v0.13.12/down.sql | 5 +++ thoth-api/migrations/v0.13.12/up.sql | 29 +++++++++++++ thoth-api/src/event/model.rs | 57 +++++++++++++++++++++++++- thoth-api/src/graphql/model.rs | 27 +++++++++++- thoth-api/src/schema.rs | 34 +++++++++++++++ 5 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 thoth-api/migrations/v0.13.12/down.sql create mode 100644 thoth-api/migrations/v0.13.12/up.sql diff --git a/thoth-api/migrations/v0.13.12/down.sql b/thoth-api/migrations/v0.13.12/down.sql new file mode 100644 index 000000000..fedc9e93d --- /dev/null +++ b/thoth-api/migrations/v0.13.12/down.sql @@ -0,0 +1,5 @@ +DROP TRIGGER set_updated_at ON publisher_webhook; +DROP TABLE IF EXISTS publisher_webhook; +DROP TRIGGER set_updated_at ON webhook; +DROP TABLE IF EXISTS webhook; +DROP TYPE IF EXISTS event_type; diff --git a/thoth-api/migrations/v0.13.12/up.sql b/thoth-api/migrations/v0.13.12/up.sql new file mode 100644 index 000000000..807da4232 --- /dev/null +++ b/thoth-api/migrations/v0.13.12/up.sql @@ -0,0 +1,29 @@ +CREATE TYPE event_type AS ENUM ( + 'WorkCreated', + 'WorkUpdated', + 'WorkPublished' +); + +CREATE TABLE webhook ( + webhook_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + endpoint TEXT NOT NULL CHECK (endpoint ~* '^[^:]*:\/\/(?:[^\/:]*:[^\/@]*@)?(?:[^\/:.]*\.)+([^:\/]+)'), + token TEXT CHECK (OCTET_LENGTH(token) >= 1), + is_published BOOLEAN NOT NULL, + event_type event_type NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +SELECT diesel_manage_updated_at('webhook'); + +CREATE INDEX idx_webhook_endpoint ON webhook (endpoint); + +CREATE TABLE publisher_webhook ( + webhook_id UUID NOT NULL REFERENCES webhook(webhook_id) ON DELETE CASCADE, + publisher_id UUID NOT NULL REFERENCES publisher(publisher_id) ON DELETE CASCADE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (webhook_id, publisher_id) +); +SELECT diesel_manage_updated_at('publisher_webhook'); + +CREATE INDEX idx_publisher_webhook_webhook_id ON publisher_webhook (webhook_id); diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index bbcfada1d..6b7be4737 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -2,7 +2,13 @@ use crate::model::Timestamp; use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Debug, Deserialize, Serialize)] +#[cfg_attr( + feature = "backend", + derive(DbEnum, juniper::GraphQLEnum), + graphql(description = "Nature of an event"), + ExistingTypePath = "crate::schema::sql_types::EventType" +)] +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] pub enum EventType { WorkCreated, WorkUpdated, @@ -17,3 +23,52 @@ pub struct Event { pub event_timestamp: Timestamp, pub thoth_version: String, } + +#[cfg_attr(feature = "backend", derive(Queryable, juniper::GraphQLObject))] +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub webhook_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, + pub created_at: Timestamp, + pub updated_at: Timestamp, +} + +#[cfg(feature = "backend")] +use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; +use thoth_errors::ThothResult; +impl Webhook { + pub fn all( + db: &crate::db::PgPool, + limit: i32, + offset: i32, + publisher_id: Option, + event_types: Vec, + published: Option, + ) -> ThothResult> { + use crate::schema::webhook::dsl::*; + let mut connection = db.get()?; + let mut query = webhook + .inner_join(crate::schema::publisher_webhook::table) + .select(crate::schema::webhook::all_columns) + .into_boxed(); + + if let Some(pid) = publisher_id { + query = query.filter(crate::schema::publisher_webhook::publisher_id.eq(pid)); + } + if !event_types.is_empty() { + query = query.filter(event_type.eq_any(event_types)); + } + if let Some(boolean) = published { + query = query.filter(is_published.eq(boolean)); + } + query + .limit(limit.into()) + .offset(offset.into()) + .load::(&mut connection) + .map_err(Into::into) + } +} diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index a934472f7..73b65c5c2 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -8,7 +8,7 @@ use crate::account::model::AccountAccess; use crate::account::model::DecodedToken; use crate::db::PgPool; use crate::event::handler::send_event; -use crate::event::model::EventType; +use crate::event::model::{EventType, Webhook}; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; @@ -3185,6 +3185,31 @@ impl Publisher { ) .map_err(|e| e.into()) } + + #[graphql(description = "Get webhooks linked to this publisher")] + pub fn webhooks( + &self, + context: &Context, + #[graphql(default = 100, description = "The number of items to return")] limit: Option, + #[graphql(default = 0, description = "The number of items to skip")] offset: Option, + #[graphql( + default = vec![], + description = "Specific types to filter by", + )] + event_types: Option>, + #[graphql(description = "Only show results where is_published is True")] + is_published: Option, + ) -> FieldResult> { + Webhook::all( + &context.db, + limit.unwrap_or_default(), + offset.unwrap_or_default(), + Some(self.publisher_id), + event_types.unwrap_or_default(), + is_published, + ) + .map_err(|e| e.into()) + } } #[juniper::graphql_object(Context = Context, description = "The brand under which a publisher issues works.")] diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index e78c5350f..e94f2cf5d 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -46,6 +46,10 @@ pub mod sql_types { #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] #[diesel(postgres_type(name = "relation_type"))] pub struct RelationType; + + #[derive(diesel::sql_types::SqlType, diesel::query_builder::QueryId)] + #[diesel(postgres_type(name = "event_type"))] + pub struct EventType; } table! { @@ -400,6 +404,17 @@ table! { } } +table! { + use diesel::sql_types::*; + + publisher_webhook (webhook_id, publisher_id) { + webhook_id -> Uuid, + publisher_id -> Uuid, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + table! { use diesel::sql_types::*; @@ -514,6 +529,21 @@ table! { } } +table! { + use diesel::sql_types::*; + use super::sql_types::EventType; + + webhook (webhook_id) { + webhook_id -> Uuid, + endpoint -> Text, + token -> Nullable, + is_published -> Bool, + event_type -> EventType, + created_at -> Timestamptz, + updated_at -> Timestamptz, + } +} + table! { use diesel::sql_types::*; use super::sql_types::WorkType; @@ -638,6 +668,8 @@ joinable!(publisher_account -> account (account_id)); joinable!(publisher_account -> publisher (publisher_id)); joinable!(publisher_history -> account (account_id)); joinable!(publisher_history -> publisher (publisher_id)); +joinable!(publisher_webhook -> webhook (webhook_id)); +joinable!(publisher_webhook -> publisher (publisher_id)); joinable!(reference -> work (work_id)); joinable!(reference_history -> account (account_id)); joinable!(reference_history -> reference (reference_id)); @@ -681,12 +713,14 @@ allow_tables_to_appear_in_same_query!( publisher, publisher_account, publisher_history, + publisher_webhook, reference, reference_history, series, series_history, subject, subject_history, + webhook, work, work_history, work_relation, From e057e30d2380b0d809ef447992b1c969b07e1ebd Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 11:00:35 +0100 Subject: [PATCH 41/69] Webhooks should be nested under publisher rather than shared between them --- thoth-api/migrations/v0.13.12/down.sql | 3 +-- thoth-api/migrations/v0.13.12/up.sql | 17 +++++++------- thoth-api/src/event/model.rs | 4 ++-- thoth-api/src/schema.rs | 31 ++++++++++++++------------ 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/thoth-api/migrations/v0.13.12/down.sql b/thoth-api/migrations/v0.13.12/down.sql index fedc9e93d..cce46d74a 100644 --- a/thoth-api/migrations/v0.13.12/down.sql +++ b/thoth-api/migrations/v0.13.12/down.sql @@ -1,5 +1,4 @@ -DROP TRIGGER set_updated_at ON publisher_webhook; -DROP TABLE IF EXISTS publisher_webhook; +DROP TABLE IF EXISTS webhook_history; DROP TRIGGER set_updated_at ON webhook; DROP TABLE IF EXISTS webhook; DROP TYPE IF EXISTS event_type; diff --git a/thoth-api/migrations/v0.13.12/up.sql b/thoth-api/migrations/v0.13.12/up.sql index 807da4232..3261bf411 100644 --- a/thoth-api/migrations/v0.13.12/up.sql +++ b/thoth-api/migrations/v0.13.12/up.sql @@ -6,6 +6,7 @@ CREATE TYPE event_type AS ENUM ( CREATE TABLE webhook ( webhook_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + publisher_id UUID NOT NULL REFERENCES publisher(publisher_id) ON DELETE CASCADE, endpoint TEXT NOT NULL CHECK (endpoint ~* '^[^:]*:\/\/(?:[^\/:]*:[^\/@]*@)?(?:[^\/:.]*\.)+([^:\/]+)'), token TEXT CHECK (OCTET_LENGTH(token) >= 1), is_published BOOLEAN NOT NULL, @@ -16,14 +17,12 @@ CREATE TABLE webhook ( SELECT diesel_manage_updated_at('webhook'); CREATE INDEX idx_webhook_endpoint ON webhook (endpoint); +CREATE INDEX idx_webhook_publisher_id ON webhook (publisher_id); -CREATE TABLE publisher_webhook ( - webhook_id UUID NOT NULL REFERENCES webhook(webhook_id) ON DELETE CASCADE, - publisher_id UUID NOT NULL REFERENCES publisher(publisher_id) ON DELETE CASCADE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (webhook_id, publisher_id) +CREATE TABLE webhook_history ( + webhook_history_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + webhook_id UUID NOT NULL REFERENCES webhook(webhook_id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES account(account_id), + data JSONB NOT NULL, + timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); -SELECT diesel_manage_updated_at('publisher_webhook'); - -CREATE INDEX idx_publisher_webhook_webhook_id ON publisher_webhook (webhook_id); diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 6b7be4737..e2a68e60a 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -29,6 +29,7 @@ pub struct Event { #[serde(rename_all = "camelCase")] pub struct Webhook { pub webhook_id: Uuid, + pub publisher_id: Uuid, pub endpoint: String, pub token: Option, pub is_published: bool, @@ -52,12 +53,11 @@ impl Webhook { use crate::schema::webhook::dsl::*; let mut connection = db.get()?; let mut query = webhook - .inner_join(crate::schema::publisher_webhook::table) .select(crate::schema::webhook::all_columns) .into_boxed(); if let Some(pid) = publisher_id { - query = query.filter(crate::schema::publisher_webhook::publisher_id.eq(pid)); + query = query.filter(publisher_id.eq(pid)); } if !event_types.is_empty() { query = query.filter(event_type.eq_any(event_types)); diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index e94f2cf5d..1405c9342 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -404,17 +404,6 @@ table! { } } -table! { - use diesel::sql_types::*; - - publisher_webhook (webhook_id, publisher_id) { - webhook_id -> Uuid, - publisher_id -> Uuid, - created_at -> Timestamptz, - updated_at -> Timestamptz, - } -} - table! { use diesel::sql_types::*; @@ -535,6 +524,7 @@ table! { webhook (webhook_id) { webhook_id -> Uuid, + publisher_id -> Uuid, endpoint -> Text, token -> Nullable, is_published -> Bool, @@ -544,6 +534,18 @@ table! { } } +table! { + use diesel::sql_types::*; + + webhook_history (webhook_history_id) { + webhook_history_id -> Uuid, + webhook_id -> Uuid, + account_id -> Uuid, + data -> Jsonb, + timestamp -> Timestamptz, + } +} + table! { use diesel::sql_types::*; use super::sql_types::WorkType; @@ -668,8 +670,6 @@ joinable!(publisher_account -> account (account_id)); joinable!(publisher_account -> publisher (publisher_id)); joinable!(publisher_history -> account (account_id)); joinable!(publisher_history -> publisher (publisher_id)); -joinable!(publisher_webhook -> webhook (webhook_id)); -joinable!(publisher_webhook -> publisher (publisher_id)); joinable!(reference -> work (work_id)); joinable!(reference_history -> account (account_id)); joinable!(reference_history -> reference (reference_id)); @@ -679,6 +679,9 @@ joinable!(series_history -> series (series_id)); joinable!(subject -> work (work_id)); joinable!(subject_history -> account (account_id)); joinable!(subject_history -> subject (subject_id)); +joinable!(webhook -> publisher (publisher_id)); +joinable!(webhook_history -> account (account_id)); +joinable!(webhook_history -> webhook (webhook_id)); joinable!(work -> imprint (imprint_id)); joinable!(work_history -> account (account_id)); joinable!(work_history -> work (work_id)); @@ -713,7 +716,6 @@ allow_tables_to_appear_in_same_query!( publisher, publisher_account, publisher_history, - publisher_webhook, reference, reference_history, series, @@ -721,6 +723,7 @@ allow_tables_to_appear_in_same_query!( subject, subject_history, webhook, + webhook_history, work, work_history, work_relation, From 56283269085da94661144e57ad80c3453c90a488 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 11:38:53 +0100 Subject: [PATCH 42/69] Use standard crud pattern for webhooks --- thoth-api/src/event/model.rs | 49 -------- thoth-api/src/graphql/model.rs | 15 ++- thoth-api/src/model/mod.rs | 1 + thoth-api/src/model/webhook/crud.rs | 175 ++++++++++++++++++++++++++++ thoth-api/src/model/webhook/mod.rs | 113 ++++++++++++++++++ 5 files changed, 302 insertions(+), 51 deletions(-) create mode 100644 thoth-api/src/model/webhook/crud.rs create mode 100644 thoth-api/src/model/webhook/mod.rs diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index e2a68e60a..139181821 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -23,52 +23,3 @@ pub struct Event { pub event_timestamp: Timestamp, pub thoth_version: String, } - -#[cfg_attr(feature = "backend", derive(Queryable, juniper::GraphQLObject))] -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct Webhook { - pub webhook_id: Uuid, - pub publisher_id: Uuid, - pub endpoint: String, - pub token: Option, - pub is_published: bool, - pub event_type: EventType, - pub created_at: Timestamp, - pub updated_at: Timestamp, -} - -#[cfg(feature = "backend")] -use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; -use thoth_errors::ThothResult; -impl Webhook { - pub fn all( - db: &crate::db::PgPool, - limit: i32, - offset: i32, - publisher_id: Option, - event_types: Vec, - published: Option, - ) -> ThothResult> { - use crate::schema::webhook::dsl::*; - let mut connection = db.get()?; - let mut query = webhook - .select(crate::schema::webhook::all_columns) - .into_boxed(); - - if let Some(pid) = publisher_id { - query = query.filter(publisher_id.eq(pid)); - } - if !event_types.is_empty() { - query = query.filter(event_type.eq_any(event_types)); - } - if let Some(boolean) = published { - query = query.filter(is_published.eq(boolean)); - } - query - .limit(limit.into()) - .offset(offset.into()) - .load::(&mut connection) - .map_err(Into::into) - } -} diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 73b65c5c2..b23ab3fae 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -8,7 +8,7 @@ use crate::account::model::AccountAccess; use crate::account::model::DecodedToken; use crate::db::PgPool; use crate::event::handler::send_event; -use crate::event::model::{EventType, Webhook}; +use crate::event::model::EventType; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; @@ -24,6 +24,7 @@ use crate::model::publisher::*; use crate::model::reference::*; use crate::model::series::*; use crate::model::subject::*; +use crate::model::webhook::*; use crate::model::work::*; use crate::model::work_relation::*; use crate::model::Convert; @@ -3192,20 +3193,30 @@ impl Publisher { context: &Context, #[graphql(default = 100, description = "The number of items to return")] limit: Option, #[graphql(default = 0, description = "The number of items to skip")] offset: Option, + #[graphql( + default = WebhookOrderBy::default(), + description = "The order in which to sort the results" + )] + order: Option, #[graphql( default = vec![], description = "Specific types to filter by", )] event_types: Option>, - #[graphql(description = "Only show results where is_published is True")] + #[graphql(description = "Only show results where IsPublished is True")] is_published: Option, ) -> FieldResult> { Webhook::all( &context.db, limit.unwrap_or_default(), offset.unwrap_or_default(), + None, + order.unwrap_or_default(), + vec![], Some(self.publisher_id), + None, event_types.unwrap_or_default(), + vec![], is_published, ) .map_err(|e| e.into()) diff --git a/thoth-api/src/model/mod.rs b/thoth-api/src/model/mod.rs index b8a087479..7a417c0c1 100644 --- a/thoth-api/src/model/mod.rs +++ b/thoth-api/src/model/mod.rs @@ -1095,5 +1095,6 @@ pub mod publisher; pub mod reference; pub mod series; pub mod subject; +pub mod webhook; pub mod work; pub mod work_relation; diff --git a/thoth-api/src/model/webhook/crud.rs b/thoth-api/src/model/webhook/crud.rs new file mode 100644 index 000000000..b7efb74bd --- /dev/null +++ b/thoth-api/src/model/webhook/crud.rs @@ -0,0 +1,175 @@ +use super::{ + NewWebhook, NewWebhookHistory, PatchWebhook, Webhook, WebhookField, WebhookHistory, + WebhookOrderBy, +}; +use crate::event::model::EventType; +use crate::graphql::utils::Direction; +use crate::model::{Crud, DbInsert, HistoryEntry}; +use crate::schema::{webhook, webhook_history}; +use crate::{crud_methods, db_insert}; +use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; +use thoth_errors::ThothResult; +use uuid::Uuid; + +impl Crud for Webhook { + type NewEntity = NewWebhook; + type PatchEntity = PatchWebhook; + type OrderByEntity = WebhookOrderBy; + type FilterParameter1 = EventType; + type FilterParameter2 = (); + type FilterParameter3 = bool; + + fn pk(&self) -> Uuid { + self.webhook_id + } + + fn all( + db: &crate::db::PgPool, + limit: i32, + offset: i32, + _: Option, + order: Self::OrderByEntity, + publishers: Vec, + parent_id_1: Option, + _: Option, + event_types: Vec, + _: Vec, + published: Option, + ) -> ThothResult> { + use crate::schema::webhook::dsl::*; + let mut connection = db.get()?; + let mut query = webhook.into_boxed(); + + query = match order.field { + WebhookField::WebhookId => match order.direction { + Direction::Asc => query.order(webhook_id.asc()), + Direction::Desc => query.order(webhook_id.desc()), + }, + WebhookField::PublisherId => match order.direction { + Direction::Asc => query.order(publisher_id.asc()), + Direction::Desc => query.order(publisher_id.desc()), + }, + WebhookField::Endpoint => match order.direction { + Direction::Asc => query.order(endpoint.asc()), + Direction::Desc => query.order(endpoint.desc()), + }, + WebhookField::Token => match order.direction { + Direction::Asc => query.order(token.asc()), + Direction::Desc => query.order(token.desc()), + }, + WebhookField::IsPublished => match order.direction { + Direction::Asc => query.order(is_published.asc()), + Direction::Desc => query.order(is_published.desc()), + }, + WebhookField::EventType => match order.direction { + Direction::Asc => query.order(event_type.asc()), + Direction::Desc => query.order(event_type.desc()), + }, + WebhookField::CreatedAt => match order.direction { + Direction::Asc => query.order(created_at.asc()), + Direction::Desc => query.order(created_at.desc()), + }, + WebhookField::UpdatedAt => match order.direction { + Direction::Asc => query.order(updated_at.asc()), + Direction::Desc => query.order(updated_at.desc()), + }, + }; + if !publishers.is_empty() { + query = query.filter(publisher_id.eq_any(publishers)); + } + if let Some(pid) = parent_id_1 { + query = query.filter(publisher_id.eq(pid)); + } + if !event_types.is_empty() { + query = query.filter(event_type.eq_any(event_types)); + } + if let Some(publ) = published { + query = query.filter(is_published.eq(publ)); + } + query + .limit(limit.into()) + .offset(offset.into()) + .load::(&mut connection) + .map_err(Into::into) + } + + fn count( + db: &crate::db::PgPool, + _: Option, + publishers: Vec, + event_types: Vec, + _: Vec, + published: Option, + ) -> ThothResult { + use crate::schema::webhook::dsl::*; + let mut connection = db.get()?; + let mut query = webhook.into_boxed(); + if !publishers.is_empty() { + query = query.filter(publisher_id.eq_any(publishers)); + } + if !event_types.is_empty() { + query = query.filter(event_type.eq_any(event_types)); + } + if let Some(publ) = published { + query = query.filter(is_published.eq(publ)); + } + + // `SELECT COUNT(*)` in postgres returns a BIGINT, which diesel parses as i64. Juniper does + // not implement i64 yet, only i32. The only sensible way, albeit shameful, to solve this + // is converting i64 to string and then parsing it as i32. This should work until we reach + // 2147483647 records - if you are fixing this bug, congratulations on book number 2147483647! + query + .count() + .get_result::(&mut connection) + .map(|t| t.to_string().parse::().unwrap()) + .map_err(Into::into) + } + + fn publisher_id(&self, _db: &crate::db::PgPool) -> ThothResult { + Ok(self.publisher_id) + } + + crud_methods!(webhook::table, webhook::dsl::webhook); +} + +impl HistoryEntry for Webhook { + type NewHistoryEntity = NewWebhookHistory; + + fn new_history_entry(&self, account_id: &Uuid) -> Self::NewHistoryEntity { + Self::NewHistoryEntity { + webhook_id: self.webhook_id, + account_id: *account_id, + data: serde_json::Value::String(serde_json::to_string(&self).unwrap()), + } + } +} + +impl DbInsert for NewWebhookHistory { + type MainEntity = WebhookHistory; + + db_insert!(webhook_history::table); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_webhook_pk() { + let webhook: Webhook = Default::default(); + assert_eq!(webhook.pk(), webhook.webhook_id); + } + + #[test] + fn test_new_webhook_history_from_webhook() { + let webhook: Webhook = Default::default(); + let account_id: Uuid = Default::default(); + let new_webhook_history = webhook.new_history_entry(&account_id); + assert_eq!(new_webhook_history.webhook_id, webhook.webhook_id); + assert_eq!(new_webhook_history.account_id, account_id); + assert_eq!( + new_webhook_history.data, + serde_json::Value::String(serde_json::to_string(&webhook).unwrap()) + ); + } +} diff --git a/thoth-api/src/model/webhook/mod.rs b/thoth-api/src/model/webhook/mod.rs new file mode 100644 index 000000000..e94b1abd7 --- /dev/null +++ b/thoth-api/src/model/webhook/mod.rs @@ -0,0 +1,113 @@ +use serde::Deserialize; +use serde::Serialize; +use uuid::Uuid; + +use crate::event::model::EventType; +use crate::graphql::utils::Direction; +use crate::model::Timestamp; +#[cfg(feature = "backend")] +use crate::schema::webhook; +#[cfg(feature = "backend")] +use crate::schema::webhook_history; + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLEnum), + graphql(description = "Field to use when sorting webhooks list") +)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum WebhookField { + WebhookId, + PublisherId, + #[default] + Endpoint, + Token, + IsPublished, + EventType, + CreatedAt, + UpdatedAt, +} + +#[cfg_attr(feature = "backend", derive(Queryable))] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub webhook_id: Uuid, + pub publisher_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, + pub created_at: Timestamp, + pub updated_at: Timestamp, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLInputObject, Insertable), + graphql(description = "Set of values required to define a new webhook"), + diesel(table_name = webhook) +)] +pub struct NewWebhook { + pub publisher_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLInputObject, AsChangeset), + graphql(description = "Set of values required to update an existing webhook"), + diesel(table_name = webhook, treat_none_as_null = true) +)] +pub struct PatchWebhook { + pub webhook_id: Uuid, + pub publisher_id: Uuid, + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, +} + +#[cfg_attr(feature = "backend", derive(Queryable))] +pub struct WebhookHistory { + pub webhook_history_id: Uuid, + pub webhook_id: Uuid, + pub account_id: Uuid, + pub data: serde_json::Value, + pub timestamp: Timestamp, +} + +#[cfg_attr( + feature = "backend", + derive(Insertable), + diesel(table_name = webhook_history) +)] +pub struct NewWebhookHistory { + pub webhook_id: Uuid, + pub account_id: Uuid, + pub data: serde_json::Value, +} + +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLInputObject), + graphql(description = "Field and order to use when sorting webhooks list") +)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct WebhookOrderBy { + pub field: WebhookField, + pub direction: Direction, +} + +#[test] +fn test_webhookfield_default() { + let webfield: WebhookField = Default::default(); + assert_eq!(webfield, WebhookField::Endpoint); +} + +#[cfg(feature = "backend")] +pub mod crud; From 7f7bde7b5890e197649295f7269788c81296e899 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 13:17:27 +0100 Subject: [PATCH 43/69] Fix build error: EventType no longer only used by backend --- thoth-api/src/event/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/thoth-api/src/event/mod.rs b/thoth-api/src/event/mod.rs index 0a88473cd..0386dfea0 100644 --- a/thoth-api/src/event/mod.rs +++ b/thoth-api/src/event/mod.rs @@ -1,4 +1,3 @@ #[cfg(feature = "backend")] pub mod handler; -#[cfg(feature = "backend")] pub mod model; From a55084a99c9aea10037fe5b125f0cf847d5bf780 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 15:09:33 +0100 Subject: [PATCH 44/69] Fill out GraphQL CRUD implementation for webhooks (fixes remaining compile errors) --- thoth-api/src/graphql/model.rs | 136 +++++++++++++++++++++++++++++ thoth-api/src/model/webhook/mod.rs | 15 ++++ 2 files changed, 151 insertions(+) diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index b23ab3fae..eb1221188 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1467,6 +1467,51 @@ impl QueryRoot { fn reference_count(context: &Context) -> FieldResult { Reference::count(&context.db, None, vec![], vec![], vec![], None).map_err(|e| e.into()) } + + #[graphql(description = "Query the full list of webhooks")] + fn webhooks( + context: &Context, + #[graphql(default = 100, description = "The number of items to return")] limit: Option, + #[graphql(default = 0, description = "The number of items to skip")] offset: Option, + #[graphql( + default = WebhookOrderBy::default(), + description = "The order in which to sort the results" + )] + order: Option, + #[graphql( + default = vec![], + description = "If set, only shows results connected to publishers with these IDs" + )] + publishers: Option>, + ) -> FieldResult> { + Webhook::all( + &context.db, + limit.unwrap_or_default(), + offset.unwrap_or_default(), + None, + order.unwrap_or_default(), + publishers.unwrap_or_default(), + None, + None, + vec![], + vec![], + None, + ) + .map_err(|e| e.into()) + } + + #[graphql(description = "Query a single webhook using its ID")] + fn webhook( + context: &Context, + #[graphql(description = "Thoth webhook ID to search on")] webhook_id: Uuid, + ) -> FieldResult { + Webhook::from_id(&context.db, &webhook_id).map_err(|e| e.into()) + } + + #[graphql(description = "Get the total number of webhooks")] + fn webhook_count(context: &Context) -> FieldResult { + Webhook::count(&context.db, None, vec![], vec![], vec![], None).map_err(|e| e.into()) + } } pub struct MutationRoot; @@ -1735,6 +1780,17 @@ impl MutationRoot { Reference::create(&context.db, &data).map_err(|e| e.into()) } + #[graphql(description = "Create a new webhook with the specified values")] + fn create_webhook( + context: &Context, + #[graphql(description = "Values for webhook to be created")] data: NewWebhook, + ) -> FieldResult { + context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; + context.account_access.can_edit(data.publisher_id)?; + + Webhook::create(&context.db, &data).map_err(|e| e.into()) + } + #[graphql(description = "Update an existing work with the specified values")] async fn update_work( context: &Context, @@ -2187,6 +2243,26 @@ impl MutationRoot { .map_err(|e| e.into()) } + #[graphql(description = "Update an existing webhook with the specified values")] + fn update_webhook( + context: &Context, + #[graphql(description = "Values to apply to existing webhook")] data: PatchWebhook, + ) -> FieldResult { + context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; + let webhook = Webhook::from_id(&context.db, &data.webhook_id).unwrap(); + context + .account_access.can_edit(webhook.publisher_id())?; + + if data.publisher_id != webhook.publisher_id { + context.account_access.can_edit(data.publisher_id)?; + } + + let account_id = context.token.jwt.as_ref().unwrap().account_id(&context.db); + webhook + .update(&context.db, &data, &account_id) + .map_err(|e| e.into()) + } + #[graphql(description = "Delete a single work using its ID")] fn delete_work( context: &Context, @@ -2437,6 +2513,18 @@ impl MutationRoot { reference.delete(&context.db).map_err(|e| e.into()) } + + #[graphql(description = "Delete a single webhook using its ID")] + fn delete_webhook( + context: &Context, + #[graphql(description = "Thoth ID of webhook to be deleted")] webhook_id: Uuid, + ) -> FieldResult { + context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; + let webhook = Webhook::from_id(&context.db, &webhook_id).unwrap(); + context.account_access.can_edit(webhook.publisher_id())?; + + webhook.delete(&context.db).map_err(|e| e.into()) + } } #[juniper::graphql_object(Context = Context, description = "A written text that can be published")] @@ -4245,6 +4333,54 @@ impl Reference { } } +#[juniper::graphql_object(Context = Context, description = "A web request made when a specified data event occurs.")] +impl Webhook { + #[graphql(description = "Thoth ID of the webhook")] + pub fn webhook_id(&self) -> Uuid { + self.webhook_id + } + + #[graphql(description = "Thoth ID of the publisher to which this webhook belongs")] + pub fn publisher_id(&self) -> Uuid { + self.publisher_id + } + + #[graphql(description = "URL which is called by the webhook")] + pub fn endpoint(&self) -> &String { + &self.endpoint + } + + #[graphql(description = "Authentication token required for the webhook")] + pub fn token(&self) -> Option<&String> { + self.token.as_ref() + } + + #[graphql(description = "Type of event which triggers the webhook")] + pub fn event_type(&self) -> &EventType { + &self.event_type + } + + #[graphql(description = "Whether the activation of the webhook depends on publication status")] + pub fn is_published(&self) -> bool { + self.is_published + } + + #[graphql(description = "Date and time at which the webhook record was created")] + pub fn created_at(&self) -> Timestamp { + self.created_at + } + + #[graphql(description = "Date and time at which the webhook record was last updated")] + pub fn updated_at(&self) -> Timestamp { + self.updated_at + } + + #[graphql(description = "Get the publisher to which this webhook belongs")] + pub fn publisher(&self, context: &Context) -> FieldResult { + Publisher::from_id(&context.db, &self.publisher_id).map_err(|e| e.into()) + } +} + pub type Schema = RootNode<'static, QueryRoot, MutationRoot, EmptySubscription>; pub fn create_schema() -> Schema { diff --git a/thoth-api/src/model/webhook/mod.rs b/thoth-api/src/model/webhook/mod.rs index e94b1abd7..441118bf9 100644 --- a/thoth-api/src/model/webhook/mod.rs +++ b/thoth-api/src/model/webhook/mod.rs @@ -103,6 +103,21 @@ pub struct WebhookOrderBy { pub direction: Direction, } +impl Default for Webhook { + fn default() -> Webhook { + Webhook { + webhook_id: Default::default(), + publisher_id: Default::default(), + endpoint: Default::default(), + token: Default::default(), + is_published: Default::default(), + event_type: EventType::WorkUpdated, + created_at: Default::default(), + updated_at: Default::default(), + } + } +} + #[test] fn test_webhookfield_default() { let webfield: WebhookField = Default::default(); From f886f4d5f7f25d8820014db9fff9080b645c3c5e Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 16:04:13 +0100 Subject: [PATCH 45/69] Tweaks to bring Webhook objects more into line with existing GraphQL pattern --- diesel.toml | 1 + thoth-api/src/event/model.rs | 11 ++++++++++- thoth-api/src/model/webhook/mod.rs | 23 ++++------------------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/diesel.toml b/diesel.toml index 752265c18..880a0107b 100644 --- a/diesel.toml +++ b/diesel.toml @@ -17,4 +17,5 @@ custom_type_derives = [ "crate::model::subject::Subject_type" "crate::model::institution::Country_code" "crate::model::work_relation::Relation_type" + "crate::event::model::Event_type" ] diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 139181821..92d8b94f2 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -1,5 +1,6 @@ use crate::model::Timestamp; use serde::{Deserialize, Serialize}; +use strum::{Display, EnumString}; use uuid::Uuid; #[cfg_attr( @@ -8,10 +9,18 @@ use uuid::Uuid; graphql(description = "Nature of an event"), ExistingTypePath = "crate::schema::sql_types::EventType" )] -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive( + Debug, Clone, Default, Copy, PartialEq, Eq, Deserialize, Serialize, EnumString, Display, +)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +#[strum(serialize_all = "title_case")] pub enum EventType { + #[cfg_attr(feature = "backend", graphql(description = "Work creation event"))] WorkCreated, + #[default] + #[cfg_attr(feature = "backend", graphql(description = "Work update event"))] WorkUpdated, + #[cfg_attr(feature = "backend", graphql(description = "Work publication event"))] WorkPublished, } diff --git a/thoth-api/src/model/webhook/mod.rs b/thoth-api/src/model/webhook/mod.rs index 441118bf9..76cb99170 100644 --- a/thoth-api/src/model/webhook/mod.rs +++ b/thoth-api/src/model/webhook/mod.rs @@ -1,5 +1,5 @@ -use serde::Deserialize; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumString}; use uuid::Uuid; use crate::event::model::EventType; @@ -15,7 +15,7 @@ use crate::schema::webhook_history; derive(juniper::GraphQLEnum), graphql(description = "Field to use when sorting webhooks list") )] -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, EnumString, Display)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum WebhookField { WebhookId, @@ -30,7 +30,7 @@ pub enum WebhookField { } #[cfg_attr(feature = "backend", derive(Queryable))] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct Webhook { pub webhook_id: Uuid, @@ -103,21 +103,6 @@ pub struct WebhookOrderBy { pub direction: Direction, } -impl Default for Webhook { - fn default() -> Webhook { - Webhook { - webhook_id: Default::default(), - publisher_id: Default::default(), - endpoint: Default::default(), - token: Default::default(), - is_published: Default::default(), - event_type: EventType::WorkUpdated, - created_at: Default::default(), - updated_at: Default::default(), - } - } -} - #[test] fn test_webhookfield_default() { let webfield: WebhookField = Default::default(); From 6ed92da0918f3726d5608b3abad7268b83886240 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 16:13:16 +0100 Subject: [PATCH 46/69] Fix formatting --- thoth-api/src/graphql/model.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index eb1221188..54d7854de 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -2250,8 +2250,7 @@ impl MutationRoot { ) -> FieldResult { context.token.jwt.as_ref().ok_or(ThothError::Unauthorised)?; let webhook = Webhook::from_id(&context.db, &data.webhook_id).unwrap(); - context - .account_access.can_edit(webhook.publisher_id())?; + context.account_access.can_edit(webhook.publisher_id())?; if data.publisher_id != webhook.publisher_id { context.account_access.can_edit(data.publisher_id)?; From 65b98b1972235ed71993f24f45670aba215e7bf1 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 17:01:05 +0100 Subject: [PATCH 47/69] Improve database error message reporting --- thoth-errors/src/database_errors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thoth-errors/src/database_errors.rs b/thoth-errors/src/database_errors.rs index 6f53b9069..4fffe8081 100644 --- a/thoth-errors/src/database_errors.rs +++ b/thoth-errors/src/database_errors.rs @@ -179,7 +179,7 @@ impl From for ThothError { ThothError::DatabaseError(info.message().to_string()) } Error::NotFound => ThothError::EntityNotFound, - _ => ThothError::InternalError("".into()), + _ => ThothError::InternalError(error.to_string()), } } } From d8488e7ecbabd7d6009c1e3e8e64cdf0b2b417f6 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 22 May 2025 17:11:19 +0100 Subject: [PATCH 48/69] Correct EventType definition to fix database error --- thoth-api/src/event/model.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 92d8b94f2..618cf5d3a 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -15,12 +15,24 @@ use uuid::Uuid; #[serde(rename_all = "SCREAMING_SNAKE_CASE")] #[strum(serialize_all = "title_case")] pub enum EventType { - #[cfg_attr(feature = "backend", graphql(description = "Work creation event"))] + #[cfg_attr( + feature = "backend", + db_rename = "WorkCreated", + graphql(description = "Work creation event") + )] WorkCreated, #[default] - #[cfg_attr(feature = "backend", graphql(description = "Work update event"))] + #[cfg_attr( + feature = "backend", + db_rename = "WorkUpdated", + graphql(description = "Work update event") + )] WorkUpdated, - #[cfg_attr(feature = "backend", graphql(description = "Work publication event"))] + #[cfg_attr( + feature = "backend", + db_rename = "WorkPublished", + graphql(description = "Work publication event") + )] WorkPublished, } From 6739efe1fc32a131dade62be3d353b78a8ddeafc Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 11 Jun 2025 14:58:03 +0100 Subject: [PATCH 49/69] Re-version migrations following new release --- thoth-api/migrations/{v0.13.12 => v0.13.14}/down.sql | 0 thoth-api/migrations/{v0.13.12 => v0.13.14}/up.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename thoth-api/migrations/{v0.13.12 => v0.13.14}/down.sql (100%) rename thoth-api/migrations/{v0.13.12 => v0.13.14}/up.sql (100%) diff --git a/thoth-api/migrations/v0.13.12/down.sql b/thoth-api/migrations/v0.13.14/down.sql similarity index 100% rename from thoth-api/migrations/v0.13.12/down.sql rename to thoth-api/migrations/v0.13.14/down.sql diff --git a/thoth-api/migrations/v0.13.12/up.sql b/thoth-api/migrations/v0.13.14/up.sql similarity index 100% rename from thoth-api/migrations/v0.13.12/up.sql rename to thoth-api/migrations/v0.13.14/up.sql From a1efdc1b69a7104b20f11dd2b6b3d95f284508cf Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 17 Jun 2025 17:11:06 +0100 Subject: [PATCH 50/69] Skeleton logic for calling into GraphQL (working, with placeholders) --- Cargo.lock | 1 + thoth-processor/Cargo.toml | 1 + thoth-processor/src/lib.rs | 1 + thoth-processor/src/requests/graphql.rs | 40 +++++++++++++++++++ thoth-processor/src/requests/mod.rs | 1 + .../src/workers/work_published_worker.rs | 3 ++ 6 files changed, 47 insertions(+) create mode 100644 thoth-processor/src/requests/graphql.rs create mode 100644 thoth-processor/src/requests/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 228546fe9..35bfa3183 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5360,6 +5360,7 @@ dependencies = [ "insta", "loco-rs", "regex", + "reqwest", "rstest", "serde", "serde_json", diff --git a/thoth-processor/Cargo.toml b/thoth-processor/Cargo.toml index 5fc65bbbc..341aed5b4 100644 --- a/thoth-processor/Cargo.toml +++ b/thoth-processor/Cargo.toml @@ -18,6 +18,7 @@ axum = { version = "0.8.1" } tracing = { version = "0.1.40" } tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] } regex = { version = "1.11.1" } +reqwest = { version = "0.12", features = ["json"] } thoth-api = { version = "=0.13.13", path = "../thoth-api" } [dev-dependencies] diff --git a/thoth-processor/src/lib.rs b/thoth-processor/src/lib.rs index b69d00408..7e5cf0ffe 100644 --- a/thoth-processor/src/lib.rs +++ b/thoth-processor/src/lib.rs @@ -1,5 +1,6 @@ pub mod app; pub mod initializers; +pub mod requests; pub mod workers; use crate::app::App; diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs new file mode 100644 index 000000000..bd63015c9 --- /dev/null +++ b/thoth-processor/src/requests/graphql.rs @@ -0,0 +1,40 @@ +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; +use thoth_api::event::model::Event; + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksQueryBody { + pub query: String, +} + +pub async fn query_webhooks(event: Event) -> Result { + let client = reqwest::Client::new(); + let url = "https://api.thoth.pub/graphql".to_string(); + let query = format!( + " +{{ + work(workId: \"{}\") {{ + workId + fullTitle + }} +}}", + event.work_id + ); + let body = WebhooksQueryBody { query }; + let token = "placeholder".to_string(); + + let response = client + .post(&url) + .json(&body) + .bearer_auth(token) + .send() + .await? + .error_for_status()?; + + tracing::info!("response: {:?}", response); + let response_text = response.text().await?; + tracing::info!("response: {:?}", response_text); + + Ok(response_text) +} diff --git a/thoth-processor/src/requests/mod.rs b/thoth-processor/src/requests/mod.rs new file mode 100644 index 000000000..50b9335ce --- /dev/null +++ b/thoth-processor/src/requests/mod.rs @@ -0,0 +1 @@ +pub mod graphql; diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index 057e9eb02..5d86aefc1 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -1,3 +1,4 @@ +use crate::requests::graphql::query_webhooks; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -21,6 +22,8 @@ impl BackgroundWorker for WorkPublishedWorker { tracing::info!("WorkPublishedWorker start"); tracing::info!("Event: {:?}", args.event); tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + let response = query_webhooks(args.event).await?; + tracing::info!("Response: {:?}", response); tracing::info!("WorkPublishedWorker end"); Ok(()) From 3bdfb6a6d62374ef12d7a295adf3143c11f337d6 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 18 Jun 2025 14:00:12 +0100 Subject: [PATCH 51/69] Correct graphql syntax to allow proper json serialisation --- Cargo.lock | 1 + thoth-processor/Cargo.toml | 1 + thoth-processor/src/requests/graphql.rs | 29 ++++++++++++++++++------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35bfa3183..2f17588b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5369,6 +5369,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "uuid", ] [[package]] diff --git a/thoth-processor/Cargo.toml b/thoth-processor/Cargo.toml index 341aed5b4..37bf24370 100644 --- a/thoth-processor/Cargo.toml +++ b/thoth-processor/Cargo.toml @@ -19,6 +19,7 @@ tracing = { version = "0.1.40" } tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] } regex = { version = "1.11.1" } reqwest = { version = "0.12", features = ["json"] } +uuid = { version = "1.16.0", features = ["serde"] } thoth-api = { version = "=0.13.13", path = "../thoth-api" } [dev-dependencies] diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index bd63015c9..9cec295e3 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -1,27 +1,40 @@ use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; +use uuid::Uuid; + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksVariables { + work_id: Uuid, +} #[derive(Deserialize, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct WebhooksQueryBody { pub query: String, + pub variables: WebhooksVariables, } pub async fn query_webhooks(event: Event) -> Result { let client = reqwest::Client::new(); let url = "https://api.thoth.pub/graphql".to_string(); - let query = format!( + let query = " -{{ - work(workId: \"{}\") {{ +query WebhooksQuery($workId: Uuid!) { + work(workId: $workId) { workId fullTitle - }} -}}", - event.work_id - ); - let body = WebhooksQueryBody { query }; + } +}".to_string(); + + let variables = WebhooksVariables { + work_id: event.work_id, + }; + let body = WebhooksQueryBody { + query, + variables, + }; let token = "placeholder".to_string(); let response = client From c1166ff47cd2a1d089d93d9d7428d4697580e8d1 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 18 Jun 2025 14:14:39 +0100 Subject: [PATCH 52/69] Replace placeholder query with correct graphql syntax for webhooks retrieval --- thoth-processor/src/requests/graphql.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index 9cec295e3..b3d1b806e 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -1,12 +1,14 @@ use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; -use thoth_api::event::model::Event; +use thoth_api::event::model::{Event, EventType}; use uuid::Uuid; #[derive(Deserialize, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct WebhooksVariables { work_id: Uuid, + event_types: Vec, + is_published: bool, } #[derive(Deserialize, Debug, Serialize)] @@ -21,15 +23,25 @@ pub async fn query_webhooks(event: Event) -> Result { let url = "https://api.thoth.pub/graphql".to_string(); let query = " -query WebhooksQuery($workId: Uuid!) { +query WebhooksQuery($workId: Uuid!, $eventTypes: [EventType!], $isPublished: Boolean!) { work(workId: $workId) { - workId - fullTitle + imprint { + publisher { + webhooks(eventTypes: $eventTypes, isPublished: $isPublished) { + endpoint + token + isPublished + eventType + } + } + } } }".to_string(); let variables = WebhooksVariables { work_id: event.work_id, + event_types: vec![event.event_type], + is_published: event.is_published, }; let body = WebhooksQueryBody { query, From c3829103800ee13ed20e6079053c576535b272c4 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 18 Jun 2025 14:29:02 +0100 Subject: [PATCH 53/69] Use appropriate structures for deserialising response --- thoth-processor/src/requests/graphql.rs | 45 +++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index b3d1b806e..3c186aa8a 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -18,7 +18,46 @@ pub struct WebhooksQueryBody { pub variables: WebhooksVariables, } -pub async fn query_webhooks(event: Event) -> Result { +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub endpoint: String, + pub token: Option, + pub is_published: bool, + pub event_type: EventType, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponsePublisher { + pub webhooks: Vec, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseImprint { + pub publisher: WebhooksResponsePublisher, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseWork { + pub imprint: WebhooksResponseImprint, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseData { + pub work: WebhooksResponseWork, +} + +#[derive(Deserialize, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksResponseBody { + pub data: WebhooksResponseData +} + +pub async fn query_webhooks(event: Event) -> Result { let client = reqwest::Client::new(); let url = "https://api.thoth.pub/graphql".to_string(); let query = @@ -58,8 +97,8 @@ query WebhooksQuery($workId: Uuid!, $eventTypes: [EventType!], $isPublished: Boo .error_for_status()?; tracing::info!("response: {:?}", response); - let response_text = response.text().await?; - tracing::info!("response: {:?}", response_text); + let response_text = response.json::().await?; + tracing::info!("response_text: {:?}", response_text); Ok(response_text) } From 5c60c1507a506205b3c9d1f60df2b611f5dbf6d3 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:18:18 +0100 Subject: [PATCH 54/69] Skeleton logic for firing a GitHub Actions webhook (working, with placeholders) --- thoth-processor/src/requests/mod.rs | 1 + thoth-processor/src/requests/target.rs | 34 +++++++++++++++++++ .../src/workers/work_published_worker.rs | 5 +++ 3 files changed, 40 insertions(+) create mode 100644 thoth-processor/src/requests/target.rs diff --git a/thoth-processor/src/requests/mod.rs b/thoth-processor/src/requests/mod.rs index 50b9335ce..eba80b67a 100644 --- a/thoth-processor/src/requests/mod.rs +++ b/thoth-processor/src/requests/mod.rs @@ -1 +1,2 @@ pub mod graphql; +pub mod target; diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs new file mode 100644 index 000000000..06cda054b --- /dev/null +++ b/thoth-processor/src/requests/target.rs @@ -0,0 +1,34 @@ +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Debug, Serialize)] +pub struct Payload { + event_type: String +} + +pub async fn fire_webhook() -> Result { + let client = reqwest::Client::new(); + // Will trigger any GitHub Action in this repo with a `repository_dispatch` option set + // (as long as the "event_type" matches it) + let url = "https://api.github.com/repos/thoth-pub/thoth-dissemination/dispatches".to_string(); + // For GitHub Actions this can be a "fine-grained access token": + // https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event--fine-grained-access-tokens + let token = "placeholder"; + + let response = client + .post(&url) + .bearer_auth(token) + // GitHub Actions repository dispatch events require a User-Agent header + .header("User-Agent", "Thoth") + // GitHub Actions repository dispatch events require a payload containing "event_type" + // (this can then be used to control which events trigger which Actions) + .json(&Payload { event_type: "test".to_string() }) + .send() + .await? + .error_for_status()?; + + tracing::info!("response: {:?}", response); + tracing::info!("response_status: {:?}", response.status()); + + Ok(response.status().to_string()) +} diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index 5d86aefc1..ab5d63355 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -1,4 +1,5 @@ use crate::requests::graphql::query_webhooks; +use crate::requests::target::fire_webhook; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -24,6 +25,10 @@ impl BackgroundWorker for WorkPublishedWorker { tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; let response = query_webhooks(args.event).await?; tracing::info!("Response: {:?}", response); + + let target_rsp = fire_webhook().await?; + tracing::info!("Target response: {:?}", target_rsp); + tracing::info!("WorkPublishedWorker end"); Ok(()) From 873f45057089f4fe1599ecaccce2ce6ce5909734 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:27:55 +0100 Subject: [PATCH 55/69] Pass in webhook parameters as arguments --- thoth-processor/src/requests/graphql.rs | 4 ++-- thoth-processor/src/requests/target.rs | 11 +++-------- thoth-processor/src/workers/work_published_worker.rs | 10 ++++++---- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index 3c186aa8a..e8138ca18 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -57,7 +57,7 @@ pub struct WebhooksResponseBody { pub data: WebhooksResponseData } -pub async fn query_webhooks(event: Event) -> Result { +pub async fn query_webhooks(event: Event) -> Result, Error> { let client = reqwest::Client::new(); let url = "https://api.thoth.pub/graphql".to_string(); let query = @@ -100,5 +100,5 @@ query WebhooksQuery($workId: Uuid!, $eventTypes: [EventType!], $isPublished: Boo let response_text = response.json::().await?; tracing::info!("response_text: {:?}", response_text); - Ok(response_text) + Ok(response_text.data.work.imprint.publisher.webhooks) } diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs index 06cda054b..ab9574bd4 100644 --- a/thoth-processor/src/requests/target.rs +++ b/thoth-processor/src/requests/target.rs @@ -6,22 +6,17 @@ pub struct Payload { event_type: String } -pub async fn fire_webhook() -> Result { +pub async fn fire_webhook(url: String, token: Option) -> Result { let client = reqwest::Client::new(); - // Will trigger any GitHub Action in this repo with a `repository_dispatch` option set - // (as long as the "event_type" matches it) - let url = "https://api.github.com/repos/thoth-pub/thoth-dissemination/dispatches".to_string(); - // For GitHub Actions this can be a "fine-grained access token": - // https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event--fine-grained-access-tokens - let token = "placeholder"; let response = client .post(&url) - .bearer_auth(token) + .bearer_auth(token.unwrap_or_default()) // GitHub Actions repository dispatch events require a User-Agent header .header("User-Agent", "Thoth") // GitHub Actions repository dispatch events require a payload containing "event_type" // (this can then be used to control which events trigger which Actions) + // (it also seems to determine the name given to any ensuing workflow runs) .json(&Payload { event_type: "test".to_string() }) .send() .await? diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index ab5d63355..a7a2850be 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -23,11 +23,13 @@ impl BackgroundWorker for WorkPublishedWorker { tracing::info!("WorkPublishedWorker start"); tracing::info!("Event: {:?}", args.event); tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; - let response = query_webhooks(args.event).await?; - tracing::info!("Response: {:?}", response); + let webhooks = query_webhooks(args.event).await?; + tracing::info!("Webhooks: {:?}", webhooks); - let target_rsp = fire_webhook().await?; - tracing::info!("Target response: {:?}", target_rsp); + for webhook in webhooks { + let target_rsp = fire_webhook(webhook.endpoint, webhook.token).await?; + tracing::info!("Target response: {:?}", target_rsp); + } tracing::info!("WorkPublishedWorker end"); From 2553832f70a1d44d0c741cb2d7c93c45dd686955 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:52:17 +0100 Subject: [PATCH 56/69] Pull async webhook firing out into separate worker --- thoth-processor/src/app.rs | 5 ++-- .../src/workers/fire_webhook_worker.rs | 30 +++++++++++++++++++ thoth-processor/src/workers/mod.rs | 1 + .../src/workers/work_published_worker.rs | 9 ++++-- 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 thoth-processor/src/workers/fire_webhook_worker.rs diff --git a/thoth-processor/src/app.rs b/thoth-processor/src/app.rs index 07c240e7b..08c90db00 100644 --- a/thoth-processor/src/app.rs +++ b/thoth-processor/src/app.rs @@ -1,8 +1,8 @@ use crate::{ initializers::handle_events::HandleEvents, workers::{ - work_created_worker::WorkCreatedWorker, work_published_worker::WorkPublishedWorker, - work_updated_worker::WorkUpdatedWorker, + fire_webhook_worker::FireWebhookWorker, work_created_worker::WorkCreatedWorker, + work_published_worker::WorkPublishedWorker, work_updated_worker::WorkUpdatedWorker, }, }; use async_trait::async_trait; @@ -50,6 +50,7 @@ impl Hooks for App { AppRoutes::with_default_routes() } async fn connect_workers(ctx: &AppContext, queue: &Queue) -> Result<()> { + queue.register(FireWebhookWorker::build(ctx)).await?; queue.register(WorkCreatedWorker::build(ctx)).await?; queue.register(WorkUpdatedWorker::build(ctx)).await?; queue.register(WorkPublishedWorker::build(ctx)).await?; diff --git a/thoth-processor/src/workers/fire_webhook_worker.rs b/thoth-processor/src/workers/fire_webhook_worker.rs new file mode 100644 index 000000000..2b3f4ee63 --- /dev/null +++ b/thoth-processor/src/workers/fire_webhook_worker.rs @@ -0,0 +1,30 @@ +use crate::requests::graphql::Webhook; +use crate::requests::target::fire_webhook; +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; + +pub struct FireWebhookWorker { + pub ctx: AppContext, +} + +#[derive(Deserialize, Debug, Serialize)] +pub struct FireWebhookWorkerArgs { + pub webhook: Webhook, +} + +#[async_trait] +impl BackgroundWorker for FireWebhookWorker { + fn build(ctx: &AppContext) -> Self { + Self { ctx: ctx.clone() } + } + + async fn perform(&self, args: FireWebhookWorkerArgs) -> Result<()> { + tracing::info!("FireWebhookWorker start"); + tracing::info!("Webhook: {:?}", args.webhook); + let target_rsp = fire_webhook(args.webhook.endpoint, args.webhook.token).await?; + tracing::info!("Target response: {:?}", target_rsp); + tracing::info!("FireWebhookWorker end"); + + Ok(()) + } +} diff --git a/thoth-processor/src/workers/mod.rs b/thoth-processor/src/workers/mod.rs index 72682cae2..0ff891e6d 100644 --- a/thoth-processor/src/workers/mod.rs +++ b/thoth-processor/src/workers/mod.rs @@ -1,3 +1,4 @@ +pub mod fire_webhook_worker; pub mod work_created_worker; pub mod work_published_worker; pub mod work_updated_worker; diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index a7a2850be..f89a82522 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -1,5 +1,5 @@ +use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; use crate::requests::graphql::query_webhooks; -use crate::requests::target::fire_webhook; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -27,8 +27,11 @@ impl BackgroundWorker for WorkPublishedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let target_rsp = fire_webhook(webhook.endpoint, webhook.token).await?; - tracing::info!("Target response: {:?}", target_rsp); + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { webhook }, + ) + .await; } tracing::info!("WorkPublishedWorker end"); From a8562f6731bec72ceb24a137677d66e71e9008d2 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:55:04 +0100 Subject: [PATCH 57/69] Extend webhook fire logic to all event type workers --- thoth-processor/src/workers/work_created_worker.rs | 14 +++++++++++++- .../src/workers/work_published_worker.rs | 1 - thoth-processor/src/workers/work_updated_worker.rs | 14 +++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs index 83ecd1740..5ea98d48f 100644 --- a/thoth-processor/src/workers/work_created_worker.rs +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -1,3 +1,5 @@ +use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; +use crate::requests::graphql::query_webhooks; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -20,7 +22,17 @@ impl BackgroundWorker for WorkCreatedWorker { async fn perform(&self, args: WorkCreatedWorkerArgs) -> Result<()> { tracing::info!("WorkCreatedWorker start"); tracing::info!("Event: {:?}", args.event); - tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + let webhooks = query_webhooks(args.event).await?; + tracing::info!("Webhooks: {:?}", webhooks); + + for webhook in webhooks { + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { webhook }, + ) + .await; + } + tracing::info!("WorkCreatedWorker end"); Ok(()) diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index f89a82522..2e625ac81 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -22,7 +22,6 @@ impl BackgroundWorker for WorkPublishedWorker { async fn perform(&self, args: WorkPublishedWorkerArgs) -> Result<()> { tracing::info!("WorkPublishedWorker start"); tracing::info!("Event: {:?}", args.event); - tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; let webhooks = query_webhooks(args.event).await?; tracing::info!("Webhooks: {:?}", webhooks); diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs index 845266dcd..e49b35b0c 100644 --- a/thoth-processor/src/workers/work_updated_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -1,3 +1,5 @@ +use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; +use crate::requests::graphql::query_webhooks; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -20,7 +22,17 @@ impl BackgroundWorker for WorkUpdatedWorker { async fn perform(&self, args: WorkUpdatedWorkerArgs) -> Result<()> { tracing::info!("WorkUpdatedWorker start"); tracing::info!("Event: {:?}", args.event); - tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + let webhooks = query_webhooks(args.event).await?; + tracing::info!("Webhooks: {:?}", webhooks); + + for webhook in webhooks { + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { webhook }, + ) + .await; + } + tracing::info!("WorkUpdatedWorker end"); Ok(()) From 998f3f4b2a2c69907f5601ef45251abe0274447b Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:57:56 +0100 Subject: [PATCH 58/69] Fix formatting --- thoth-processor/src/requests/graphql.rs | 13 +++++-------- thoth-processor/src/requests/target.rs | 6 ++++-- thoth-processor/src/workers/work_created_worker.rs | 7 ++----- .../src/workers/work_published_worker.rs | 7 ++----- thoth-processor/src/workers/work_updated_worker.rs | 7 ++----- 5 files changed, 15 insertions(+), 25 deletions(-) diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index e8138ca18..aa1cb98cc 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -54,14 +54,13 @@ pub struct WebhooksResponseData { #[derive(Deserialize, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct WebhooksResponseBody { - pub data: WebhooksResponseData + pub data: WebhooksResponseData, } pub async fn query_webhooks(event: Event) -> Result, Error> { let client = reqwest::Client::new(); let url = "https://api.thoth.pub/graphql".to_string(); - let query = - " + let query = " query WebhooksQuery($workId: Uuid!, $eventTypes: [EventType!], $isPublished: Boolean!) { work(workId: $workId) { imprint { @@ -75,17 +74,15 @@ query WebhooksQuery($workId: Uuid!, $eventTypes: [EventType!], $isPublished: Boo } } } -}".to_string(); +}" + .to_string(); let variables = WebhooksVariables { work_id: event.work_id, event_types: vec![event.event_type], is_published: event.is_published, }; - let body = WebhooksQueryBody { - query, - variables, - }; + let body = WebhooksQueryBody { query, variables }; let token = "placeholder".to_string(); let response = client diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs index ab9574bd4..c99a17094 100644 --- a/thoth-processor/src/requests/target.rs +++ b/thoth-processor/src/requests/target.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; #[derive(Deserialize, Debug, Serialize)] pub struct Payload { - event_type: String + event_type: String, } pub async fn fire_webhook(url: String, token: Option) -> Result { @@ -17,7 +17,9 @@ pub async fn fire_webhook(url: String, token: Option) -> Result for WorkCreatedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let _ = FireWebhookWorker::perform_later( - &self.ctx, - FireWebhookWorkerArgs { webhook }, - ) - .await; + let _ = FireWebhookWorker::perform_later(&self.ctx, FireWebhookWorkerArgs { webhook }) + .await; } tracing::info!("WorkCreatedWorker end"); diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index 2e625ac81..6438cfb8e 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -26,11 +26,8 @@ impl BackgroundWorker for WorkPublishedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let _ = FireWebhookWorker::perform_later( - &self.ctx, - FireWebhookWorkerArgs { webhook }, - ) - .await; + let _ = FireWebhookWorker::perform_later(&self.ctx, FireWebhookWorkerArgs { webhook }) + .await; } tracing::info!("WorkPublishedWorker end"); diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs index e49b35b0c..04584051f 100644 --- a/thoth-processor/src/workers/work_updated_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -26,11 +26,8 @@ impl BackgroundWorker for WorkUpdatedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let _ = FireWebhookWorker::perform_later( - &self.ctx, - FireWebhookWorkerArgs { webhook }, - ) - .await; + let _ = FireWebhookWorker::perform_later(&self.ctx, FireWebhookWorkerArgs { webhook }) + .await; } tracing::info!("WorkUpdatedWorker end"); From a7d9fb2c2e15f690ad61c239c560518f8dc106fa Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:28:08 +0100 Subject: [PATCH 59/69] Back loco with Postgres instead of Redis (inc. sqlx issue workaround and hard-coded URL) --- Cargo.lock | 799 +++++++++++++----- thoth-processor/Cargo.toml | 3 +- thoth-processor/config/development.yaml | 6 +- thoth-processor/config/test.yaml | 6 +- .../src/initializers/handle_events.rs | 5 +- 5 files changed, 612 insertions(+), 207 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f17588b4..f8d0e5c43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -489,6 +489,15 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -693,18 +702,6 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" -[[package]] -name = "bb8" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" -dependencies = [ - "async-trait", - "futures-util", - "parking_lot 0.12.3", - "tokio", -] - [[package]] name = "bincode" version = "1.3.3" @@ -725,6 +722,9 @@ name = "bitflags" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +dependencies = [ + "serde", +] [[package]] name = "blake2" @@ -1014,6 +1014,15 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.11" @@ -1038,19 +1047,16 @@ dependencies = [ ] [[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "convert_case" -version = "0.6.0" +name = "constant_time_eq" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" -dependencies = [ - "unicode-segmentation", -] +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "cookie" @@ -1106,6 +1112,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -1126,17 +1147,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "cron_clock" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a8699d8ed16e3db689f8ae04d8dc3c6666a4ba7e724e5a157884b7cc385d16b" -dependencies = [ - "chrono", - "nom 7.1.3", - "once_cell", -] - [[package]] name = "crossbeam-channel" version = "0.5.14" @@ -1165,6 +1175,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1308,7 +1327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c136f185b3ca9d1f4e4e19c11570e1002f4bfdd592d589053e225716d613851f" dependencies = [ "deadpool", - "redis 0.29.2", + "redis", ] [[package]] @@ -1320,6 +1339,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.4.1" @@ -1498,31 +1528,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -1540,6 +1550,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dsl_auto_type" version = "0.1.3" @@ -1601,6 +1617,9 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] [[package]] name = "email-encoding" @@ -1681,6 +1700,28 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1703,6 +1744,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1800,6 +1852,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.12.3", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -1872,16 +1935,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "gethostname" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "getopts" version = "0.2.21" @@ -2266,6 +2319,20 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.2", +] [[package]] name = "heck" @@ -2285,12 +2352,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "hermit-abi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" - [[package]] name = "hex" version = "0.4.3" @@ -2315,6 +2376,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "hostname" version = "0.4.0" @@ -2768,17 +2838,6 @@ dependencies = [ "serde", ] -[[package]] -name = "is-terminal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" -dependencies = [ - "hermit-abi 0.5.0", - "libc", - "windows-sys 0.59.0", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2913,6 +2972,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lettre" @@ -2956,13 +3018,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] -name = "libredox" -version = "0.1.3" +name = "libsqlite3-sys" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ - "bitflags 2.9.0", - "libc", + "cc", + "pkg-config", + "vcpkg", ] [[package]] @@ -3042,7 +3105,6 @@ dependencies = [ "axum-extra", "axum-test", "backtrace_printer", - "bb8", "byte-unit", "bytes", "cfg-if", @@ -3066,13 +3128,13 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest", - "rusty-sidekiq", "scraper", "semver", "serde", "serde_json", "serde_variant", "serde_yaml", + "sqlx", "tera", "thiserror 1.0.69", "thousands", @@ -3085,6 +3147,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "ulid", "uuid", "validator", ] @@ -3298,6 +3361,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -3324,6 +3404,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3331,6 +3422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3339,7 +3431,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", ] @@ -3540,6 +3632,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" @@ -3618,6 +3716,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -3753,6 +3860,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -4056,26 +4184,6 @@ dependencies = [ "getrandom 0.3.2", ] -[[package]] -name = "redis" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8455fa3621f6b41c514946de66ea0531f57ca017b2e6c7cc368035ea5b46df" -dependencies = [ - "async-trait", - "bytes", - "combine", - "futures-util", - "itoa", - "percent-encoding", - "pin-project-lite", - "ryu", - "sha1_smol", - "tokio", - "tokio-util", - "url", -] - [[package]] name = "redis" version = "0.29.2" @@ -4115,17 +4223,6 @@ dependencies = [ "bitflags 2.9.0", ] -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.15", - "libredox", - "thiserror 1.0.69", -] - [[package]] name = "regex" version = "1.11.1" @@ -4337,6 +4434,26 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "rsa" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rstest" version = "0.21.0" @@ -4468,44 +4585,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] -name = "rusty-sidekiq" -version = "0.11.1" +name = "ryu" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15544f047600b602c7b11ff7ee0882f9034f9cbe2c205693edd5615e2a6c03ee" -dependencies = [ - "async-trait", - "bb8", - "chrono", - "convert_case", - "cron_clock", - "gethostname", - "hex", - "num_cpus", - "rand 0.8.5", - "redis 0.22.3", - "serde", - "serde_json", - "serial_test", - "sha2", - "slog-term", - "thiserror 1.0.69", - "tokio", - "tokio-util", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ "winapi-util", ] @@ -4778,12 +4867,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha1_smol" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" - [[package]] name = "sha2" version = "0.10.8" @@ -4835,6 +4918,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "similar" version = "2.7.0" @@ -4868,25 +4961,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-term" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" -dependencies = [ - "is-terminal", - "slog", - "term", - "thread_local", - "time", -] - [[package]] name = "slug" version = "0.1.6" @@ -4902,6 +4976,9 @@ name = "smallvec" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +dependencies = [ + "serde", +] [[package]] name = "smartstring" @@ -4924,6 +5001,217 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.2", + "hashlink", + "indexmap 2.8.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.12", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.100", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.100", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.12", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.12", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.12", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -4974,6 +5262,17 @@ dependencies = [ "quote", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -5136,17 +5435,6 @@ dependencies = [ "unic-segment", ] -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -5365,6 +5653,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "sqlx", "thoth-api", "tokio", "tracing", @@ -5508,6 +5797,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.14" @@ -5726,6 +6026,16 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.0", + "web-time", +] + [[package]] name = "unic-char-property" version = "0.9.0" @@ -5782,6 +6092,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -5789,10 +6105,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] -name = "unicode-segmentation" -version = "1.12.0" +name = "unicode-normalization" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-width" @@ -5976,6 +6301,12 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -6117,6 +6448,16 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall 0.5.10", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6202,6 +6543,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -6220,6 +6570,21 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -6252,6 +6617,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6264,6 +6635,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -6276,6 +6653,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -6300,6 +6683,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6312,6 +6701,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6324,6 +6719,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6336,6 +6737,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/thoth-processor/Cargo.toml b/thoth-processor/Cargo.toml index 37bf24370..34d49bdff 100644 --- a/thoth-processor/Cargo.toml +++ b/thoth-processor/Cargo.toml @@ -7,9 +7,10 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } +loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_pg"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } +sqlx = { version = "0.8.6", features = ["runtime-tokio"] } tokio = { version = "1.33.0", default-features = false, features = [ "rt-multi-thread", ] } diff --git a/thoth-processor/config/development.yaml b/thoth-processor/config/development.yaml index 05a86bb54..eae8de85d 100644 --- a/thoth-processor/config/development.yaml +++ b/thoth-processor/config/development.yaml @@ -41,7 +41,7 @@ workers: mode: BackgroundQueue queue: - kind: Redis - # Redis connection URI - uri: {{ get_env(name="REDIS_URL") }} + kind: Postgres + # Postgres connection URI + uri: {{ get_env(name="DATABASE_URL") }} dangerously_flush: false diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml index a4fbaa956..eae8de85d 100644 --- a/thoth-processor/config/test.yaml +++ b/thoth-processor/config/test.yaml @@ -41,7 +41,7 @@ workers: mode: BackgroundQueue queue: - kind: Redis - # Redis connection URI - uri: {{ get_env(name="TEST_REDIS_URL") }} + kind: Postgres + # Postgres connection URI + uri: {{ get_env(name="DATABASE_URL") }} dangerously_flush: false diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs index 5c5ee3140..fe5338449 100644 --- a/thoth-processor/src/initializers/handle_events.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -23,10 +23,7 @@ impl Initializer for HandleEvents { async fn before_run(&self, ctx: &AppContext) -> Result<()> { let ctx = ctx.clone(); - let redis_url = match ctx.config.queue.as_ref().unwrap() { - QueueConfig::Redis(queue) => &queue.uri, - _ => unreachable!(), - }; + let redis_url = "redis://localhost:6379"; let redis = init_pool(redis_url); tokio::spawn(async move { From 031a8781d95128e5a4798533913c56e91eee01a2 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:06:28 +0100 Subject: [PATCH 60/69] Revert "Back loco with Postgres instead of Redis (inc. sqlx issue workaround and hard-coded URL)" This reverts commit a7d9fb2c2e15f690ad61c239c560518f8dc106fa. --- Cargo.lock | 801 +++++------------- thoth-processor/Cargo.toml | 3 +- thoth-processor/config/development.yaml | 6 +- thoth-processor/config/test.yaml | 6 +- .../src/initializers/handle_events.rs | 5 +- 5 files changed, 208 insertions(+), 613 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8d0e5c43..2f17588b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -489,15 +489,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -702,6 +693,18 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +[[package]] +name = "bb8" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" +dependencies = [ + "async-trait", + "futures-util", + "parking_lot 0.12.3", + "tokio", +] + [[package]] name = "bincode" version = "1.3.3" @@ -722,9 +725,6 @@ name = "bitflags" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" -dependencies = [ - "serde", -] [[package]] name = "blake2" @@ -1014,15 +1014,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console" version = "0.15.11" @@ -1046,18 +1037,21 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "constant_time_eq" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1112,21 +1106,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1147,6 +1126,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "cron_clock" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8699d8ed16e3db689f8ae04d8dc3c6666a4ba7e724e5a157884b7cc385d16b" +dependencies = [ + "chrono", + "nom 7.1.3", + "once_cell", +] + [[package]] name = "crossbeam-channel" version = "0.5.14" @@ -1175,15 +1165,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1327,7 +1308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c136f185b3ca9d1f4e4e19c11570e1002f4bfdd592d589053e225716d613851f" dependencies = [ "deadpool", - "redis", + "redis 0.29.2", ] [[package]] @@ -1339,17 +1320,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - [[package]] name = "deranged" version = "0.4.1" @@ -1528,11 +1498,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", "crypto-common", "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1550,12 +1540,6 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "dsl_auto_type" version = "0.1.3" @@ -1617,9 +1601,6 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -dependencies = [ - "serde", -] [[package]] name = "email-encoding" @@ -1700,28 +1681,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -1744,17 +1703,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "spin", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1852,17 +1800,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot 0.12.3", -] - [[package]] name = "futures-io" version = "0.3.31" @@ -1935,6 +1872,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "getopts" version = "0.2.21" @@ -2319,20 +2266,6 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown 0.15.2", -] [[package]] name = "heck" @@ -2352,6 +2285,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + [[package]] name = "hex" version = "0.4.3" @@ -2376,15 +2315,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "hostname" version = "0.4.0" @@ -2838,6 +2768,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -2972,9 +2913,6 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin", -] [[package]] name = "lettre" @@ -3018,14 +2956,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] -name = "libsqlite3-sys" -version = "0.30.1" +name = "libredox" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "bitflags 2.9.0", + "libc", ] [[package]] @@ -3105,6 +3042,7 @@ dependencies = [ "axum-extra", "axum-test", "backtrace_printer", + "bb8", "byte-unit", "bytes", "cfg-if", @@ -3128,13 +3066,13 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest", + "rusty-sidekiq", "scraper", "semver", "serde", "serde_json", "serde_variant", "serde_yaml", - "sqlx", "tera", "thiserror 1.0.69", "thousands", @@ -3147,7 +3085,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "ulid", "uuid", "validator", ] @@ -3361,23 +3298,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "smallvec", - "zeroize", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -3404,17 +3324,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -3422,7 +3331,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -3431,7 +3339,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -3632,12 +3540,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.11.2" @@ -3716,15 +3618,6 @@ dependencies = [ "serde", ] -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -3860,27 +3753,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkg-config" version = "0.3.32" @@ -4184,6 +4056,26 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "redis" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8455fa3621f6b41c514946de66ea0531f57ca017b2e6c7cc368035ea5b46df" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "redis" version = "0.29.2" @@ -4223,6 +4115,17 @@ dependencies = [ "bitflags 2.9.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -4434,26 +4337,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "rsa" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "signature", - "spki", - "subtle", - "zeroize", -] - [[package]] name = "rstest" version = "0.21.0" @@ -4585,18 +4468,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" +name = "rusty-sidekiq" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "15544f047600b602c7b11ff7ee0882f9034f9cbe2c205693edd5615e2a6c03ee" dependencies = [ - "winapi-util", + "async-trait", + "bb8", + "chrono", + "convert_case", + "cron_clock", + "gethostname", + "hex", + "num_cpus", + "rand 0.8.5", + "redis 0.22.3", + "serde", + "serde_json", + "serial_test", + "sha2", + "slog-term", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", ] [[package]] @@ -4867,6 +4778,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.8" @@ -4918,16 +4835,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - [[package]] name = "similar" version = "2.7.0" @@ -4961,6 +4868,25 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-term" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +dependencies = [ + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "slug" version = "0.1.6" @@ -4976,9 +4902,6 @@ name = "smallvec" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" -dependencies = [ - "serde", -] [[package]] name = "smartstring" @@ -5001,217 +4924,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlx" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" -dependencies = [ - "base64 0.22.1", - "bytes", - "chrono", - "crc", - "crossbeam-queue", - "either", - "event-listener", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashbrown 0.15.2", - "hashlink", - "indexmap 2.8.0", - "log", - "memchr", - "once_cell", - "percent-encoding", - "serde", - "serde_json", - "sha2", - "smallvec", - "thiserror 2.0.12", - "tokio", - "tokio-stream", - "tracing", - "url", -] - -[[package]] -name = "sqlx-macros" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 2.0.100", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" -dependencies = [ - "dotenvy", - "either", - "heck 0.5.0", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn 2.0.100", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" -dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags 2.9.0", - "byteorder", - "bytes", - "chrono", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand 0.8.5", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror 2.0.12", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" -dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags 2.9.0", - "byteorder", - "chrono", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand 0.8.5", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror 2.0.12", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" -dependencies = [ - "atoi", - "chrono", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "serde_urlencoded", - "sqlx-core", - "thiserror 2.0.12", - "tracing", - "url", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -5262,17 +4974,6 @@ dependencies = [ "quote", ] -[[package]] -name = "stringprep" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", - "unicode-properties", -] - [[package]] name = "strsim" version = "0.11.1" @@ -5435,6 +5136,17 @@ dependencies = [ "unic-segment", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -5653,7 +5365,6 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sqlx", "thoth-api", "tokio", "tracing", @@ -5797,17 +5508,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.14" @@ -6026,16 +5726,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" -[[package]] -name = "ulid" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" -dependencies = [ - "rand 0.9.0", - "web-time", -] - [[package]] name = "unic-char-property" version = "0.9.0" @@ -6092,12 +5782,6 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.18" @@ -6105,19 +5789,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-properties" -version = "0.1.3" +name = "unicode-segmentation" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" @@ -6301,12 +5976,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -6448,16 +6117,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "whoami" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" -dependencies = [ - "redox_syscall 0.5.10", - "wasite", -] - [[package]] name = "winapi" version = "0.3.9" @@ -6543,15 +6202,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -6570,21 +6220,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -6617,12 +6252,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6635,12 +6264,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -6653,12 +6276,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -6683,12 +6300,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6701,12 +6312,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6719,12 +6324,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6737,12 +6336,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/thoth-processor/Cargo.toml b/thoth-processor/Cargo.toml index 34d49bdff..37bf24370 100644 --- a/thoth-processor/Cargo.toml +++ b/thoth-processor/Cargo.toml @@ -7,10 +7,9 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_pg"] } +loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } -sqlx = { version = "0.8.6", features = ["runtime-tokio"] } tokio = { version = "1.33.0", default-features = false, features = [ "rt-multi-thread", ] } diff --git a/thoth-processor/config/development.yaml b/thoth-processor/config/development.yaml index eae8de85d..05a86bb54 100644 --- a/thoth-processor/config/development.yaml +++ b/thoth-processor/config/development.yaml @@ -41,7 +41,7 @@ workers: mode: BackgroundQueue queue: - kind: Postgres - # Postgres connection URI - uri: {{ get_env(name="DATABASE_URL") }} + kind: Redis + # Redis connection URI + uri: {{ get_env(name="REDIS_URL") }} dangerously_flush: false diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml index eae8de85d..a4fbaa956 100644 --- a/thoth-processor/config/test.yaml +++ b/thoth-processor/config/test.yaml @@ -41,7 +41,7 @@ workers: mode: BackgroundQueue queue: - kind: Postgres - # Postgres connection URI - uri: {{ get_env(name="DATABASE_URL") }} + kind: Redis + # Redis connection URI + uri: {{ get_env(name="TEST_REDIS_URL") }} dangerously_flush: false diff --git a/thoth-processor/src/initializers/handle_events.rs b/thoth-processor/src/initializers/handle_events.rs index fe5338449..5c5ee3140 100644 --- a/thoth-processor/src/initializers/handle_events.rs +++ b/thoth-processor/src/initializers/handle_events.rs @@ -23,7 +23,10 @@ impl Initializer for HandleEvents { async fn before_run(&self, ctx: &AppContext) -> Result<()> { let ctx = ctx.clone(); - let redis_url = "redis://localhost:6379"; + let redis_url = match ctx.config.queue.as_ref().unwrap() { + QueueConfig::Redis(queue) => &queue.uri, + _ => unreachable!(), + }; let redis = init_pool(redis_url); tokio::spawn(async move { From 47639ef0f1b6e4f980a6b2f21849854942c617f7 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:50:27 +0100 Subject: [PATCH 61/69] Draft (inline) logic for retrieving Loco Redis queue items via GraphQL (tested, working) --- thoth-api/src/event/model.rs | 6 +++ thoth-api/src/graphql/model.rs | 97 ++++++++++++++++++++++++++++++++++ thoth-api/src/redis.rs | 10 ++++ 3 files changed, 113 insertions(+) diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 618cf5d3a..55dafabf0 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -36,6 +36,7 @@ pub enum EventType { WorkPublished, } +#[cfg_attr(feature = "backend", derive(juniper::GraphQLObject))] #[derive(Debug, Deserialize, Serialize)] pub struct Event { pub event_type: EventType, @@ -44,3 +45,8 @@ pub struct Event { pub event_timestamp: Timestamp, pub thoth_version: String, } +#[cfg_attr(feature = "backend", derive(juniper::GraphQLObject))] +#[derive(Debug, Deserialize, Serialize)] +pub struct EventWrapper { + pub event: Event, +} diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 54d7854de..554c3db3f 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -1512,6 +1512,103 @@ impl QueryRoot { fn webhook_count(context: &Context) -> FieldResult { Webhook::count(&context.db, None, vec![], vec![], vec![], None).map_err(|e| e.into()) } + + #[graphql(description = "Get information about retried jobs")] + async fn retried_jobs(context: &Context) -> FieldResult> { + get_jobs(&context.redis).await.map_err(|e| e.into()) + } +} + +use serde::{Deserialize, Serialize}; +#[derive(Debug, Deserialize, Serialize)] +pub struct Job { + pub queue: String, + pub args: Vec, + pub retry: bool, + pub class: String, + pub jid: String, + pub created_at: f64, + pub enqueued_at: f64, + pub failed_at: f64, + pub error_message: String, + pub error_class: Option, + pub retry_count: i32, + pub retried_at: f64, +} + +pub const RETRY_QUEUE_KEY: &str = "retry"; + +pub async fn get_jobs(redis: &RedisPool) -> ThothResult> { + use crate::redis::zrange; + let jobs_vec_string = zrange(redis, RETRY_QUEUE_KEY, 0, -1).await?; + let jobs_vec_json_result = jobs_vec_string + .into_iter() + .map(|j| serde_json::from_str(&j).map_err(|e| e.into())) + .collect::>>(); + jobs_vec_json_result +} + +#[juniper::graphql_object(Context = Context, description = "")] +impl Job { + #[graphql(description = "")] + pub fn queue(&self) -> &String { + &self.queue + } + + #[graphql(description = "")] + pub fn args(&self) -> &Vec { + &self.args + } + + #[graphql(description = "")] + pub fn retry(&self) -> bool { + self.retry + } + + #[graphql(description = "")] + pub fn class(&self) -> &String { + &self.class + } + + #[graphql(description = "")] + pub fn jid(&self) -> &String { + &self.jid + } + + #[graphql(description = "")] + pub fn created_at(&self) -> f64 { + self.created_at + } + + #[graphql(description = "")] + pub fn enqueued_at(&self) -> f64 { + self.enqueued_at + } + + #[graphql(description = "")] + pub fn failed_at(&self) -> f64 { + self.failed_at + } + + #[graphql(description = "")] + pub fn error_message(&self) -> &String { + &self.error_message + } + + #[graphql(description = "")] + pub fn error_class(&self) -> Option<&String> { + self.error_class.as_ref() + } + + #[graphql(description = "")] + pub fn retry_count(&self) -> i32 { + self.retry_count + } + + #[graphql(description = "")] + pub fn retried_at(&self) -> f64 { + self.retried_at + } } pub struct MutationRoot; diff --git a/thoth-api/src/redis.rs b/thoth-api/src/redis.rs index 89ea4aa44..9f73a59b5 100644 --- a/thoth-api/src/redis.rs +++ b/thoth-api/src/redis.rs @@ -49,6 +49,16 @@ pub async fn blpop(pool: &RedisPool, key: &str) -> ThothResult { Ok(value) } +pub async fn zrange( + pool: &RedisPool, + key: &str, + start: isize, + stop: isize, +) -> ThothResult> { + let mut con = create_connection(pool).await?; + con.zrange(key, start, stop).await.map_err(Into::into) +} + #[cfg(test)] mod tests { use super::*; From b0fffcd291332bca9bf2643ff40dc314c8796726 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:54:16 +0100 Subject: [PATCH 62/69] Restructure queue retrieval and add GraphQL doc strings --- thoth-api/src/event/model.rs | 7 +- thoth-api/src/graphql/model.rs | 165 ++++++++++++++------------------- thoth-api/src/job/handler.rs | 14 +++ thoth-api/src/job/mod.rs | 2 + thoth-api/src/job/model.rs | 18 ++++ thoth-api/src/lib.rs | 2 + 6 files changed, 113 insertions(+), 95 deletions(-) create mode 100644 thoth-api/src/job/handler.rs create mode 100644 thoth-api/src/job/mod.rs create mode 100644 thoth-api/src/job/model.rs diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 55dafabf0..55e30d52a 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -36,7 +36,11 @@ pub enum EventType { WorkPublished, } -#[cfg_attr(feature = "backend", derive(juniper::GraphQLObject))] +#[cfg_attr( + feature = "backend", + derive(juniper::GraphQLObject), + graphql(description = "Details of a change made to a record which may require follow-up processing"), +)] #[derive(Debug, Deserialize, Serialize)] pub struct Event { pub event_type: EventType, @@ -45,6 +49,7 @@ pub struct Event { pub event_timestamp: Timestamp, pub thoth_version: String, } + #[cfg_attr(feature = "backend", derive(juniper::GraphQLObject))] #[derive(Debug, Deserialize, Serialize)] pub struct EventWrapper { diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 554c3db3f..52d58b981 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -8,7 +8,9 @@ use crate::account::model::AccountAccess; use crate::account::model::DecodedToken; use crate::db::PgPool; use crate::event::handler::send_event; -use crate::event::model::EventType; +use crate::event::model::{EventType, EventWrapper}; +use crate::job::handler::get_jobs; +use crate::job::model::Job; use crate::model::affiliation::*; use crate::model::contribution::*; use crate::model::contributor::*; @@ -1519,98 +1521,6 @@ impl QueryRoot { } } -use serde::{Deserialize, Serialize}; -#[derive(Debug, Deserialize, Serialize)] -pub struct Job { - pub queue: String, - pub args: Vec, - pub retry: bool, - pub class: String, - pub jid: String, - pub created_at: f64, - pub enqueued_at: f64, - pub failed_at: f64, - pub error_message: String, - pub error_class: Option, - pub retry_count: i32, - pub retried_at: f64, -} - -pub const RETRY_QUEUE_KEY: &str = "retry"; - -pub async fn get_jobs(redis: &RedisPool) -> ThothResult> { - use crate::redis::zrange; - let jobs_vec_string = zrange(redis, RETRY_QUEUE_KEY, 0, -1).await?; - let jobs_vec_json_result = jobs_vec_string - .into_iter() - .map(|j| serde_json::from_str(&j).map_err(|e| e.into())) - .collect::>>(); - jobs_vec_json_result -} - -#[juniper::graphql_object(Context = Context, description = "")] -impl Job { - #[graphql(description = "")] - pub fn queue(&self) -> &String { - &self.queue - } - - #[graphql(description = "")] - pub fn args(&self) -> &Vec { - &self.args - } - - #[graphql(description = "")] - pub fn retry(&self) -> bool { - self.retry - } - - #[graphql(description = "")] - pub fn class(&self) -> &String { - &self.class - } - - #[graphql(description = "")] - pub fn jid(&self) -> &String { - &self.jid - } - - #[graphql(description = "")] - pub fn created_at(&self) -> f64 { - self.created_at - } - - #[graphql(description = "")] - pub fn enqueued_at(&self) -> f64 { - self.enqueued_at - } - - #[graphql(description = "")] - pub fn failed_at(&self) -> f64 { - self.failed_at - } - - #[graphql(description = "")] - pub fn error_message(&self) -> &String { - &self.error_message - } - - #[graphql(description = "")] - pub fn error_class(&self) -> Option<&String> { - self.error_class.as_ref() - } - - #[graphql(description = "")] - pub fn retry_count(&self) -> i32 { - self.retry_count - } - - #[graphql(description = "")] - pub fn retried_at(&self) -> f64 { - self.retried_at - } -} - pub struct MutationRoot; #[juniper::graphql_object(Context = Context)] @@ -4429,7 +4339,7 @@ impl Reference { } } -#[juniper::graphql_object(Context = Context, description = "A web request made when a specified data event occurs.")] +#[juniper::graphql_object(Context = Context, description = "A web request to be made when a specified data event occurs")] impl Webhook { #[graphql(description = "Thoth ID of the webhook")] pub fn webhook_id(&self) -> Uuid { @@ -4477,6 +4387,73 @@ impl Webhook { } } +#[juniper::graphql_object(Context = Context, description = "A task generated when a webhook is triggered")] +impl Job { + #[graphql(description = "Name of the processing queue where this job was initially created")] + pub fn queue(&self) -> &String { + &self.queue + } + + #[graphql(description = "Arguments which were supplied when generating the job")] + pub fn args(&self) -> &Vec { + &self.args + } + + #[graphql(description = "Whether or not the job should be retried on failure")] + pub fn retry(&self) -> bool { + self.retry + } + + #[graphql(description = "Type of the job (defined by the name of the worker processing it)")] + pub fn class(&self) -> &String { + &self.class + } + + #[graphql(description = "Unique identifier of the job")] + pub fn jid(&self) -> &String { + &self.jid + } + + #[graphql(description = "Date and time at which the job was created (in unix epoch format)")] + pub fn created_at(&self) -> f64 { + self.created_at + } + + #[graphql( + description = "Date and time at which the job was added to the queue (in unix epoch format)" + )] + pub fn enqueued_at(&self) -> f64 { + self.enqueued_at + } + + #[graphql(description = "Date and time at which the job failed (in unix epoch format)")] + pub fn failed_at(&self) -> f64 { + self.failed_at + } + + #[graphql(description = "Error message returned on failure of the job")] + pub fn error_message(&self) -> &String { + &self.error_message + } + + #[graphql(description = "Type of error with which the job failed")] + pub fn error_class(&self) -> Option<&String> { + self.error_class.as_ref() + } + + #[graphql(description = "Number of times the job has been retried")] + pub fn retry_count(&self) -> i32 { + self.retry_count + } + + #[graphql( + description = "Date and time at which the job was last retried (in unix epoch format)" + )] + pub fn retried_at(&self) -> f64 { + self.retried_at + } +} + pub type Schema = RootNode<'static, QueryRoot, MutationRoot, EmptySubscription>; pub fn create_schema() -> Schema { diff --git a/thoth-api/src/job/handler.rs b/thoth-api/src/job/handler.rs new file mode 100644 index 000000000..25d01ca07 --- /dev/null +++ b/thoth-api/src/job/handler.rs @@ -0,0 +1,14 @@ +use super::model::Job; +use crate::redis::{zrange, RedisPool}; +use thoth_errors::ThothResult; + +pub const RETRY_QUEUE_KEY: &str = "retry"; + +pub async fn get_jobs(redis: &RedisPool) -> ThothResult> { + let jobs_vec_string = zrange(redis, RETRY_QUEUE_KEY, 0, -1).await?; + let jobs_vec_json_result = jobs_vec_string + .into_iter() + .map(|j| serde_json::from_str(&j).map_err(|e| e.into())) + .collect::>>(); + jobs_vec_json_result +} diff --git a/thoth-api/src/job/mod.rs b/thoth-api/src/job/mod.rs new file mode 100644 index 000000000..8583a1b23 --- /dev/null +++ b/thoth-api/src/job/mod.rs @@ -0,0 +1,2 @@ +pub mod handler; +pub mod model; diff --git a/thoth-api/src/job/model.rs b/thoth-api/src/job/model.rs new file mode 100644 index 000000000..6c038862e --- /dev/null +++ b/thoth-api/src/job/model.rs @@ -0,0 +1,18 @@ +use crate::event::model::EventWrapper; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub struct Job { + pub queue: String, + pub args: Vec, + pub retry: bool, + pub class: String, + pub jid: String, + pub created_at: f64, + pub enqueued_at: f64, + pub failed_at: f64, + pub error_message: String, + pub error_class: Option, + pub retry_count: i32, + pub retried_at: f64, +} diff --git a/thoth-api/src/lib.rs b/thoth-api/src/lib.rs index b7e400ec9..030c52f5a 100644 --- a/thoth-api/src/lib.rs +++ b/thoth-api/src/lib.rs @@ -19,6 +19,8 @@ pub mod account; pub mod db; pub mod event; pub mod graphql; +#[cfg(feature = "backend")] +pub mod job; #[macro_use] pub mod model; #[cfg(feature = "backend")] From b399e0cfebfb40db4311a91737d43054f082fb24 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 8 Jul 2025 12:12:17 +0100 Subject: [PATCH 63/69] Small formatting fixes --- thoth-api/src/event/model.rs | 4 +++- thoth-api/src/job/handler.rs | 7 +++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/thoth-api/src/event/model.rs b/thoth-api/src/event/model.rs index 55e30d52a..76ce9b8a0 100644 --- a/thoth-api/src/event/model.rs +++ b/thoth-api/src/event/model.rs @@ -39,7 +39,9 @@ pub enum EventType { #[cfg_attr( feature = "backend", derive(juniper::GraphQLObject), - graphql(description = "Details of a change made to a record which may require follow-up processing"), + graphql( + description = "Details of a change made to a record which may require follow-up processing" + ) )] #[derive(Debug, Deserialize, Serialize)] pub struct Event { diff --git a/thoth-api/src/job/handler.rs b/thoth-api/src/job/handler.rs index 25d01ca07..42449f53e 100644 --- a/thoth-api/src/job/handler.rs +++ b/thoth-api/src/job/handler.rs @@ -5,10 +5,9 @@ use thoth_errors::ThothResult; pub const RETRY_QUEUE_KEY: &str = "retry"; pub async fn get_jobs(redis: &RedisPool) -> ThothResult> { - let jobs_vec_string = zrange(redis, RETRY_QUEUE_KEY, 0, -1).await?; - let jobs_vec_json_result = jobs_vec_string + let jobs_string = zrange(redis, RETRY_QUEUE_KEY, 0, -1).await?; + jobs_string .into_iter() .map(|j| serde_json::from_str(&j).map_err(|e| e.into())) - .collect::>>(); - jobs_vec_json_result + .collect::>>() } From 06cee92cf4a9796b60390608209222be14eca60c Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 10 Jul 2025 10:43:32 +0100 Subject: [PATCH 64/69] Minor improvements for consistency etc --- Cargo.lock | 2 +- Cargo.toml | 2 +- thoth-processor/Cargo.toml | 10 +++-- thoth-processor/config/development.yaml | 2 +- thoth-processor/config/test.yaml | 2 +- thoth-processor/src/requests/graphql.rs | 42 +++++++++++-------- .../src/workers/fire_webhook_worker.rs | 2 - .../src/workers/work_created_worker.rs | 3 -- .../src/workers/work_published_worker.rs | 3 -- .../src/workers/work_updated_worker.rs | 3 -- .../tests/workers/fire_webhook_worker.rs | 29 +++++++++++++ thoth-processor/tests/workers/mod.rs | 1 + 12 files changed, 65 insertions(+), 36 deletions(-) create mode 100644 thoth-processor/tests/workers/fire_webhook_worker.rs diff --git a/Cargo.lock b/Cargo.lock index 2f17588b4..c16047fd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5353,7 +5353,7 @@ dependencies = [ [[package]] name = "thoth-processor" -version = "0.1.0" +version = "0.13.13" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index 8c47f4c63..c523fc172 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ thoth-api-server = { version = "=0.13.13", path = "thoth-api-server" } thoth-app-server = { version = "=0.13.13", path = "thoth-app-server" } thoth-errors = { version = "=0.13.13", path = "thoth-errors" } thoth-export-server = { version = "=0.13.13", path = "thoth-export-server" } -thoth-processor = { version = "=0.1.0", path = "thoth-processor" } +thoth-processor = { version = "=0.13.13", path = "thoth-processor" } clap = { version = "4.5.32", features = ["cargo", "env"] } dialoguer = { version = "0.11.0", features = ["password"] } dotenv = "0.15.0" diff --git a/thoth-processor/Cargo.toml b/thoth-processor/Cargo.toml index 37bf24370..8a336f29e 100644 --- a/thoth-processor/Cargo.toml +++ b/thoth-processor/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "thoth-processor" -version = "0.1.0" +version = "0.13.13" +authors = ["Javier Arias ", "Ross Higman ", "Brendan O'Connell "] edition = "2021" -publish = false - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +license = "Apache-2.0" +description = "Event processor for Thoth" +repository = "https://github.com/thoth-pub/thoth" +readme = "README.md" [dependencies] loco-rs = { version = "0.14.0", default-features = false, features = ["cli", "bg_redis"] } diff --git a/thoth-processor/config/development.yaml b/thoth-processor/config/development.yaml index 05a86bb54..2220162f3 100644 --- a/thoth-processor/config/development.yaml +++ b/thoth-processor/config/development.yaml @@ -17,7 +17,7 @@ logger: # Web server configuration server: # Port on which the server will listen. the server binding is 0.0.0.0:{PORT} - port: 5150 + port: 8282 # Binding for the server (which interface to bind to) binding: localhost # The UI hostname or IP address that mailers will point to. diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml index a4fbaa956..2d3f4129b 100644 --- a/thoth-processor/config/test.yaml +++ b/thoth-processor/config/test.yaml @@ -17,7 +17,7 @@ logger: # Web server configuration server: # Port on which the server will listen. the server binding is 0.0.0.0:{PORT} - port: 5150 + port: 8282 # Binding for the server (which interface to bind to) binding: localhost # The UI hostname or IP address that mailers will point to. diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index aa1cb98cc..c2d3445fb 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -3,6 +3,27 @@ use serde::{Deserialize, Serialize}; use thoth_api::event::model::{Event, EventType}; use uuid::Uuid; +const WEBHOOKS_QUERY: &str = " + query WebhooksQuery( + $workId: Uuid!, + $eventTypes: [EventType!], + $isPublished: Boolean!, + ) { + work(workId: $workId) { + imprint { + publisher { + webhooks(eventTypes: $eventTypes, isPublished: $isPublished) { + endpoint + token + isPublished + eventType + } + } + } + } + } +"; + #[derive(Deserialize, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct WebhooksVariables { @@ -60,29 +81,16 @@ pub struct WebhooksResponseBody { pub async fn query_webhooks(event: Event) -> Result, Error> { let client = reqwest::Client::new(); let url = "https://api.thoth.pub/graphql".to_string(); - let query = " -query WebhooksQuery($workId: Uuid!, $eventTypes: [EventType!], $isPublished: Boolean!) { - work(workId: $workId) { - imprint { - publisher { - webhooks(eventTypes: $eventTypes, isPublished: $isPublished) { - endpoint - token - isPublished - eventType - } - } - } - } -}" - .to_string(); let variables = WebhooksVariables { work_id: event.work_id, event_types: vec![event.event_type], is_published: event.is_published, }; - let body = WebhooksQueryBody { query, variables }; + let body = WebhooksQueryBody { + query: WEBHOOKS_QUERY.to_string(), + variables, + }; let token = "placeholder".to_string(); let response = client diff --git a/thoth-processor/src/workers/fire_webhook_worker.rs b/thoth-processor/src/workers/fire_webhook_worker.rs index 2b3f4ee63..b285c0dff 100644 --- a/thoth-processor/src/workers/fire_webhook_worker.rs +++ b/thoth-processor/src/workers/fire_webhook_worker.rs @@ -19,11 +19,9 @@ impl BackgroundWorker for FireWebhookWorker { } async fn perform(&self, args: FireWebhookWorkerArgs) -> Result<()> { - tracing::info!("FireWebhookWorker start"); tracing::info!("Webhook: {:?}", args.webhook); let target_rsp = fire_webhook(args.webhook.endpoint, args.webhook.token).await?; tracing::info!("Target response: {:?}", target_rsp); - tracing::info!("FireWebhookWorker end"); Ok(()) } diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs index 4fa1149ec..fbec2b1a9 100644 --- a/thoth-processor/src/workers/work_created_worker.rs +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -20,7 +20,6 @@ impl BackgroundWorker for WorkCreatedWorker { } async fn perform(&self, args: WorkCreatedWorkerArgs) -> Result<()> { - tracing::info!("WorkCreatedWorker start"); tracing::info!("Event: {:?}", args.event); let webhooks = query_webhooks(args.event).await?; tracing::info!("Webhooks: {:?}", webhooks); @@ -30,8 +29,6 @@ impl BackgroundWorker for WorkCreatedWorker { .await; } - tracing::info!("WorkCreatedWorker end"); - Ok(()) } } diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index 6438cfb8e..1d73fc8c9 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -20,7 +20,6 @@ impl BackgroundWorker for WorkPublishedWorker { } async fn perform(&self, args: WorkPublishedWorkerArgs) -> Result<()> { - tracing::info!("WorkPublishedWorker start"); tracing::info!("Event: {:?}", args.event); let webhooks = query_webhooks(args.event).await?; tracing::info!("Webhooks: {:?}", webhooks); @@ -30,8 +29,6 @@ impl BackgroundWorker for WorkPublishedWorker { .await; } - tracing::info!("WorkPublishedWorker end"); - Ok(()) } } diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs index 04584051f..121730933 100644 --- a/thoth-processor/src/workers/work_updated_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -20,7 +20,6 @@ impl BackgroundWorker for WorkUpdatedWorker { } async fn perform(&self, args: WorkUpdatedWorkerArgs) -> Result<()> { - tracing::info!("WorkUpdatedWorker start"); tracing::info!("Event: {:?}", args.event); let webhooks = query_webhooks(args.event).await?; tracing::info!("Webhooks: {:?}", webhooks); @@ -30,8 +29,6 @@ impl BackgroundWorker for WorkUpdatedWorker { .await; } - tracing::info!("WorkUpdatedWorker end"); - Ok(()) } } diff --git a/thoth-processor/tests/workers/fire_webhook_worker.rs b/thoth-processor/tests/workers/fire_webhook_worker.rs new file mode 100644 index 000000000..cbf19d231 --- /dev/null +++ b/thoth-processor/tests/workers/fire_webhook_worker.rs @@ -0,0 +1,29 @@ +use loco_rs::{bgworker::BackgroundWorker, testing::prelude::*}; +use serial_test::serial; +use thoth_processor::{ + app::App, + requests::graphql::Webhook, + workers::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}, +}; + +#[tokio::test] +#[serial] +async fn test_run_work_created_worker_worker() { + let boot = boot_test::().await.unwrap(); + + // Execute the worker ensuring that it operates in 'ForegroundBlocking' mode, which prevents the addition of your worker to the background + assert!(FireWebhookWorker::perform_later( + &boot.app_context, + FireWebhookWorkerArgs { + webhook: Webhook { + endpoint: Default::default(), + token: Default::default(), + is_published: Default::default(), + event_type: Default::default(), + } + } + ) + .await + .is_ok()); + // Include additional assert validations after the execution of the worker +} diff --git a/thoth-processor/tests/workers/mod.rs b/thoth-processor/tests/workers/mod.rs index cc74d4c06..fd6a481d6 100644 --- a/thoth-processor/tests/workers/mod.rs +++ b/thoth-processor/tests/workers/mod.rs @@ -1,3 +1,4 @@ +mod fire_webhook_worker; mod work_created_worker; mod work_published_worker; mod work_updated_worker; From 3aba604bf440963747bb6d264c6525423367a42c Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:01:50 +0100 Subject: [PATCH 65/69] Retrieve GraphQL API URL from config instead of hard-coding --- thoth-processor/config/development.yaml | 3 +++ thoth-processor/config/test.yaml | 3 +++ thoth-processor/src/common/mod.rs | 1 + thoth-processor/src/common/settings.rs | 13 +++++++++++++ thoth-processor/src/lib.rs | 1 + thoth-processor/src/requests/graphql.rs | 4 +--- thoth-processor/src/workers/work_created_worker.rs | 11 +++++++++-- .../src/workers/work_published_worker.rs | 11 +++++++++-- thoth-processor/src/workers/work_updated_worker.rs | 11 +++++++++-- 9 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 thoth-processor/src/common/mod.rs create mode 100644 thoth-processor/src/common/settings.rs diff --git a/thoth-processor/config/development.yaml b/thoth-processor/config/development.yaml index 2220162f3..1c93ff406 100644 --- a/thoth-processor/config/development.yaml +++ b/thoth-processor/config/development.yaml @@ -45,3 +45,6 @@ queue: # Redis connection URI uri: {{ get_env(name="REDIS_URL") }} dangerously_flush: false + +settings: + thoth_graphql_api: {{ get_env(name="THOTH_GRAPHQL_API") }} diff --git a/thoth-processor/config/test.yaml b/thoth-processor/config/test.yaml index 2d3f4129b..82a82f992 100644 --- a/thoth-processor/config/test.yaml +++ b/thoth-processor/config/test.yaml @@ -45,3 +45,6 @@ queue: # Redis connection URI uri: {{ get_env(name="TEST_REDIS_URL") }} dangerously_flush: false + +settings: + thoth_graphql_api: {{ get_env(name="THOTH_GRAPHQL_API") }} diff --git a/thoth-processor/src/common/mod.rs b/thoth-processor/src/common/mod.rs new file mode 100644 index 000000000..6e98cefd0 --- /dev/null +++ b/thoth-processor/src/common/mod.rs @@ -0,0 +1 @@ +pub mod settings; diff --git a/thoth-processor/src/common/settings.rs b/thoth-processor/src/common/settings.rs new file mode 100644 index 000000000..e0939d1f0 --- /dev/null +++ b/thoth-processor/src/common/settings.rs @@ -0,0 +1,13 @@ +use loco_rs::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Default, Debug)] +pub struct Settings { + pub thoth_graphql_api: String, +} + +impl Settings { + pub fn from_json(value: &serde_json::Value) -> Result { + Ok(serde_json::from_value(value.clone())?) + } +} diff --git a/thoth-processor/src/lib.rs b/thoth-processor/src/lib.rs index 7e5cf0ffe..8cbec65c8 100644 --- a/thoth-processor/src/lib.rs +++ b/thoth-processor/src/lib.rs @@ -1,4 +1,5 @@ pub mod app; +pub mod common; pub mod initializers; pub mod requests; pub mod workers; diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index c2d3445fb..e65ce0079 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -78,10 +78,8 @@ pub struct WebhooksResponseBody { pub data: WebhooksResponseData, } -pub async fn query_webhooks(event: Event) -> Result, Error> { +pub async fn query_webhooks(url: String, event: Event) -> Result, Error> { let client = reqwest::Client::new(); - let url = "https://api.thoth.pub/graphql".to_string(); - let variables = WebhooksVariables { work_id: event.work_id, event_types: vec![event.event_type], diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs index fbec2b1a9..391bd0d6a 100644 --- a/thoth-processor/src/workers/work_created_worker.rs +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -1,5 +1,5 @@ use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; -use crate::requests::graphql::query_webhooks; +use crate::{common::settings::Settings, requests::graphql::query_webhooks}; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -21,7 +21,14 @@ impl BackgroundWorker for WorkCreatedWorker { async fn perform(&self, args: WorkCreatedWorkerArgs) -> Result<()> { tracing::info!("Event: {:?}", args.event); - let webhooks = query_webhooks(args.event).await?; + let webhooks = query_webhooks( + format!( + "{}/graphql", + Settings::from_json(&self.ctx.config.settings.as_ref().unwrap())?.thoth_graphql_api + ), + args.event, + ) + .await?; tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index 1d73fc8c9..3488e88be 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -1,5 +1,5 @@ use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; -use crate::requests::graphql::query_webhooks; +use crate::{common::settings::Settings, requests::graphql::query_webhooks}; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -21,7 +21,14 @@ impl BackgroundWorker for WorkPublishedWorker { async fn perform(&self, args: WorkPublishedWorkerArgs) -> Result<()> { tracing::info!("Event: {:?}", args.event); - let webhooks = query_webhooks(args.event).await?; + let webhooks = query_webhooks( + format!( + "{}/graphql", + Settings::from_json(&self.ctx.config.settings.as_ref().unwrap())?.thoth_graphql_api + ), + args.event, + ) + .await?; tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs index 121730933..260ed5010 100644 --- a/thoth-processor/src/workers/work_updated_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -1,5 +1,5 @@ use super::fire_webhook_worker::{FireWebhookWorker, FireWebhookWorkerArgs}; -use crate::requests::graphql::query_webhooks; +use crate::{common::settings::Settings, requests::graphql::query_webhooks}; use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use thoth_api::event::model::Event; @@ -21,7 +21,14 @@ impl BackgroundWorker for WorkUpdatedWorker { async fn perform(&self, args: WorkUpdatedWorkerArgs) -> Result<()> { tracing::info!("Event: {:?}", args.event); - let webhooks = query_webhooks(args.event).await?; + let webhooks = query_webhooks( + format!( + "{}/graphql", + Settings::from_json(&self.ctx.config.settings.as_ref().unwrap())?.thoth_graphql_api + ), + args.event, + ) + .await?; tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { From 743a8fe7c3b98b4bef9ff591c7b7e22ac75a2f18 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 22 Jul 2025 15:16:48 +0100 Subject: [PATCH 66/69] Add client payload and pass down work_id to send to GitHub Action --- thoth-processor/src/requests/target.rs | 14 +++++++++++++- thoth-processor/src/workers/fire_webhook_worker.rs | 5 ++++- thoth-processor/src/workers/work_created_worker.rs | 8 ++++++-- .../src/workers/work_published_worker.rs | 8 ++++++-- thoth-processor/src/workers/work_updated_worker.rs | 8 ++++++-- .../tests/workers/fire_webhook_worker.rs | 1 + 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs index c99a17094..0da2bcc45 100644 --- a/thoth-processor/src/requests/target.rs +++ b/thoth-processor/src/requests/target.rs @@ -1,12 +1,23 @@ use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Deserialize, Debug, Serialize)] +pub struct ClientPayload { + work_id: Uuid, +} #[derive(Deserialize, Debug, Serialize)] pub struct Payload { event_type: String, + client_payload: ClientPayload, } -pub async fn fire_webhook(url: String, token: Option) -> Result { +pub async fn fire_webhook( + url: String, + token: Option, + work_id: Uuid, +) -> Result { let client = reqwest::Client::new(); let response = client @@ -19,6 +30,7 @@ pub async fn fire_webhook(url: String, token: Option) -> Result for FireWebhookWorker { async fn perform(&self, args: FireWebhookWorkerArgs) -> Result<()> { tracing::info!("Webhook: {:?}", args.webhook); - let target_rsp = fire_webhook(args.webhook.endpoint, args.webhook.token).await?; + let target_rsp = + fire_webhook(args.webhook.endpoint, args.webhook.token, args.work_id).await?; tracing::info!("Target response: {:?}", target_rsp); Ok(()) diff --git a/thoth-processor/src/workers/work_created_worker.rs b/thoth-processor/src/workers/work_created_worker.rs index 391bd0d6a..68611a5de 100644 --- a/thoth-processor/src/workers/work_created_worker.rs +++ b/thoth-processor/src/workers/work_created_worker.rs @@ -21,6 +21,7 @@ impl BackgroundWorker for WorkCreatedWorker { async fn perform(&self, args: WorkCreatedWorkerArgs) -> Result<()> { tracing::info!("Event: {:?}", args.event); + let work_id = args.event.work_id; let webhooks = query_webhooks( format!( "{}/graphql", @@ -32,8 +33,11 @@ impl BackgroundWorker for WorkCreatedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let _ = FireWebhookWorker::perform_later(&self.ctx, FireWebhookWorkerArgs { webhook }) - .await; + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { work_id, webhook }, + ) + .await; } Ok(()) diff --git a/thoth-processor/src/workers/work_published_worker.rs b/thoth-processor/src/workers/work_published_worker.rs index 3488e88be..da80c44a1 100644 --- a/thoth-processor/src/workers/work_published_worker.rs +++ b/thoth-processor/src/workers/work_published_worker.rs @@ -21,6 +21,7 @@ impl BackgroundWorker for WorkPublishedWorker { async fn perform(&self, args: WorkPublishedWorkerArgs) -> Result<()> { tracing::info!("Event: {:?}", args.event); + let work_id = args.event.work_id; let webhooks = query_webhooks( format!( "{}/graphql", @@ -32,8 +33,11 @@ impl BackgroundWorker for WorkPublishedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let _ = FireWebhookWorker::perform_later(&self.ctx, FireWebhookWorkerArgs { webhook }) - .await; + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { work_id, webhook }, + ) + .await; } Ok(()) diff --git a/thoth-processor/src/workers/work_updated_worker.rs b/thoth-processor/src/workers/work_updated_worker.rs index 260ed5010..1e806553e 100644 --- a/thoth-processor/src/workers/work_updated_worker.rs +++ b/thoth-processor/src/workers/work_updated_worker.rs @@ -21,6 +21,7 @@ impl BackgroundWorker for WorkUpdatedWorker { async fn perform(&self, args: WorkUpdatedWorkerArgs) -> Result<()> { tracing::info!("Event: {:?}", args.event); + let work_id = args.event.work_id; let webhooks = query_webhooks( format!( "{}/graphql", @@ -32,8 +33,11 @@ impl BackgroundWorker for WorkUpdatedWorker { tracing::info!("Webhooks: {:?}", webhooks); for webhook in webhooks { - let _ = FireWebhookWorker::perform_later(&self.ctx, FireWebhookWorkerArgs { webhook }) - .await; + let _ = FireWebhookWorker::perform_later( + &self.ctx, + FireWebhookWorkerArgs { work_id, webhook }, + ) + .await; } Ok(()) diff --git a/thoth-processor/tests/workers/fire_webhook_worker.rs b/thoth-processor/tests/workers/fire_webhook_worker.rs index cbf19d231..e4ab79401 100644 --- a/thoth-processor/tests/workers/fire_webhook_worker.rs +++ b/thoth-processor/tests/workers/fire_webhook_worker.rs @@ -15,6 +15,7 @@ async fn test_run_work_created_worker_worker() { assert!(FireWebhookWorker::perform_later( &boot.app_context, FireWebhookWorkerArgs { + work_id: Default::default(), webhook: Webhook { endpoint: Default::default(), token: Default::default(), From f0264971e8633cb7c2e7ae86ea6b63cc64326bb2 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Tue, 22 Jul 2025 15:44:25 +0100 Subject: [PATCH 67/69] Add platform to webhook data model so that it can be passed to GitHub Action --- thoth-api/migrations/v0.13.14/up.sql | 1 + thoth-api/src/graphql/model.rs | 5 +++++ thoth-api/src/model/webhook/crud.rs | 4 ++++ thoth-api/src/model/webhook/mod.rs | 4 ++++ thoth-api/src/schema.rs | 1 + thoth-processor/src/requests/graphql.rs | 2 ++ thoth-processor/src/requests/target.rs | 7 ++++++- thoth-processor/src/workers/fire_webhook_worker.rs | 7 ++++++- thoth-processor/tests/workers/fire_webhook_worker.rs | 1 + 9 files changed, 30 insertions(+), 2 deletions(-) diff --git a/thoth-api/migrations/v0.13.14/up.sql b/thoth-api/migrations/v0.13.14/up.sql index 3261bf411..62f0543a0 100644 --- a/thoth-api/migrations/v0.13.14/up.sql +++ b/thoth-api/migrations/v0.13.14/up.sql @@ -11,6 +11,7 @@ CREATE TABLE webhook ( token TEXT CHECK (OCTET_LENGTH(token) >= 1), is_published BOOLEAN NOT NULL, event_type event_type NOT NULL, + platform TEXT CHECK (OCTET_LENGTH(platform) >= 1), created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 52d58b981..802ff12d9 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -4371,6 +4371,11 @@ impl Webhook { self.is_published } + #[graphql(description = "Platform which is targeted by the webhook")] + pub fn platform(&self) -> Option<&String> { + self.platform.as_ref() + } + #[graphql(description = "Date and time at which the webhook record was created")] pub fn created_at(&self) -> Timestamp { self.created_at diff --git a/thoth-api/src/model/webhook/crud.rs b/thoth-api/src/model/webhook/crud.rs index b7efb74bd..6efd88189 100644 --- a/thoth-api/src/model/webhook/crud.rs +++ b/thoth-api/src/model/webhook/crud.rs @@ -65,6 +65,10 @@ impl Crud for Webhook { Direction::Asc => query.order(event_type.asc()), Direction::Desc => query.order(event_type.desc()), }, + WebhookField::Platform => match order.direction { + Direction::Asc => query.order(platform.asc()), + Direction::Desc => query.order(platform.desc()), + }, WebhookField::CreatedAt => match order.direction { Direction::Asc => query.order(created_at.asc()), Direction::Desc => query.order(created_at.desc()), diff --git a/thoth-api/src/model/webhook/mod.rs b/thoth-api/src/model/webhook/mod.rs index 76cb99170..78fa7d543 100644 --- a/thoth-api/src/model/webhook/mod.rs +++ b/thoth-api/src/model/webhook/mod.rs @@ -25,6 +25,7 @@ pub enum WebhookField { Token, IsPublished, EventType, + Platform, CreatedAt, UpdatedAt, } @@ -39,6 +40,7 @@ pub struct Webhook { pub token: Option, pub is_published: bool, pub event_type: EventType, + pub platform: Option, pub created_at: Timestamp, pub updated_at: Timestamp, } @@ -55,6 +57,7 @@ pub struct NewWebhook { pub token: Option, pub is_published: bool, pub event_type: EventType, + pub platform: Option, } #[cfg_attr( @@ -70,6 +73,7 @@ pub struct PatchWebhook { pub token: Option, pub is_published: bool, pub event_type: EventType, + pub platform: Option, } #[cfg_attr(feature = "backend", derive(Queryable))] diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index 1405c9342..041328971 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -529,6 +529,7 @@ table! { token -> Nullable, is_published -> Bool, event_type -> EventType, + platform -> Nullable, created_at -> Timestamptz, updated_at -> Timestamptz, } diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index e65ce0079..b2f95685d 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -17,6 +17,7 @@ const WEBHOOKS_QUERY: &str = " token isPublished eventType + platform } } } @@ -46,6 +47,7 @@ pub struct Webhook { pub token: Option, pub is_published: bool, pub event_type: EventType, + pub platform: Option, } #[derive(Deserialize, Debug, Serialize)] diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs index 0da2bcc45..bff15e5cd 100644 --- a/thoth-processor/src/requests/target.rs +++ b/thoth-processor/src/requests/target.rs @@ -5,6 +5,7 @@ use uuid::Uuid; #[derive(Deserialize, Debug, Serialize)] pub struct ClientPayload { work_id: Uuid, + platform: String, } #[derive(Deserialize, Debug, Serialize)] @@ -17,6 +18,7 @@ pub async fn fire_webhook( url: String, token: Option, work_id: Uuid, + platform: Option, ) -> Result { let client = reqwest::Client::new(); @@ -30,7 +32,10 @@ pub async fn fire_webhook( // (it also seems to determine the name given to any ensuing workflow runs) .json(&Payload { event_type: "test".to_string(), - client_payload: ClientPayload { work_id: work_id }, + client_payload: ClientPayload { + work_id: work_id, + platform: platform.unwrap_or_default(), + }, }) .send() .await? diff --git a/thoth-processor/src/workers/fire_webhook_worker.rs b/thoth-processor/src/workers/fire_webhook_worker.rs index 3c3e03151..4e5f29e2c 100644 --- a/thoth-processor/src/workers/fire_webhook_worker.rs +++ b/thoth-processor/src/workers/fire_webhook_worker.rs @@ -23,7 +23,12 @@ impl BackgroundWorker for FireWebhookWorker { async fn perform(&self, args: FireWebhookWorkerArgs) -> Result<()> { tracing::info!("Webhook: {:?}", args.webhook); let target_rsp = - fire_webhook(args.webhook.endpoint, args.webhook.token, args.work_id).await?; + fire_webhook( + args.webhook.endpoint, + args.webhook.token, + args.work_id, + args.webhook.platform, + ).await?; tracing::info!("Target response: {:?}", target_rsp); Ok(()) diff --git a/thoth-processor/tests/workers/fire_webhook_worker.rs b/thoth-processor/tests/workers/fire_webhook_worker.rs index e4ab79401..f059ccbc2 100644 --- a/thoth-processor/tests/workers/fire_webhook_worker.rs +++ b/thoth-processor/tests/workers/fire_webhook_worker.rs @@ -21,6 +21,7 @@ async fn test_run_work_created_worker_worker() { token: Default::default(), is_published: Default::default(), event_type: Default::default(), + platform: Default::default(), } } ) From 746d3c9b3c69d8222fa6fd2603d8b4789f6ef0fe Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 23 Jul 2025 16:49:58 +0100 Subject: [PATCH 68/69] Make fire_webhook generic, store raw payload instead of platform --- thoth-api/migrations/v0.13.14/up.sql | 2 +- thoth-api/src/graphql/model.rs | 6 +-- thoth-api/src/model/webhook/crud.rs | 6 +-- thoth-api/src/model/webhook/mod.rs | 8 ++-- thoth-api/src/schema.rs | 2 +- thoth-processor/src/requests/graphql.rs | 4 +- thoth-processor/src/requests/target.rs | 44 +++++++------------ .../src/workers/fire_webhook_worker.rs | 14 +++--- .../tests/workers/fire_webhook_worker.rs | 2 +- 9 files changed, 39 insertions(+), 49 deletions(-) diff --git a/thoth-api/migrations/v0.13.14/up.sql b/thoth-api/migrations/v0.13.14/up.sql index 62f0543a0..1e5571a78 100644 --- a/thoth-api/migrations/v0.13.14/up.sql +++ b/thoth-api/migrations/v0.13.14/up.sql @@ -11,7 +11,7 @@ CREATE TABLE webhook ( token TEXT CHECK (OCTET_LENGTH(token) >= 1), is_published BOOLEAN NOT NULL, event_type event_type NOT NULL, - platform TEXT CHECK (OCTET_LENGTH(platform) >= 1), + payload TEXT CHECK (OCTET_LENGTH(payload) >= 1), created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/thoth-api/src/graphql/model.rs b/thoth-api/src/graphql/model.rs index 802ff12d9..41c8a05ef 100644 --- a/thoth-api/src/graphql/model.rs +++ b/thoth-api/src/graphql/model.rs @@ -4371,9 +4371,9 @@ impl Webhook { self.is_published } - #[graphql(description = "Platform which is targeted by the webhook")] - pub fn platform(&self) -> Option<&String> { - self.platform.as_ref() + #[graphql(description = "Payload to be sent in the webhook request body (usually JSON)")] + pub fn payload(&self) -> Option<&String> { + self.payload.as_ref() } #[graphql(description = "Date and time at which the webhook record was created")] diff --git a/thoth-api/src/model/webhook/crud.rs b/thoth-api/src/model/webhook/crud.rs index 6efd88189..d62a28946 100644 --- a/thoth-api/src/model/webhook/crud.rs +++ b/thoth-api/src/model/webhook/crud.rs @@ -65,9 +65,9 @@ impl Crud for Webhook { Direction::Asc => query.order(event_type.asc()), Direction::Desc => query.order(event_type.desc()), }, - WebhookField::Platform => match order.direction { - Direction::Asc => query.order(platform.asc()), - Direction::Desc => query.order(platform.desc()), + WebhookField::Payload => match order.direction { + Direction::Asc => query.order(payload.asc()), + Direction::Desc => query.order(payload.desc()), }, WebhookField::CreatedAt => match order.direction { Direction::Asc => query.order(created_at.asc()), diff --git a/thoth-api/src/model/webhook/mod.rs b/thoth-api/src/model/webhook/mod.rs index 78fa7d543..e0d09fa35 100644 --- a/thoth-api/src/model/webhook/mod.rs +++ b/thoth-api/src/model/webhook/mod.rs @@ -25,7 +25,7 @@ pub enum WebhookField { Token, IsPublished, EventType, - Platform, + Payload, CreatedAt, UpdatedAt, } @@ -40,7 +40,7 @@ pub struct Webhook { pub token: Option, pub is_published: bool, pub event_type: EventType, - pub platform: Option, + pub payload: Option, pub created_at: Timestamp, pub updated_at: Timestamp, } @@ -57,7 +57,7 @@ pub struct NewWebhook { pub token: Option, pub is_published: bool, pub event_type: EventType, - pub platform: Option, + pub payload: Option, } #[cfg_attr( @@ -73,7 +73,7 @@ pub struct PatchWebhook { pub token: Option, pub is_published: bool, pub event_type: EventType, - pub platform: Option, + pub payload: Option, } #[cfg_attr(feature = "backend", derive(Queryable))] diff --git a/thoth-api/src/schema.rs b/thoth-api/src/schema.rs index 041328971..294422da0 100644 --- a/thoth-api/src/schema.rs +++ b/thoth-api/src/schema.rs @@ -529,7 +529,7 @@ table! { token -> Nullable, is_published -> Bool, event_type -> EventType, - platform -> Nullable, + payload -> Nullable, created_at -> Timestamptz, updated_at -> Timestamptz, } diff --git a/thoth-processor/src/requests/graphql.rs b/thoth-processor/src/requests/graphql.rs index b2f95685d..141f55abc 100644 --- a/thoth-processor/src/requests/graphql.rs +++ b/thoth-processor/src/requests/graphql.rs @@ -17,7 +17,7 @@ const WEBHOOKS_QUERY: &str = " token isPublished eventType - platform + payload } } } @@ -47,7 +47,7 @@ pub struct Webhook { pub token: Option, pub is_published: bool, pub event_type: EventType, - pub platform: Option, + pub payload: Option, } #[derive(Deserialize, Debug, Serialize)] diff --git a/thoth-processor/src/requests/target.rs b/thoth-processor/src/requests/target.rs index bff15e5cd..ff56f0f5b 100644 --- a/thoth-processor/src/requests/target.rs +++ b/thoth-processor/src/requests/target.rs @@ -1,42 +1,32 @@ use loco_rs::prelude::*; -use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Deserialize, Debug, Serialize)] -pub struct ClientPayload { - work_id: Uuid, - platform: String, -} - -#[derive(Deserialize, Debug, Serialize)] -pub struct Payload { - event_type: String, - client_payload: ClientPayload, -} - pub async fn fire_webhook( url: String, token: Option, work_id: Uuid, - platform: Option, + payload: Option, ) -> Result { let client = reqwest::Client::new(); - let response = client + let mut request = client .post(&url) - .bearer_auth(token.unwrap_or_default()) // GitHub Actions repository dispatch events require a User-Agent header - .header("User-Agent", "Thoth") - // GitHub Actions repository dispatch events require a payload containing "event_type" - // (this can then be used to control which events trigger which Actions) - // (it also seems to determine the name given to any ensuing workflow runs) - .json(&Payload { - event_type: "test".to_string(), - client_payload: ClientPayload { - work_id: work_id, - platform: platform.unwrap_or_default(), - }, - }) + .header("User-Agent", "Thoth"); + + if let Some(token_value) = token { + request = request.bearer_auth(token_value); + } + + // References for constructing payloads: + // GitHub Actions: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#repository_dispatch + // Mattermost: https://developers.mattermost.com/integrate/webhooks/incoming/ + if let Some(payload_value) = payload { + let interpolated_payload = payload_value.replace("${work_id}", &work_id.to_string()); + request = request.body(interpolated_payload); + } + + let response = request .send() .await? .error_for_status()?; diff --git a/thoth-processor/src/workers/fire_webhook_worker.rs b/thoth-processor/src/workers/fire_webhook_worker.rs index 4e5f29e2c..050b0723f 100644 --- a/thoth-processor/src/workers/fire_webhook_worker.rs +++ b/thoth-processor/src/workers/fire_webhook_worker.rs @@ -22,13 +22,13 @@ impl BackgroundWorker for FireWebhookWorker { async fn perform(&self, args: FireWebhookWorkerArgs) -> Result<()> { tracing::info!("Webhook: {:?}", args.webhook); - let target_rsp = - fire_webhook( - args.webhook.endpoint, - args.webhook.token, - args.work_id, - args.webhook.platform, - ).await?; + let target_rsp = fire_webhook( + args.webhook.endpoint, + args.webhook.token, + args.work_id, + args.webhook.payload, + ) + .await?; tracing::info!("Target response: {:?}", target_rsp); Ok(()) diff --git a/thoth-processor/tests/workers/fire_webhook_worker.rs b/thoth-processor/tests/workers/fire_webhook_worker.rs index f059ccbc2..848e8e3bf 100644 --- a/thoth-processor/tests/workers/fire_webhook_worker.rs +++ b/thoth-processor/tests/workers/fire_webhook_worker.rs @@ -21,7 +21,7 @@ async fn test_run_work_created_worker_worker() { token: Default::default(), is_published: Default::default(), event_type: Default::default(), - platform: Default::default(), + payload: Default::default(), } } ) From a4e64a04426c39a9312348d6f82034941e435d93 Mon Sep 17 00:00:00 2001 From: rhigman <73792779+rhigman@users.noreply.github.com> Date: Wed, 23 Jul 2025 17:42:20 +0100 Subject: [PATCH 69/69] Review markup: hardcode Loco config path --- .env.example | 2 -- thoth-processor/src/lib.rs | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index edab17732..311a47a38 100644 --- a/.env.example +++ b/.env.example @@ -12,8 +12,6 @@ REDIS_URL=redis://localhost:6379 SECRET_KEY=an_up_to_255_bytes_random_key # Logging level RUST_LOG=info -# Location of Loco (thoth-processor) config folder -LOCO_CONFIG_FOLDER=thoth-processor/config/ # Uncomment the following if running with docker # DATABASE_URL=postgres://thoth:thoth@db/thoth diff --git a/thoth-processor/src/lib.rs b/thoth-processor/src/lib.rs index 8cbec65c8..60a20ded2 100644 --- a/thoth-processor/src/lib.rs +++ b/thoth-processor/src/lib.rs @@ -8,9 +8,10 @@ use crate::app::App; use loco_rs::{ app::Hooks, boot::{start, ServeParams, StartMode}, - environment::resolve_from_env, + environment::{resolve_from_env, Environment}, logger, Result, }; +use std::path::Path; #[tokio::main] #[allow(clippy::result_large_err)] @@ -19,8 +20,8 @@ pub async fn start_server() -> Result<()> { } pub async fn start_app() -> Result<()> { - let environment = resolve_from_env().into(); - let config = H::load_config(&environment).await?; + let environment: Environment = resolve_from_env().into(); + let config = environment.load_from_folder(Path::new("thoth-processor/config/"))?; if !H::init_logger(&config, &environment)? { logger::init::(&config.logger)?;