From 47f17e0900d6e1cbe3230b54a702436a03d106cd Mon Sep 17 00:00:00 2001 From: Parnell Springmeyer Date: Wed, 28 Feb 2024 10:37:36 -0800 Subject: [PATCH 1/3] gc.rs: LIMIT number of `orphan_chunk_ids`; fixes #115 --- server/src/gc.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/gc.rs b/server/src/gc.rs index 9b60887..71d4c4c 100644 --- a/server/src/gc.rs +++ b/server/src/gc.rs @@ -158,6 +158,15 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> { let db = state.database().await?; let storage = state.storage().await?; + let orphan_chunk_id_limit = match db.get_database_backend() { + // Default value of --max-allowed-packet https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet + sea_orm::DatabaseBackend::MySql => 67108864, + // Panic limit set by sqlx for postgresql: https://github.com/launchbadge/sqlx/issues/671#issuecomment-687043510 + sea_orm::DatabaseBackend::Postgres => u64::from(u16::MAX), + // Default statement limit imposed by sqlite: https://www.sqlite.org/limits.html#max_variable_number + sea_orm::DatabaseBackend::Sqlite => 500, + }; + // find all orphan chunks... let orphan_chunk_ids = Query::select() .from(Chunk) @@ -171,6 +180,7 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> { .and_where(chunkref::Column::Id.is_null()) .and_where(chunk::Column::State.eq(ChunkState::Valid)) .and_where(chunk::Column::HoldersCount.eq(0)) + .limit(orphan_chunk_id_limit) .lock_with_tables_behavior(LockType::Update, [Chunk], LockBehavior::SkipLocked) .to_owned(); From 2705d1d90b153fb855307344bec4c48cb1e78c38 Mon Sep 17 00:00:00 2001 From: Parnell Springmeyer Date: Wed, 28 Feb 2024 12:53:27 -0800 Subject: [PATCH 2/3] Limit the chunk finding query, not the GC mark query --- server/src/gc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/gc.rs b/server/src/gc.rs index 71d4c4c..073858b 100644 --- a/server/src/gc.rs +++ b/server/src/gc.rs @@ -158,7 +158,7 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> { let db = state.database().await?; let storage = state.storage().await?; - let orphan_chunk_id_limit = match db.get_database_backend() { + let orphan_chunk_limit = match db.get_database_backend() { // Default value of --max-allowed-packet https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet sea_orm::DatabaseBackend::MySql => 67108864, // Panic limit set by sqlx for postgresql: https://github.com/launchbadge/sqlx/issues/671#issuecomment-687043510 @@ -180,7 +180,6 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> { .and_where(chunkref::Column::Id.is_null()) .and_where(chunk::Column::State.eq(ChunkState::Valid)) .and_where(chunk::Column::HoldersCount.eq(0)) - .limit(orphan_chunk_id_limit) .lock_with_tables_behavior(LockType::Update, [Chunk], LockBehavior::SkipLocked) .to_owned(); @@ -200,6 +199,7 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> { let orphan_chunks: Vec = Chunk::find() .filter(chunk::Column::State.eq(ChunkState::Deleted)) + .limit(orphan_chunk_limit) .all(db) .await?; From d3ffcf885c1f2e69de5c07f6ae7ac58514b9570e Mon Sep 17 00:00:00 2001 From: Parnell Springmeyer Date: Wed, 28 Feb 2024 14:06:31 -0800 Subject: [PATCH 3/3] Choose a more sensible limit for MySQL, suggested by @baloo --- server/src/gc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/gc.rs b/server/src/gc.rs index 073858b..a06bdd9 100644 --- a/server/src/gc.rs +++ b/server/src/gc.rs @@ -159,8 +159,8 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> { let storage = state.storage().await?; let orphan_chunk_limit = match db.get_database_backend() { - // Default value of --max-allowed-packet https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet - sea_orm::DatabaseBackend::MySql => 67108864, + // Arbitrarily chosen sensible value since there's no good default to choose from for MySQL + sea_orm::DatabaseBackend::MySql => 1000, // Panic limit set by sqlx for postgresql: https://github.com/launchbadge/sqlx/issues/671#issuecomment-687043510 sea_orm::DatabaseBackend::Postgres => u64::from(u16::MAX), // Default statement limit imposed by sqlite: https://www.sqlite.org/limits.html#max_variable_number