From fde43774b43649a71bb942554726a7ee43df442f Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 24 Sep 2026 15:51:06 -0700 Subject: [PATCH] PostgreSQL search GIN indexes without a pending list A three-node rehearsal on PostgreSQL saw searches take about 185 ms with 80 to 260 pages in the full-text indexes' pending lists, 2 to 6 ms right after gin_clean_pending_list() or VACUUM, then creep back up as mail came in. The search tables' GIN indexes were created with the default fastupdate=on: new entries wait in an unindexed pending list that every search scans in full until VACUUM (or 4 MB of backlog) merges it, and autovacuum only visits an insert-only table after thousands of inserts. The search GIN indexes are now created WITH (fastupdate = off), so an insert pays its index update at once. The schema step runs at every startup (create_search_tables, via SearchStore::create_indexes), so indexes made before this change are switched there: when an index's reloptions don't already turn fastupdate off, ALTER INDEX ... SET (fastupdate = off) and one gin_clean_pending_list() merge its backlog. The ALTER takes a SHARE UPDATE EXCLUSIVE lock, which blocks neither reads nor writes; after the first startup the step is one catalog read per index. A failure is logged and startup goes on (search still works, only slower). Per-table autovacuum settings for the search tables are left alone. The pending list was the only reason the insert threshold mattered for search; dead tuples and freezing are served by the defaults, and table settings would override whatever tuning the DBA has done. MySQL is unaffected: InnoDB FULLTEXT keeps new entries in an in-memory cache that queries read directly, with no setting like fastupdate. store::search_gin::postgres_gin_fastupdate (new, PostgreSQL) builds the search schema in a schema of its own and checks pg_class.reloptions: fastupdate=off on every GIN index of a fresh schema; then, with the option reset to the default and 500 rows pending, one startup turns it off everywhere and leaves no pending tuples (pgstatginindex); a second startup changes nothing. On main it fails at the first check. --- crates/store/src/backend/postgres/main.rs | 74 +++++++++- tests/src/store/mod.rs | 2 + tests/src/store/search_gin.rs | 159 ++++++++++++++++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 tests/src/store/search_gin.rs diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 6dbd853..3e97aab 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -265,12 +265,21 @@ async fn create_search_tables( for field in T::all_fields() { if field.is_text() || field.is_json() { let column_name = field.column(); + // inbuxa: with GIN's default fastupdate=on, new entries wait in + // an unindexed pending list that every search scans in full + // until a VACUUM (or 4 MB of backlog) merges it. On a mailbox + // taking steady mail that list never drains and searches slow + // from milliseconds to hundreds of them. Pay the index update + // at insert time instead. + let index_name = format!("gin_{table_name}_{column_name}"); let create_index_query = format!( - "CREATE INDEX IF NOT EXISTS gin_{table_name}_{column_name} ON {table_name} USING GIN({column_name})", + "CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} USING GIN({column_name}) WITH (fastupdate = off)", ); conn.execute(&create_index_query, &[]) .await .map_err(into_error)?; + // Indexes made before this change keep fastupdate=on + disable_gin_fastupdate(conn, &index_name).await; } if field.is_indexed() { @@ -287,6 +296,69 @@ async fn create_search_tables( Ok(()) } +/// inbuxa: turns fastupdate off on a GIN index made with the default and +/// merges the pending list it has built up. Idempotent: an index that already +/// has the option is left alone, so this costs one catalog read per index at +/// startup. A failure is logged and startup goes on, since search still works, +/// only slower. +async fn disable_gin_fastupdate(conn: &Object, index_name: &str) { + if let Err(err) = try_disable_gin_fastupdate(conn, index_name).await { + trc::event!( + Store(trc::StoreEvent::PostgresqlError), + Details = format!("Failed to turn off fastupdate on search index {index_name}"), + Reason = err.to_string(), + ); + } +} + +async fn try_disable_gin_fastupdate(conn: &Object, index_name: &str) -> trc::Result<()> { + let options = conn + .query_opt( + "SELECT COALESCE(reloptions, '{}')::text[] FROM pg_class WHERE oid = to_regclass($1)", + &[&index_name], + ) + .await + .map_err(into_error)? + .map(|row| row.try_get::<_, Vec>(0)) + .transpose() + .map_err(into_error)?; + let Some(options) = options else { + return Ok(()); + }; + if gin_fastupdate_is_off(&options) { + return Ok(()); + } + // SET (fastupdate) takes a SHARE UPDATE EXCLUSIVE lock, which doesn't + // block reads or writes. Turning it off stops new entries going to the + // pending list but doesn't flush the entries already there. + conn.execute( + &format!("ALTER INDEX {index_name} SET (fastupdate = off)"), + &[], + ) + .await + .map_err(into_error)?; + conn.query_one( + "SELECT gin_clean_pending_list($1::text::regclass)", + &[&index_name], + ) + .await + .map_err(into_error)?; + Ok(()) +} + +/// Whether a relation's reloptions turn GIN's fastupdate off. +fn gin_fastupdate_is_off(options: &[String]) -> bool { + options.iter().any(|option| { + option.split_once('=').is_some_and(|(name, value)| { + name.trim().eq_ignore_ascii_case("fastupdate") + && matches!( + value.trim().to_ascii_lowercase().as_str(), + "off" | "false" | "no" | "0" | "f" | "n" + ) + }) + }) +} + async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> { let mut ts_configs = AHashSet::from_iter([PG_FALLBACK_LANG, PG_UNSTEMMED_LANG]); diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 490bb4f..1c1a577 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -21,6 +21,8 @@ pub mod replica_mysql; // inbuxa: read replicas on MySQL #[cfg(all(feature = "postgres", feature = "redis"))] pub mod replica_cluster; // inbuxa: read replicas across nodes pub mod scaleout; // inbuxa: scale-out storage +#[cfg(feature = "postgres")] +pub mod search_gin; // inbuxa: GIN indexes without a pending list #[cfg(any(feature = "postgres", feature = "mysql"))] pub mod sql_timeout; pub mod task_locks; // inbuxa: task locks across nodes diff --git a/tests/src/store/search_gin.rs b/tests/src/store/search_gin.rs new file mode 100644 index 0000000..97c671c --- /dev/null +++ b/tests/src/store/search_gin.rs @@ -0,0 +1,159 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! PostgreSQL full-text GIN indexes are built with fastupdate off, and an +//! index made earlier with the default is switched over at startup. With +//! fastupdate on, new entries wait in a pending list that every search scans +//! in full until VACUUM merges it. + +use crate::utils::storage::build_data_store; +use registry::schema::structs::DataStore; +use store::{Rows, SearchStore, Store}; + +const SCHEMA: &str = "gin_fastupdate_test"; + +#[tokio::test(flavor = "multi_thread")] +pub async fn postgres_gin_fastupdate() { + println!("Running PostgreSQL GIN fastupdate test..."); + + // Work in a schema of our own so the shared search tables are untouched + let admin = Store::build(build_data_store("PostgreSql", "").await) + .await + .expect("Failed to connect to PostgreSQL"); + for query in [ + format!("DROP SCHEMA IF EXISTS {SCHEMA} CASCADE"), + format!("CREATE SCHEMA {SCHEMA}"), + ] { + admin.sql_query::(&query, vec![]).await.unwrap(); + } + let DataStore::PostgreSql(mut config) = build_data_store("PostgreSql", "").await else { + unreachable!() + }; + config.options = Some(format!("-c search_path={SCHEMA}")); + let store = Store::build(DataStore::PostgreSql(config)) + .await + .expect("Failed to connect to PostgreSQL"); + let search = SearchStore::Store(store.clone()); + + // A fresh schema + search.create_indexes().await.unwrap(); + let indexes = gin_indexes(&admin).await; + assert!( + indexes.len() >= 4, + "expected the search GIN indexes, found {indexes:?}" + ); + for (name, options) in &indexes { + assert!( + options.contains("fastupdate=off"), + "fresh index {name} has options {options:?}" + ); + } + + // A schema from before the change: the same indexes, made with the + // default fastupdate=on, and a pending list with something in it + for (name, _) in &indexes { + admin + .sql_query::( + &format!("ALTER INDEX {SCHEMA}.{name} RESET (fastupdate)"), + vec![], + ) + .await + .unwrap(); + } + for (name, options) in gin_indexes(&admin).await { + assert!( + !options.contains("fastupdate"), + "index {name} still has options {options:?}" + ); + } + admin + .sql_query::( + &format!( + "INSERT INTO {SCHEMA}.s_email (accid, docid, subj, body) \ + SELECT 1, n, to_tsvector('simple', 'pending subject ' || n), \ + to_tsvector('simple', 'pending body text ' || n) \ + FROM generate_series(1, 500) n" + ), + vec![], + ) + .await + .unwrap(); + assert!( + pending_tuples(&admin, "gin_s_email_body").await > 0, + "no pending list to merge" + ); + + // Startup on the existing schema switches every index over and merges + // what was pending + search.create_indexes().await.unwrap(); + for (name, options) in gin_indexes(&admin).await { + assert!( + options.contains("fastupdate=off"), + "existing index {name} has options {options:?} after startup" + ); + } + assert_eq!(pending_tuples(&admin, "gin_s_email_body").await, 0); + + // And a second startup changes nothing + search.create_indexes().await.unwrap(); + for (name, options) in gin_indexes(&admin).await { + assert!(options.contains("fastupdate=off"), "{name}: {options:?}"); + } + + admin + .sql_query::(&format!("DROP SCHEMA {SCHEMA} CASCADE"), vec![]) + .await + .unwrap(); +} + +/// The GIN indexes in the test schema with their reloptions. +async fn gin_indexes(admin: &Store) -> Vec<(String, String)> { + admin + .sql_query::( + &format!( + "SELECT c.relname::text, COALESCE(array_to_string(c.reloptions, ','), '') \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + JOIN pg_am a ON a.oid = c.relam \ + WHERE n.nspname = '{SCHEMA}' AND c.relkind = 'i' AND a.amname = 'gin' \ + ORDER BY 1" + ), + vec![], + ) + .await + .unwrap() + .rows + .into_iter() + .map(|row| { + let mut values = row.values.into_iter(); + ( + values.next().unwrap().to_str().into_owned(), + values.next().unwrap().to_str().into_owned(), + ) + }) + .collect() +} + +/// Tuples waiting in a GIN index's pending list (pgstattuple is a contrib +/// extension the test database has). +async fn pending_tuples(admin: &Store, index: &str) -> i64 { + admin + .sql_query::("CREATE EXTENSION IF NOT EXISTS pgstattuple", vec![]) + .await + .unwrap(); + admin + .sql_query::( + &format!("SELECT pending_tuples FROM pgstatginindex('{SCHEMA}.{index}'::regclass)"), + vec![], + ) + .await + .unwrap() + .rows + .into_iter() + .next() + .and_then(|row| row.values.into_iter().next()) + .map(|value| value.to_str().parse::().unwrap()) + .unwrap() +}