feat: create sqlx.toml format (#3383)

* feat: create `sqlx.toml` format

* feat: add support for ignored_chars config to sqlx_core::migrate

* chore: test ignored_chars with `U+FEFF` (ZWNBSP/BOM)

https://en.wikipedia.org/wiki/Byte_order_mark

* refactor: make `Config` always compiled

simplifies usage while still making parsing optional for less generated code

* refactor: add origin information to `Column`

* feat(macros): implement `type_override` and `column_override` from `sqlx.toml`

* refactor(sqlx.toml): make all keys kebab-case, create `macros.preferred-crates`

* feat: make macros aware of `macros.preferred-crates`

* feat: make `sqlx-cli` aware of `database-url-var`

* feat: teach macros about `migrate.table-name`, `migrations-dir`

* feat: teach macros about `migrate.ignored-chars`

* chore: delete unused source file `sqlx-cli/src/migration.rs`

* feat: teach `sqlx-cli` about `migrate.defaults`

* feat: teach `sqlx-cli` about `migrate.migrations-dir`

* feat: teach `sqlx-cli` about `migrate.table-name`

* feat: introduce `migrate.create-schemas`

* WIP feat: create multi-tenant database example

* fix(postgres): don't fetch `ColumnOrigin` for transparently-prepared statements

* feat: progress on axum-multi-tenant example

* feat(config): better errors for mislabeled fields

* WIP feat: filling out axum-multi-tenant example

* feat: multi-tenant example

No longer Axum-based because filling out the request routes would have distracted from the purpose of the example.

* chore(ci): test multi-tenant example

* fixup after merge

* fix(ci): enable `sqlx-toml` in CLI build for examples

* fix: CI, README for `multi-tenant`

* fix: clippy warnings

* fix: multi-tenant README

* fix: sequential versioning inference for migrations

* fix: migration versioning with explicit overrides

* fix: only warn on ambiguous crates if the invocation relies on it

* fix: remove unused imports

* fix: doctest

* fix: `sqlx mig add` behavior and tests

* fix: restore original type-checking order

* fix: deprecation warning in `tests/postgres/macros.rs`

* feat: create postgres/multi-database example

* fix: examples/postgres/multi-database

* fix: cargo fmt

* chore: add tests for config `migrate.defaults`

* fix: sqlx-cli/tests/add.rs

* feat(cli): add `--config` override to all relevant commands

* chore: run `sqlx mig add` test with `RUST_BACKTRACE=1`

* fix: properly canonicalize config path for `sqlx mig add` test

* fix: get `sqlx mig add` test passing

* fix(cli): test `migrate.ignored-chars`, fix bugs

* feat: create `macros.preferred-crates` example

* fix(examples): use workspace `sqlx`

* fix: examples

* fix(sqlite): unexpected feature flags in `type_checking.rs`

* fix: run `cargo fmt`

* fix: more example fixes

* fix(ci): preferred-crates setup

* fix(examples): enable default-features for workspace `sqlx`

* fix(examples): issues in `preferred-crates`

* chore: adjust error message for missing param type in `query!()`

* doc: mention new `sqlx.toml` configuration

* chore: add `CHANGELOG` entry

Normally I generate these when cutting the release, but I wanted to take time to editorialize this one.

* doc: fix new example titles

* refactor: make `sqlx-toml` feature non-default, improve errors

* refactor: eliminate panics in `Config` read path

* chore: remove unused `axum` dependency from new examples

* fix(config): restore fallback to default config for macros

* chore(config): remove use of `once_cell` (to match `main`)

sqlx-sqlite/Cargo.toml

@@ -27,6 +27,10 @@ preupdate-hook = ["libsqlite3-sys/preupdate_hook"]
 bundled = ["libsqlite3-sys/bundled"]
 unbundled = ["libsqlite3-sys/buildtime_bindgen"]
 
+# Note: currently unused, only to satisfy "unexpected `cfg` condition" lint
+bigdecimal = []
+rust_decimal = []
+
 [dependencies]
 futures-core = { version = "0.3.19", default-features = false }
 futures-channel = { version = "0.3.19", default-features = false, features = ["sink", "alloc", "std"] }
@@ -73,4 +77,4 @@ sqlx = { workspace = true, default-features = false, features = ["macros", "runt
 workspace = true
 
 [package.metadata.docs.rs]
-features = ["bundled", "any", "json", "chrono", "time", "uuid"]
+features = ["bundled", "any", "json", "chrono", "time", "uuid"]

sqlx-sqlite/src/column.rs

@@ -9,6 +9,9 @@ pub struct SqliteColumn {
     pub(crate) name: UStr,
     pub(crate) ordinal: usize,
     pub(crate) type_info: SqliteTypeInfo,
+
+    #[cfg_attr(feature = "offline", serde(default))]
+    pub(crate) origin: ColumnOrigin,
 }
 
 impl Column for SqliteColumn {
@@ -25,4 +28,8 @@ impl Column for SqliteColumn {
     fn type_info(&self) -> &SqliteTypeInfo {
         &self.type_info
     }
+
+    fn origin(&self) -> ColumnOrigin {
+        self.origin.clone()
+    }
 }

sqlx-sqlite/src/connection/describe.rs

@@ -50,6 +50,8 @@ pub(crate) fn describe(conn: &mut ConnectionState, query: &str) -> Result<Descri
         for col in 0..num {
             let name = stmt.handle.column_name(col).to_owned();
 
+            let origin = stmt.handle.column_origin(col);
+
             let type_info = if let Some(ty) = stmt.handle.column_decltype(col) {
                 ty
             } else {
@@ -82,6 +84,7 @@ pub(crate) fn describe(conn: &mut ConnectionState, query: &str) -> Result<Descri
                 name: name.into(),
                 type_info,
                 ordinal: col,
+                origin,
             });
         }
     }

sqlx-sqlite/src/migrate.rs

@@ -15,6 +15,7 @@ use std::time::Duration;
 use std::time::Instant;
 
 pub(crate) use sqlx_core::migrate::*;
+use sqlx_core::query_scalar::query_scalar;
 
 impl MigrateDatabase for Sqlite {
     fn create_database(url: &str) -> BoxFuture<'_, Result<(), Error>> {
@@ -64,12 +65,36 @@ impl MigrateDatabase for Sqlite {
 }
 
 impl Migrate for SqliteConnection {
-    fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> {
+    fn create_schema_if_not_exists<'e>(
+        &'e mut self,
+        schema_name: &'e str,
+    ) -> BoxFuture<'e, Result<(), MigrateError>> {
+        Box::pin(async move {
+            // Check if the schema already exists; if so, don't error.
+            let schema_version: Option<i64> =
+                query_scalar(&format!("PRAGMA {schema_name}.schema_version"))
+                    .fetch_optional(&mut *self)
+                    .await?;
+
+            if schema_version.is_some() {
+                return Ok(());
+            }
+
+            Err(MigrateError::CreateSchemasNotSupported(
+                format!("cannot create new schema {schema_name}; creation of additional schemas in SQLite requires attaching extra database files"),
+            ))
+        })
+    }
+
+    fn ensure_migrations_table<'e>(
+        &'e mut self,
+        table_name: &'e str,
+    ) -> BoxFuture<'e, Result<(), MigrateError>> {
         Box::pin(async move {
             // language=SQLite
-            self.execute(
+            self.execute(&*format!(
                 r#"
-CREATE TABLE IF NOT EXISTS _sqlx_migrations (
+CREATE TABLE IF NOT EXISTS {table_name} (
     version BIGINT PRIMARY KEY,
     description TEXT NOT NULL,
     installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -77,20 +102,23 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
     checksum BLOB NOT NULL,
     execution_time BIGINT NOT NULL
 );
-                "#,
-            )
+                "#
+            ))
             .await?;
 
             Ok(())
         })
     }
 
-    fn dirty_version(&mut self) -> BoxFuture<'_, Result<Option<i64>, MigrateError>> {
+    fn dirty_version<'e>(
+        &'e mut self,
+        table_name: &'e str,
+    ) -> BoxFuture<'e, Result<Option<i64>, MigrateError>> {
         Box::pin(async move {
             // language=SQLite
-            let row: Option<(i64,)> = query_as(
-                "SELECT version FROM _sqlx_migrations WHERE success = false ORDER BY version LIMIT 1",
-            )
+            let row: Option<(i64,)> = query_as(&format!(
+                "SELECT version FROM {table_name} WHERE success = false ORDER BY version LIMIT 1"
+            ))
             .fetch_optional(self)
             .await?;
 
@@ -98,15 +126,17 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
         })
     }
 
-    fn list_applied_migrations(
-        &mut self,
-    ) -> BoxFuture<'_, Result<Vec<AppliedMigration>, MigrateError>> {
+    fn list_applied_migrations<'e>(
+        &'e mut self,
+        table_name: &'e str,
+    ) -> BoxFuture<'e, Result<Vec<AppliedMigration>, MigrateError>> {
         Box::pin(async move {
             // language=SQLite
-            let rows: Vec<(i64, Vec<u8>)> =
-                query_as("SELECT version, checksum FROM _sqlx_migrations ORDER BY version")
-                    .fetch_all(self)
-                    .await?;
+            let rows: Vec<(i64, Vec<u8>)> = query_as(&format!(
+                "SELECT version, checksum FROM {table_name} ORDER BY version"
+            ))
+            .fetch_all(self)
+            .await?;
 
             let migrations = rows
                 .into_iter()
@@ -128,10 +158,11 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
         Box::pin(async move { Ok(()) })
     }
 
-    fn apply<'e: 'm, 'm>(
+    fn apply<'e>(
         &'e mut self,
-        migration: &'m Migration,
-    ) -> BoxFuture<'m, Result<Duration, MigrateError>> {
+        table_name: &'e str,
+        migration: &'e Migration,
+    ) -> BoxFuture<'e, Result<Duration, MigrateError>> {
         Box::pin(async move {
             let mut tx = self.begin().await?;
             let start = Instant::now();
@@ -147,12 +178,12 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
                 .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?;
 
             // language=SQL
-            let _ = query(
+            let _ = query(&format!(
                 r#"
-    INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time )
+    INSERT INTO {table_name} ( version, description, success, checksum, execution_time )
     VALUES ( ?1, ?2, TRUE, ?3, -1 )
-                "#,
-            )
+                "#
+            ))
             .bind(migration.version)
             .bind(&*migration.description)
             .bind(&*migration.checksum)
@@ -169,13 +200,13 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
 
             // language=SQL
             #[allow(clippy::cast_possible_truncation)]
-            let _ = query(
+            let _ = query(&format!(
                 r#"
-    UPDATE _sqlx_migrations
+    UPDATE {table_name}
     SET execution_time = ?1
     WHERE version = ?2
-                "#,
-            )
+                "#
+            ))
             .bind(elapsed.as_nanos() as i64)
             .bind(migration.version)
             .execute(self)
@@ -185,10 +216,11 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
         })
     }
 
-    fn revert<'e: 'm, 'm>(
+    fn revert<'e>(
         &'e mut self,
-        migration: &'m Migration,
-    ) -> BoxFuture<'m, Result<Duration, MigrateError>> {
+        table_name: &'e str,
+        migration: &'e Migration,
+    ) -> BoxFuture<'e, Result<Duration, MigrateError>> {
         Box::pin(async move {
             // Use a single transaction for the actual migration script and the essential bookeeping so we never
             // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966.
@@ -197,8 +229,8 @@ CREATE TABLE IF NOT EXISTS _sqlx_migrations (
 
             let _ = tx.execute(&*migration.sql).await?;
 
-            // language=SQL
-            let _ = query(r#"DELETE FROM _sqlx_migrations WHERE version = ?1"#)
+            // language=SQLite
+            let _ = query(&format!(r#"DELETE FROM {table_name} WHERE version = ?1"#))
                 .bind(migration.version)
                 .execute(&mut *tx)
                 .await?;

sqlx-sqlite/src/statement/handle.rs

@@ -1,12 +1,9 @@
 use std::ffi::c_void;
 use std::ffi::CStr;
 
-use std::os::raw::{c_char, c_int};
-use std::ptr;
-use std::ptr::NonNull;
-use std::slice::from_raw_parts;
-use std::str::{from_utf8, from_utf8_unchecked};
-
+use crate::error::{BoxDynError, Error};
+use crate::type_info::DataType;
+use crate::{SqliteError, SqliteTypeInfo};
 use libsqlite3_sys::{
     sqlite3, sqlite3_bind_blob64, sqlite3_bind_double, sqlite3_bind_int, sqlite3_bind_int64,
     sqlite3_bind_null, sqlite3_bind_parameter_count, sqlite3_bind_parameter_name,
@@ -19,10 +16,13 @@ use libsqlite3_sys::{
     sqlite3_value, SQLITE_DONE, SQLITE_LOCKED_SHAREDCACHE, SQLITE_MISUSE, SQLITE_OK, SQLITE_ROW,
     SQLITE_TRANSIENT, SQLITE_UTF8,
 };
-
-use crate::error::{BoxDynError, Error};
-use crate::type_info::DataType;
-use crate::{SqliteError, SqliteTypeInfo};
+use sqlx_core::column::{ColumnOrigin, TableColumn};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::ptr::NonNull;
+use std::slice::from_raw_parts;
+use std::str::{from_utf8, from_utf8_unchecked};
+use std::sync::Arc;
 
 use super::unlock_notify;
 
@@ -34,6 +34,9 @@ pub(crate) struct StatementHandle(NonNull<sqlite3_stmt>);
 
 unsafe impl Send for StatementHandle {}
 
+// Most of the getters below allocate internally, and unsynchronized access is undefined.
+// unsafe impl !Sync for StatementHandle {}
+
 macro_rules! expect_ret_valid {
     ($fn_name:ident($($args:tt)*)) => {{
         let val = $fn_name($($args)*);
@@ -110,6 +113,65 @@ impl StatementHandle {
         }
     }
 
+    pub(crate) fn column_origin(&self, index: usize) -> ColumnOrigin {
+        if let Some((table, name)) = self
+            .column_table_name(index)
+            .zip(self.column_origin_name(index))
+        {
+            let table: Arc<str> = self
+                .column_db_name(index)
+                .filter(|&db| db != "main")
+                .map_or_else(
+                    || table.into(),
+                    // TODO: check that SQLite returns the names properly quoted if necessary
+                    |db| format!("{db}.{table}").into(),
+                );
+
+            ColumnOrigin::Table(TableColumn {
+                table,
+                name: name.into(),
+            })
+        } else {
+            ColumnOrigin::Expression
+        }
+    }
+
+    fn column_db_name(&self, index: usize) -> Option<&str> {
+        unsafe {
+            let db_name = sqlite3_column_database_name(self.0.as_ptr(), check_col_idx!(index));
+
+            if !db_name.is_null() {
+                Some(from_utf8_unchecked(CStr::from_ptr(db_name).to_bytes()))
+            } else {
+                None
+            }
+        }
+    }
+
+    fn column_table_name(&self, index: usize) -> Option<&str> {
+        unsafe {
+            let table_name = sqlite3_column_table_name(self.0.as_ptr(), check_col_idx!(index));
+
+            if !table_name.is_null() {
+                Some(from_utf8_unchecked(CStr::from_ptr(table_name).to_bytes()))
+            } else {
+                None
+            }
+        }
+    }
+
+    fn column_origin_name(&self, index: usize) -> Option<&str> {
+        unsafe {
+            let origin_name = sqlite3_column_origin_name(self.0.as_ptr(), check_col_idx!(index));
+
+            if !origin_name.is_null() {
+                Some(from_utf8_unchecked(CStr::from_ptr(origin_name).to_bytes()))
+            } else {
+                None
+            }
+        }
+    }
+
     pub(crate) fn column_type_info(&self, index: usize) -> SqliteTypeInfo {
         SqliteTypeInfo(DataType::from_code(self.column_type(index)))
     }

sqlx-sqlite/src/statement/virtual.rs

@@ -104,6 +104,7 @@ impl VirtualStatement {
                         ordinal: i,
                         name: name.clone(),
                         type_info,
+                        origin: statement.column_origin(i),
                     });
 
                     column_names.insert(name, i);

sqlx-sqlite/src/type_checking.rs

@@ -1,8 +1,7 @@
+use crate::Sqlite;
 #[allow(unused_imports)]
 use sqlx_core as sqlx;
 
-use crate::Sqlite;
-
 // f32 is not included below as REAL represents a floating point value
 // stored as an 8-byte IEEE floating point number (i.e. an f64)
 // For more info see: https://www.sqlite.org/datatype3.html#storage_classes_and_datatypes
@@ -20,24 +19,6 @@ impl_type_checking!(
         String,
         Vec<u8>,
 
-        #[cfg(all(feature = "chrono", not(feature = "time")))]
-        sqlx::types::chrono::NaiveDate,
-
-        #[cfg(all(feature = "chrono", not(feature = "time")))]
-        sqlx::types::chrono::NaiveDateTime,
-
-        #[cfg(all(feature = "chrono", not(feature = "time")))]
-        sqlx::types::chrono::DateTime<sqlx::types::chrono::Utc> | sqlx::types::chrono::DateTime<_>,
-
-        #[cfg(feature = "time")]
-        sqlx::types::time::OffsetDateTime,
-
-        #[cfg(feature = "time")]
-        sqlx::types::time::PrimitiveDateTime,
-
-        #[cfg(feature = "time")]
-        sqlx::types::time::Date,
-
         #[cfg(feature = "uuid")]
         sqlx::types::Uuid,
     },
@@ -48,4 +29,28 @@ impl_type_checking!(
     // The type integrations simply allow the user to skip some intermediate representation,
     // which is usually TEXT.
     feature-types: _info => None,
+
+    // The expansion of the macro automatically applies the correct feature name
+    // and checks `[macros.preferred-crates]`
+    datetime-types: {
+        chrono: {
+            sqlx::types::chrono::NaiveDate,
+
+            sqlx::types::chrono::NaiveDateTime,
+
+            sqlx::types::chrono::DateTime<sqlx::types::chrono::Utc>
+                | sqlx::types::chrono::DateTime<_>,
+        },
+        time: {
+            sqlx::types::time::OffsetDateTime,
+
+            sqlx::types::time::PrimitiveDateTime,
+
+            sqlx::types::time::Date,
+        },
+    },
+    numeric-types: {
+        bigdecimal: { },
+        rust_decimal: { },
+    },
 );