itsscb/sqlx
1
0
Fork 0
mirror of https://github.com/launchbadge/sqlx.git synced 2026-03-23 18:40:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(macros): move to one-file-per-query for offline mode

Browse Source 
Query data is now stored in .sqlx/{query_hash}.json directly by the macro
invocations, rather than first writing to target/sqlx/{input_span_hash}.json
and then collecting those into sqlx-data.json separately.
This commit is contained in:
Jonas Platte
2020-12-29 20:06:23 +01:00
committed by Austin Bonander
parent babd353c2c
commit 1fa2381015
10 changed files with 501 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
sqlx-cli/src/cargo.rs Normal file
View File

@@ -0,0 +1,47 @@
use anyhow::Context;
use serde::Deserialize;
use std::env;
use std::ffi::{OsStr, OsString};
use std::path::PathBuf;
use std::process::Command;
use std::str;
#[derive(Deserialize)]
pub struct CargoMetadata {
pub target_directory: PathBuf,
pub workspace_root: PathBuf,
}
/// Path to the `cargo` executable
pub fn cargo_path() -> anyhow::Result<OsString> {
env::var_os("CARGO").context("Failed to obtain value of `CARGO`")
}
pub fn manifest_dir(cargo: &OsStr) -> anyhow::Result<PathBuf> {
let stdout = Command::new(&cargo)
.args(&["locate-project", "--message-format=plain"])
.output()
.context("could not locate manifest dir")?
.stdout;
let mut manifest_path: PathBuf = str::from_utf8(&stdout)
.context("output of `cargo locate-project` was not valid UTF-8")?
// get rid of the trailing newline
.trim()
.into();
manifest_path.pop();
Ok(manifest_path)
}
pub fn metadata(cargo: &OsStr) -> anyhow::Result<CargoMetadata> {
let output = Command::new(&cargo)
.args(&["metadata", "--format-version=1"])
.output()
.context("Could not fetch metadata")?;
serde_json::from_slice(&output.stdout)
.context("Invalid `cargo metadata` output")
.map_err(Into::into)
}

44
sqlx-cli/src/lib.rs
View File

@@ -2,6 +2,12 @@ use anyhow::Result;
use crate::opt::{Command, DatabaseCommand, MigrateCommand};
use anyhow::{anyhow, Context};
use dotenv::dotenv;
use prepare::PrepareCtx;
use std::env;
mod cargo;
mod database;
// mod migration;
// mod migrator;
@@ -74,18 +80,36 @@ pub async fn run(opt: Opt) -> Result<()> {
},
Command::Prepare {
check: false,
merged,
args,
check,
workspace,
database_url,
} => prepare::run(&database_url, merged, args)?,
args,
} => {
let cargo_path = cargo::cargo_path()?;
println!("cargo path: {:?}", cargo_path);
Command::Prepare {
check: true,
merged,
args,
database_url,
} => prepare::check(&database_url, merged, args)?,
let manifest_dir = cargo::manifest_dir(&cargo_path)?;
let metadata = cargo::metadata(&cargo_path)
.context("`prepare` subcommand may only be invoked as `cargo sqlx prepare`")?;
let ctx = PrepareCtx {
workspace,
cargo: cargo_path,
cargo_args: args,
manifest_dir,
target_dir: metadata.target_directory,
workspace_root: metadata.workspace_root,
database_url,
};
println!("{:?}", ctx);
if check {
prepare::check(&ctx)?
} else {
prepare::run(&ctx)?
}
}
};
Ok(())

13
sqlx-cli/src/opt.rs
View File

@@ -29,16 +29,21 @@ pub enum Command {
#[clap(long)]
check: bool,
/// Generate a single top-level `sqlx-data.json` file when using a cargo workspace.
/// Do a clean build of all crates in the workspace.
///
/// This option is intended for workspaces where multiple crates use SQLx; if there is only
/// one, it is better to run `cargo sqlx prepare` without this option inside of that crate.
#[clap(long)]
merged: bool,
workspace: bool,
/// Arguments to be passed to `cargo rustc ...`.
#[clap(last = true)]
args: Vec<String>,
#[clap(flatten)]
database_url: DatabaseUrl,
// `DatabaseUrl` doesn't allow it to be optional
/// Location of the DB, by default will be read from the DATABASE_URL env var
#[clap(long, short = 'D', env)]
database_url: Option<String>,
},
#[clap(alias = "mig")]

275
sqlx-cli/src/prepare.rs
View File

@@ -1,137 +1,82 @@
use anyhow::{bail, Context};
use console::style;
use remove_dir_all::remove_dir_all;
use serde::Deserialize;
use sqlx::any::{AnyConnectOptions, AnyKind};
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use anyhow::bail;
use std::ffi::OsString;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
use std::str::FromStr;
use std::time::SystemTime;
use std::{env, fs};
type QueryData = BTreeMap<String, serde_json::Value>;
type JsonObject = serde_json::Map<String, serde_json::Value>;
#[derive(serde::Serialize, serde::Deserialize)]
struct DataFile {
db: String,
#[serde(flatten)]
data: QueryData,
#[derive(Debug)]
pub struct PrepareCtx {
pub workspace: bool,
pub cargo: OsString,
pub cargo_args: Vec<String>,
pub manifest_dir: PathBuf,
pub target_dir: PathBuf,
pub workspace_root: PathBuf,
pub database_url: Option<String>,
}
pub fn run(url: &str, merge: bool, cargo_args: Vec<String>) -> anyhow::Result<()> {
let db_kind = get_db_kind(url)?;
let data = run_prepare_step(url, merge, cargo_args)?;
pub fn run(ctx: &PrepareCtx) -> anyhow::Result<()> {
let root = if ctx.workspace {
&ctx.workspace_root
} else {
&ctx.manifest_dir
};
if data.is_empty() {
println!(
"{} no queries found; please ensure that the `offline` feature is enabled in sqlx",
style("warning:").yellow()
);
}
serde_json::to_writer_pretty(
BufWriter::new(
File::create("sqlx-data.json").context("failed to create/open `sqlx-data.json`")?,
),
&DataFile {
db: db_kind.to_owned(),
data,
},
)
.context("failed to write to `sqlx-data.json`")?;
run_prepare_step(ctx, &root.join(".sqlx"))?;
println!(
"query data written to `sqlx-data.json` in the current directory; \
"query data written to `.sqlx` in the current directory; \
please check this into version control"
);
Ok(())
}
pub fn check(url: &str, merge: bool, cargo_args: Vec<String>) -> anyhow::Result<()> {
let db_kind = get_db_kind(url)?;
let data = run_prepare_step(url, merge, cargo_args)?;
pub fn check(ctx: &PrepareCtx) -> anyhow::Result<()> {
let cache_dir = ctx.target_dir.join("sqlx");
run_prepare_step(ctx, &cache_dir)?;
let data_file = File::open("sqlx-data.json").context(
"failed to open `sqlx-data.json`; you may need to run `cargo sqlx prepare` first",
)?;
let DataFile {
db: expected_db,
data: saved_data,
} = serde_json::from_reader(BufReader::new(data_file))?;
if db_kind != expected_db {
bail!(
"saved prepare data is for {}, not {} (inferred from `DATABASE_URL`)",
expected_db,
db_kind
)
}
if data != saved_data {
bail!("`cargo sqlx prepare` needs to be rerun")
}
// TODO: Compare .sqlx to target/sqlx
// * For files thta are only in the former, raise a warning
// * For files that are only in the latter, raise an error
Ok(())
}
fn run_prepare_step(url: &str, merge: bool, cargo_args: Vec<String>) -> anyhow::Result<QueryData> {
fn run_prepare_step(ctx: &PrepareCtx, cache_dir: &Path) -> anyhow::Result<()> {
anyhow::ensure!(
Path::new("Cargo.toml").exists(),
r#"Failed to read `Cargo.toml`.
hint: This command only works in the manifest directory of a Cargo package."#
);
// path to the Cargo executable
let cargo = env::var("CARGO")
.context("`prepare` subcommand may only be invoked as `cargo sqlx prepare`")?;
let output = Command::new(&cargo)
.args(&["metadata", "--format-version=1"])
.output()
.context("Could not fetch metadata")?;
#[derive(Deserialize)]
struct Metadata {
target_directory: PathBuf,
if cache_dir.exists() {
clear_cache_dir(cache_dir)?;
} else {
fs::create_dir(cache_dir)?;
}
let metadata: Metadata =
serde_json::from_slice(&output.stdout).context("Invalid `cargo metadata` output")?;
// try removing the target/sqlx directory before running, as stale files
// have repeatedly caused issues in the past.
let _ = remove_dir_all(metadata.target_directory.join("sqlx"));
let check_status = if merge {
let check_status = Command::new(&cargo).arg("clean").status()?;
let mut check_cmd = Command::new(&ctx.cargo);
if ctx.workspace {
let check_status = Command::new(&ctx.cargo).arg("clean").status()?;
if !check_status.success() {
bail!("`cargo clean` failed with status: {}", check_status);
}
let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();
rustflags.push_str(&format!(
" --cfg __sqlx_recompile_trigger=\"{}\"",
SystemTime::UNIX_EPOCH.elapsed()?.as_millis()
));
Command::new(&cargo)
.arg("check")
.args(cargo_args)
.env("RUSTFLAGS", rustflags)
.env("SQLX_OFFLINE", "false")
.env("DATABASE_URL", url)
.status()?
check_cmd.arg("check").args(&ctx.cargo_args).env(
"RUSTFLAGS",
format!(
"--cfg __sqlx_recompile_trigger=\"{}\"",
SystemTime::UNIX_EPOCH.elapsed()?.as_millis()
),
);
} else {
Command::new(&cargo)
check_cmd
.arg("rustc")
.args(cargo_args)
.args(&ctx.cargo_args)
.arg("--")
.arg("--emit")
.arg("dep-info,metadata")
@@ -140,121 +85,33 @@ hint: This command only works in the manifest directory of a Cargo package."#
.arg(format!(
"__sqlx_recompile_trigger=\"{}\"",
SystemTime::UNIX_EPOCH.elapsed()?.as_millis()
))
.env("SQLX_OFFLINE", "false")
.env("DATABASE_URL", url)
.status()?
};
));
}
// override database url
if let Some(database_url) = &ctx.database_url {
check_cmd.env("DATABASE_URL", database_url);
}
check_cmd
.env("SQLX_OFFLINE", "false")
.env("SQLX_OFFLINE_DIR", cache_dir);
println!("executing {:?}", check_cmd);
let check_status = check_cmd.status()?;
if !check_status.success() {
bail!("`cargo check` failed with status: {}", check_status);
}
let pattern = metadata.target_directory.join("sqlx/query-*.json");
let mut data = BTreeMap::new();
for path in glob::glob(
pattern
.to_str()
.context("CARGO_TARGET_DIR not valid UTF-8")?,
)? {
let path = path?;
let contents = fs::read(&*path)?;
let mut query_data: JsonObject = serde_json::from_slice(&contents)?;
// we lift the `hash` key to the outer map
let hash = query_data
.remove("hash")
.context("expected key `hash` in query data")?;
if let serde_json::Value::String(hash) = hash {
data.insert(hash, serde_json::Value::Object(query_data));
} else {
bail!(
"expected key `hash` in query data to be string, was {:?} instead; file: {}",
hash,
path.display()
)
}
// lazily remove the file, we don't care too much if we can't
let _ = fs::remove_file(&path);
}
Ok(data)
Ok(())
}
fn get_db_kind(url: &str) -> anyhow::Result<&'static str> {
let options = AnyConnectOptions::from_str(&url)?;
// these should match the values of `DatabaseExt::NAME` in `sqlx-macros`
match options.kind() {
#[cfg(feature = "postgres")]
AnyKind::Postgres => Ok("PostgreSQL"),
#[cfg(feature = "mysql")]
AnyKind::MySql => Ok("MySQL"),
#[cfg(feature = "sqlite")]
AnyKind::Sqlite => Ok("SQLite"),
#[cfg(feature = "mssql")]
AnyKind::Mssql => Ok("MSSQL"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use std::assert_eq;
#[test]
fn data_file_serialization_works() {
let data_file = DataFile {
db: "mysql".to_owned(),
data: {
let mut data = BTreeMap::new();
data.insert("a".to_owned(), json!({"key1": "value1"}));
data.insert("z".to_owned(), json!({"key2": "value2"}));
data
},
};
let data_file = serde_json::to_string(&data_file).expect("Data file serialized.");
assert_eq!(
data_file,
"{\"db\":\"mysql\",\"a\":{\"key1\":\"value1\"},\"z\":{\"key2\":\"value2\"}}"
);
}
#[test]
fn data_file_deserialization_works() {
let data_file =
"{\"db\":\"mysql\",\"a\":{\"key1\":\"value1\"},\"z\":{\"key2\":\"value2\"}}";
let data_file: DataFile = serde_json::from_str(data_file).expect("Data file deserialized.");
let DataFile { db, data } = data_file;
assert_eq!(db, "mysql");
assert_eq!(data.len(), 2);
assert_eq!(data.get("a"), Some(&json!({"key1": "value1"})));
assert_eq!(data.get("z"), Some(&json!({"key2": "value2"})));
}
#[test]
fn data_file_deserialization_works_for_ordered_keys() {
let data_file =
"{\"a\":{\"key1\":\"value1\"},\"db\":\"mysql\",\"z\":{\"key2\":\"value2\"}}";
let data_file: DataFile = serde_json::from_str(data_file).expect("Data file deserialized.");
let DataFile { db, data } = data_file;
assert_eq!(db, "mysql");
assert_eq!(data.len(), 2);
assert_eq!(data.get("a"), Some(&json!({"key1": "value1"})));
assert_eq!(data.get("z"), Some(&json!({"key2": "value2"})));
fn clear_cache_dir(path: &Path) -> anyhow::Result<()> {
for entry in fs::read_dir(path)? {
fs::remove_file(entry?.path())?;
}
Ok(())
}