#![allow(clippy::multiple_crate_versions)] //! The `sqliteshape` command inspects `SQLite` databases. use std::ffi::OsString; use std::fmt::Write as _; use std::io::{self, Read}; use std::path::PathBuf; use std::time::Duration; use common::{ CliError, CommonArgs, ExitCode, InputFormat, RenderMode, map_result_count, parse_color_choice, parse_format_choice, parse_input_format, print_error, print_json, print_quick_help_error, print_structured, should_read_stdin, }; use lexopt::prelude::{Long, Short, Value as ArgValue}; use rusqlite::{Connection, OpenFlags}; use serde::Serialize; const SQLITE_BUSY_TIMEOUT: Duration = Duration::from_millis(100); const SQLITE_EXPRESSION_INDEX_COLUMN: &str = ""; const HELP: &str = "\ Inspect SQLite schema and table stats without leaving the terminal. Usage: sqliteshape [OPTIONS] [PATH...] sqliteshape [OPTIONS] diff Options: --format Structured output format: text, json, toon --json Shortcut for --format json --toon Shortcut for --format toon --input-format Override stdin parsing mode: auto, lines, jsonl --color Control ANSI color output: auto, never --quiet Suppress non-essential status output --table Limit output to one or more table names --sample-rows Maximum rows to sample per table --include-indexes Include index metadata in the summary --count-rows Use exact COUNT(*) for each table instead of estimates/unknown -h, --help Show this help text -V, --version Show the command version Examples: sqliteshape .\\fixtures\\sqliteshape\\sample.db 'C:\\data\\events.db' | sqliteshape --input-format lines --json | ConvertFrom-Json sqliteshape .\\fixtures\\sqliteshape\\sample.db --table users --include-indexes sqliteshape diff before.db after.db --json | ConvertFrom-Json "; #[derive(Debug, Clone)] struct Cli { common: CommonArgs, command: CommandMode, tables: Vec, sample_rows: usize, include_indexes: bool, count_rows: bool, paths: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ParseOutcome { Help, Version, Run, } #[derive(Debug, Clone, PartialEq, Eq)] enum CommandMode { Summary, Diff { before: PathBuf, after: PathBuf }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct DatabaseSummary { path: String, page_size: i64, page_count: i64, journal_mode: String, table_count: usize, requested_tables: Vec, unmatched_tables: Vec, tables: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] struct TableSelection { matched: Vec, unmatched: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct TableSummary { name: String, row_count: Option, row_count_mode: String, columns: Vec, sample_rows: Vec, indexes: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct ColumnSummary { name: String, declared_type: String, not_null: bool, primary_key_position: i64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct IndexSummary { name: String, unique: bool, columns: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct DatabaseDiff { path_before: String, path_after: String, page_size_before: i64, page_size_after: i64, page_count_before: i64, page_count_after: i64, journal_mode_before: String, journal_mode_after: String, added_tables: Vec, removed_tables: Vec, changed_tables: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct TableDiff { name: String, row_count_before: Option, row_count_after: Option, row_count_mode_before: String, row_count_mode_after: String, added_columns: Vec, removed_columns: Vec, changed_columns: Vec, added_indexes: Vec, removed_indexes: Vec, } const ROW_COUNT_MODE_EXACT: &str = "exact"; const ROW_COUNT_MODE_ESTIMATED: &str = "estimated"; const ROW_COUNT_MODE_UNKNOWN: &str = "unknown"; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct ColumnChange { name: String, declared_type_before: String, declared_type_after: String, not_null_before: bool, not_null_after: bool, primary_key_position_before: i64, primary_key_position_after: i64, } /// Parses CLI arguments and returns a process exit code. #[must_use] pub fn main_entry() -> i32 { match parse_cli_from(std::env::args_os()) { Ok((ParseOutcome::Help, _)) => { print!("{HELP}"); ExitCode::Success.as_i32() } Ok((ParseOutcome::Version, _)) => { println!("sqliteshape {}", env!("CARGO_PKG_VERSION")); ExitCode::Success.as_i32() } Ok((ParseOutcome::Run, cli)) => match run(&cli) { Ok(code) => code.as_i32(), Err(error) => { match error { CliError::Usage(_) => print_quick_help_error(&error, HELP), CliError::Runtime(_) => print_error(&error), } error.exit_code().as_i32() } }, Err(error) => { print_quick_help_error(&error, HELP); error.exit_code().as_i32() } } } fn parse_cli_from(args: I) -> Result<(ParseOutcome, Cli), CliError> where I: IntoIterator, T: Into, { let mut parser = lexopt::Parser::from_iter(args); let mut cli = Cli { common: CommonArgs::default(), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 3, include_indexes: false, count_rows: false, paths: Vec::new(), }; let mut diff_paths = Vec::::new(); while let Some(argument) = parser .next() .map_err(|error| CliError::usage(error.to_string()))? { match argument { Long("help") | Short('h') => return Ok((ParseOutcome::Help, cli)), Long("version") | Short('V') => return Ok((ParseOutcome::Version, cli)), Long("json") => cli.common.set_render_mode(RenderMode::Json), Long("toon") => cli.common.set_render_mode(RenderMode::Toon), Long("format") => { let value = parser_value_string(&mut parser, "--format")?; cli.common.set_render_mode(parse_format_choice(&value)?); } Long("input-format") => { cli.common.input_format = parse_input_format(&parser_value_string(&mut parser, "--input-format")?)?; } Long("color") => { cli.common.color = parse_color_choice(&parser_value_string(&mut parser, "--color")?)?; } Long("quiet") => cli.common.quiet = true, Long("table") => cli .tables .push(parser_value_string(&mut parser, "--table")?), Long("sample-rows") => { cli.sample_rows = parse_usize_flag( "--sample-rows", &parser_value_string(&mut parser, "--sample-rows")?, )?; } Long("include-indexes") => cli.include_indexes = true, Long("count-rows") => cli.count_rows = true, ArgValue(path) => { if matches!(cli.command, CommandMode::Summary) && cli.paths.is_empty() && diff_paths.is_empty() && path == "diff" { cli.command = CommandMode::Diff { before: PathBuf::new(), after: PathBuf::new(), }; } else if matches!(cli.command, CommandMode::Diff { .. }) { diff_paths.push(PathBuf::from(path)); } else { cli.paths.push(PathBuf::from(path)); } } _ => { return Err(CliError::usage( "unsupported argument; use --help to see available options", )); } } } if matches!(cli.command, CommandMode::Diff { .. }) { if diff_paths.len() != 2 { return Err(CliError::usage("diff expects exactly two SQLite paths")); } cli.command = CommandMode::Diff { before: diff_paths[0].clone(), after: diff_paths[1].clone(), }; } Ok((ParseOutcome::Run, cli)) } fn parser_value_string(parser: &mut lexopt::Parser, flag: &str) -> Result { let value = parser .value() .map_err(|error| CliError::usage(error.to_string()))?; value.into_string().map_err(|invalid| { CliError::usage(format!( "{flag} expects UTF-8 text, got '{}'", invalid.to_string_lossy() )) }) } fn parse_usize_flag(flag: &str, value: &str) -> Result { value .parse::() .map_err(|error| CliError::usage(format!("invalid {flag} value '{value}': {error}"))) } fn run(cli: &Cli) -> Result { if cli.sample_rows == 0 { return Err(CliError::usage("--sample-rows must be greater than 0")); } match &cli.command { CommandMode::Summary => { let paths = collect_paths(cli)?; if paths.is_empty() { return Err(CliError::usage( "provide at least one SQLite path or pipe paths into stdin", )); } let mut summaries = paths .iter() .map(|path| inspect_database(path, cli)) .collect::, _>>()?; let summary_count = summaries.len(); let matched_table_count = summaries .iter() .map(|summary| summary.table_count) .sum::(); match cli.common.render_mode() { RenderMode::Json => { if summaries.len() == 1 { print_json(&summaries.remove(0))?; } else { print_json(&summaries)?; } } RenderMode::Toon => { if summaries.len() == 1 { print_structured(&summaries.remove(0), RenderMode::Toon)?; } else { print_structured(&summaries, RenderMode::Toon)?; } } RenderMode::Text => { for summary in &summaries { print!("{}", render_summary(summary)); } } } if !cli.tables.is_empty() && matched_table_count == 0 { Ok(ExitCode::NoResults) } else { Ok(map_result_count(summary_count)) } } CommandMode::Diff { before, after } => { let before_path = common::require_exactly_one_input_path( &common::expand_input_patterns(std::slice::from_ref(before), "sqliteshape")?, "sqliteshape diff before", )?; let after_path = common::require_exactly_one_input_path( &common::expand_input_patterns(std::slice::from_ref(after), "sqliteshape")?, "sqliteshape diff after", )?; let before_summary = inspect_database(&before_path, cli)?; let after_summary = inspect_database(&after_path, cli)?; let diff = diff_databases(&before_summary, &after_summary); match cli.common.render_mode() { RenderMode::Json => print_json(&diff)?, RenderMode::Toon => print_structured(&diff, RenderMode::Toon)?, RenderMode::Text => print!("{}", render_diff(&diff)), } Ok(map_result_count( diff.added_tables.len() + diff.removed_tables.len() + diff.changed_tables.len(), )) } } } fn collect_paths(cli: &Cli) -> Result, CliError> { if should_read_stdin(!cli.paths.is_empty(), cli.common.stdin_is_terminal()) { let mut buffer = String::new(); io::stdin() .read_to_string(&mut buffer) .map_err(|error| CliError::runtime(format!("failed to read stdin: {error}")))?; let parsed = parse_paths_from_string(&buffer, cli.common.input_format)?; if !parsed.is_empty() { return Ok(parsed); } } common::expand_input_patterns(&cli.paths, "sqliteshape") } fn parse_paths_from_string( buffer: &str, input_format: InputFormat, ) -> Result, CliError> { common::read_existing_stdin_path_records(buffer, input_format, "sqliteshape")? .map_or_else(|| Ok(Vec::new()), Ok) } fn inspect_database(path: &PathBuf, cli: &Cli) -> Result { if !path.exists() { return Err(CliError::runtime(format!( "database path does not exist: {}", path.display() ))); } let connection = Connection::open_with_flags( path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, ) .map_err(|error| CliError::runtime(format!("failed to open {}: {error}", path.display())))?; connection .busy_timeout(SQLITE_BUSY_TIMEOUT) .map_err(|error| { CliError::runtime(format!( "failed to configure sqlite busy timeout for {}: {error}", path.display() )) })?; let page_size = pragma_i64(&connection, "page_size")?; let page_count = pragma_i64(&connection, "page_count")?; let journal_mode = pragma_string(&connection, "journal_mode")?; let table_selection = load_table_selection(&connection, &cli.tables)?; let tables = table_selection .matched .iter() .map(|name| inspect_table(&connection, name, cli)) .collect::, _>>()?; Ok(DatabaseSummary { path: path.display().to_string(), page_size, page_count, journal_mode, table_count: tables.len(), requested_tables: cli.tables.clone(), unmatched_tables: table_selection.unmatched, tables, }) } fn pragma_i64(connection: &Connection, pragma: &str) -> Result { connection .query_row(&format!("PRAGMA {pragma};"), [], |row| row.get::<_, i64>(0)) .map_err(|error| CliError::runtime(format!("failed to read PRAGMA {pragma}: {error}"))) } fn pragma_string(connection: &Connection, pragma: &str) -> Result { connection .query_row(&format!("PRAGMA {pragma};"), [], |row| { row.get::<_, String>(0) }) .map_err(|error| CliError::runtime(format!("failed to read PRAGMA {pragma}: {error}"))) } fn load_table_selection( connection: &Connection, filter: &[String], ) -> Result { let mut statement = connection .prepare( "SELECT name FROM sqlite_master \ WHERE type = 'table' AND name NOT LIKE 'sqlite_%' \ ORDER BY name", ) .map_err(|error| CliError::runtime(format!("failed to list tables: {error}")))?; let rows = statement .query_map([], |row| row.get::<_, String>(0)) .map_err(|error| CliError::runtime(format!("failed to query tables: {error}")))?; let mut names = Vec::new(); for row in rows { names.push( row.map_err(|error| CliError::runtime(format!("failed to read table name: {error}")))?, ); } if filter.is_empty() { return Ok(TableSelection { matched: names, unmatched: Vec::new(), }); } let filtered = names .iter() .filter(|name| filter.iter().any(|wanted| wanted == *name)) .cloned() .collect::>(); let unmatched = filter .iter() .filter(|wanted| !names.iter().any(|name| name == *wanted)) .cloned() .collect::>(); Ok(TableSelection { matched: filtered, unmatched, }) } fn inspect_table( connection: &Connection, table: &str, cli: &Cli, ) -> Result { let columns = load_columns(connection, table)?; let sample_rows = load_sample_rows(connection, table, cli.sample_rows)?; let (row_count, row_count_mode) = if cli.count_rows { ( Some(count_rows(connection, table)?), ROW_COUNT_MODE_EXACT.to_string(), ) } else { let estimated = estimate_row_count(connection, table)?; ( estimated, if estimated.is_some() { ROW_COUNT_MODE_ESTIMATED.to_string() } else { ROW_COUNT_MODE_UNKNOWN.to_string() }, ) }; let indexes = if cli.include_indexes { load_indexes(connection, table)? } else { Vec::new() }; Ok(TableSummary { name: table.to_string(), row_count, row_count_mode, columns, sample_rows, indexes, }) } fn load_columns(connection: &Connection, table: &str) -> Result, CliError> { let escaped = escape_identifier(table); let sql = format!("PRAGMA table_info({escaped});"); let mut statement = connection.prepare(&sql).map_err(|error| { CliError::runtime(format!("failed to inspect columns for {table}: {error}")) })?; let rows = statement .query_map([], |row| { Ok(ColumnSummary { name: row.get(1)?, declared_type: row.get::<_, Option>(2)?.unwrap_or_default(), not_null: row.get::<_, i64>(3)? != 0, primary_key_position: row.get(5)?, }) }) .map_err(|error| { CliError::runtime(format!("failed to query columns for {table}: {error}")) })?; let mut columns = Vec::new(); for row in rows { columns.push(row.map_err(|error| { CliError::runtime(format!( "failed to read column metadata for {table}: {error}" )) })?); } Ok(columns) } fn load_sample_rows( connection: &Connection, table: &str, sample_rows: usize, ) -> Result, CliError> { let escaped = escape_identifier(table); let sql = format!("SELECT * FROM {escaped} LIMIT {sample_rows}"); let mut statement = connection.prepare(&sql).map_err(|error| { CliError::runtime(format!("failed to sample rows for {table}: {error}")) })?; let names = statement .column_names() .iter() .map(ToString::to_string) .collect::>(); let rows = statement .query_map([], |row| { let mut object = serde_json::Map::new(); for (index, name) in names.iter().enumerate() { let value = row.get_ref(index)?; object.insert(name.clone(), sqlite_value_to_json(value)); } Ok(serde_json::Value::Object(object)) }) .map_err(|error| { CliError::runtime(format!("failed to read sample rows for {table}: {error}")) })?; let mut samples = Vec::new(); for row in rows { samples.push(row.map_err(|error| { CliError::runtime(format!("failed to decode sample row for {table}: {error}")) })?); } Ok(samples) } fn count_rows(connection: &Connection, table: &str) -> Result { let escaped = escape_identifier(table); connection .query_row(&format!("SELECT COUNT(*) FROM {escaped}"), [], |row| { row.get(0) }) .map_err(|error| CliError::runtime(format!("failed to count rows for {table}: {error}"))) } fn estimate_row_count(connection: &Connection, table: &str) -> Result, CliError> { let mut statement = match connection.prepare("SELECT stat FROM sqlite_stat1 WHERE tbl = ?1 LIMIT 1") { Ok(statement) => statement, Err(rusqlite::Error::SqliteFailure(_, Some(message))) if message.contains("no such table: sqlite_stat1") => { return Ok(None); } Err(error) => { return Err(CliError::runtime(format!( "failed to inspect sqlite_stat1: {error}" ))); } }; let result = statement.query_row([table], |row| row.get::<_, String>(0)); match result { Ok(stat) => Ok(stat .split_whitespace() .next() .and_then(|part| part.parse::().ok())), Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), Err(rusqlite::Error::SqliteFailure(_, Some(message))) if message.contains("no such table: sqlite_stat1") => { Ok(None) } Err(error) => Err(CliError::runtime(format!( "failed to estimate rows for {table}: {error}" ))), } } fn load_indexes(connection: &Connection, table: &str) -> Result, CliError> { let escaped = escape_identifier(table); let sql = format!("PRAGMA index_list({escaped});"); let mut statement = connection.prepare(&sql).map_err(|error| { CliError::runtime(format!("failed to inspect indexes for {table}: {error}")) })?; let rows = statement .query_map([], |row| { Ok((row.get::<_, String>(1)?, row.get::<_, i64>(2)? != 0)) }) .map_err(|error| { CliError::runtime(format!("failed to query indexes for {table}: {error}")) })?; let mut indexes = Vec::new(); for row in rows { let (name, unique) = row.map_err(|error| { CliError::runtime(format!( "failed to read index metadata for {table}: {error}" )) })?; indexes.push(IndexSummary { columns: load_index_columns(connection, &name)?, name, unique, }); } Ok(indexes) } fn load_index_columns(connection: &Connection, index: &str) -> Result, CliError> { let escaped = escape_identifier(index); let sql = format!("PRAGMA index_info({escaped});"); let mut statement = connection .prepare(&sql) .map_err(|error| CliError::runtime(format!("failed to inspect index {index}: {error}")))?; let rows = statement .query_map([], |row| { let position = row.get::<_, i64>(0)?; let name = row.get::<_, Option>(2)?; Ok(( position, name.unwrap_or_else(|| SQLITE_EXPRESSION_INDEX_COLUMN.to_string()), )) }) .map_err(|error| { CliError::runtime(format!( "failed to query index columns for {index}: {error}" )) })?; let mut columns = Vec::new(); for row in rows { columns.push(row.map_err(|error| { CliError::runtime(format!("failed to read index column for {index}: {error}")) })?); } columns.sort_by_key(|(position, _)| *position); Ok(columns.into_iter().map(|(_, name)| name).collect()) } fn escape_identifier(value: &str) -> String { format!("\"{}\"", value.replace('"', "\"\"")) } fn sqlite_value_to_json(value: rusqlite::types::ValueRef<'_>) -> serde_json::Value { match value { rusqlite::types::ValueRef::Null => serde_json::Value::Null, rusqlite::types::ValueRef::Integer(number) => serde_json::Value::Number(number.into()), rusqlite::types::ValueRef::Real(number) => serde_json::Number::from_f64(number) .map_or(serde_json::Value::Null, serde_json::Value::Number), rusqlite::types::ValueRef::Text(text) => { serde_json::Value::String(String::from_utf8_lossy(text).to_string()) } rusqlite::types::ValueRef::Blob(blob) => { serde_json::Value::String(format!("", blob.len())) } } } fn render_summary(summary: &DatabaseSummary) -> String { let mut rendered = String::new(); writeln!( rendered, "path={} page_size={} page_count={} journal_mode={} tables={}", summary.path, summary.page_size, summary.page_count, summary.journal_mode, summary.table_count ) .expect("writing to a String cannot fail"); if !summary.requested_tables.is_empty() { writeln!( rendered, "requested_tables={} unmatched_tables={}", summary.requested_tables.join("|"), summary.unmatched_tables.join("|") ) .expect("writing to a String cannot fail"); } for table in &summary.tables { writeln!( rendered, "table={} rows={} columns={} sample_rows={} indexes={}", table.name, render_row_count(table.row_count, &table.row_count_mode), table.columns.len(), table.sample_rows.len(), table.indexes.len() ) .expect("writing to a String cannot fail"); for column in &table.columns { writeln!( rendered, "column={} declared_type={} not_null={} pk={}", column.name, column.declared_type, column.not_null, column.primary_key_position ) .expect("writing to a String cannot fail"); } } rendered } fn render_row_count(row_count: Option, row_count_mode: &str) -> String { match (row_count_mode, row_count) { (ROW_COUNT_MODE_EXACT, Some(count)) => format!("exact:{count}"), (ROW_COUNT_MODE_ESTIMATED, Some(count)) => format!("estimate:{count}"), _ => ROW_COUNT_MODE_UNKNOWN.to_string(), } } fn diff_databases(before: &DatabaseSummary, after: &DatabaseSummary) -> DatabaseDiff { let before_tables = before .tables .iter() .map(|table| (table.name.clone(), table)) .collect::>(); let after_tables = after .tables .iter() .map(|table| (table.name.clone(), table)) .collect::>(); let mut added_tables = Vec::new(); let mut removed_tables = Vec::new(); let mut changed_tables = Vec::new(); for name in before_tables.keys().chain(after_tables.keys()) { match (before_tables.get(name), after_tables.get(name)) { (None, Some(table)) => added_tables.push((*table).clone()), (Some(table), None) => removed_tables.push((*table).clone()), (Some(before_table), Some(after_table)) if before_table != after_table => { changed_tables.push(diff_tables(before_table, after_table)); } _ => {} } } dedup_tables(&mut added_tables); dedup_tables(&mut removed_tables); dedup_table_diffs(&mut changed_tables); DatabaseDiff { path_before: before.path.clone(), path_after: after.path.clone(), page_size_before: before.page_size, page_size_after: after.page_size, page_count_before: before.page_count, page_count_after: after.page_count, journal_mode_before: before.journal_mode.clone(), journal_mode_after: after.journal_mode.clone(), added_tables, removed_tables, changed_tables, } } fn diff_tables(before: &TableSummary, after: &TableSummary) -> TableDiff { let before_columns = before .columns .iter() .map(|column| (column.name.clone(), column)) .collect::>(); let after_columns = after .columns .iter() .map(|column| (column.name.clone(), column)) .collect::>(); let before_indexes = before .indexes .iter() .map(|index| (index.name.clone(), index)) .collect::>(); let after_indexes = after .indexes .iter() .map(|index| (index.name.clone(), index)) .collect::>(); let mut added_columns = Vec::new(); let mut removed_columns = Vec::new(); let mut changed_columns = Vec::new(); let mut added_indexes = Vec::new(); let mut removed_indexes = Vec::new(); for name in before_columns.keys().chain(after_columns.keys()) { match (before_columns.get(name), after_columns.get(name)) { (None, Some(column)) => added_columns.push((*column).clone()), (Some(column), None) => removed_columns.push((*column).clone()), (Some(before_column), Some(after_column)) if before_column != after_column => { changed_columns.push(ColumnChange { name: name.clone(), declared_type_before: before_column.declared_type.clone(), declared_type_after: after_column.declared_type.clone(), not_null_before: before_column.not_null, not_null_after: after_column.not_null, primary_key_position_before: before_column.primary_key_position, primary_key_position_after: after_column.primary_key_position, }); } _ => {} } } for name in before_indexes.keys().chain(after_indexes.keys()) { match (before_indexes.get(name), after_indexes.get(name)) { (None, Some(index)) => added_indexes.push((*index).clone()), (Some(index), None) => removed_indexes.push((*index).clone()), _ => {} } } dedup_columns(&mut added_columns); dedup_columns(&mut removed_columns); dedup_column_changes(&mut changed_columns); dedup_indexes(&mut added_indexes); dedup_indexes(&mut removed_indexes); TableDiff { name: before.name.clone(), row_count_before: before.row_count, row_count_after: after.row_count, row_count_mode_before: before.row_count_mode.clone(), row_count_mode_after: after.row_count_mode.clone(), added_columns, removed_columns, changed_columns, added_indexes, removed_indexes, } } fn dedup_tables(tables: &mut Vec) { let mut seen = std::collections::BTreeSet::::new(); tables.retain(|table| seen.insert(table.name.clone())); } fn dedup_table_diffs(tables: &mut Vec) { let mut seen = std::collections::BTreeSet::::new(); tables.retain(|table| seen.insert(table.name.clone())); } fn dedup_columns(columns: &mut Vec) { let mut seen = std::collections::BTreeSet::::new(); columns.retain(|column| seen.insert(column.name.clone())); } fn dedup_column_changes(columns: &mut Vec) { let mut seen = std::collections::BTreeSet::::new(); columns.retain(|column| seen.insert(column.name.clone())); } fn dedup_indexes(indexes: &mut Vec) { let mut seen = std::collections::BTreeSet::::new(); indexes.retain(|index| seen.insert(index.name.clone())); } fn render_diff(diff: &DatabaseDiff) -> String { let mut rendered = String::new(); writeln!( rendered, "before={} after={} page_size={}=>{} page_count={}=>{} journal_mode={}=>{} added_tables={} removed_tables={} changed_tables={}", diff.path_before, diff.path_after, diff.page_size_before, diff.page_size_after, diff.page_count_before, diff.page_count_after, diff.journal_mode_before, diff.journal_mode_after, diff.added_tables.len(), diff.removed_tables.len(), diff.changed_tables.len() ) .expect("writing to a String cannot fail"); for table in &diff.added_tables { writeln!( rendered, "table={} change=added rows={} columns={} indexes={}", table.name, render_row_count(table.row_count, &table.row_count_mode), table.columns.len(), table.indexes.len() ) .expect("writing to a String cannot fail"); } for table in &diff.removed_tables { writeln!( rendered, "table={} change=removed rows={} columns={} indexes={}", table.name, render_row_count(table.row_count, &table.row_count_mode), table.columns.len(), table.indexes.len() ) .expect("writing to a String cannot fail"); } for table in &diff.changed_tables { writeln!( rendered, "table={} change=changed rows={}=>{} added_columns={} removed_columns={} changed_columns={} added_indexes={} removed_indexes={}", table.name, render_row_count(table.row_count_before, &table.row_count_mode_before), render_row_count(table.row_count_after, &table.row_count_mode_after), table.added_columns.len(), table.removed_columns.len(), table.changed_columns.len(), table.added_indexes.len(), table.removed_indexes.len() ) .expect("writing to a String cannot fail"); } rendered } #[cfg(test)] mod tests { use std::fmt::Write as _; use std::fs; use std::time::Instant; use common::{ColorChoice, InputFormat}; use tempfile::tempdir; use super::*; fn common_args(json: bool, input_format: InputFormat) -> CommonArgs { CommonArgs { json, format: None, input_format, color: ColorChoice::Never, quiet: false, } } #[test] fn path_parsing_supports_lines_jsonl_and_auto() { let temp = tempdir().expect("tempdir"); let first = temp.path().join("one.db"); let second = temp.path().join("two.db"); fs::write(&first, "one").expect("first"); fs::write(&second, "two").expect("second"); assert!(matches!( parse_paths_from_string( &format!( "{}\n{{\"path\":{}}}\n", serde_json::to_string(&first.display().to_string()).expect("json path"), serde_json::to_string(&second.display().to_string()).expect("json path"), ), InputFormat::Jsonl, ), Ok(paths) if paths == vec![first.clone(), second.clone()] )); assert_eq!( parse_paths_from_string( &format!("{}\n{}\n", first.display(), second.display()), InputFormat::Auto, ) .expect("paths"), vec![first, second] ); } #[test] fn inspect_database_reads_schema_and_samples() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("sample.db"); let connection = Connection::open(&path).expect("db"); connection .execute_batch( "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL, city TEXT); CREATE UNIQUE INDEX idx_users_name ON users(name); INSERT INTO users(name, city) VALUES ('Ada', 'London'), ('Bob', NULL);", ) .expect("schema"); drop(connection); let summary = inspect_database( &path, &Cli { common: common_args(true, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 2, include_indexes: true, count_rows: true, paths: Vec::new(), }, ) .expect("summary"); assert_eq!(summary.table_count, 1); assert_eq!(summary.tables.len(), 1); assert_eq!(summary.tables[0].name, "users"); assert_eq!(summary.tables[0].row_count, Some(2)); assert_eq!(summary.tables[0].row_count_mode, ROW_COUNT_MODE_EXACT); assert_eq!(summary.tables[0].columns[1].declared_type, "TEXT"); assert_eq!(summary.tables[0].indexes[0].name, "idx_users_name"); assert_eq!(summary.tables[0].sample_rows.len(), 2); } #[test] fn inspect_database_handles_many_tables_without_sampling_or_counts() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("many_tables.db"); let connection = Connection::open(&path).expect("db"); let mut schema = String::from("BEGIN;\n"); for index in 0..96 { writeln!( schema, "CREATE TABLE t_{index:03}(id INTEGER PRIMARY KEY, c1 TEXT, c2 INTEGER, c3 REAL, c4 BLOB, c5 TEXT);" ) .expect("write schema"); } schema.push_str("COMMIT;\n"); connection.execute_batch(&schema).expect("schema"); drop(connection); let summary = inspect_database( &path, &Cli { common: common_args(true, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 0, include_indexes: false, count_rows: false, paths: Vec::new(), }, ) .expect("summary"); assert_eq!(summary.table_count, 96); assert_eq!(summary.tables.len(), 96); assert_eq!(summary.tables[0].name, "t_000"); assert_eq!(summary.tables[95].name, "t_095"); assert_eq!(summary.tables[0].columns.len(), 6); assert!( summary .tables .iter() .all(|table| table.sample_rows.is_empty()) ); assert!(summary.tables.iter().all(|table| table.indexes.is_empty())); } #[test] fn corrupt_sqlite_file_reports_database_parse_error() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("corrupt.db"); fs::write(&path, b"not a sqlite database").expect("corrupt fixture"); let error = inspect_database( &path, &Cli { common: common_args(true, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 1, include_indexes: false, count_rows: false, paths: Vec::new(), }, ) .expect_err("corrupt sqlite should fail"); assert!( matches!(&error, CliError::Runtime(message) if message.contains("file is not a database") || message.contains("database disk image is malformed")), "unexpected corrupt database error: {error}" ); } #[test] fn malicious_sqlite_catalog_null_table_name_fails_closed() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("malicious_catalog.db"); let connection = Connection::open(&path).expect("db"); connection .execute_batch( "CREATE TABLE users(id INTEGER PRIMARY KEY); PRAGMA writable_schema = ON; UPDATE sqlite_schema SET name = NULL WHERE type = 'table' AND name = 'users'; PRAGMA writable_schema = OFF;", ) .expect("malicious catalog"); drop(connection); let error = inspect_database( &path, &Cli { common: common_args(true, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 1, include_indexes: true, count_rows: false, paths: Vec::new(), }, ) .expect_err("malicious sqlite catalog should fail closed"); assert!( matches!(&error, CliError::Runtime(message) if message.contains("failed to read table name") || message.contains("malformed")), "unexpected malicious catalog error: {error}" ); } #[test] fn malicious_sqlite_catalog_invalid_table_sql_fails_closed() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("malicious_catalog_invalid_sql.db"); let connection = Connection::open(&path).expect("db"); connection .execute_batch( "CREATE TABLE users(id INTEGER PRIMARY KEY); PRAGMA writable_schema = ON; UPDATE sqlite_schema SET sql = 'CREATE TABLE users(' WHERE type = 'table' AND name = 'users'; PRAGMA writable_schema = OFF;", ) .expect("malicious catalog"); drop(connection); let error = inspect_database( &path, &Cli { common: common_args(true, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 1, include_indexes: true, count_rows: false, paths: Vec::new(), }, ) .expect_err("malicious sqlite catalog should fail closed"); let message = error.to_string(); assert!( message.contains("malformed") || message.contains("failed to list tables") || message.contains("failed to inspect columns"), "unexpected malicious catalog error: {error}" ); assert!( message.len() < 512, "unbounded sqlite catalog error: {error}" ); } #[test] fn locked_sqlite_file_reports_busy_without_long_wait() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("locked.db"); let writer = Connection::open(&path).expect("sqlite"); writer .execute_batch("CREATE TABLE users(id INTEGER PRIMARY KEY); BEGIN EXCLUSIVE;") .expect("exclusive transaction"); let started = Instant::now(); let error = inspect_database( &path, &Cli { common: common_args(true, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 1, include_indexes: false, count_rows: false, paths: Vec::new(), }, ) .expect_err("locked sqlite should fail"); assert!( started.elapsed().as_secs_f32() < 2.0, "locked sqlite inspection waited too long" ); assert!( matches!(&error, CliError::Runtime(message) if message.contains("locked") || message.contains("busy")), "unexpected locked database error: {error}" ); } #[test] fn render_summary_is_compact() { let text = render_summary(&DatabaseSummary { path: "sample.db".to_string(), page_size: 4096, page_count: 2, journal_mode: "delete".to_string(), table_count: 1, requested_tables: Vec::new(), unmatched_tables: Vec::new(), tables: vec![TableSummary { name: "users".to_string(), row_count: Some(2), row_count_mode: ROW_COUNT_MODE_EXACT.to_string(), columns: vec![ColumnSummary { name: "id".to_string(), declared_type: "INTEGER".to_string(), not_null: false, primary_key_position: 1, }], sample_rows: vec![], indexes: vec![], }], }); assert!(text.contains("path=sample.db")); assert!(text.contains("table=users rows=exact:2")); assert!(text.contains("column=id declared_type=INTEGER")); } #[test] fn parse_cli_and_helpers_cover_indexes_counts_and_identifiers() { let (outcome, _) = parse_cli_from(["sqliteshape", "--help"]).expect("help"); assert_eq!(outcome, ParseOutcome::Help); let (_, cli) = parse_cli_from([ "sqliteshape", "--json", "--table", "users", "--sample-rows", "5", "--include-indexes", "--count-rows", "sample.db", ]) .expect("cli"); assert!(cli.common.json); assert_eq!(cli.tables, vec!["users".to_string()]); assert_eq!(cli.sample_rows, 5); assert!(cli.include_indexes); assert!(cli.count_rows); assert_eq!(escape_identifier("a\"b"), "\"a\"\"b\""); } #[test] fn sqlite_helpers_cover_value_and_estimate_paths() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("estimate.db"); let connection = Connection::open(&path).expect("db"); connection .execute_batch( "CREATE TABLE items(id INTEGER PRIMARY KEY, score REAL, payload BLOB); INSERT INTO items(score, payload) VALUES (1.5, x'0102'); ANALYZE;", ) .expect("schema"); assert_eq!(count_rows(&connection, "items").expect("count"), 1); assert!( estimate_row_count(&connection, "items") .expect("estimate") .is_some() ); assert_eq!( sqlite_value_to_json(rusqlite::types::ValueRef::Blob(&[1, 2])), serde_json::Value::String("".to_string()) ); assert_eq!( sqlite_value_to_json(rusqlite::types::ValueRef::Integer(7)), serde_json::Value::Number(7.into()) ); assert_eq!( sqlite_value_to_json(rusqlite::types::ValueRef::Null), serde_json::Value::Null ); } #[test] fn parse_cli_supports_diff_subcommand() { let (_, cli) = parse_cli_from([ "sqliteshape", "--json", "diff", "--include-indexes", "before.db", "after.db", ]) .expect("cli"); assert!(cli.common.json); assert!(cli.include_indexes); assert_eq!( cli.command, CommandMode::Diff { before: PathBuf::from("before.db"), after: PathBuf::from("after.db"), } ); } #[test] fn diff_summary_reports_table_and_column_changes() { let before = DatabaseSummary { path: "before.db".to_string(), page_size: 4096, page_count: 2, journal_mode: "delete".to_string(), table_count: 1, requested_tables: Vec::new(), unmatched_tables: Vec::new(), tables: vec![TableSummary { name: "users".to_string(), row_count: Some(2), row_count_mode: ROW_COUNT_MODE_EXACT.to_string(), columns: vec![ColumnSummary { name: "name".to_string(), declared_type: "TEXT".to_string(), not_null: true, primary_key_position: 0, }], sample_rows: vec![], indexes: vec![], }], }; let after = DatabaseSummary { path: "after.db".to_string(), page_size: 4096, page_count: 3, journal_mode: "wal".to_string(), table_count: 2, requested_tables: Vec::new(), unmatched_tables: Vec::new(), tables: vec![ TableSummary { name: "users".to_string(), row_count: Some(3), row_count_mode: ROW_COUNT_MODE_EXACT.to_string(), columns: vec![ ColumnSummary { name: "name".to_string(), declared_type: "TEXT".to_string(), not_null: false, primary_key_position: 0, }, ColumnSummary { name: "city".to_string(), declared_type: "TEXT".to_string(), not_null: false, primary_key_position: 0, }, ], sample_rows: vec![], indexes: vec![], }, TableSummary { name: "events".to_string(), row_count: Some(1), row_count_mode: ROW_COUNT_MODE_EXACT.to_string(), columns: vec![], sample_rows: vec![], indexes: vec![], }, ], }; let diff = diff_databases(&before, &after); assert_eq!(diff.added_tables.len(), 1); assert_eq!(diff.added_tables[0].name, "events"); assert_eq!(diff.changed_tables.len(), 1); assert_eq!(diff.changed_tables[0].name, "users"); assert_eq!(diff.changed_tables[0].added_columns.len(), 1); assert_eq!(diff.changed_tables[0].added_columns[0].name, "city"); assert_eq!(diff.changed_tables[0].changed_columns.len(), 1); assert_eq!(diff.changed_tables[0].changed_columns[0].name, "name"); assert_eq!(diff.journal_mode_before, "delete"); assert_eq!(diff.journal_mode_after, "wal"); } #[test] fn schema_helpers_cover_filters_indexes_and_rendering() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("helpers.db"); let connection = Connection::open(&path).expect("db"); connection .execute_batch( "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL, city TEXT); CREATE INDEX idx_users_city ON users(city); CREATE TABLE events(id INTEGER PRIMARY KEY, kind TEXT); INSERT INTO users(name, city) VALUES ('Ada', 'London');", ) .expect("schema"); assert_eq!( load_table_selection(&connection, &[]) .expect("tables") .matched, vec!["events".to_string(), "users".to_string()] ); assert_eq!( load_table_selection(&connection, &[String::from("users")]) .expect("filtered") .matched, vec!["users".to_string()] ); let columns = load_columns(&connection, "users").expect("columns"); assert_eq!(columns.len(), 3); assert!(columns.iter().any(|column| column.name == "city")); let indexes = load_indexes(&connection, "users").expect("indexes"); assert_eq!(indexes.len(), 1); assert_eq!(indexes[0].name, "idx_users_city"); assert_eq!(indexes[0].columns, vec!["city".to_string()]); let table = inspect_table( &connection, "users", &Cli { common: common_args(false, InputFormat::Auto), command: CommandMode::Summary, tables: Vec::new(), sample_rows: 1, include_indexes: true, count_rows: true, paths: Vec::new(), }, ) .expect("table"); assert_eq!(table.row_count, Some(1)); assert_eq!(table.sample_rows.len(), 1); assert_eq!(table.indexes.len(), 1); } #[test] fn table_selection_tracks_unmatched_requested_tables() { let temp = tempdir().expect("tempdir"); let path = temp.path().join("helpers.db"); let connection = Connection::open(&path).expect("db"); connection .execute_batch("CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT);") .expect("schema"); let selection = load_table_selection( &connection, &[String::from("users"), String::from("missing")], ) .expect("selection"); assert_eq!(selection.matched, vec!["users".to_string()]); assert_eq!(selection.unmatched, vec!["missing".to_string()]); } #[test] fn diff_and_dedup_helpers_cover_removed_indexes_and_render_output() { let before = TableSummary { name: "users".to_string(), row_count: Some(2), row_count_mode: ROW_COUNT_MODE_EXACT.to_string(), columns: vec![ ColumnSummary { name: "id".to_string(), declared_type: "INTEGER".to_string(), not_null: false, primary_key_position: 1, }, ColumnSummary { name: "name".to_string(), declared_type: "TEXT".to_string(), not_null: true, primary_key_position: 0, }, ], sample_rows: vec![], indexes: vec![IndexSummary { name: "idx_users_name".to_string(), unique: true, columns: vec!["name".to_string()], }], }; let after = TableSummary { name: "users".to_string(), row_count: Some(2), row_count_mode: ROW_COUNT_MODE_ESTIMATED.to_string(), columns: vec![ColumnSummary { name: "id".to_string(), declared_type: "INTEGER".to_string(), not_null: false, primary_key_position: 1, }], sample_rows: vec![], indexes: vec![], }; let diff = diff_tables(&before, &after); assert_eq!(diff.removed_columns.len(), 1); assert_eq!(diff.removed_columns[0].name, "name"); assert_eq!(diff.removed_indexes.len(), 1); assert_eq!(diff.removed_indexes[0].name, "idx_users_name"); let mut duplicate_tables = vec![before.clone(), before]; dedup_tables(&mut duplicate_tables); assert_eq!(duplicate_tables.len(), 1); let mut duplicate_diffs = vec![diff.clone(), diff]; dedup_table_diffs(&mut duplicate_diffs); assert_eq!(duplicate_diffs.len(), 1); let rendered = render_diff(&DatabaseDiff { path_before: "before.db".to_string(), path_after: "after.db".to_string(), page_size_before: 4096, page_size_after: 4096, page_count_before: 1, page_count_after: 2, journal_mode_before: "delete".to_string(), journal_mode_after: "wal".to_string(), added_tables: vec![], removed_tables: vec![], changed_tables: duplicate_diffs, }); assert!(rendered.contains("before=before.db")); assert!(rendered.contains("table=users change=changed")); } }