//! The `ison` command converts between JSON and a compact ISON subset. use std::ffi::OsString; use std::fmt::Write as _; use std::fs; use std::io::{self, Read, Write}; use std::path::PathBuf; use common::{ CliError, CommonArgs, ExitCode, InputFormat, RenderMode, parse_color_choice, parse_format_choice, parse_input_format, print_quick_help_error, print_structured, read_existing_stdin_paths, should_read_stdin, }; use lexopt::prelude::{Long, Short, Value as ArgValue}; use serde_json::{Map, Number, Value, json}; const HELP: &str = "\ Convert between JSON and ISON. Usage: ison [OPTIONS] [PATH] Options: --format Structured output format: text, json, toon --json Shortcut for --format json --toon Shortcut for --format toon --from Force source syntax: auto, json, ison --to Force target syntax: auto, json, ison --input-format Override stdin path mode: auto, lines, jsonl --color Control ANSI color output: auto, never --quiet Suppress non-essential status output -h, --help Show this help text -V, --version Show the command version ISON block syntax: table.users id:int name:str active:bool 1 Ada true Examples: ison .\\fixtures\\json-family\\ison\\users.json ison --from ison --to json .\\fixtures\\json-family\\ison\\users.ison "; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Syntax { Auto, Json, Ison, } #[derive(Debug, Clone)] struct Cli { common: CommonArgs, from: Syntax, to: Syntax, path: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ParseOutcome { Help, Version, Run, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ResolvedSyntax { Json, Ison, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BlockKind { Table, Object, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum FieldType { Int, Float, Bool, Str, Null, } #[derive(Debug, Clone, PartialEq, Eq)] struct Field { name: String, kind: FieldType, } #[derive(Debug, Clone, PartialEq, Eq)] struct Token { value: String, quoted: bool, } /// Parses CLI arguments and returns a process exit code. #[must_use] pub fn main_entry() -> i32 { match parse_cli_from(std::env::args_os()) { Ok((ParseOutcome::Help, _)) => { print!("{HELP}"); ExitCode::Success.as_i32() } Ok((ParseOutcome::Version, _)) => { println!("ison {}", env!("CARGO_PKG_VERSION")); ExitCode::Success.as_i32() } Ok((ParseOutcome::Run, cli)) => match run(&cli) { Ok(code) => code.as_i32(), Err(error) => { print_quick_help_error(&error, HELP); error.exit_code().as_i32() } }, Err(error) => { print_quick_help_error(&error, HELP); error.exit_code().as_i32() } } } fn parse_cli_from(args: I) -> Result<(ParseOutcome, Cli), CliError> where I: IntoIterator, T: Into, { let mut parser = lexopt::Parser::from_iter(args); let mut cli = Cli { common: CommonArgs::default(), from: Syntax::Auto, to: Syntax::Auto, path: None, }; while let Some(argument) = parser .next() .map_err(|error| CliError::usage(error.to_string()))? { match argument { Long("help") | Short('h') => return Ok((ParseOutcome::Help, cli)), Long("version") | Short('V') => return Ok((ParseOutcome::Version, cli)), Long("input-format") => { let value = parser_value_string(&mut parser, "--input-format")?; cli.common.input_format = parse_input_format(&value)?; } Long("format") => { let value = parser_value_string(&mut parser, "--format")?; cli.common.set_render_mode(parse_format_choice(&value)?); } Long("json") => cli.common.set_render_mode(RenderMode::Json), Long("toon") => cli.common.set_render_mode(RenderMode::Toon), Long("color") => { let value = parser_value_string(&mut parser, "--color")?; cli.common.color = parse_color_choice(&value)?; } Long("quiet") => cli.common.quiet = true, Long("from") => { cli.from = parse_syntax("--from", &parser_value_string(&mut parser, "--from")?)?; } Long("to") => { cli.to = parse_syntax("--to", &parser_value_string(&mut parser, "--to")?)?; } ArgValue(path) => { if cli.path.replace(PathBuf::from(path)).is_some() { return Err(CliError::usage("ison accepts at most one explicit path")); } } _ => { return Err(CliError::usage( "unsupported argument; use --help to see available options", )); } } } Ok((ParseOutcome::Run, cli)) } fn parser_value_string(parser: &mut lexopt::Parser, flag: &str) -> Result { let value = parser .value() .map_err(|error| CliError::usage(error.to_string()))?; value.into_string().map_err(|invalid| { CliError::usage(format!( "{flag} expects UTF-8 text, got '{}'", invalid.to_string_lossy() )) }) } fn parse_syntax(flag: &str, value: &str) -> Result { match value { "auto" => Ok(Syntax::Auto), "json" => Ok(Syntax::Json), "ison" => Ok(Syntax::Ison), other => Err(CliError::usage(format!( "invalid {flag} value '{other}'; expected auto, json, or ison" ))), } } fn run(cli: &Cli) -> Result { let input = load_input(cli)?; let source = resolve_source(cli.from, &input); let target = resolve_target(cli.to, source); let value = match source { ResolvedSyntax::Json => serde_json::from_str::(input.trim()) .map_err(|error| CliError::usage(format!("invalid JSON input: {error}")))?, ResolvedSyntax::Ison => decode_document(&input).map_err(CliError::usage)?, }; let (format, text) = match target { ResolvedSyntax::Json => { let text = serde_json::to_string_pretty(&value) .map_err(|error| CliError::runtime(format!("failed to render JSON: {error}")))?; ("json", format!("{text}\n")) } ResolvedSyntax::Ison => ("ison", encode_document(&value).map_err(CliError::usage)?), }; match cli.common.render_mode() { RenderMode::Text => write_stdout(&text)?, RenderMode::Json | RenderMode::Toon => { print_structured( &json!({ "format": format, "text": text, }), cli.common.render_mode(), )?; } } Ok(ExitCode::Success) } fn load_input(cli: &Cli) -> Result { if should_read_stdin(cli.path.is_some(), cli.common.stdin_is_terminal()) { let mut buffer = String::new(); io::stdin() .read_to_string(&mut buffer) .map_err(|error| CliError::runtime(format!("failed to read stdin: {error}")))?; if !buffer.is_empty() { if cli.common.input_format != InputFormat::Jsonl && let Some(paths) = read_existing_stdin_paths(&buffer, cli.common.input_format, "ison")? { return read_single_stdin_path(&paths, "ison"); } return Ok(buffer); } } let Some(path) = &cli.path else { return Err(CliError::usage( "provide one JSON/ISON path or pipe input into stdin", )); }; let path = common::require_exactly_one_input_path( &common::expand_input_patterns(std::slice::from_ref(path), "ison")?, "ison", )?; fs::read_to_string(&path) .map_err(|error| CliError::runtime(format!("failed to read {}: {error}", path.display()))) } fn read_single_stdin_path(paths: &[PathBuf], command_name: &str) -> Result { if paths.len() != 1 { return Err(CliError::usage(format!( "{command_name} accepts exactly one stdin path, got {}", paths.len() ))); } fs::read_to_string(&paths[0]).map_err(|error| { CliError::runtime(format!("failed to read {}: {error}", paths[0].display())) }) } fn resolve_source(source: Syntax, input: &str) -> ResolvedSyntax { match source { Syntax::Json => ResolvedSyntax::Json, Syntax::Ison => ResolvedSyntax::Ison, Syntax::Auto => { if serde_json::from_str::(input.trim()).is_ok() { ResolvedSyntax::Json } else { ResolvedSyntax::Ison } } } } const fn resolve_target(target: Syntax, source: ResolvedSyntax) -> ResolvedSyntax { match target { Syntax::Json => ResolvedSyntax::Json, Syntax::Ison => ResolvedSyntax::Ison, Syntax::Auto => match source { ResolvedSyntax::Json => ResolvedSyntax::Ison, ResolvedSyntax::Ison => ResolvedSyntax::Json, }, } } fn write_stdout(text: &str) -> Result<(), CliError> { let mut stdout = io::stdout().lock(); stdout .write_all(text.as_bytes()) .map_err(|error| CliError::runtime(format!("failed to write stdout: {error}"))) } /// Encodes one JSON document into deterministic ISON block text. /// /// # Errors /// /// Returns an error when the document cannot be represented by the v1 compact /// subset, for example a top-level non-object or an empty array whose fields /// cannot be inferred. pub fn encode_document(value: &Value) -> Result { let object = value .as_object() .ok_or_else(|| "ISON v1 expects a top-level JSON object".to_string())?; let mut output = String::with_capacity(object.len().saturating_mul(128)); for (name, child) in object { if !output.is_empty() { output.push('\n'); } match child { Value::Array(items) => encode_table_block(name, items, &mut output)?, Value::Object(map) => encode_object_block(name, map, &mut output)?, scalar => { let mut map = Map::new(); map.insert("value".to_string(), scalar.clone()); encode_object_block(name, &map, &mut output)?; } } } output.push('\n'); Ok(output) } fn encode_table_block(name: &str, items: &[Value], output: &mut String) -> Result<(), String> { let Some(first) = items.first() else { return Err(format!( "table '{name}' is empty; fields cannot be inferred" )); }; let first_object = first .as_object() .ok_or_else(|| format!("table '{name}' expects JSON object rows"))?; let fields = infer_fields(first_object); output.push_str("table."); output.push_str(name); output.push('\n'); push_fields(output, &fields); output.push('\n'); for (index, item) in items.iter().enumerate() { let object = item .as_object() .ok_or_else(|| format!("table '{name}' row {} is not an object", index + 1))?; if index > 0 { output.push('\n'); } push_row(output, object, &fields); } Ok(()) } fn encode_object_block( name: &str, object: &Map, output: &mut String, ) -> Result<(), String> { if object.is_empty() { return Err(format!( "object '{name}' is empty; fields cannot be inferred" )); } let fields = infer_fields(object); output.push_str("object."); output.push_str(name); output.push('\n'); push_fields(output, &fields); output.push('\n'); push_row(output, object, &fields); Ok(()) } /// Encodes one JSON object into an ISONL record line. /// /// # Errors /// /// Returns an error when `record` is not a JSON object or has no fields. pub fn encode_record(record: &Value, name: &str) -> Result { let object = record .as_object() .ok_or_else(|| "ISONL v1 expects each JSONL line to be an object".to_string())?; if object.is_empty() { return Err("ISONL v1 cannot infer fields from an empty object".to_string()); } let fields = infer_fields(object); let mut output = String::with_capacity(name.len() + fields.len().saturating_mul(24) + 16); output.push_str("object."); output.push_str(name); output.push('|'); push_fields(&mut output, &fields); output.push('|'); push_row(&mut output, object, &fields); Ok(output) } fn infer_fields(object: &Map) -> Vec { object .iter() .map(|(name, value)| Field { name: name.clone(), kind: infer_type(value), }) .collect() } fn infer_type(value: &Value) -> FieldType { match value { Value::Bool(_) => FieldType::Bool, Value::Number(number) if number.is_i64() || number.is_u64() => FieldType::Int, Value::Number(_) => FieldType::Float, Value::String(_) | Value::Array(_) | Value::Object(_) => FieldType::Str, Value::Null => FieldType::Null, } } fn push_fields(output: &mut String, fields: &[Field]) { for (index, field) in fields.iter().enumerate() { if index > 0 { output.push(' '); } output.push_str(&field.name); output.push(':'); output.push_str(field.kind.as_str()); } } fn push_row(output: &mut String, object: &Map, fields: &[Field]) { for (index, field) in fields.iter().enumerate() { if index > 0 { output.push(' '); } push_value(output, object.get(&field.name).unwrap_or(&Value::Null)); } } fn push_value(output: &mut String, value: &Value) { match value { Value::Null => output.push_str("null"), Value::Bool(value) => output.push_str(if *value { "true" } else { "false" }), Value::Number(value) => { let _ = write!(output, "{value}"); } Value::String(value) => push_string(output, value), Value::Array(_) | Value::Object(_) => push_string(output, &value.to_string()), } } fn push_string(output: &mut String, value: &str) { if value.is_empty() || value .chars() .any(|character| character.is_whitespace() || matches!(character, '"' | '\\' | '|')) { output.push_str(&serde_json::to_string(value).unwrap_or_else(|_| "\"\"".to_string())); } else { output.push_str(value); } } /// Decodes ISON block text into a JSON document. /// /// # Errors /// /// Returns an error with a line number when block headers, field definitions, /// or row values are malformed. pub fn decode_document(input: &str) -> Result { let lines = input .lines() .enumerate() .filter_map(|(index, line)| { let trimmed = line.trim(); (!trimmed.is_empty()).then_some((index + 1, trimmed)) }) .collect::>(); let mut index = 0; let mut root = Map::new(); while index < lines.len() { let (header_line, header) = lines[index]; let (kind, name) = parse_header(header, header_line)?; index += 1; let Some((fields_line, fields_text)) = lines.get(index).copied() else { return Err(format!("line {header_line}: missing field definition")); }; let fields = parse_fields(fields_text, fields_line)?; index += 1; let mut rows = Vec::new(); while let Some((line_number, line)) = lines.get(index).copied() { if looks_like_header(line) { break; } rows.push(parse_row(line, line_number, &fields)?); index += 1; } if rows.is_empty() { return Err(format!("line {header_line}: block '{name}' has no rows")); } let value = match kind { BlockKind::Table => Value::Array(rows.into_iter().map(Value::Object).collect()), BlockKind::Object => Value::Object(rows.remove(0)), }; root.insert(name, value); } Ok(Value::Object(root)) } /// Decodes one ISONL record line into a JSON object. /// /// # Errors /// /// Returns an error with `line_number` when the ISONL record is malformed. pub fn decode_record_line(line: &str, line_number: usize) -> Result { let mut parts = line.splitn(3, '|'); let Some(header) = parts.next() else { return Err(format!( "line {line_number}: ISONL record expects 3 pipe-delimited segments" )); }; let Some(fields_text) = parts.next() else { return Err(format!( "line {line_number}: ISONL record expects 3 pipe-delimited segments" )); }; let Some(row_text) = parts.next() else { return Err(format!( "line {line_number}: ISONL record expects 3 pipe-delimited segments" )); }; parse_header(header.trim(), line_number)?; let fields = parse_fields(fields_text.trim(), line_number)?; let row = parse_row(row_text.trim(), line_number, &fields)?; Ok(Value::Object(row)) } fn parse_header(line: &str, line_number: usize) -> Result<(BlockKind, String), String> { let Some((kind, name)) = line.split_once('.') else { return Err(format!( "line {line_number}: expected block header kind.name" )); }; if name.is_empty() { return Err(format!("line {line_number}: block name must not be empty")); } let kind = match kind { "table" => BlockKind::Table, "object" => BlockKind::Object, other => { return Err(format!( "line {line_number}: unsupported block kind '{other}'; expected table or object" )); } }; Ok((kind, name.to_string())) } fn looks_like_header(line: &str) -> bool { line.starts_with("table.") || line.starts_with("object.") } fn parse_fields(line: &str, line_number: usize) -> Result, String> { let fields = line .split_whitespace() .map(|part| parse_field(part, line_number)) .collect::, _>>()?; if fields.is_empty() { Err(format!("line {line_number}: expected at least one field")) } else { Ok(fields) } } fn parse_field(part: &str, line_number: usize) -> Result { let Some((name, kind)) = part.split_once(':') else { return Err(format!( "line {line_number}: field '{part}' must use name:type syntax" )); }; if name.is_empty() { return Err(format!("line {line_number}: field name must not be empty")); } Ok(Field { name: name.to_string(), kind: parse_field_type(kind, line_number)?, }) } fn parse_field_type(kind: &str, line_number: usize) -> Result { match kind { "int" => Ok(FieldType::Int), "float" => Ok(FieldType::Float), "bool" => Ok(FieldType::Bool), "str" | "string" => Ok(FieldType::Str), "null" => Ok(FieldType::Null), other => Err(format!( "line {line_number}: unsupported field type '{other}'" )), } } fn parse_row( line: &str, line_number: usize, fields: &[Field], ) -> Result, String> { let tokens = tokenize_values(line, line_number)?; if tokens.len() != fields.len() { return Err(format!( "line {line_number}: expected {} values, got {}", fields.len(), tokens.len() )); } fields .iter() .zip(tokens) .map(|(field, token)| { parse_value(&token, field.kind, line_number).map(|value| (field.name.clone(), value)) }) .collect() } fn tokenize_values(line: &str, line_number: usize) -> Result, String> { let mut tokens = Vec::new(); let mut chars = line.char_indices().peekable(); while let Some((_, character)) = chars.peek().copied() { if character.is_whitespace() { chars.next(); continue; } if character == '"' { let start = chars.next().map_or(0, |(index, _)| index); let mut escaped = false; let mut end = None; for (index, current) in chars.by_ref() { if escaped { escaped = false; continue; } if current == '\\' { escaped = true; continue; } if current == '"' { end = Some(index + current.len_utf8()); break; } } let Some(end) = end else { return Err(format!("line {line_number}: unterminated string value")); }; tokens.push(Token { value: line[start..end].to_string(), quoted: true, }); continue; } let start = chars.next().map_or(0, |(index, _)| index); let mut end = line.len(); while let Some((index, current)) = chars.peek().copied() { if current.is_whitespace() { end = index; break; } chars.next(); } tokens.push(Token { value: line[start..end].to_string(), quoted: false, }); } Ok(tokens) } fn parse_value(token: &Token, kind: FieldType, line_number: usize) -> Result { match kind { FieldType::Int => parse_number_value(token, line_number), FieldType::Float => token .value .parse::() .map_err(|error| { format!( "line {line_number}: invalid float '{}': {error}", token.value ) }) .and_then(|value| { Number::from_f64(value) .map(Value::Number) .ok_or_else(|| format!("line {line_number}: float must be finite")) }), FieldType::Bool => token .value .parse::() .map(Value::Bool) .map_err(|error| { format!( "line {line_number}: invalid bool '{}': {error}", token.value ) }), FieldType::Str => { if token.quoted { serde_json::from_str::(&token.value) .map(Value::String) .map_err(|error| { format!( "line {line_number}: invalid quoted string '{}': {error}", token.value ) }) } else { Ok(Value::String(token.value.clone())) } } FieldType::Null => Ok(Value::Null), } } fn parse_number_value(token: &Token, line_number: usize) -> Result { if let Ok(value) = token.value.parse::() { return Ok(Value::Number(Number::from(value))); } token .value .parse::() .map(Number::from) .map(Value::Number) .map_err(|error| format!("line {line_number}: invalid int '{}': {error}", token.value)) } impl FieldType { const fn as_str(self) -> &'static str { match self { Self::Int => "int", Self::Float => "float", Self::Bool => "bool", Self::Str => "str", Self::Null => "null", } } } #[cfg(test)] mod tests { use super::*; use tempfile::tempdir; #[test] fn parse_cli_covers_flags_and_rejects_duplicate_paths() { let (outcome, cli) = parse_cli_from([ "ison", "--from", "json", "--to", "ison", "--format", "json", "--input-format", "lines", "input.json", ]) .expect("valid cli"); assert_eq!(outcome, ParseOutcome::Run); assert_eq!(cli.from, Syntax::Json); assert_eq!(cli.to, Syntax::Ison); assert_eq!(cli.common.render_mode(), RenderMode::Json); assert_eq!(cli.common.input_format, InputFormat::Lines); assert_eq!(cli.path, Some(PathBuf::from("input.json"))); assert!(parse_cli_from(["ison", "--from", "yaml"]).is_err()); assert!(parse_cli_from(["ison", "a.json", "b.json"]).is_err()); } #[test] fn json_document_roundtrips_tables_objects_and_scalars() { let value = json!({ "users": [ {"id": 1, "name": "Ada Lovelace", "active": true}, {"id": 2, "name": "Grace", "active": false} ], "meta": {"score": 1.5, "note": "pipe|quote\"", "missing": null}, "count": 2 }); let encoded = encode_document(&value).expect("encode document"); assert!(encoded.contains("table.users")); assert!(encoded.contains("object.meta")); assert!(encoded.contains("object.count")); let decoded = decode_document(&encoded).expect("decode document"); assert_eq!( decoded, json!({ "users": value["users"].clone(), "meta": value["meta"].clone(), "count": {"value": 2} }) ); assert_eq!(resolve_source(Syntax::Auto, &encoded), ResolvedSyntax::Ison); assert_eq!( resolve_target(Syntax::Auto, ResolvedSyntax::Ison), ResolvedSyntax::Json ); } #[test] fn record_lines_handle_quoted_values_and_type_errors() { let record = json!({ "id": 7, "name": "Ada Byron", "active": true, "payload": {"role": "math"} }); let encoded = encode_record(&record, "user").expect("encode record"); assert!(encoded.starts_with("object.user|")); let decoded = decode_record_line(&encoded, 4).expect("decode record"); assert_eq!( decoded, json!({ "id": 7, "name": "Ada Byron", "active": true, "payload": "{\"role\":\"math\"}" }) ); let bad = decode_record_line("object.user|id:int active:bool|oops maybe", 9) .expect_err("invalid fields should fail"); assert!(bad.contains("line 9")); assert_eq!( decode_record_line("object.counter|value:int|18446744073709551615", 10) .expect("u64 int") .get("value"), Some(&Value::Number(Number::from(u64::MAX))) ); assert!(encode_record(&json!([]), "record").is_err()); } #[test] fn decoder_reports_malformed_headers_fields_and_rows() { assert!(decode_document("plain\nid:int\n1\n").is_err()); assert!(decode_document("object.\nid:int\n1\n").is_err()); assert!(decode_document("object.user\nid:weird\n1\n").is_err()); assert!(decode_document("object.user\nid:int name:str\n1\n").is_err()); assert!(decode_document("object.user\nid:int\nnot-an-int\n").is_err()); assert!(decode_document("object.user\nname:str\n\"unterminated\n").is_err()); } #[test] fn run_reads_path_inputs_and_supports_wrapper_modes() { let directory = tempdir().expect("tempdir"); let json_path = directory.path().join("users.json"); fs::write( &json_path, r#"{"users":[{"id":1,"name":"Ada"},{"id":2,"name":"Grace"}]}"#, ) .expect("json fixture"); let (_, encode_cli) = parse_cli_from([ "ison", "--from", "json", "--to", "ison", "--json", json_path.to_str().expect("utf8 path"), ]) .expect("encode cli"); assert_eq!(run(&encode_cli).expect("encode run"), ExitCode::Success); let ison_path = directory.path().join("users.ison"); fs::write(&ison_path, "table.users\nid:int name:str\n1 Ada\n2 Grace\n") .expect("ison fixture"); let (_, decode_cli) = parse_cli_from([ "ison", "--from", "ison", "--to", "json", "--toon", ison_path.to_str().expect("utf8 path"), ]) .expect("decode cli"); assert_eq!(run(&decode_cli).expect("decode run"), ExitCode::Success); let (_, missing_cli) = parse_cli_from(["ison"]).expect("missing input cli"); assert!(run(&missing_cli).is_err()); } }