chore(release): prepare public source release

This commit is contained in:
MercuryToolbox Release
2026-07-18 15:33:01 +08:00
commit 34d6a57f38
510 changed files with 163501 additions and 0 deletions
+26
View File
@@ -0,0 +1,26 @@
[package]
name = "isonl"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
authors.workspace = true
readme.workspace = true
publish.workspace = true
description = "Convert between JSONL and ISONL line-oriented records."
keywords.workspace = true
categories.workspace = true
[lints]
workspace = true
[dependencies]
common = { path = "../common", default-features = false }
ison = { path = "../ison", default-features = false }
lexopt.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
[dev-dependencies]
assert_cmd.workspace = true
predicates.workspace = true
tempfile.workspace = true
+452
View File
@@ -0,0 +1,452 @@
//! The `isonl` command converts between JSONL and ISONL records.
use std::ffi::OsString;
use std::fs;
use std::io::{self, Read, Write};
use std::path::PathBuf;
use common::{
CliError, CommonArgs, ExitCode, InputFormat, RenderMode, formats::ison as common_ison,
parse_color_choice, parse_format_choice, parse_input_format, print_quick_help_error,
print_structured, read_existing_stdin_paths, should_read_stdin,
};
use lexopt::prelude::{Long, Short, Value as ArgValue};
use serde_json::{Value, json};
const HELP: &str = "\
Convert between JSONL and ISONL line-oriented records.
Usage:
isonl [OPTIONS] [PATH]
Options:
--format <FORMAT> Structured output format: text, json, toon
--json Shortcut for --format json
--toon Shortcut for --format toon
--from <FORMAT> Force source syntax: auto, jsonl, isonl
--to <FORMAT> Force target syntax: auto, jsonl, isonl
--record-name <NAME> ISONL record name when encoding JSONL
--input-format <FORMAT> Override stdin path mode: auto, lines, jsonl
--color <WHEN> Control ANSI color output: auto, never
--quiet Suppress non-essential status output
-h, --help Show this help text
-V, --version Show the command version
Examples:
isonl --from jsonl --to isonl .\\records.jsonl
isonl --from isonl --to jsonl .\\fixtures\\json-family\\ison\\users.isonl
";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Syntax {
Auto,
Jsonl,
Isonl,
}
#[derive(Debug, Clone)]
struct Cli {
common: CommonArgs,
from: Syntax,
to: Syntax,
record_name: String,
path: Option<PathBuf>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ParseOutcome {
Help,
Version,
Run,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ResolvedSyntax {
Jsonl,
Isonl,
}
/// Parses CLI arguments and returns a process exit code.
#[must_use]
pub fn main_entry() -> i32 {
match parse_cli_from(std::env::args_os()) {
Ok((ParseOutcome::Help, _)) => {
print!("{HELP}");
ExitCode::Success.as_i32()
}
Ok((ParseOutcome::Version, _)) => {
println!("isonl {}", env!("CARGO_PKG_VERSION"));
ExitCode::Success.as_i32()
}
Ok((ParseOutcome::Run, cli)) => match run(&cli) {
Ok(code) => code.as_i32(),
Err(error) => {
print_quick_help_error(&error, HELP);
error.exit_code().as_i32()
}
},
Err(error) => {
print_quick_help_error(&error, HELP);
error.exit_code().as_i32()
}
}
}
fn parse_cli_from<I, T>(args: I) -> Result<(ParseOutcome, Cli), CliError>
where
I: IntoIterator<Item = T>,
T: Into<OsString>,
{
let mut parser = lexopt::Parser::from_iter(args);
let mut cli = Cli {
common: CommonArgs::default(),
from: Syntax::Auto,
to: Syntax::Auto,
record_name: "record".to_string(),
path: None,
};
while let Some(argument) = parser
.next()
.map_err(|error| CliError::usage(error.to_string()))?
{
match argument {
Long("help") | Short('h') => return Ok((ParseOutcome::Help, cli)),
Long("version") | Short('V') => return Ok((ParseOutcome::Version, cli)),
Long("input-format") => {
let value = parser_value_string(&mut parser, "--input-format")?;
cli.common.input_format = parse_input_format(&value)?;
}
Long("format") => {
let value = parser_value_string(&mut parser, "--format")?;
cli.common.set_render_mode(parse_format_choice(&value)?);
}
Long("json") => cli.common.set_render_mode(RenderMode::Json),
Long("toon") => cli.common.set_render_mode(RenderMode::Toon),
Long("color") => {
let value = parser_value_string(&mut parser, "--color")?;
cli.common.color = parse_color_choice(&value)?;
}
Long("quiet") => cli.common.quiet = true,
Long("from") => {
cli.from = parse_syntax("--from", &parser_value_string(&mut parser, "--from")?)?;
}
Long("to") => {
cli.to = parse_syntax("--to", &parser_value_string(&mut parser, "--to")?)?;
}
Long("record-name") => {
cli.record_name = parser_value_string(&mut parser, "--record-name")?;
if cli.record_name.is_empty() {
return Err(CliError::usage("--record-name must not be empty"));
}
}
ArgValue(path) => {
if cli.path.replace(PathBuf::from(path)).is_some() {
return Err(CliError::usage("isonl accepts at most one explicit path"));
}
}
_ => {
return Err(CliError::usage(
"unsupported argument; use --help to see available options",
));
}
}
}
Ok((ParseOutcome::Run, cli))
}
fn parser_value_string(parser: &mut lexopt::Parser, flag: &str) -> Result<String, CliError> {
let value = parser
.value()
.map_err(|error| CliError::usage(error.to_string()))?;
value.into_string().map_err(|invalid| {
CliError::usage(format!(
"{flag} expects UTF-8 text, got '{}'",
invalid.to_string_lossy()
))
})
}
fn parse_syntax(flag: &str, value: &str) -> Result<Syntax, CliError> {
match value {
"auto" => Ok(Syntax::Auto),
"jsonl" => Ok(Syntax::Jsonl),
"isonl" => Ok(Syntax::Isonl),
other => Err(CliError::usage(format!(
"invalid {flag} value '{other}'; expected auto, jsonl, or isonl"
))),
}
}
fn run(cli: &Cli) -> Result<ExitCode, CliError> {
let input = load_input(cli)?;
let source = resolve_source(cli.from, &input);
let target = resolve_target(cli.to, source);
let values = match source {
ResolvedSyntax::Jsonl => parse_jsonl(&input)?,
ResolvedSyntax::Isonl => parse_isonl(&input)?,
};
let (format, text) = match target {
ResolvedSyntax::Jsonl => ("jsonl", render_jsonl(&values)?),
ResolvedSyntax::Isonl => ("isonl", render_isonl(&values, &cli.record_name)?),
};
match cli.common.render_mode() {
RenderMode::Text => write_stdout(&text)?,
RenderMode::Json | RenderMode::Toon => {
print_structured(
&json!({
"format": format,
"records": values.len(),
"text": text,
}),
cli.common.render_mode(),
)?;
}
}
Ok(ExitCode::Success)
}
fn load_input(cli: &Cli) -> Result<String, CliError> {
if should_read_stdin(cli.path.is_some(), cli.common.stdin_is_terminal()) {
let mut buffer = String::new();
io::stdin()
.read_to_string(&mut buffer)
.map_err(|error| CliError::runtime(format!("failed to read stdin: {error}")))?;
if !buffer.is_empty() {
if cli.common.input_format != InputFormat::Jsonl
&& let Some(paths) =
read_existing_stdin_paths(&buffer, cli.common.input_format, "isonl")?
{
return read_single_stdin_path(&paths, "isonl");
}
return Ok(buffer);
}
}
let Some(path) = &cli.path else {
return Err(CliError::usage(
"provide one JSONL/ISONL path or pipe input into stdin",
));
};
let path = common::require_exactly_one_input_path(
&common::expand_input_patterns(std::slice::from_ref(path), "isonl")?,
"isonl",
)?;
fs::read_to_string(&path)
.map_err(|error| CliError::runtime(format!("failed to read {}: {error}", path.display())))
}
fn read_single_stdin_path(paths: &[PathBuf], command_name: &str) -> Result<String, CliError> {
if paths.len() != 1 {
return Err(CliError::usage(format!(
"{command_name} accepts exactly one stdin path, got {}",
paths.len()
)));
}
fs::read_to_string(&paths[0]).map_err(|error| {
CliError::runtime(format!("failed to read {}: {error}", paths[0].display()))
})
}
fn resolve_source(source: Syntax, input: &str) -> ResolvedSyntax {
match source {
Syntax::Jsonl => ResolvedSyntax::Jsonl,
Syntax::Isonl => ResolvedSyntax::Isonl,
Syntax::Auto => {
if input
.lines()
.map(str::trim)
.find(|line| !line.is_empty())
.is_some_and(|line| line.contains('|'))
{
ResolvedSyntax::Isonl
} else {
ResolvedSyntax::Jsonl
}
}
}
}
const fn resolve_target(target: Syntax, source: ResolvedSyntax) -> ResolvedSyntax {
match target {
Syntax::Jsonl => ResolvedSyntax::Jsonl,
Syntax::Isonl => ResolvedSyntax::Isonl,
Syntax::Auto => match source {
ResolvedSyntax::Jsonl => ResolvedSyntax::Isonl,
ResolvedSyntax::Isonl => ResolvedSyntax::Jsonl,
},
}
}
fn parse_jsonl(input: &str) -> Result<Vec<Value>, CliError> {
let mut values = Vec::new();
for (index, line) in input.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let value = serde_json::from_str::<Value>(trimmed).map_err(|error| {
CliError::usage(format!("invalid JSONL at line {}: {error}", index + 1))
})?;
values.push(value);
}
if values.is_empty() {
Err(CliError::usage("JSONL input is empty"))
} else {
Ok(values)
}
}
fn parse_isonl(input: &str) -> Result<Vec<Value>, CliError> {
let values = common_ison::decode_records(input)?;
if values.is_empty() {
Err(CliError::usage("ISONL input is empty"))
} else {
Ok(values)
}
}
fn render_jsonl(values: &[Value]) -> Result<String, CliError> {
let mut output = String::new();
for value in values {
let line = serde_json::to_string(value)
.map_err(|error| CliError::runtime(format!("failed to render JSONL: {error}")))?;
output.push_str(&line);
output.push('\n');
}
Ok(output)
}
fn render_isonl(values: &[Value], record_name: &str) -> Result<String, CliError> {
let mut output = String::new();
for value in values {
let line = ison::encode_record(value, record_name).map_err(CliError::usage)?;
output.push_str(&line);
output.push('\n');
}
Ok(output)
}
fn write_stdout(text: &str) -> Result<(), CliError> {
let mut stdout = io::stdout().lock();
stdout
.write_all(text.as_bytes())
.map_err(|error| CliError::runtime(format!("failed to write stdout: {error}")))
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn parse_cli_covers_record_name_and_render_flags() {
let (outcome, cli) = parse_cli_from([
"isonl",
"--from",
"jsonl",
"--to",
"isonl",
"--record-name",
"event",
"--toon",
"records.jsonl",
])
.expect("valid cli");
assert_eq!(outcome, ParseOutcome::Run);
assert_eq!(cli.from, Syntax::Jsonl);
assert_eq!(cli.to, Syntax::Isonl);
assert_eq!(cli.record_name, "event");
assert_eq!(cli.common.render_mode(), RenderMode::Toon);
assert_eq!(cli.path, Some(PathBuf::from("records.jsonl")));
assert!(parse_cli_from(["isonl", "--record-name", ""]).is_err());
assert!(parse_cli_from(["isonl", "a", "b"]).is_err());
}
#[test]
fn jsonl_and_isonl_roundtrip_records() {
let input = "{\"id\":1,\"name\":\"Ada\"}\n\n{\"id\":2,\"name\":\"Grace\"}\n";
let records = parse_jsonl(input).expect("jsonl records");
assert_eq!(records.len(), 2);
let isonl = render_isonl(&records, "user").expect("isonl render");
assert!(isonl.contains("object.user|"));
assert_eq!(resolve_source(Syntax::Auto, &isonl), ResolvedSyntax::Isonl);
assert_eq!(
resolve_target(Syntax::Auto, ResolvedSyntax::Isonl),
ResolvedSyntax::Jsonl
);
let decoded = parse_isonl(&isonl).expect("isonl parse");
assert_eq!(decoded, records);
let jsonl = render_jsonl(&decoded).expect("jsonl render");
assert!(jsonl.contains("\"Ada\""));
assert!(jsonl.ends_with('\n'));
assert_eq!(
parse_isonl("object.counter|value:int|18446744073709551615\n").expect("u64 isonl")[0]
.get("value"),
Some(&Value::Number(serde_json::Number::from(u64::MAX)))
);
}
#[test]
fn parsers_report_empty_and_invalid_inputs() {
assert!(parse_jsonl("\n \n").is_err());
assert!(parse_jsonl("{\"ok\":true}\nnot-json\n").is_err());
assert!(parse_isonl("\n").is_err());
assert!(parse_syntax("--from", "yaml").is_err());
}
#[test]
fn run_reads_files_and_wraps_conversions() {
let directory = tempdir().expect("tempdir");
let jsonl_path = directory.path().join("records.jsonl");
fs::write(
&jsonl_path,
"{\"id\":1,\"name\":\"Ada\"}\n{\"id\":2,\"name\":\"Grace\"}\n",
)
.expect("jsonl fixture");
let (_, encode_cli) = parse_cli_from([
"isonl",
"--from",
"jsonl",
"--to",
"isonl",
"--json",
"--record-name",
"user",
jsonl_path.to_str().expect("utf8 path"),
])
.expect("encode cli");
assert_eq!(run(&encode_cli).expect("encode run"), ExitCode::Success);
let isonl_path = directory.path().join("records.isonl");
fs::write(
&isonl_path,
"object.user|id:int name:str|1 Ada\nobject.user|id:int name:str|2 Grace\n",
)
.expect("isonl fixture");
let (_, decode_cli) = parse_cli_from([
"isonl",
"--from",
"isonl",
"--to",
"jsonl",
"--toon",
isonl_path.to_str().expect("utf8 path"),
])
.expect("decode cli");
assert_eq!(run(&decode_cli).expect("decode run"), ExitCode::Success);
let (_, missing_cli) = parse_cli_from(["isonl"]).expect("missing input cli");
assert!(run(&missing_cli).is_err());
}
}
+5
View File
@@ -0,0 +1,5 @@
//! Binary entry point for `isonl`.
fn main() {
std::process::exit(isonl::main_entry());
}
+108
View File
@@ -0,0 +1,108 @@
//! Integration tests for the `isonl` command.
//!
//! The pipe-delimited record expectations track public examples on
//! <https://ison.dev> and <https://ison.dev/spec.html>. The v1 CLI tests
//! JSONL/ISONL interop for scalar object records first.
use assert_cmd::Command;
use predicates::prelude::*;
use std::path::{Path, PathBuf};
fn cargo_command() -> Command {
Command::cargo_bin("isonl").expect("binary")
}
fn fixture_path(relative: &str) -> PathBuf {
let fixture = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(Path::parent)
.expect("workspace root")
.join(relative);
assert!(
fixture.exists(),
"missing fixture `{relative}` at {}",
fixture.display()
);
fixture
}
#[test]
fn help_mentions_line_oriented_json_interop() {
cargo_command()
.arg("--help")
.assert()
.success()
.stdout(predicate::str::contains("Convert between JSONL and ISONL"))
.stdout(predicate::str::contains("--from <FORMAT>"))
.stdout(predicate::str::contains("--to <FORMAT>"));
}
#[test]
fn jsonl_stdin_encodes_to_isonl_records() {
cargo_command()
.args(["--from", "jsonl", "--to", "isonl"])
.write_stdin("{\"id\":1,\"name\":\"Ada\",\"active\":true}\n{\"id\":2,\"name\":\"Bob\",\"active\":false}\n")
.assert()
.success()
.stdout(predicate::str::contains(
"object.record|id:int name:str active:bool|1 Ada true\n",
))
.stdout(predicate::str::contains(
"object.record|id:int name:str active:bool|2 Bob false\n",
));
}
#[test]
fn isonl_path_decodes_to_jsonl() {
cargo_command()
.args(["--from", "isonl", "--to", "jsonl"])
.arg(fixture_path("fixtures/json-family/ison/users.isonl"))
.assert()
.success()
.stdout(predicate::str::contains(
"{\"id\":1,\"name\":\"Ada\",\"active\":true}\n",
))
.stdout(predicate::str::contains(
"{\"id\":2,\"name\":\"Bob\",\"active\":false}\n",
));
}
#[test]
fn format_toon_wraps_jsonl_conversion_for_ai_pipelines() {
cargo_command()
.args(["--format", "toon", "--from", "isonl", "--to", "jsonl"])
.arg(fixture_path("fixtures/json-family/ison/users.isonl"))
.assert()
.success()
.stdout(predicate::str::contains("format: jsonl"))
.stdout(predicate::str::contains("records: 2"))
.stdout(predicate::str::contains("text:"));
}
#[test]
fn invalid_isonl_reports_line_number() {
cargo_command()
.args(["--from", "isonl", "--to", "jsonl"])
.write_stdin("object.record|id:int name:str|1\n")
.assert()
.failure()
.code(2)
.stderr(predicate::str::contains("line 1"))
.stderr(predicate::str::contains("expected 2 values"));
}
#[test]
fn empty_stdin_reports_usage_diagnostic() {
cargo_command()
.args(["--from", "jsonl", "--to", "isonl"])
.write_stdin("")
.assert()
.failure()
.code(2)
.stderr(
predicate::str::contains("input is empty").or(predicate::str::contains(
"provide one JSONL/ISONL path or pipe input into stdin",
)),
)
.stderr(predicate::str::contains("Usage:"));
}