chore(release): prepare public source release
This commit is contained in:
MercuryToolbox Release
@@ -0,0 +1,921 @@
|
||||
//! The `fileprobe` command classifies files with lightweight heuristics.
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::fmt::Write as _;
|
||||
use std::fs;
|
||||
use std::io::{self, Read};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use common::{
|
||||
CliError, CommonArgs, ExitCode, InputFormat, RenderMode, map_result_count, parse_color_choice,
|
||||
parse_format_choice, parse_input_format, print_json, print_quick_help_error, print_structured,
|
||||
should_read_stdin,
|
||||
};
|
||||
use lexopt::prelude::{Long, Short, Value as ArgValue};
|
||||
use serde::Serialize;
|
||||
|
||||
const HELP: &str = "\
|
||||
Probe file type and usefulness heuristics for AI-friendly workflows.
|
||||
|
||||
Usage:
|
||||
fileprobe [OPTIONS] [PATH...]
|
||||
|
||||
Options:
|
||||
--format <FORMAT> Structured output format: text, json, toon
|
||||
--json Shortcut for --format json
|
||||
--toon Shortcut for --format toon
|
||||
--input-format <FORMAT> Override stdin parsing mode: auto, lines, jsonl
|
||||
--color <WHEN> Control ANSI color output: auto, never
|
||||
--quiet Suppress non-essential status output
|
||||
-h, --help Show this help text
|
||||
-V, --version Show the command version
|
||||
|
||||
Examples:
|
||||
fileprobe .\\src\\main.rs
|
||||
fileprobe .\\dist\\bundle.min.js --json | ConvertFrom-Json
|
||||
fileprobe .\\samples\\*.json --toon
|
||||
fd -t f . .\\src | fileprobe --input-format lines --json | ConvertFrom-Json
|
||||
fd -t f . .\\samples | fileprobe --input-format lines --toon
|
||||
";
|
||||
|
||||
/// CLI arguments for the `fileprobe` binary.
|
||||
#[derive(Debug, Clone)]
|
||||
struct Cli {
|
||||
/// Shared output and stdin policy flags.
|
||||
common: CommonArgs,
|
||||
/// Explicit files to inspect when stdin is empty.
|
||||
paths: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ParseOutcome {
|
||||
Help,
|
||||
Version,
|
||||
Run,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum FileFamily {
|
||||
Directory,
|
||||
Source,
|
||||
Config,
|
||||
Data,
|
||||
Text,
|
||||
Binary,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
struct TextStats {
|
||||
line_count: usize,
|
||||
blank_lines: usize,
|
||||
longest_line: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
struct LineEndingCounts {
|
||||
lf: usize,
|
||||
crlf: usize,
|
||||
cr: usize,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::struct_excessive_bools,
|
||||
reason = "the JSON contract intentionally exposes fixed heuristic toggles as stable booleans"
|
||||
)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
struct FileReport {
|
||||
path: String,
|
||||
extension: Option<String>,
|
||||
size_bytes: u64,
|
||||
modified_rfc3339: String,
|
||||
family: FileFamily,
|
||||
is_directory: bool,
|
||||
language_hint: Option<String>,
|
||||
container_hint: Option<String>,
|
||||
is_binary: bool,
|
||||
encoding_hint: Option<String>,
|
||||
bom: Option<String>,
|
||||
newline_style: Option<String>,
|
||||
mixed_newlines: Option<bool>,
|
||||
line_ending_counts: Option<LineEndingCounts>,
|
||||
line_count: Option<usize>,
|
||||
blank_lines: Option<usize>,
|
||||
longest_line: Option<usize>,
|
||||
likely_generated: bool,
|
||||
likely_minified: bool,
|
||||
likely_lockfile: bool,
|
||||
likely_test: bool,
|
||||
likely_vendor: bool,
|
||||
}
|
||||
|
||||
/// Parses CLI arguments and returns a process exit code.
|
||||
#[must_use]
|
||||
pub fn main_entry() -> i32 {
|
||||
match parse_cli_from(std::env::args_os()) {
|
||||
Ok((ParseOutcome::Help, _)) => {
|
||||
print!("{HELP}");
|
||||
ExitCode::Success.as_i32()
|
||||
}
|
||||
Ok((ParseOutcome::Version, _)) => {
|
||||
println!("fileprobe {}", env!("CARGO_PKG_VERSION"));
|
||||
ExitCode::Success.as_i32()
|
||||
}
|
||||
Ok((ParseOutcome::Run, cli)) => match run(&cli) {
|
||||
Ok(code) => code.as_i32(),
|
||||
Err(error) => {
|
||||
print_quick_help_error(&error, HELP);
|
||||
error.exit_code().as_i32()
|
||||
}
|
||||
},
|
||||
Err(error) => {
|
||||
print_quick_help_error(&error, HELP);
|
||||
error.exit_code().as_i32()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_cli_from<I, T>(args: I) -> Result<(ParseOutcome, Cli), CliError>
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: Into<OsString>,
|
||||
{
|
||||
let mut parser = lexopt::Parser::from_iter(args);
|
||||
let mut cli = Cli {
|
||||
common: CommonArgs::default(),
|
||||
paths: Vec::new(),
|
||||
};
|
||||
|
||||
while let Some(argument) = parser
|
||||
.next()
|
||||
.map_err(|error| CliError::usage(error.to_string()))?
|
||||
{
|
||||
match argument {
|
||||
Long("help") | Short('h') => return Ok((ParseOutcome::Help, cli)),
|
||||
Long("version") | Short('V') => return Ok((ParseOutcome::Version, cli)),
|
||||
Long("json") => cli.common.set_render_mode(RenderMode::Json),
|
||||
Long("toon") => cli.common.set_render_mode(RenderMode::Toon),
|
||||
Long("format") => {
|
||||
let value = parser_value_string(&mut parser, "--format")?;
|
||||
cli.common.set_render_mode(parse_format_choice(&value)?);
|
||||
}
|
||||
Long("input-format") => {
|
||||
let value = parser_value_string(&mut parser, "--input-format")?;
|
||||
cli.common.input_format = parse_input_format(&value)?;
|
||||
}
|
||||
Long("color") => {
|
||||
let value = parser_value_string(&mut parser, "--color")?;
|
||||
cli.common.color = parse_color_choice(&value)?;
|
||||
}
|
||||
Long("quiet") => cli.common.quiet = true,
|
||||
ArgValue(path) => cli.paths.push(PathBuf::from(path)),
|
||||
_ => {
|
||||
return Err(CliError::usage(
|
||||
"unsupported argument; use --help to see available options",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((ParseOutcome::Run, cli))
|
||||
}
|
||||
|
||||
fn parser_value_string(parser: &mut lexopt::Parser, flag: &str) -> Result<String, CliError> {
|
||||
let value = parser
|
||||
.value()
|
||||
.map_err(|error| CliError::usage(error.to_string()))?;
|
||||
value.into_string().map_err(|invalid| {
|
||||
CliError::usage(format!(
|
||||
"{flag} expects UTF-8 text, got '{}'",
|
||||
invalid.to_string_lossy()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn run(cli: &Cli) -> Result<ExitCode, CliError> {
|
||||
let paths = collect_paths(cli)?;
|
||||
if paths.is_empty() {
|
||||
return Err(CliError::usage(
|
||||
"provide at least one path or pipe paths into stdin",
|
||||
));
|
||||
}
|
||||
|
||||
let mut reports = Vec::new();
|
||||
for path in &paths {
|
||||
reports.push(inspect_path(path)?);
|
||||
}
|
||||
|
||||
match cli.common.render_mode() {
|
||||
RenderMode::Json => print_json(&reports)?,
|
||||
RenderMode::Toon => print_structured(&reports, RenderMode::Toon)?,
|
||||
RenderMode::Text => {
|
||||
for report in &reports {
|
||||
println!("{}", render_text_report(report));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(map_result_count(reports.len()))
|
||||
}
|
||||
|
||||
fn collect_paths(cli: &Cli) -> Result<Vec<PathBuf>, CliError> {
|
||||
if should_read_stdin(!cli.paths.is_empty(), cli.common.stdin_is_terminal()) {
|
||||
let mut buffer = String::new();
|
||||
io::stdin()
|
||||
.read_to_string(&mut buffer)
|
||||
.map_err(|error| CliError::runtime(format!("failed to read stdin: {error}")))?;
|
||||
let paths = parse_paths_from_string(&buffer, cli.common.input_format)?;
|
||||
if !paths.is_empty() {
|
||||
return Ok(paths);
|
||||
}
|
||||
}
|
||||
|
||||
common::expand_input_patterns(&cli.paths, "fileprobe")
|
||||
}
|
||||
|
||||
fn parse_paths_from_string(
|
||||
buffer: &str,
|
||||
input_format: InputFormat,
|
||||
) -> Result<Vec<PathBuf>, CliError> {
|
||||
common::read_existing_stdin_path_records(buffer, input_format, "fileprobe")?
|
||||
.map_or_else(|| Ok(Vec::new()), Ok)
|
||||
}
|
||||
|
||||
fn inspect_path(path: &Path) -> Result<FileReport, CliError> {
|
||||
let path_text = path.display().to_string();
|
||||
let metadata = fs::metadata(path).map_err(|error| {
|
||||
CliError::runtime(format!("failed to read metadata for {path_text}: {error}"))
|
||||
})?;
|
||||
let modified = metadata.modified().map_err(|error| {
|
||||
CliError::runtime(format!(
|
||||
"failed to read modified time for {path_text}: {error}"
|
||||
))
|
||||
})?;
|
||||
if metadata.is_dir() {
|
||||
return Ok(FileReport {
|
||||
path: path_text,
|
||||
extension: None,
|
||||
size_bytes: 0,
|
||||
modified_rfc3339: humantime::format_rfc3339_seconds(modified).to_string(),
|
||||
family: FileFamily::Directory,
|
||||
is_directory: true,
|
||||
language_hint: None,
|
||||
container_hint: Some("directory".to_string()),
|
||||
is_binary: false,
|
||||
encoding_hint: None,
|
||||
bom: None,
|
||||
newline_style: None,
|
||||
mixed_newlines: None,
|
||||
line_ending_counts: None,
|
||||
line_count: None,
|
||||
blank_lines: None,
|
||||
longest_line: None,
|
||||
likely_generated: false,
|
||||
likely_minified: false,
|
||||
likely_lockfile: false,
|
||||
likely_test: detect_test_path(path),
|
||||
likely_vendor: detect_vendor_path(path),
|
||||
});
|
||||
}
|
||||
let bytes = fs::read(path)
|
||||
.map_err(|error| CliError::runtime(format!("failed to read {path_text}: {error}")))?;
|
||||
|
||||
let extension = extension_label(path);
|
||||
let container_hint = detect_container_hint(&bytes);
|
||||
let bom = detect_bom(&bytes);
|
||||
let is_binary = is_binary_blob(&bytes, container_hint.as_deref(), bom.as_deref());
|
||||
let language_hint = detect_language_hint(path);
|
||||
let family = classify_family(is_binary, language_hint.as_deref());
|
||||
let decoded_text = (!is_binary)
|
||||
.then(|| decode_text(&bytes, bom.as_deref()))
|
||||
.flatten();
|
||||
let text_stats = decoded_text.as_deref().map(summarize_text);
|
||||
let encoding_hint = decoded_text
|
||||
.as_ref()
|
||||
.and_then(|_| detect_encoding_hint(&bytes, bom.as_deref()));
|
||||
let line_ending_counts = decoded_text.as_deref().map(count_line_endings);
|
||||
let newline_style = line_ending_counts.as_ref().and_then(detect_newline_style);
|
||||
let mixed_newlines = line_ending_counts
|
||||
.as_ref()
|
||||
.map(|counts| distinct_line_endings(counts) > 1);
|
||||
let likely_lockfile = detect_lockfile(path);
|
||||
let likely_generated = detect_generated(path, &bytes, likely_lockfile);
|
||||
let likely_minified = text_stats
|
||||
.as_ref()
|
||||
.zip(decoded_text.as_deref())
|
||||
.is_some_and(|(stats, text)| detect_minified(text, stats));
|
||||
|
||||
Ok(FileReport {
|
||||
path: path_text,
|
||||
extension,
|
||||
size_bytes: metadata.len(),
|
||||
modified_rfc3339: humantime::format_rfc3339_seconds(modified).to_string(),
|
||||
family,
|
||||
is_directory: false,
|
||||
language_hint,
|
||||
container_hint,
|
||||
is_binary,
|
||||
encoding_hint,
|
||||
bom,
|
||||
newline_style,
|
||||
mixed_newlines,
|
||||
line_ending_counts,
|
||||
line_count: text_stats.as_ref().map(|stats| stats.line_count),
|
||||
blank_lines: text_stats.as_ref().map(|stats| stats.blank_lines),
|
||||
longest_line: text_stats.as_ref().map(|stats| stats.longest_line),
|
||||
likely_generated,
|
||||
likely_minified,
|
||||
likely_lockfile,
|
||||
likely_test: detect_test_path(path),
|
||||
likely_vendor: detect_vendor_path(path),
|
||||
})
|
||||
}
|
||||
|
||||
fn extension_label(path: &Path) -> Option<String> {
|
||||
path.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(str::to_ascii_lowercase)
|
||||
}
|
||||
|
||||
fn detect_container_hint(bytes: &[u8]) -> Option<String> {
|
||||
magic_container_hint(bytes).map(str::to_owned)
|
||||
}
|
||||
|
||||
fn magic_container_hint(bytes: &[u8]) -> Option<&'static str> {
|
||||
if bytes.starts_with(b"MZ") || bytes.windows(2).take(8).any(|window| window == b"MZ") {
|
||||
Some("pe")
|
||||
} else if bytes.starts_with(&[0x7f, b'E', b'L', b'F']) {
|
||||
Some("elf")
|
||||
} else if bytes.starts_with(&[0xfe, 0xed, 0xfa, 0xce])
|
||||
|| bytes.starts_with(&[0xfe, 0xed, 0xfa, 0xcf])
|
||||
|| bytes.starts_with(&[0xce, 0xfa, 0xed, 0xfe])
|
||||
|| bytes.starts_with(&[0xcf, 0xfa, 0xed, 0xfe])
|
||||
|| bytes.starts_with(&[0xca, 0xfe, 0xba, 0xbe])
|
||||
{
|
||||
Some("mach")
|
||||
} else if bytes.starts_with(b"!<arch>\n") {
|
||||
Some("archive")
|
||||
} else if bytes.starts_with(b"PK\x03\x04") {
|
||||
Some("zip")
|
||||
} else if bytes.starts_with(b"SQLite format 3\0") {
|
||||
Some("sqlite")
|
||||
} else if bytes.starts_with(b"%PDF-") {
|
||||
Some("pdf")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn is_binary_blob(bytes: &[u8], container_hint: Option<&str>, bom: Option<&str>) -> bool {
|
||||
if container_hint.is_some() {
|
||||
return true;
|
||||
}
|
||||
if bom.is_some() {
|
||||
return false;
|
||||
}
|
||||
if bytes.contains(&0) {
|
||||
return true;
|
||||
}
|
||||
std::str::from_utf8(bytes).is_err()
|
||||
}
|
||||
|
||||
fn detect_bom(bytes: &[u8]) -> Option<String> {
|
||||
if bytes.starts_with(&[0xef, 0xbb, 0xbf]) {
|
||||
Some("utf-8".to_owned())
|
||||
} else if bytes.starts_with(&[0xff, 0xfe]) {
|
||||
Some("utf-16le".to_owned())
|
||||
} else if bytes.starts_with(&[0xfe, 0xff]) {
|
||||
Some("utf-16be".to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn detect_encoding_hint(bytes: &[u8], bom: Option<&str>) -> Option<String> {
|
||||
if let Some(bom) = bom {
|
||||
return Some(bom.to_string());
|
||||
}
|
||||
std::str::from_utf8(bytes).ok().map(|_| "utf-8".to_string())
|
||||
}
|
||||
|
||||
fn detect_language_hint(path: &Path) -> Option<String> {
|
||||
match path.extension().and_then(|value| value.to_str()) {
|
||||
Some("rs") => Some("rust".to_string()),
|
||||
Some("cs") => Some("csharp".to_string()),
|
||||
Some("js" | "mjs" | "cjs") => Some("javascript".to_string()),
|
||||
Some("ts" | "tsx") => Some("typescript".to_string()),
|
||||
Some("json") => Some("json".to_string()),
|
||||
Some("jsonl") => Some("jsonl".to_string()),
|
||||
Some("csv") => Some("csv".to_string()),
|
||||
Some("tsv") => Some("tsv".to_string()),
|
||||
Some("toml") => Some("toml".to_string()),
|
||||
Some("yaml" | "yml") => Some("yaml".to_string()),
|
||||
Some("ps1") => Some("powershell".to_string()),
|
||||
Some("py") => Some("python".to_string()),
|
||||
Some("md") => Some("markdown".to_string()),
|
||||
Some("xml") => Some("xml".to_string()),
|
||||
Some("html" | "htm") => Some("html".to_string()),
|
||||
Some("css") => Some("css".to_string()),
|
||||
Some("lock") => Some("lockfile".to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_family(is_binary: bool, language_hint: Option<&str>) -> FileFamily {
|
||||
if is_binary {
|
||||
return FileFamily::Binary;
|
||||
}
|
||||
|
||||
match language_hint {
|
||||
Some(
|
||||
"rust" | "csharp" | "javascript" | "typescript" | "powershell" | "python" | "xml"
|
||||
| "html" | "css",
|
||||
) => FileFamily::Source,
|
||||
Some("json" | "toml" | "yaml" | "lockfile") => FileFamily::Config,
|
||||
Some("jsonl" | "csv" | "tsv") => FileFamily::Data,
|
||||
Some(_) | None => FileFamily::Text,
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_text(bytes: &[u8], bom: Option<&str>) -> Option<String> {
|
||||
match bom {
|
||||
Some("utf-8") => Some(String::from_utf8_lossy(&bytes[3..]).to_string()),
|
||||
Some("utf-16le") => Some(decode_utf16(&bytes[2..], true)),
|
||||
Some("utf-16be") => Some(decode_utf16(&bytes[2..], false)),
|
||||
Some(_) => None,
|
||||
None => Some(String::from_utf8_lossy(bytes).to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_utf16(bytes: &[u8], little_endian: bool) -> String {
|
||||
let units = bytes
|
||||
.chunks_exact(2)
|
||||
.map(|chunk| {
|
||||
if little_endian {
|
||||
u16::from_le_bytes([chunk[0], chunk[1]])
|
||||
} else {
|
||||
u16::from_be_bytes([chunk[0], chunk[1]])
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
String::from_utf16_lossy(&units)
|
||||
}
|
||||
|
||||
fn summarize_text(text: &str) -> TextStats {
|
||||
let mut line_count = 0_usize;
|
||||
let mut blank_lines = 0_usize;
|
||||
let mut longest_line = 0_usize;
|
||||
|
||||
for line in text.lines() {
|
||||
line_count += 1;
|
||||
if line.trim().is_empty() {
|
||||
blank_lines += 1;
|
||||
}
|
||||
longest_line = longest_line.max(line.len());
|
||||
}
|
||||
|
||||
TextStats {
|
||||
line_count,
|
||||
blank_lines,
|
||||
longest_line,
|
||||
}
|
||||
}
|
||||
|
||||
fn detect_generated(path: &Path, bytes: &[u8], likely_lockfile: bool) -> bool {
|
||||
if likely_lockfile {
|
||||
return true;
|
||||
}
|
||||
|
||||
let file_name = file_name_lower(path);
|
||||
if is_known_generated_artifact(path, &file_name) {
|
||||
return true;
|
||||
}
|
||||
if file_name.contains(".designer.") || file_name.contains(".generated.") {
|
||||
return true;
|
||||
}
|
||||
|
||||
contains_generated_marker(&bytes[..bytes.len().min(512)])
|
||||
}
|
||||
|
||||
fn is_known_generated_artifact(path: &Path, file_name: &str) -> bool {
|
||||
if matches!(
|
||||
file_name,
|
||||
"project.assets.json"
|
||||
| "project.nuget.cache"
|
||||
| ".netcoreapp,version=v1.0.assemblyattributes.cs"
|
||||
| ".netframework,version=v4.8.assemblyattributes.cs"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let in_obj_dir = path.components().any(|component| match component {
|
||||
Component::Normal(value) => value.to_string_lossy().eq_ignore_ascii_case("obj"),
|
||||
_ => false,
|
||||
});
|
||||
in_obj_dir
|
||||
&& (file_name.ends_with(".assemblyinfo.cs")
|
||||
|| file_name.ends_with(".assemblyattributes.cs"))
|
||||
}
|
||||
|
||||
fn contains_generated_marker(preview: &[u8]) -> bool {
|
||||
const MARKERS: [&[u8]; 4] = [
|
||||
b"@generated",
|
||||
b"generated by",
|
||||
b"auto-generated",
|
||||
b"automatically generated",
|
||||
];
|
||||
MARKERS
|
||||
.iter()
|
||||
.any(|marker| contains_ascii_case_insensitive(preview, marker))
|
||||
}
|
||||
|
||||
fn contains_ascii_case_insensitive(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.any(|window| window.eq_ignore_ascii_case(needle))
|
||||
}
|
||||
|
||||
fn count_line_endings(text: &str) -> LineEndingCounts {
|
||||
let mut counts = LineEndingCounts {
|
||||
lf: 0,
|
||||
crlf: 0,
|
||||
cr: 0,
|
||||
};
|
||||
let bytes = text.as_bytes();
|
||||
let mut index = 0_usize;
|
||||
while index < bytes.len() {
|
||||
match bytes[index] {
|
||||
b'\r' if bytes.get(index + 1) == Some(&b'\n') => {
|
||||
counts.crlf += 1;
|
||||
index += 2;
|
||||
}
|
||||
b'\r' => {
|
||||
counts.cr += 1;
|
||||
index += 1;
|
||||
}
|
||||
b'\n' => {
|
||||
counts.lf += 1;
|
||||
index += 1;
|
||||
}
|
||||
_ => index += 1,
|
||||
}
|
||||
}
|
||||
counts
|
||||
}
|
||||
|
||||
fn detect_newline_style(counts: &LineEndingCounts) -> Option<String> {
|
||||
if counts.crlf > 0 && counts.lf == 0 && counts.cr == 0 {
|
||||
Some("crlf".to_owned())
|
||||
} else if counts.lf > 0 && counts.crlf == 0 && counts.cr == 0 {
|
||||
Some("lf".to_owned())
|
||||
} else if counts.cr > 0 && counts.lf == 0 && counts.crlf == 0 {
|
||||
Some("cr".to_owned())
|
||||
} else if distinct_line_endings(counts) > 1 {
|
||||
Some("mixed".to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn distinct_line_endings(counts: &LineEndingCounts) -> usize {
|
||||
usize::from(counts.lf > 0) + usize::from(counts.crlf > 0) + usize::from(counts.cr > 0)
|
||||
}
|
||||
|
||||
fn detect_minified(text: &str, stats: &TextStats) -> bool {
|
||||
if stats.line_count == 0 || stats.line_count > 3 || stats.longest_line < 80 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut non_newline_len = 0usize;
|
||||
let mut whitespace_chars = 0usize;
|
||||
let mut has_open_brace = false;
|
||||
let mut has_semicolon = false;
|
||||
|
||||
for ch in text.chars() {
|
||||
if matches!(ch, '\n' | '\r') {
|
||||
continue;
|
||||
}
|
||||
non_newline_len += 1;
|
||||
if ch.is_whitespace() {
|
||||
whitespace_chars += 1;
|
||||
}
|
||||
has_open_brace |= ch == '{';
|
||||
has_semicolon |= ch == ';';
|
||||
}
|
||||
|
||||
if non_newline_len == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
whitespace_chars.saturating_mul(100) < non_newline_len.saturating_mul(12)
|
||||
&& has_open_brace
|
||||
&& has_semicolon
|
||||
}
|
||||
|
||||
fn detect_lockfile(path: &Path) -> bool {
|
||||
let file_name = file_name_lower(path);
|
||||
Path::new(&file_name)
|
||||
.extension()
|
||||
.is_some_and(|extension| extension.eq_ignore_ascii_case("lock"))
|
||||
|| matches!(
|
||||
file_name.as_str(),
|
||||
"package-lock.json"
|
||||
| "pnpm-lock.yaml"
|
||||
| "yarn.lock"
|
||||
| "bun.lockb"
|
||||
| "composer.lock"
|
||||
| "poetry.lock"
|
||||
| "uv.lock"
|
||||
)
|
||||
}
|
||||
|
||||
fn detect_test_path(path: &Path) -> bool {
|
||||
let lower_path = path.display().to_string().to_ascii_lowercase();
|
||||
lower_path.contains("\\tests\\")
|
||||
|| lower_path.contains("/tests/")
|
||||
|| lower_path.contains(".test.")
|
||||
|| lower_path.contains("_test.")
|
||||
|| lower_path.contains(".spec.")
|
||||
|| lower_path.contains("_spec.")
|
||||
}
|
||||
|
||||
fn detect_vendor_path(path: &Path) -> bool {
|
||||
path.components().any(|component| {
|
||||
let Component::Normal(segment) = component else {
|
||||
return false;
|
||||
};
|
||||
let lower = segment.to_string_lossy().to_ascii_lowercase();
|
||||
matches!(
|
||||
lower.as_str(),
|
||||
"vendor" | "node_modules" | "third_party" | "packages"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn file_name_lower(path: &Path) -> String {
|
||||
path.file_name()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn render_text_report(report: &FileReport) -> String {
|
||||
let mut line = format!(
|
||||
"path={} family={} language={} binary={} size={} mtime={}",
|
||||
report.path,
|
||||
family_label(report.family),
|
||||
report.language_hint.as_deref().unwrap_or("-"),
|
||||
report.is_binary,
|
||||
report.size_bytes,
|
||||
report.modified_rfc3339
|
||||
);
|
||||
|
||||
if let Some(extension) = &report.extension {
|
||||
write!(line, " ext={extension}").expect("writing to a String cannot fail");
|
||||
}
|
||||
write!(line, " directory={}", report.is_directory).expect("writing to a String cannot fail");
|
||||
if let Some(container_hint) = &report.container_hint {
|
||||
write!(line, " container={container_hint}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(encoding_hint) = &report.encoding_hint {
|
||||
write!(line, " encoding={encoding_hint}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(bom) = &report.bom {
|
||||
write!(line, " bom={bom}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(newline_style) = &report.newline_style {
|
||||
write!(line, " newline={newline_style}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(mixed_newlines) = report.mixed_newlines {
|
||||
write!(line, " mixed_newlines={mixed_newlines}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(line_count) = report.line_count {
|
||||
write!(line, " lines={line_count}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(blank_lines) = report.blank_lines {
|
||||
write!(line, " blank={blank_lines}").expect("writing to a String cannot fail");
|
||||
}
|
||||
if let Some(longest_line) = report.longest_line {
|
||||
write!(line, " longest={longest_line}").expect("writing to a String cannot fail");
|
||||
}
|
||||
write!(
|
||||
line,
|
||||
" generated={} minified={} lockfile={} test={} vendor={}",
|
||||
report.likely_generated,
|
||||
report.likely_minified,
|
||||
report.likely_lockfile,
|
||||
report.likely_test,
|
||||
report.likely_vendor
|
||||
)
|
||||
.expect("writing to a String cannot fail");
|
||||
|
||||
line
|
||||
}
|
||||
|
||||
const fn family_label(family: FileFamily) -> &'static str {
|
||||
match family {
|
||||
FileFamily::Directory => "directory",
|
||||
FileFamily::Source => "source",
|
||||
FileFamily::Config => "config",
|
||||
FileFamily::Data => "data",
|
||||
FileFamily::Text => "text",
|
||||
FileFamily::Binary => "binary",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
|
||||
use common::ColorChoice;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn common_args(json: bool, input_format: InputFormat) -> CommonArgs {
|
||||
CommonArgs {
|
||||
json,
|
||||
format: None,
|
||||
input_format,
|
||||
color: ColorChoice::Never,
|
||||
quiet: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_paths_supports_line_and_json_inputs() {
|
||||
let temp = tempdir().expect("tempdir");
|
||||
let line_path = temp.path().join("a.rs");
|
||||
let jsonl_one = temp.path().join("b.cs");
|
||||
let jsonl_two = temp.path().join("c.js");
|
||||
let auto_path = temp.path().join("d.toml");
|
||||
fs::write(&line_path, "a").expect("line file");
|
||||
fs::write(&jsonl_one, "b").expect("jsonl one");
|
||||
fs::write(&jsonl_two, "c").expect("jsonl two");
|
||||
fs::write(&auto_path, "d").expect("auto file");
|
||||
|
||||
assert_eq!(
|
||||
parse_paths_from_string(&format!("{}\n", line_path.display()), InputFormat::Lines)
|
||||
.expect("lines"),
|
||||
vec![line_path]
|
||||
);
|
||||
assert_eq!(
|
||||
parse_paths_from_string(
|
||||
&format!(
|
||||
"{}\n{{\"path\":{}}}\n",
|
||||
serde_json::to_string(&jsonl_one.display().to_string()).expect("jsonl one"),
|
||||
serde_json::to_string(&jsonl_two.display().to_string()).expect("jsonl two"),
|
||||
),
|
||||
InputFormat::Jsonl,
|
||||
)
|
||||
.expect("jsonl"),
|
||||
vec![jsonl_one, jsonl_two]
|
||||
);
|
||||
assert_eq!(
|
||||
parse_paths_from_string(&format!("{}\n", auto_path.display()), InputFormat::Auto)
|
||||
.expect("auto"),
|
||||
vec![auto_path]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_paths_reports_invalid_jsonl() {
|
||||
let error =
|
||||
parse_paths_from_string("nope\n", InputFormat::Jsonl).expect_err("invalid jsonl");
|
||||
assert!(matches!(
|
||||
error,
|
||||
CliError::Usage(message)
|
||||
if message.contains("stdin JSONL path line 1 is not valid JSON")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn container_language_and_path_heuristics_are_stable() {
|
||||
assert_eq!(
|
||||
detect_container_hint(b"MZ\x00\x01payload"),
|
||||
Some("pe".to_string())
|
||||
);
|
||||
assert_eq!(detect_container_hint(b"%PDF-1.7"), Some("pdf".to_string()));
|
||||
assert!(is_binary_blob(b"\x00\x01\xff", None, None));
|
||||
assert!(!is_binary_blob(b"plain text", None, None));
|
||||
assert_eq!(
|
||||
detect_language_hint(Path::new("demo.rs")),
|
||||
Some("rust".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
detect_language_hint(Path::new("events.jsonl")),
|
||||
Some("jsonl".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_family(false, Some("javascript")),
|
||||
FileFamily::Source
|
||||
);
|
||||
assert_eq!(family_label(FileFamily::Directory), "directory");
|
||||
assert_eq!(classify_family(false, Some("toml")), FileFamily::Config);
|
||||
assert_eq!(classify_family(false, Some("jsonl")), FileFamily::Data);
|
||||
assert_eq!(classify_family(true, Some("rust")), FileFamily::Binary);
|
||||
assert!(detect_lockfile(Path::new("Cargo.lock")));
|
||||
assert!(detect_test_path(Path::new("C:\\repo\\tests\\probe.rs")));
|
||||
assert!(detect_vendor_path(Path::new("C:\\repo\\vendor\\lib.rs")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_and_minified_heuristics_use_content_markers() {
|
||||
let stats = summarize_text(
|
||||
"function boot(){const state={ready:true,mode:\"fast\"};if(state.ready){console.log(state.mode);}}\n",
|
||||
);
|
||||
assert!(detect_minified(
|
||||
"function boot(){const state={ready:true,mode:\"fast\"};if(state.ready){console.log(state.mode);}}\n",
|
||||
&stats,
|
||||
));
|
||||
|
||||
assert!(detect_generated(
|
||||
Path::new("generated.lock"),
|
||||
b"# This file is automatically @generated by the build system.\n",
|
||||
true,
|
||||
));
|
||||
assert!(detect_generated(
|
||||
Path::new("demo.rs"),
|
||||
b"// auto-generated by tool\nfn run() {}\n",
|
||||
false,
|
||||
));
|
||||
assert!(detect_generated(
|
||||
Path::new("C:\\repo\\obj\\project.assets.json"),
|
||||
b"{\"version\":3}\n",
|
||||
false,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_path_and_render_text_cover_text_and_binary_reports() {
|
||||
let temp = tempdir().expect("tempdir");
|
||||
let source_path = temp.path().join("sample.rs");
|
||||
let binary_path = temp.path().join("sample.bin");
|
||||
fs::write(&source_path, b"pub fn run() {}\n").expect("source fixture");
|
||||
fs::write(&binary_path, b"MZ\x00\x01\xff").expect("binary fixture");
|
||||
|
||||
let source_report = inspect_path(&source_path).expect("source report");
|
||||
assert_eq!(source_report.family, FileFamily::Source);
|
||||
assert_eq!(source_report.language_hint.as_deref(), Some("rust"));
|
||||
assert_eq!(source_report.line_count, Some(1));
|
||||
assert_eq!(source_report.encoding_hint.as_deref(), Some("utf-8"));
|
||||
assert!(render_text_report(&source_report).contains("family=source"));
|
||||
|
||||
let binary_report = inspect_path(&binary_path).expect("binary report");
|
||||
assert_eq!(binary_report.family, FileFamily::Binary);
|
||||
assert_eq!(binary_report.container_hint.as_deref(), Some("pe"));
|
||||
assert!(binary_report.is_binary);
|
||||
assert!(render_text_report(&binary_report).contains("container=pe"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_path_reports_directories_without_raw_os_errors() {
|
||||
let temp = tempdir().expect("tempdir");
|
||||
let report = inspect_path(temp.path()).expect("directory report");
|
||||
assert_eq!(report.family, FileFamily::Directory);
|
||||
assert!(report.is_directory);
|
||||
assert_eq!(report.container_hint.as_deref(), Some("directory"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_bom_and_mixed_newlines() {
|
||||
let temp = tempdir().expect("tempdir");
|
||||
let path = temp.path().join("bom.txt");
|
||||
fs::write(&path, [0xef, 0xbb, 0xbf, b'a', b'\r', b'\n', b'b', b'\n']).expect("fixture");
|
||||
|
||||
let report = inspect_path(&path).expect("report");
|
||||
|
||||
assert_eq!(report.bom.as_deref(), Some("utf-8"));
|
||||
assert_eq!(report.newline_style.as_deref(), Some("mixed"));
|
||||
assert_eq!(report.mixed_newlines, Some(true));
|
||||
assert_eq!(
|
||||
report.line_ending_counts,
|
||||
Some(LineEndingCounts {
|
||||
lf: 1,
|
||||
crlf: 1,
|
||||
cr: 0,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_accepts_explicit_paths_for_text_and_json_output() {
|
||||
let temp = tempdir().expect("tempdir");
|
||||
let path = temp.path().join("sample.cs");
|
||||
fs::write(&path, b"class Demo {}\n").expect("fixture");
|
||||
|
||||
let text_exit = run(&Cli {
|
||||
common: common_args(false, InputFormat::Auto),
|
||||
paths: vec![path.clone()],
|
||||
})
|
||||
.expect("text run");
|
||||
assert_eq!(text_exit, ExitCode::Success);
|
||||
|
||||
let json_exit = run(&Cli {
|
||||
common: common_args(true, InputFormat::Lines),
|
||||
paths: vec![path],
|
||||
})
|
||||
.expect("json run");
|
||||
assert_eq!(json_exit, ExitCode::Success);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user