From 3a983c399aecc348c5f01a6dcaa03f4f8ac0ea7e Mon Sep 17 00:00:00 2001 From: Alex Macleod Date: Sun, 16 Jun 2024 18:28:00 +0000 Subject: [PATCH] Merge lintcheck popular-crates bin as a subcommand --- lintcheck/Cargo.toml | 13 +------ lintcheck/README.md | 6 +-- lintcheck/src/config.rs | 9 +++++ lintcheck/src/main.rs | 20 ++++++---- lintcheck/src/popular-crates.rs | 65 --------------------------------- lintcheck/src/popular_crates.rs | 52 ++++++++++++++++++++++++++ 6 files changed, 77 insertions(+), 88 deletions(-) delete mode 100644 lintcheck/src/popular-crates.rs create mode 100644 lintcheck/src/popular_crates.rs diff --git a/lintcheck/Cargo.toml b/lintcheck/Cargo.toml index 623af922e2a8..e0e94d7fec3e 100644 --- a/lintcheck/Cargo.toml +++ b/lintcheck/Cargo.toml @@ -11,30 +11,19 @@ publish = false default-run = "lintcheck" [dependencies] -anyhow = "1.0.69" cargo_metadata = "0.15.3" clap = { version = "4.4", features = ["derive", "env"] } -crates_io_api = "0.8.1" crossbeam-channel = "0.5.6" diff = "0.1.13" flate2 = "1.0" -indicatif = "0.17.3" rayon = "1.5.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.85" strip-ansi-escapes = "0.1.1" tar = "0.4" toml = "0.7.3" -ureq = "2.2" +ureq = { version = "2.2", features = ["json"] } walkdir = "2.3" [features] deny-warnings = [] - -[[bin]] -name = "lintcheck" -path = "src/main.rs" - -[[bin]] -name = "popular-crates" -path = "src/popular-crates.rs" diff --git a/lintcheck/README.md b/lintcheck/README.md index 61b581ba0fae..2d6039caeef0 100644 --- a/lintcheck/README.md +++ b/lintcheck/README.md @@ -26,11 +26,11 @@ the repo root. The results will then be saved to `lintcheck-logs/custom_logs.toml`. The `custom.toml` file may be built using recently most -downloaded crates by using the `popular-crates` binary from the `lintcheck` -directory. For example, to retrieve the 100 recently most downloaded crates: +downloaded crates by using `cargo lintcheck popular`. For example, to retrieve +the 200 recently most downloaded crates: ``` -cargo run --release --bin popular-crates -- -n 100 custom.toml +cargo lintcheck popular -n 200 custom.toml ``` diff --git a/lintcheck/src/config.rs b/lintcheck/src/config.rs index c3540bbe4ce7..e6cd7c9fdc2b 100644 --- a/lintcheck/src/config.rs +++ b/lintcheck/src/config.rs @@ -48,7 +48,16 @@ pub(crate) struct LintcheckConfig { #[derive(Subcommand, Clone, Debug)] pub(crate) enum Commands { + /// Display a markdown diff between two lintcheck log files in JSON format Diff { old: PathBuf, new: PathBuf }, + /// Create a lintcheck crates TOML file containing the top N popular crates + Popular { + /// Output TOML file name + output: PathBuf, + /// Number of crate names to download + #[clap(short, long, default_value_t = 100)] + number: usize, + }, } #[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] diff --git a/lintcheck/src/main.rs b/lintcheck/src/main.rs index 985df9647d82..c246883c7045 100644 --- a/lintcheck/src/main.rs +++ b/lintcheck/src/main.rs @@ -17,6 +17,7 @@ mod config; mod driver; mod json; +mod popular_crates; mod recursive; use crate::config::{Commands, LintcheckConfig, OutputFormat}; @@ -43,21 +44,21 @@ const LINTCHECK_DOWNLOADS: &str = "target/lintcheck/downloads"; const LINTCHECK_SOURCES: &str = "target/lintcheck/sources"; /// List of sources to check, loaded from a .toml file -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct SourceList { crates: HashMap, #[serde(default)] recursive: RecursiveOptions, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Deserialize, Default)] struct RecursiveOptions { ignore: HashSet, } /// A crate source stored inside the .toml /// will be translated into on one of the `CrateSource` variants -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct TomlCrate { name: String, versions: Option>, @@ -69,7 +70,7 @@ struct TomlCrate { /// Represents an archive we download from crates.io, or a git repo, or a local repo/folder /// Once processed (downloaded/extracted/cloned/copied...), this will be translated into a `Crate` -#[derive(Debug, Serialize, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)] +#[derive(Debug, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)] enum CrateSource { CratesIo { name: String, @@ -609,7 +610,6 @@ fn gather_stats(warnings: &[ClippyWarning]) -> (String, HashMap<&String, usize>) (stats_string, counter) } -#[allow(clippy::too_many_lines)] fn main() { // We're being executed as a `RUSTC_WRAPPER` as part of `--recursive` if let Ok(addr) = env::var("LINTCHECK_SERVER") { @@ -624,11 +624,15 @@ fn main() { let config = LintcheckConfig::new(); - if let Some(Commands::Diff { old, new }) = config.subcommand { - json::diff(&old, &new); - return; + match config.subcommand { + Some(Commands::Diff { old, new }) => json::diff(&old, &new), + Some(Commands::Popular { output, number }) => popular_crates::fetch(output, number).unwrap(), + None => lintcheck(config), } +} +#[allow(clippy::too_many_lines)] +fn lintcheck(config: LintcheckConfig) { println!("Compiling clippy..."); build_clippy(); println!("Done compiling"); diff --git a/lintcheck/src/popular-crates.rs b/lintcheck/src/popular-crates.rs deleted file mode 100644 index fdab984ad86c..000000000000 --- a/lintcheck/src/popular-crates.rs +++ /dev/null @@ -1,65 +0,0 @@ -#![deny(clippy::pedantic)] - -use clap::Parser; -use crates_io_api::{CratesQueryBuilder, Sort, SyncClient}; -use indicatif::ProgressBar; -use std::collections::HashSet; -use std::fs::File; -use std::io::{BufWriter, Write}; -use std::path::PathBuf; -use std::time::Duration; - -#[derive(Parser)] -struct Opts { - /// Output TOML file name - output: PathBuf, - /// Number of crate names to download - #[clap(short, long, default_value_t = 100)] - number: usize, - /// Do not output progress - #[clap(short, long)] - quiet: bool, -} - -fn main() -> anyhow::Result<()> { - let opts = Opts::parse(); - let mut output = BufWriter::new(File::create(opts.output)?); - output.write_all(b"[crates]\n")?; - let client = SyncClient::new( - "clippy/lintcheck (github.com/rust-lang/rust-clippy/)", - Duration::from_secs(1), - )?; - let mut seen_crates = HashSet::new(); - let pb = if opts.quiet { - None - } else { - Some(ProgressBar::new(opts.number as u64)) - }; - let mut query = CratesQueryBuilder::new() - .sort(Sort::RecentDownloads) - .page_size(100) - .build(); - while seen_crates.len() < opts.number { - let retrieved = client.crates(query.clone())?.crates; - if retrieved.is_empty() { - eprintln!("No more than {} crates available from API", seen_crates.len()); - break; - } - for c in retrieved { - if seen_crates.insert(c.name.clone()) { - output.write_all( - format!( - "{} = {{ name = '{}', versions = ['{}'] }}\n", - c.name, c.name, c.max_version - ) - .as_bytes(), - )?; - if let Some(pb) = &pb { - pb.inc(1); - } - } - } - query.set_page(query.page() + 1); - } - Ok(()) -} diff --git a/lintcheck/src/popular_crates.rs b/lintcheck/src/popular_crates.rs new file mode 100644 index 000000000000..880a8bd81f08 --- /dev/null +++ b/lintcheck/src/popular_crates.rs @@ -0,0 +1,52 @@ +use serde::Deserialize; +use std::error::Error; +use std::fmt::Write; +use std::fs; +use std::path::PathBuf; + +#[derive(Deserialize, Debug)] +struct Page { + crates: Vec, + meta: Meta, +} + +#[derive(Deserialize, Debug)] +struct Crate { + name: String, + max_version: String, +} + +#[derive(Deserialize, Debug)] +struct Meta { + next_page: String, +} + +pub(crate) fn fetch(output: PathBuf, number: usize) -> Result<(), Box> { + let agent = ureq::builder() + .user_agent("clippy/lintcheck (github.com/rust-lang/rust-clippy/)") + .build(); + + let mut crates = Vec::with_capacity(number); + let mut query = "?sort=recent-downloads&per_page=100".to_string(); + while crates.len() < number { + let page: Page = agent + .get(&format!("https://crates.io/api/v1/crates{query}")) + .call()? + .into_json()?; + + query = page.meta.next_page; + crates.extend(page.crates); + crates.truncate(number); + + let width = number.ilog10() as usize + 1; + println!("Fetched {:>width$}/{number} crates", crates.len()); + } + + let mut out = "[crates]\n".to_string(); + for Crate { name, max_version } in crates { + writeln!(out, "{name} = {{ name = '{name}', versions = ['{max_version}'] }}").unwrap(); + } + fs::write(output, out)?; + + Ok(()) +}