Починил поиск по хэшу и ускорил поиск по хэшу

Cargo.lock

@@ -691,11 +691,11 @@ dependencies = [
  "md5",
  "rayon",
  "reqwest",
+ "semver",
  "serde",
  "serde_json",
  "sha1",
  "sha2",
- "tokio",
 ]
 
 [[package]]
@@ -998,6 +998,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+
 [[package]]
 name = "serde"
 version = "1.0.228"
@@ -1202,21 +1208,9 @@ dependencies = [
  "mio",
  "pin-project-lite",
  "socket2 0.6.1",
- "tokio-macros",
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "tokio-macros"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "tokio-native-tls"
 version = "0.3.1"

Cargo.toml

@@ -4,13 +4,13 @@ version = "0.1.1"
 edition = "2021"
 
 [dependencies]
+reqwest = { version = "0.11", features = ["blocking", "json"] }
+rayon = "1.9"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+log = "0.4"
+env_logger = "0.11"
 sha2 = "0.10"
 sha1 = "0.10"
 md5 = "0.7"
-rayon = "1.10"
+semver = "1.0"   # только для Semaphore (rayon)
-log = "0.4"
-env_logger = "0.11"
-reqwest = { version = "0.11", features = ["json", "blocking"] }
-tokio = { version = "1", features = ["rt-multi-thread", "macros"] }

src/main.rs

@@ -6,10 +6,11 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use sha1::Sha1;
 use sha2::{Digest, Sha256, Sha512};
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::fs::{self, File};
 use std::io::{self, BufReader, BufWriter, Read};
 use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::Duration;
 
@@ -86,7 +87,7 @@ fn main() {
 
 fn load_json() -> ModList {
     match File::open(JSON_FILE) {
-        Ok(file) => serde_json::from_reader(BufReader::new(file)).unwrap_or_else(|e| {
+        Ok(f) => serde_json::from_reader(BufReader::new(f)).unwrap_or_else(|e| {
             warn!("Ошибка чтения JSON: {}. Создаём пустой.", e);
             ModList {
                 client_only: vec![],
@@ -103,39 +104,38 @@ fn load_json() -> ModList {
 }
 
 fn save_json(data: &ModList) -> io::Result<()> {
-    let file = File::create(JSON_FILE)?;
+    let f = File::create(JSON_FILE)?;
-    serde_json::to_writer_pretty(BufWriter::new(file), data)?;
+    serde_json::to_writer_pretty(BufWriter::new(f), data)?;
     Ok(())
 }
 
-fn compute_hashes_for_file(path: &PathBuf) -> Hashes {
+fn compute_hashes_for_file(p: &PathBuf) -> Hashes {
-    let file = File::open(path).expect("Не удалось открыть файл");
+    let mut r = BufReader::new(File::open(p).expect("open"));
-    let mut reader = BufReader::new(file);
+    let mut s512 = Sha512::new();
-    let mut sha512 = Sha512::new();
+    let mut s256 = Sha256::new();
-    let mut sha256 = Sha256::new();
+    let mut s1 = Sha1::new();
-    let mut sha1 = Sha1::new();
+    let mut md = md5::Context::new();
-    let mut md5_ctx = md5::Context::new();
+    let mut buf = [0u8; 8192];
-    let mut buffer = [0u8; 8192];
     loop {
-        match reader.read(&mut buffer) {
+        match r.read(&mut buf) {
             Ok(0) => break,
             Ok(n) => {
-                sha512.update(&buffer[..n]);
+                s512.update(&buf[..n]);
-                sha256.update(&buffer[..n]);
+                s256.update(&buf[..n]);
-                sha1.update(&buffer[..n]);
+                s1.update(&buf[..n]);
-                md5_ctx.consume(&buffer[..n]);
+                md.consume(&buf[..n]);
             }
             Err(e) => {
-                eprintln!("Ошибка чтения файла {}: {}", path.display(), e);
+                eprintln!("read {}: {}", p.display(), e);
                 break;
             }
         }
     }
     Hashes {
-        sha512: format!("{:x}", sha512.finalize()),
+        sha512: format!("{:x}", s512.finalize()),
-        sha256: format!("{:x}", sha256.finalize()),
+        sha256: format!("{:x}", s256.finalize()),
-        sha1: format!("{:x}", sha1.finalize()),
+        sha1: format!("{:x}", s1.finalize()),
-        md5: format!("{:x}", md5_ctx.compute()),
+        md5: format!("{:x}", md.compute()),
     }
 }
 
@@ -146,16 +146,16 @@ fn compute_hashes(paths: &[PathBuf]) -> Vec<(PathBuf, Hashes)> {
         .collect()
 }
 
-fn ask_category(mod_name: &str) -> &'static str {
+fn ask_category(name: &str) -> &'static str {
-    println!("\nМод: {}", mod_name);
+    println!("\nМод: {}", name);
     println!("Выберите категорию:");
-    for (i, cat) in CATEGORIES.iter().enumerate() {
+    for (i, c) in CATEGORIES.iter().enumerate() {
-        println!("[{}] {}", i + 1, cat);
+        println!("[{}] {}", i + 1, c);
     }
     loop {
-        let mut input = String::new();
+        let mut inp = String::new();
-        if io::stdin().read_line(&mut input).is_ok() {
+        if io::stdin().read_line(&mut inp).is_ok() {
-            if let Ok(n) = input.trim().parse::<usize>() {
+            if let Ok(n) = inp.trim().parse::<usize>() {
                 if n > 0 && n <= CATEGORIES.len() {
                     return CATEGORIES[n - 1];
                 }
@@ -166,38 +166,35 @@ fn ask_category(mod_name: &str) -> &'static str {
 }
 
 fn get_jar_files(dir: &PathBuf) -> Vec<PathBuf> {
-    match fs::read_dir(dir) {
+    fs::read_dir(dir)
-        Ok(entries) => entries
+        .ok()
-            .filter_map(|e| e.ok().map(|e| e.path()))
+        .into_iter()
-            .filter(|p| p.extension().map(|e| e == "jar").unwrap_or(false))
+        .flatten()
-            .collect(),
+        .filter_map(|e| e.ok().map(|e| e.path()))
-        Err(e) => {
+        .filter(|p| p.extension().map(|e| e == "jar").unwrap_or(false))
-            eprintln!("Не удалось прочитать папку {}: {}", MODS_DIR, e);
+        .collect()
-            vec![]
-        }
-    }
 }
 
 fn lookup_side(sha1: &str, sha512: &str) -> Option<&'static str> {
     let client = Client::new();
     for (hash, algo) in [(sha1, "sha1"), (sha512, "sha512")] {
         let url = if algo == "sha512" {
-            format!("{}/version_file/{}?algorithm=sha512", MODRINTH_API, hash)
+            format!("{MODRINTH_API}/version_file/{hash}?algorithm=sha512")
         } else {
-            format!("{}/version_file/{}", MODRINTH_API, hash)
+            format!("{MODRINTH_API}/version_file/{hash}")
         };
         if let Ok(resp) = client.get(&url).send() {
-            if resp.status() == 200 {
+            if resp.status().is_success() {
-                if let Ok(version) = resp.json::<Value>() {
+                if let Ok(v) = resp.json::<Value>() {
-                    if let Some(project_id) = version["project_id"].as_str() {
+                    if let Some(pid) = v["project_id"].as_str() {
-                        if let Ok(project) = client
+                        if let Ok(p) = client
-                            .get(&format!("{}/project/{}", MODRINTH_API, project_id))
+                            .get(&format!("{MODRINTH_API}/project/{pid}"))
                             .send()
                             .and_then(|r| r.json::<Value>())
                         {
-                            let client_side = project["client_side"].as_str()?;
+                            let cs = p["client_side"].as_str()?;
-                            let server_side = project["server_side"].as_str()?;
+                            let ss = p["server_side"].as_str()?;
-                            return match (client_side, server_side) {
+                            return match (cs, ss) {
                                 ("required", "required") => Some("universal"),
                                 ("required", "unsupported") => Some("client-only"),
                                 ("unsupported", "required") => Some("server-only"),
@@ -208,7 +205,7 @@ fn lookup_side(sha1: &str, sha512: &str) -> Option<&'static str> {
                 }
             }
         }
-        thread::sleep(Duration::from_millis(200));
+        thread::sleep(Duration::from_millis(150));
     }
     None
 }
@@ -217,76 +214,61 @@ fn process_new_mods(
     paths: &[PathBuf],
     existing_sha512: &HashSet<String>,
 ) -> Vec<(ModEntry, &'static str)> {
-    let mut new_entries = vec![];
+    let mut res = vec![];
-    let file_hashes = compute_hashes(paths);
+    let hashes = compute_hashes(paths);
-    for (path, hashes) in file_hashes {
+    for (p, h) in hashes {
-        if existing_sha512.contains(&hashes.sha512) {
+        if existing_sha512.contains(&h.sha512) {
             continue;
         }
-        let name = path
+        let name = p.file_name().unwrap().to_string_lossy().into_owned();
-            .file_name()
-            .expect("Нет имени файла")
-            .to_string_lossy()
-            .into_owned();
         info!("Новый мод: {}", name);
-        let cat = lookup_side(&hashes.sha1, &hashes.sha512).unwrap_or_else(|| ask_category(&name));
+        let cat = lookup_side(&h.sha1, &h.sha512).unwrap_or_else(|| ask_category(&name));
-        let entry = ModEntry {
+        res.push((
-            name,
+            ModEntry {
-            sha512: hashes.sha512,
+                name,
-            sha256: hashes.sha256,
+                sha512: h.sha512,
-            sha1: hashes.sha1,
+                sha256: h.sha256,
-            md5: hashes.md5,
+                sha1: h.sha1,
-        };
+                md5: h.md5,
-        new_entries.push((entry, cat));
+            },
+            cat,
+        ));
     }
-    new_entries
+    res
 }
 
-fn build(
+fn build(side: &[ModEntry], uni: &[ModEntry], src_dir: &PathBuf, dst_dir: &PathBuf, name: &str) {
-    side_only: &[ModEntry],
+    if fs::create_dir_all(dst_dir).is_err() {
-    universal: &[ModEntry],
+        error!("Не создать {}", name);
-    mods_dir: &PathBuf,
-    build_dir: &PathBuf,
-    name: &str,
-) {
-    if let Err(e) = fs::create_dir_all(build_dir) {
-        error!("Не удалось создать папку {}: {}", name, e);
         return;
     }
-    let mods: Vec<_> = side_only.iter().chain(universal).map(|e| &e.name).collect();
+    let files: Vec<_> = side.iter().chain(uni).map(|e| &e.name).collect();
-    let count = mods.len();
+    let count = files.len();
-    for file_name in &mods {
+    for f in &files {
-        let src = mods_dir.join(file_name);
+        let s = src_dir.join(f);
-        let dst = build_dir.join(file_name);
+        let d = dst_dir.join(f);
-        if src.exists() {
+        if s.exists() && fs::copy(&s, &d).is_err() {
-            if let Err(e) = fs::copy(&src, &dst) {
+            warn!("Не скопировать {}", f);
-                warn!("Не удалось скопировать {}: {}", file_name, e);
-            }
         }
     }
-    info!(
+    info!("{}: {} модов в {}", name, count, dst_dir.display());
-        "{} сборка создана: {} модов в {}",
-        name,
-        count,
-        build_dir.display()
-    );
 }
 
-fn update_mods(mods_dir: &PathBuf) {
+fn update_mods(dir: &PathBuf) {
     let mut data = load_json();
-    let existing_sha512: HashSet<String> = data
+    let existing: HashSet<String> = data
         .client_only
         .iter()
         .chain(&data.server_only)
         .chain(&data.universal)
         .map(|e| e.sha512.clone())
         .collect();
-    if !mods_dir.exists() {
+    if !dir.exists() {
         error!("Папка {} не найдена!", MODS_DIR);
         return;
     }
-    let all_files = get_jar_files(mods_dir);
+    let jars = get_jar_files(dir);
-    let current_names: HashSet<_> = all_files
+    let names: HashSet<_> = jars
         .iter()
         .filter_map(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
         .collect();
@@ -296,72 +278,74 @@ fn update_mods(mods_dir: &PathBuf) {
         &mut data.universal,
     ] {
         cat.retain(|e| {
-            if !current_names.contains(&e.name) {
+            if !names.contains(&e.name) {
-                info!("Удалён устаревший мод: {}", e.name);
+                info!("Удалён: {}", e.name);
                 false
             } else {
                 true
             }
         });
     }
-    let new_entries = process_new_mods(&all_files, &existing_sha512);
+    for (e, c) in process_new_mods(&jars, &existing) {
-    for (entry, cat) in new_entries {
+        match c {
-        match cat {
+            "client-only" => data.client_only.push(e),
-            "client-only" => data.client_only.push(entry),
+            "server-only" => data.server_only.push(e),
-            "server-only" => data.server_only.push(entry),
+            "universal" => data.universal.push(e),
-            "universal" => data.universal.push(entry),
             _ => {}
         }
     }
-    if let Err(e) = save_json(&data) {
+    if save_json(&data).is_err() {
-        error!("Не удалось сохранить JSON: {}", e);
+        error!("Не сохранить JSON");
     } else {
         info!("Обновление завершено");
     }
 }
 
-fn reassign_categories(mods_dir: &PathBuf) {
+fn reassign_categories(dir: &PathBuf) {
-    if !mods_dir.exists() {
+    if !dir.exists() {
         error!("Папка {} не найдена!", MODS_DIR);
         return;
     }
-    let all_files = get_jar_files(mods_dir);
+    let _ = fs::remove_file(JSON_FILE);
-    if PathBuf::from(JSON_FILE).exists() {
+    let jars = get_jar_files(dir);
-        if let Err(e) = fs::remove_file(JSON_FILE) {
-            warn!("Не удалось удалить старый JSON: {}", e);
-        } else {
-            info!("Старый mods-list.json удалён");
-        }
-    }
     let mut data = ModList {
         client_only: vec![],
         server_only: vec![],
         universal: vec![],
     };
-    let empty_sha512: HashSet<String> = HashSet::new();
+
-    let file_hashes = compute_hashes(&all_files);
+    let hashes = compute_hashes(&jars);
-    let mut hash_map: HashMap<String, Hashes> = HashMap::new();
+    let results = Arc::new(Mutex::new(vec![]));
-    for (path, hashes) in &file_hashes {
+    let throttle = Arc::new(Mutex::new(0u32));
-        let name = path
+    hashes.par_iter().for_each(|(p, h)| {
-            .file_name()
+        {
-            .expect("Нет имени файла")
+            let mut cnt = throttle.lock().unwrap();
-            .to_string_lossy()
+            while *cnt >= 8 {
-            .into_owned();
+                drop(cnt);
-        hash_map.insert(name, hashes.clone());
+                thread::sleep(Duration::from_millis(50));
-    }
+                cnt = throttle.lock().unwrap();
-    for (path, hashes) in file_hashes {
+            }
-        let name = path
+            *cnt += 1;
-            .file_name()
+        }
-            .expect("Нет имени файла")
+        let name = p.file_name().unwrap().to_string_lossy().into_owned();
-            .to_string_lossy()
+        let cat = lookup_side(&h.sha1, &h.sha512);
-            .into_owned();
+        {
-        let cat = lookup_side(&hashes.sha1, &hashes.sha512).unwrap_or_else(|| ask_category(&name));
+            let mut res = results.lock().unwrap();
+            if let Some(c) = cat {
+                res.push((name, h.clone(), c));
+            }
+            let mut cnt = throttle.lock().unwrap();
+            *cnt -= 1;
+        }
+    });
+
+    for (name, h, cat) in results.lock().unwrap().drain(..) {
         let entry = ModEntry {
             name: name.clone(),
-            sha512: hashes.sha512,
+            sha512: h.sha512,
-            sha256: hashes.sha256,
+            sha256: h.sha256,
-            sha1: hashes.sha1,
+            sha1: h.sha1,
-            md5: hashes.md5,
+            md5: h.md5,
         };
         match cat {
             "client-only" => data.client_only.push(entry),
@@ -370,8 +354,51 @@ fn reassign_categories(mods_dir: &PathBuf) {
             _ => {}
         }
     }
-    if let Err(e) = save_json(&data) {
+
-        error!("Не удалось сохранить JSON: {}", e);
+    let total = jars.len();
+    let found_cnt = data.client_only.len() + data.server_only.len() + data.universal.len();
+    let not_modrinth = total - found_cnt;
+
+    if not_modrinth > 0 {
+        println!(
+            "{} модов не из Modrinth. Выбрать вручную оставшиеся моды? [y/n]",
+            not_modrinth
+        );
+        let mut inp = String::new();
+        io::stdin().read_line(&mut inp).ok();
+        if inp.trim().to_lowercase() == "y" {
+            let name_set: HashSet<String> = data
+                .client_only
+                .iter()
+                .chain(&data.server_only)
+                .chain(&data.universal)
+                .map(|e| e.name.clone())
+                .collect();
+            for (p, h) in hashes {
+                let name = p.file_name().unwrap().to_string_lossy().into_owned();
+                if name_set.contains(&name) {
+                    continue;
+                }
+                let cat = ask_category(&name);
+                let entry = ModEntry {
+                    name: name.clone(),
+                    sha512: h.sha512,
+                    sha256: h.sha256,
+                    sha1: h.sha1,
+                    md5: h.md5,
+                };
+                match cat {
+                    "client-only" => data.client_only.push(entry),
+                    "server-only" => data.server_only.push(entry),
+                    "universal" => data.universal.push(entry),
+                    _ => {}
+                }
+            }
+        }
+    }
+
+    if save_json(&data).is_err() {
+        error!("Не сохранить JSON");
     } else {
         info!("Все моды перераспределены");
     }