Починил поиск по хэшу и ускорил поиск по хэшу
This commit is contained in:
parent
6b931bdba2
commit
c8e2e01b46
20
Cargo.lock
generated
20
Cargo.lock
generated
@ -691,11 +691,11 @@ dependencies = [
|
||||
"md5",
|
||||
"rayon",
|
||||
"reqwest",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1",
|
||||
"sha2",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -998,6 +998,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
@ -1202,21 +1208,9 @@ dependencies = [
|
||||
"mio",
|
||||
"pin-project-lite",
|
||||
"socket2 0.6.1",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.1"
|
||||
|
||||
10
Cargo.toml
10
Cargo.toml
@ -4,13 +4,13 @@ version = "0.1.1"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
reqwest = { version = "0.11", features = ["blocking", "json"] }
|
||||
rayon = "1.9"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
log = "0.4"
|
||||
env_logger = "0.11"
|
||||
sha2 = "0.10"
|
||||
sha1 = "0.10"
|
||||
md5 = "0.7"
|
||||
rayon = "1.10"
|
||||
log = "0.4"
|
||||
env_logger = "0.11"
|
||||
reqwest = { version = "0.11", features = ["json", "blocking"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
|
||||
semver = "1.0" # только для Semaphore (rayon)
|
||||
|
||||
311
src/main.rs
311
src/main.rs
@ -6,10 +6,11 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha1::Sha1;
|
||||
use sha2::{Digest, Sha256, Sha512};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashSet;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, BufReader, BufWriter, Read};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
@ -86,7 +87,7 @@ fn main() {
|
||||
|
||||
fn load_json() -> ModList {
|
||||
match File::open(JSON_FILE) {
|
||||
Ok(file) => serde_json::from_reader(BufReader::new(file)).unwrap_or_else(|e| {
|
||||
Ok(f) => serde_json::from_reader(BufReader::new(f)).unwrap_or_else(|e| {
|
||||
warn!("Ошибка чтения JSON: {}. Создаём пустой.", e);
|
||||
ModList {
|
||||
client_only: vec![],
|
||||
@ -103,39 +104,38 @@ fn load_json() -> ModList {
|
||||
}
|
||||
|
||||
fn save_json(data: &ModList) -> io::Result<()> {
|
||||
let file = File::create(JSON_FILE)?;
|
||||
serde_json::to_writer_pretty(BufWriter::new(file), data)?;
|
||||
let f = File::create(JSON_FILE)?;
|
||||
serde_json::to_writer_pretty(BufWriter::new(f), data)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_hashes_for_file(path: &PathBuf) -> Hashes {
|
||||
let file = File::open(path).expect("Не удалось открыть файл");
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut sha512 = Sha512::new();
|
||||
let mut sha256 = Sha256::new();
|
||||
let mut sha1 = Sha1::new();
|
||||
let mut md5_ctx = md5::Context::new();
|
||||
let mut buffer = [0u8; 8192];
|
||||
fn compute_hashes_for_file(p: &PathBuf) -> Hashes {
|
||||
let mut r = BufReader::new(File::open(p).expect("open"));
|
||||
let mut s512 = Sha512::new();
|
||||
let mut s256 = Sha256::new();
|
||||
let mut s1 = Sha1::new();
|
||||
let mut md = md5::Context::new();
|
||||
let mut buf = [0u8; 8192];
|
||||
loop {
|
||||
match reader.read(&mut buffer) {
|
||||
match r.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
sha512.update(&buffer[..n]);
|
||||
sha256.update(&buffer[..n]);
|
||||
sha1.update(&buffer[..n]);
|
||||
md5_ctx.consume(&buffer[..n]);
|
||||
s512.update(&buf[..n]);
|
||||
s256.update(&buf[..n]);
|
||||
s1.update(&buf[..n]);
|
||||
md.consume(&buf[..n]);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Ошибка чтения файла {}: {}", path.display(), e);
|
||||
eprintln!("read {}: {}", p.display(), e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Hashes {
|
||||
sha512: format!("{:x}", sha512.finalize()),
|
||||
sha256: format!("{:x}", sha256.finalize()),
|
||||
sha1: format!("{:x}", sha1.finalize()),
|
||||
md5: format!("{:x}", md5_ctx.compute()),
|
||||
sha512: format!("{:x}", s512.finalize()),
|
||||
sha256: format!("{:x}", s256.finalize()),
|
||||
sha1: format!("{:x}", s1.finalize()),
|
||||
md5: format!("{:x}", md.compute()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -146,16 +146,16 @@ fn compute_hashes(paths: &[PathBuf]) -> Vec<(PathBuf, Hashes)> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn ask_category(mod_name: &str) -> &'static str {
|
||||
println!("\nМод: {}", mod_name);
|
||||
fn ask_category(name: &str) -> &'static str {
|
||||
println!("\nМод: {}", name);
|
||||
println!("Выберите категорию:");
|
||||
for (i, cat) in CATEGORIES.iter().enumerate() {
|
||||
println!("[{}] {}", i + 1, cat);
|
||||
for (i, c) in CATEGORIES.iter().enumerate() {
|
||||
println!("[{}] {}", i + 1, c);
|
||||
}
|
||||
loop {
|
||||
let mut input = String::new();
|
||||
if io::stdin().read_line(&mut input).is_ok() {
|
||||
if let Ok(n) = input.trim().parse::<usize>() {
|
||||
let mut inp = String::new();
|
||||
if io::stdin().read_line(&mut inp).is_ok() {
|
||||
if let Ok(n) = inp.trim().parse::<usize>() {
|
||||
if n > 0 && n <= CATEGORIES.len() {
|
||||
return CATEGORIES[n - 1];
|
||||
}
|
||||
@ -166,38 +166,35 @@ fn ask_category(mod_name: &str) -> &'static str {
|
||||
}
|
||||
|
||||
fn get_jar_files(dir: &PathBuf) -> Vec<PathBuf> {
|
||||
match fs::read_dir(dir) {
|
||||
Ok(entries) => entries
|
||||
.filter_map(|e| e.ok().map(|e| e.path()))
|
||||
.filter(|p| p.extension().map(|e| e == "jar").unwrap_or(false))
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
eprintln!("Не удалось прочитать папку {}: {}", MODS_DIR, e);
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
fs::read_dir(dir)
|
||||
.ok()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|e| e.ok().map(|e| e.path()))
|
||||
.filter(|p| p.extension().map(|e| e == "jar").unwrap_or(false))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn lookup_side(sha1: &str, sha512: &str) -> Option<&'static str> {
|
||||
let client = Client::new();
|
||||
for (hash, algo) in [(sha1, "sha1"), (sha512, "sha512")] {
|
||||
let url = if algo == "sha512" {
|
||||
format!("{}/version_file/{}?algorithm=sha512", MODRINTH_API, hash)
|
||||
format!("{MODRINTH_API}/version_file/{hash}?algorithm=sha512")
|
||||
} else {
|
||||
format!("{}/version_file/{}", MODRINTH_API, hash)
|
||||
format!("{MODRINTH_API}/version_file/{hash}")
|
||||
};
|
||||
if let Ok(resp) = client.get(&url).send() {
|
||||
if resp.status() == 200 {
|
||||
if let Ok(version) = resp.json::<Value>() {
|
||||
if let Some(project_id) = version["project_id"].as_str() {
|
||||
if let Ok(project) = client
|
||||
.get(&format!("{}/project/{}", MODRINTH_API, project_id))
|
||||
if resp.status().is_success() {
|
||||
if let Ok(v) = resp.json::<Value>() {
|
||||
if let Some(pid) = v["project_id"].as_str() {
|
||||
if let Ok(p) = client
|
||||
.get(&format!("{MODRINTH_API}/project/{pid}"))
|
||||
.send()
|
||||
.and_then(|r| r.json::<Value>())
|
||||
{
|
||||
let client_side = project["client_side"].as_str()?;
|
||||
let server_side = project["server_side"].as_str()?;
|
||||
return match (client_side, server_side) {
|
||||
let cs = p["client_side"].as_str()?;
|
||||
let ss = p["server_side"].as_str()?;
|
||||
return match (cs, ss) {
|
||||
("required", "required") => Some("universal"),
|
||||
("required", "unsupported") => Some("client-only"),
|
||||
("unsupported", "required") => Some("server-only"),
|
||||
@ -208,7 +205,7 @@ fn lookup_side(sha1: &str, sha512: &str) -> Option<&'static str> {
|
||||
}
|
||||
}
|
||||
}
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
thread::sleep(Duration::from_millis(150));
|
||||
}
|
||||
None
|
||||
}
|
||||
@ -217,76 +214,61 @@ fn process_new_mods(
|
||||
paths: &[PathBuf],
|
||||
existing_sha512: &HashSet<String>,
|
||||
) -> Vec<(ModEntry, &'static str)> {
|
||||
let mut new_entries = vec![];
|
||||
let file_hashes = compute_hashes(paths);
|
||||
for (path, hashes) in file_hashes {
|
||||
if existing_sha512.contains(&hashes.sha512) {
|
||||
let mut res = vec![];
|
||||
let hashes = compute_hashes(paths);
|
||||
for (p, h) in hashes {
|
||||
if existing_sha512.contains(&h.sha512) {
|
||||
continue;
|
||||
}
|
||||
let name = path
|
||||
.file_name()
|
||||
.expect("Нет имени файла")
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let name = p.file_name().unwrap().to_string_lossy().into_owned();
|
||||
info!("Новый мод: {}", name);
|
||||
let cat = lookup_side(&hashes.sha1, &hashes.sha512).unwrap_or_else(|| ask_category(&name));
|
||||
let entry = ModEntry {
|
||||
name,
|
||||
sha512: hashes.sha512,
|
||||
sha256: hashes.sha256,
|
||||
sha1: hashes.sha1,
|
||||
md5: hashes.md5,
|
||||
};
|
||||
new_entries.push((entry, cat));
|
||||
let cat = lookup_side(&h.sha1, &h.sha512).unwrap_or_else(|| ask_category(&name));
|
||||
res.push((
|
||||
ModEntry {
|
||||
name,
|
||||
sha512: h.sha512,
|
||||
sha256: h.sha256,
|
||||
sha1: h.sha1,
|
||||
md5: h.md5,
|
||||
},
|
||||
cat,
|
||||
));
|
||||
}
|
||||
new_entries
|
||||
res
|
||||
}
|
||||
|
||||
fn build(
|
||||
side_only: &[ModEntry],
|
||||
universal: &[ModEntry],
|
||||
mods_dir: &PathBuf,
|
||||
build_dir: &PathBuf,
|
||||
name: &str,
|
||||
) {
|
||||
if let Err(e) = fs::create_dir_all(build_dir) {
|
||||
error!("Не удалось создать папку {}: {}", name, e);
|
||||
fn build(side: &[ModEntry], uni: &[ModEntry], src_dir: &PathBuf, dst_dir: &PathBuf, name: &str) {
|
||||
if fs::create_dir_all(dst_dir).is_err() {
|
||||
error!("Не создать {}", name);
|
||||
return;
|
||||
}
|
||||
let mods: Vec<_> = side_only.iter().chain(universal).map(|e| &e.name).collect();
|
||||
let count = mods.len();
|
||||
for file_name in &mods {
|
||||
let src = mods_dir.join(file_name);
|
||||
let dst = build_dir.join(file_name);
|
||||
if src.exists() {
|
||||
if let Err(e) = fs::copy(&src, &dst) {
|
||||
warn!("Не удалось скопировать {}: {}", file_name, e);
|
||||
}
|
||||
let files: Vec<_> = side.iter().chain(uni).map(|e| &e.name).collect();
|
||||
let count = files.len();
|
||||
for f in &files {
|
||||
let s = src_dir.join(f);
|
||||
let d = dst_dir.join(f);
|
||||
if s.exists() && fs::copy(&s, &d).is_err() {
|
||||
warn!("Не скопировать {}", f);
|
||||
}
|
||||
}
|
||||
info!(
|
||||
"{} сборка создана: {} модов в {}",
|
||||
name,
|
||||
count,
|
||||
build_dir.display()
|
||||
);
|
||||
info!("{}: {} модов в {}", name, count, dst_dir.display());
|
||||
}
|
||||
|
||||
fn update_mods(mods_dir: &PathBuf) {
|
||||
fn update_mods(dir: &PathBuf) {
|
||||
let mut data = load_json();
|
||||
let existing_sha512: HashSet<String> = data
|
||||
let existing: HashSet<String> = data
|
||||
.client_only
|
||||
.iter()
|
||||
.chain(&data.server_only)
|
||||
.chain(&data.universal)
|
||||
.map(|e| e.sha512.clone())
|
||||
.collect();
|
||||
if !mods_dir.exists() {
|
||||
if !dir.exists() {
|
||||
error!("Папка {} не найдена!", MODS_DIR);
|
||||
return;
|
||||
}
|
||||
let all_files = get_jar_files(mods_dir);
|
||||
let current_names: HashSet<_> = all_files
|
||||
let jars = get_jar_files(dir);
|
||||
let names: HashSet<_> = jars
|
||||
.iter()
|
||||
.filter_map(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
|
||||
.collect();
|
||||
@ -296,72 +278,74 @@ fn update_mods(mods_dir: &PathBuf) {
|
||||
&mut data.universal,
|
||||
] {
|
||||
cat.retain(|e| {
|
||||
if !current_names.contains(&e.name) {
|
||||
info!("Удалён устаревший мод: {}", e.name);
|
||||
if !names.contains(&e.name) {
|
||||
info!("Удалён: {}", e.name);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
let new_entries = process_new_mods(&all_files, &existing_sha512);
|
||||
for (entry, cat) in new_entries {
|
||||
match cat {
|
||||
"client-only" => data.client_only.push(entry),
|
||||
"server-only" => data.server_only.push(entry),
|
||||
"universal" => data.universal.push(entry),
|
||||
for (e, c) in process_new_mods(&jars, &existing) {
|
||||
match c {
|
||||
"client-only" => data.client_only.push(e),
|
||||
"server-only" => data.server_only.push(e),
|
||||
"universal" => data.universal.push(e),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let Err(e) = save_json(&data) {
|
||||
error!("Не удалось сохранить JSON: {}", e);
|
||||
if save_json(&data).is_err() {
|
||||
error!("Не сохранить JSON");
|
||||
} else {
|
||||
info!("Обновление завершено");
|
||||
}
|
||||
}
|
||||
|
||||
fn reassign_categories(mods_dir: &PathBuf) {
|
||||
if !mods_dir.exists() {
|
||||
fn reassign_categories(dir: &PathBuf) {
|
||||
if !dir.exists() {
|
||||
error!("Папка {} не найдена!", MODS_DIR);
|
||||
return;
|
||||
}
|
||||
let all_files = get_jar_files(mods_dir);
|
||||
if PathBuf::from(JSON_FILE).exists() {
|
||||
if let Err(e) = fs::remove_file(JSON_FILE) {
|
||||
warn!("Не удалось удалить старый JSON: {}", e);
|
||||
} else {
|
||||
info!("Старый mods-list.json удалён");
|
||||
}
|
||||
}
|
||||
let _ = fs::remove_file(JSON_FILE);
|
||||
let jars = get_jar_files(dir);
|
||||
let mut data = ModList {
|
||||
client_only: vec![],
|
||||
server_only: vec![],
|
||||
universal: vec![],
|
||||
};
|
||||
let empty_sha512: HashSet<String> = HashSet::new();
|
||||
let file_hashes = compute_hashes(&all_files);
|
||||
let mut hash_map: HashMap<String, Hashes> = HashMap::new();
|
||||
for (path, hashes) in &file_hashes {
|
||||
let name = path
|
||||
.file_name()
|
||||
.expect("Нет имени файла")
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
hash_map.insert(name, hashes.clone());
|
||||
}
|
||||
for (path, hashes) in file_hashes {
|
||||
let name = path
|
||||
.file_name()
|
||||
.expect("Нет имени файла")
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let cat = lookup_side(&hashes.sha1, &hashes.sha512).unwrap_or_else(|| ask_category(&name));
|
||||
|
||||
let hashes = compute_hashes(&jars);
|
||||
let results = Arc::new(Mutex::new(vec![]));
|
||||
let throttle = Arc::new(Mutex::new(0u32));
|
||||
hashes.par_iter().for_each(|(p, h)| {
|
||||
{
|
||||
let mut cnt = throttle.lock().unwrap();
|
||||
while *cnt >= 8 {
|
||||
drop(cnt);
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
cnt = throttle.lock().unwrap();
|
||||
}
|
||||
*cnt += 1;
|
||||
}
|
||||
let name = p.file_name().unwrap().to_string_lossy().into_owned();
|
||||
let cat = lookup_side(&h.sha1, &h.sha512);
|
||||
{
|
||||
let mut res = results.lock().unwrap();
|
||||
if let Some(c) = cat {
|
||||
res.push((name, h.clone(), c));
|
||||
}
|
||||
let mut cnt = throttle.lock().unwrap();
|
||||
*cnt -= 1;
|
||||
}
|
||||
});
|
||||
|
||||
for (name, h, cat) in results.lock().unwrap().drain(..) {
|
||||
let entry = ModEntry {
|
||||
name: name.clone(),
|
||||
sha512: hashes.sha512,
|
||||
sha256: hashes.sha256,
|
||||
sha1: hashes.sha1,
|
||||
md5: hashes.md5,
|
||||
sha512: h.sha512,
|
||||
sha256: h.sha256,
|
||||
sha1: h.sha1,
|
||||
md5: h.md5,
|
||||
};
|
||||
match cat {
|
||||
"client-only" => data.client_only.push(entry),
|
||||
@ -370,8 +354,51 @@ fn reassign_categories(mods_dir: &PathBuf) {
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let Err(e) = save_json(&data) {
|
||||
error!("Не удалось сохранить JSON: {}", e);
|
||||
|
||||
let total = jars.len();
|
||||
let found_cnt = data.client_only.len() + data.server_only.len() + data.universal.len();
|
||||
let not_modrinth = total - found_cnt;
|
||||
|
||||
if not_modrinth > 0 {
|
||||
println!(
|
||||
"{} модов не из Modrinth. Выбрать вручную оставшиеся моды? [y/n]",
|
||||
not_modrinth
|
||||
);
|
||||
let mut inp = String::new();
|
||||
io::stdin().read_line(&mut inp).ok();
|
||||
if inp.trim().to_lowercase() == "y" {
|
||||
let name_set: HashSet<String> = data
|
||||
.client_only
|
||||
.iter()
|
||||
.chain(&data.server_only)
|
||||
.chain(&data.universal)
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for (p, h) in hashes {
|
||||
let name = p.file_name().unwrap().to_string_lossy().into_owned();
|
||||
if name_set.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
let cat = ask_category(&name);
|
||||
let entry = ModEntry {
|
||||
name: name.clone(),
|
||||
sha512: h.sha512,
|
||||
sha256: h.sha256,
|
||||
sha1: h.sha1,
|
||||
md5: h.md5,
|
||||
};
|
||||
match cat {
|
||||
"client-only" => data.client_only.push(entry),
|
||||
"server-only" => data.server_only.push(entry),
|
||||
"universal" => data.universal.push(entry),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if save_json(&data).is_err() {
|
||||
error!("Не сохранить JSON");
|
||||
} else {
|
||||
info!("Все моды перераспределены");
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user