diff --git a/README.md b/README.md index 342e9e7..e66dde7 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ Rusted Iron Ring ================ -Low-level library for exploring From Software games files. Currently only -supports Dark Souls 1 (PTDE). +Low-level library for exploring From Software games files. This project is mainly to play with the Rust language, Nom parser, FFI, etc; if you need an actually used and tested library, see [SoulsFormats][soulsformats]. +The main target has been Dark Souls 1 PTDE, but checkout the features section +below. [soulsformats]: https://github.com/JKAnderson/SoulsFormats @@ -16,13 +17,9 @@ Usage The project contains 2 artefacts: -- `ironring`, a library with all the parsing/unpacking features implemented. +- `ironring`, a library with all the projects features implemented. - `rir`, an executable to use main lib features from the CLI. -The goal is to make the lib compatible with FFI tools such as Python's ctypes, -to ship a dynamic lib accessible for any language to easily script tasks and -ideas, but we're not there yet. - Ironring usage: ``` @@ -41,6 +38,7 @@ SUBCOMMANDS: bhf Extracts BHF/BDT contents bnd Extracts BND contents dat Extracts King's Field IV DAT contents + dat-pack Packs files in a King's Field IV DAT dcx Extracts and decompress DCX data hash Calculates hash for a string help Prints this message or the help of the given subcommand(s) @@ -53,28 +51,28 @@ SUBCOMMANDS: Features -------- -### Containers +### Format support + +| Type | Games | Features | +|----------|-------|------------------------------------------| +| BHD5/BDT | DS1 | Load, extract | +| DCX | DS1 | Load, extract | +| BND3 | DS1 | Load, extract | +| BHF3 | DS1 | Load, extract | +| DAT | KF4 | Load, extract, repack (untested) | +| PARAMDEF | DS1 | Pretty-print | +| PARAM | DS1 | Pretty-print, optionally with a PARAMDEF | -- BHD5 / BDT: extraction from disk to disk. -- DCX: decompression from disk to disk/memory. -- BND (v3): extraction from disk/memory to disk/memory, optionally decompress - from DCX. -- BHF (v3): extraction from disk/memory to disk/memory. -- DAT (King's Field IV): extraction from disk/memory to disk/memory. +Formats typically found within DCX files can usually be decompressed on the fly. -Repacking is not supported, maybe one day. It is not that useful when using -[UDSFM][udsfm] and [Yabber][yabber], but if you really need it you can check out -[SiegLib][sieglib]. +Repacking is mostly not supported, maybe one day. It is not that useful when +using [UDSFM][udsfm] and [Yabber][yabber], but if you really need it you can +check out [SiegLib][sieglib]. [udsfm]: https://github.com/HotPocketRemix/UnpackDarkSoulsForModding [yabber]: https://github.com/JKAnderson/Yabber [sieglib]: https://github.com/Dece/DarkSoulsDev/tree/master/Programs/SiegLib -### Files - -- PARAMDEF: parsing -- PARAM: parsing, optionally with a PARAMDEF - ### Misc - Encrypted archive name hasher. @@ -90,6 +88,6 @@ build. Credits ------- -All the fat cats involved in the scene and the [wiki][smwiki]. +TKGP and all the fat cats involved in the scene and the [wiki][smwiki]. [smwiki]: http://soulsmodding.wikidot.com/ diff --git a/src/bin/rir.rs b/src/bin/rir.rs index 3dee414..547fa2d 100644 --- a/src/bin/rir.rs +++ b/src/bin/rir.rs @@ -5,7 +5,7 @@ use std::process; use clap::{App, AppSettings, Arg, ArgMatches, SubCommand}; -use ironring::{name_hashes, unpackers}; +use ironring::{name_hashes, repackers, unpackers}; fn main() { let default_namefilepath: &str = &get_default_namefilepath(); @@ -94,19 +94,28 @@ fn main() { .arg(Arg::with_name("output") .help("Output directory") .short("o").long("output").takes_value(true).required(true))) + .subcommand(SubCommand::with_name("dat-pack") + .about("Pack files in a King's Field IV DAT") + .arg(Arg::with_name("files") + .help("Directory containing files to pack") + .takes_value(true).required(true)) + .arg(Arg::with_name("output") + .help("Output file") + .takes_value(true).required(true))) .get_matches(); process::exit(match matches.subcommand() { - ("bhd", Some(s)) => { cmd_bhd(s) } - ("bhds", Some(s)) => { cmd_bhds(s) } - ("hash", Some(s)) => { cmd_hash(s) } - ("dcx", Some(s)) => { cmd_dcx(s) } - ("bnd", Some(s)) => { cmd_bnd(s) } - ("bhf", Some(s)) => { cmd_bhf(s) } - ("paramdef", Some(s)) => { cmd_paramdef(s) } - ("param", Some(s)) => { cmd_param(s) } - ("dat", Some(s)) => { cmd_dat(s) } - _ => { 0 } + ("bhd", Some(s)) => cmd_bhd(s), + ("bhds", Some(s)) => cmd_bhds(s), + ("hash", Some(s)) => cmd_hash(s), + ("dcx", Some(s)) => cmd_dcx(s), + ("bnd", Some(s)) => cmd_bnd(s), + ("bhf", Some(s)) => cmd_bhf(s), + ("paramdef", Some(s)) => cmd_paramdef(s), + ("param", Some(s)) => cmd_param(s), + ("dat", Some(s)) => cmd_dat(s), + ("dat-pack", Some(s)) => cmd_dat_pack(s), + _ => 0, }) } @@ -257,3 +266,12 @@ fn cmd_dat(args: &ArgMatches) -> i32 { _ => 0 } } + +fn cmd_dat_pack(args: &ArgMatches) -> i32 { + let files_path: &str = args.value_of("files").unwrap(); + let output_path: &str = args.value_of("output").unwrap(); + match repackers::dat::pack_dat(files_path, output_path) { + Err(e) => { eprintln!("Failed to pack DAT: {:?}", e); 1 } + _ => 0 + } +} diff --git a/src/formats/common.rs b/src/formats/common.rs index aaf55de..17e9982 100644 --- a/src/formats/common.rs +++ b/src/formats/common.rs @@ -1,9 +1,15 @@ use std::fmt; +use std::io; use encoding_rs::SHIFT_JIS; use nom::IResult; use nom::bytes::complete::take_while; +pub trait Pack { + /// Write the entirety of `self` as bytes to the write buffer `f`. + fn write(&self, f: &mut dyn io::Write) -> io::Result; +} + /// Parse a zero-terminated string from the slice. pub fn take_cstring(i: &[u8]) -> IResult<&[u8], &[u8]> { take_while(|c| c != b'\0')(i) diff --git a/src/formats/dat.rs b/src/formats/dat.rs index e5918f3..288ad8d 100644 --- a/src/formats/dat.rs +++ b/src/formats/dat.rs @@ -1,9 +1,16 @@ +use std::io; + use nom::IResult; use nom::multi::count; use nom::number::complete::*; use nom::sequence::tuple; -use crate::formats::common::take_cstring_from; +use crate::formats::common::{Pack, take_cstring_from}; +use crate::utils::bin as utils_bin; + +pub const HEADER_SIZE: usize = 0x40; +pub const MAGIC: u32 = 0x1E048000; // Maybe it's 2 shorts and the 1st is padding? +pub const HEADER_PAD: usize = 0x38; // Padding after the header. #[derive(Debug)] pub struct DatHeader { @@ -11,11 +18,22 @@ pub struct DatHeader { pub num_files: u32, } +impl Pack for DatHeader { + fn write(&self, f: &mut dyn io::Write) -> io::Result { + f.write_all(&self.unk00.to_le_bytes())?; + f.write_all(&self.num_files.to_le_bytes())?; + Ok(0x8usize) + } +} + fn parse_header(i: &[u8]) -> IResult<&[u8], DatHeader> { let (i, (unk00, num_files)) = tuple((le_u32, le_u32))(i)?; Ok((i, DatHeader { unk00, num_files })) } +pub const FILE_ENTRY_SIZE: usize = 0x40; +pub const FILE_ENTRY_NAME_MAXLEN: usize = 0x34; + #[derive(Debug)] pub struct DatFileEntry { pub name: String, @@ -24,13 +42,28 @@ pub struct DatFileEntry { pub ofs_data: u32, } +impl Pack for DatFileEntry { + fn write(&self, f: &mut dyn io::Write) -> io::Result { + let name_bytes = self.name.as_bytes(); + f.write_all(name_bytes)?; + f.write_all(&vec![0u8; utils_bin::pad(name_bytes.len(), FILE_ENTRY_NAME_MAXLEN)])?; + f.write_all(&self.size.to_le_bytes())?; + f.write_all(&self.padded_size.to_le_bytes())?; + f.write_all(&self.ofs_data.to_le_bytes())?; + Ok(FILE_ENTRY_SIZE) + } +} + fn parse_file_entry(i: &[u8]) -> IResult<&[u8], DatFileEntry> { - let (i, name) = take_cstring_from(i, 0x34)?; + let (i, name) = take_cstring_from(i, FILE_ENTRY_NAME_MAXLEN)?; let name = String::from_utf8_lossy(name).to_string(); let (i, (size, padded_size, ofs_data)) = tuple((le_u32, le_u32, le_u32))(i)?; Ok((i, DatFileEntry { name, size, padded_size, ofs_data })) } +pub const INTERNAL_PATH_SEP: char = '/'; +pub const DATA_ALIGN: usize = 0x8000; + #[derive(Debug)] pub struct Dat { pub header: DatHeader, @@ -41,7 +74,7 @@ pub struct Dat { pub fn parse(i: &[u8]) -> IResult<&[u8], Dat> { let full_file = i; let (_, header) = parse_header(i)?; - let i = &full_file[0x40..]; + let i = &full_file[HEADER_SIZE..]; let (_, files) = count(parse_file_entry, header.num_files as usize)(i)?; Ok((full_file, Dat { header, files })) } diff --git a/src/lib.rs b/src/lib.rs index aa304e6..85082cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,9 @@ #![allow(non_snake_case)] pub mod name_hashes; +pub mod repackers { + pub mod dat; +} pub mod formats { pub mod bhd; pub mod bhf; diff --git a/src/repackers/dat.rs b/src/repackers/dat.rs new file mode 100644 index 0000000..82a0822 --- /dev/null +++ b/src/repackers/dat.rs @@ -0,0 +1,100 @@ +use std::fs; +use std::io::{self, Write}; +use std::path; + +use crate::formats::common::Pack; +use crate::formats::dat; +use crate::utils::bin as utils_bin; +use crate::utils::fs as utils_fs; + +/// Pack a directory as a DAT archive. +/// +/// Walks recursively in `files_path` to build all file entries. +/// For performance and laziness, the archive is built directly in RAM. +pub fn pack_dat(files_path: &str, output_path: &str) -> Result<(), io::Error> { + // Pack all files and entries description in memory. + let files_path = path::Path::new(files_path); + let mut entries = vec!(); + let mut files_data = vec!(); + pack_dat_dir(files_path, "", &mut entries, &mut files_data)?; + + let mut output_file = fs::File::create(output_path)?; + let mut ofs = 0usize; + + // Write header. + let header = dat::DatHeader { unk00: dat::MAGIC, num_files: entries.len() as u32 }; + header.write(&mut output_file)?; + output_file.write_all(&vec![0u8; dat::HEADER_PAD])?; + ofs += dat::HEADER_SIZE; + + // Write entries, but shift their data offset beforehand. + let entries_size = entries.len() * dat::FILE_ENTRY_SIZE; + let entries_pad = utils_bin::pad(ofs + entries_size, dat::DATA_ALIGN); + let ofs_data = ofs + entries_size + entries_pad; + for entry in &mut entries { + entry.ofs_data += ofs_data as u32; + entry.write(&mut output_file)?; + } + output_file.write_all(&vec![0u8; entries_pad])?; + + // Finally, write files data. + output_file.write_all(&files_data)?; + + Ok(()) +} + +/// Recursively walks in `dir` to create `DatFileEntry`s. +/// +/// `prefix` is initially "" and will contain current relative dir with +/// separator suffixed during walks, e.g. "param/". +fn pack_dat_dir( + dir: &path::Path, + prefix: &str, + entries: &mut Vec, + files_data: &mut Vec, +) -> Result<(), io::Error> { + for entry in fs::read_dir(dir)? { + let entry = entry?.path(); + if entry.is_dir() { + if let Some(dir_name) = entry.file_name().and_then(|n| n.to_str()) { + let mut prefix = String::from(prefix); + prefix.push_str(dir_name); + prefix.push(dat::INTERNAL_PATH_SEP); + pack_dat_dir(&entry, &prefix, entries, files_data)?; + } + } else if entry.is_file() /* No symlink support. */ { + if let Some(name) = entry.file_name().and_then(|n| n.to_str()) { + if let Ok(metadata) = entry.metadata() { + let mut entry_name = String::from(prefix); + entry_name.push_str(name); + pack_dat_entry(&entry, entry_name, &metadata, entries, files_data)?; + } + } + } + } + Ok(()) +} + +/// Pack the file in `files_data` and update `entries` accordingly. +fn pack_dat_entry( + file_entry: &path::PathBuf, + internal_name: String, + metadata: &fs::Metadata, + entries: &mut Vec, + files_data: &mut Vec, +) -> Result<(), io::Error> { + let file_size = metadata.len() as u32; + let padding = utils_bin::pad(file_size as usize, dat::DATA_ALIGN); + entries.push(dat::DatFileEntry { + name: internal_name, + size: file_size, + padded_size: file_size + padding as u32, + ofs_data: files_data.len() as u32, // Data will be pushed at the current end of file. + }); + + let mut data = utils_fs::open_file_to_vec(file_entry)?; + files_data.append(&mut data); + let mut padding_data = vec![0u8; padding]; + files_data.append(&mut padding_data); + Ok(()) +} diff --git a/src/unpackers/dat.rs b/src/unpackers/dat.rs index 48443fd..6ba06a7 100644 --- a/src/unpackers/dat.rs +++ b/src/unpackers/dat.rs @@ -8,11 +8,15 @@ use crate::formats::dat; use crate::unpackers::errors::UnpackError; use crate::utils::fs as utils_fs; +/// Extract DAT file contents to `output_path`. +/// +/// Wraps around `extract_dat` to load the DAT from disk. pub fn extract_dat_file(dat_path: &str, output_path: &str) -> Result<(), UnpackError> { let (dat, dat_data) = load_dat_file(dat_path)?; extract_dat(&dat, dat_data, output_path) } +/// Extract DAT contents to `output_path`. pub fn extract_dat( dat: &dat::Dat, dat_data: Vec, @@ -29,6 +33,7 @@ pub fn extract_dat( Ok(()) } +/// Extract one `DatFileEntry`, preserving internal dir structure. fn extract_file( file_entry: &dat::DatFileEntry, data: &Vec, @@ -54,11 +59,13 @@ fn extract_file( Ok(()) } +/// Load a DAT file from disk. pub fn load_dat_file(dat_path: &str) -> Result<(dat::Dat, Vec), UnpackError> { let dat_data = utils_fs::open_file_to_vec(path::Path::new(dat_path))?; Ok((load_dat(&dat_data)?, dat_data)) } +/// Load a DAT file from a bytes slice. pub fn load_dat(dat_data: &[u8]) -> Result { match dat::parse(&dat_data) { Ok((_, dat)) => Ok(dat), diff --git a/src/utils/bin.rs b/src/utils/bin.rs index e0ef7d3..57e3e61 100644 --- a/src/utils/bin.rs +++ b/src/utils/bin.rs @@ -11,6 +11,11 @@ pub fn mask(bit_size: usize) -> usize { (1 << bit_size) - 1 } +/// Return the number of bytes to pad from ofs to alignment. +pub fn pad(ofs: usize, alignment: usize) -> usize { + (alignment - (ofs % alignment)) % alignment +} + #[cfg(test)] mod test { use super::*; @@ -30,4 +35,16 @@ mod test { assert_eq!(mask(8), 0b11111111); assert_eq!(mask(15), 0b01111111_11111111); } + + #[test] + fn test_pad() { + assert_eq!(pad(0, 4), 0); + assert_eq!(pad(1, 4), 3); + assert_eq!(pad(3, 4), 1); + assert_eq!(pad(4, 4), 0); + assert_eq!(pad(4, 16), 12); + assert_eq!(pad(15, 16), 1); + assert_eq!(pad(16, 16), 0); + assert_eq!(pad(17, 16), 15); + } }