assembly-data: split into assembly-fdb, assembly-xml

Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+members = [
+    "modules/*"
+]

assembly.code-workspace

@@ -0,0 +1,8 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {}
+}

modules/data/Cargo.toml

@@ -3,7 +3,7 @@ name = "assembly-data"
 version = "0.3.0"
 authors = ["Xiphoseer"]
 edition = "2018"
-homepage = "https://xiphoseer.github.io"
+homepage = "https://xiphoseer.de"
 repository = "https://github.com/xiphoseer/assembly_rs"
 description = "Database module for the assembly crate"
 license = "MIT"
@@ -11,39 +11,17 @@ readme = "README.md"
 
 [features]
 default = ["sqlite", "serde-derives"]
-sqlite = ["rusqlite"]
-serde-derives = ["serde", "quick-xml/serialize"]
+sqlite = ["assembly-fdb/sqlite"]
+serde-derives = ["assembly-fdb/serde-derives", "assembly-xml/serialize"]
 
-[dependencies]
-hsieh-hash = "0.1"
-thiserror = "1.0"
-memchr = "2.3"
-encoding_rs = "0.8"
-derive-new = "0.5"
-bytemuck = "1.4"
-bytemuck_derive = "1"
+[dependencies.assembly-fdb]
+version = "0.1.0"
+path = "../fdb"
 
-[dependencies.assembly-core]
-version = "0.2.1"
-path = "../core"
-
-[dependencies.quick-xml]
-version = "0.20"
-features = ["encoding"]
-
-[dependencies.rusqlite]
-version = "0.21.0"
-features = ["bundled"]
-optional = true
-
-[dependencies.serde]
-version = "1"
-optional = true
-features = ["derive"]
+[dependencies.assembly-xml]
+version = "0.1.0"
+path = "../xml"
 
 [dev-dependencies]
-prettytable-rs = "0.8"
-mapr = "0.8"
 structopt = "0.3"
-color-eyre = "0.5"
-serde_json = "1.0.61"
+color-eyre = "0.5"

modules/data/examples/xmldb-to-fdb.rs

@@ -9,6 +9,7 @@ use assembly_data::{
             expect_column_or_end_columns, expect_columns, expect_database, expect_row_or_end_rows,
             expect_rows, expect_table, ValueType,
         },
+        quick::Reader,
     },
 };
 use color_eyre::eyre::WrapErr;
@@ -19,8 +20,6 @@ use std::{
     path::PathBuf,
     time::Instant,
 };
-
-use quick_xml::Reader;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
@@ -132,7 +131,8 @@ fn main() -> color_eyre::Result<()> {
                             pk = Some((*i % 128) as usize);
                         }
                         core::Field::Text(text) => {
-                            pk = Some((hsieh_hash::digest(text.as_bytes()) % 128) as usize);
+                            let lat1 = Latin1String::encode(&text);
+                            pk = Some((lat1.hash() % 128) as usize);
                         }
                         _ => panic!("Can't use {:?} as PK", &dest_value),
                     }

modules/data/src/lib.rs

@@ -1,4 +1,4 @@
 //! The Database parts of `assembly`
 
-pub mod fdb;
-pub mod xml;
+pub use assembly_fdb as fdb;
+pub use assembly_xml as xml;

modules/fdb/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "assembly-fdb"
+version = "0.1.0"
+edition = "2018"
+
+[features]
+default = ["sqlite", "serde-derives"]
+sqlite = ["rusqlite"]
+serde-derives = ["serde"]
+
+[dependencies]
+hsieh-hash = "0.1"
+thiserror = "1.0"
+memchr = "2.3"
+encoding_rs = "0.8"
+derive-new = "0.5"
+bytemuck = "1.4"
+bytemuck_derive = "1"
+
+[dependencies.assembly-core]
+version = "0.2.1"
+path = "../core"
+
+[dependencies.rusqlite]
+version = "0.21.0"
+features = ["bundled"]
+optional = true
+
+[dependencies.serde]
+version = "1"
+optional = true
+features = ["derive"]
+
+[dev-dependencies]
+prettytable-rs = "0.8"
+mapr = "0.8"
+structopt = "0.3"
+color-eyre = "0.5"
+serde_json = "1.0.61"

modules/fdb/examples/check-skills.rs

@@ -5,7 +5,7 @@ use std::{
     path::PathBuf,
 };
 
-use assembly_data::fdb::{
+use assembly_fdb::{
     common::Latin1Str,
     mem::{Database, Row, RowHeaderIter, Table, Tables},
 };

modules/fdb/examples/fdb-columns.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::mem::{Database, Table};
+use assembly_fdb::mem::{Database, Table};
 use color_eyre::eyre::{eyre, WrapErr};
 use mapr::Mmap;
 use prettytable::{Cell as PCell, Row as PRow, Table as PTable};

modules/fdb/examples/fdb-copy.rs

@@ -1,6 +1,6 @@
 use std::{fs::File, io::BufWriter, path::PathBuf, time::Instant};
 
-use assembly_data::fdb::{core::Field, mem, store};
+use assembly_fdb::{core::Field, mem, store};
 use mapr::Mmap;
 use structopt::StructOpt;
 

modules/fdb/examples/fdb-diff.rs

@@ -1,6 +1,6 @@
 use std::{fs::File, path::PathBuf, time::Instant};
 
-use assembly_data::fdb::mem;
+use assembly_fdb::mem;
 use mapr::Mmap;
 use structopt::StructOpt;
 

modules/fdb/examples/fdb-index.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::{
+use assembly_fdb::{
     core,
     mem::{self, Database, Field, Table},
     query::pk_filter,

modules/fdb/examples/fdb-stat.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::{
+use assembly_fdb::{
     common::{Value, ValueType},
     mem::{Database, Tables},
 };

modules/fdb/examples/fdb-tables.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::mem::Database;
+use assembly_fdb::mem::Database;
 use mapr::Mmap;
 use prettytable::{Cell as PCell, Row as PRow, Table as PTable};
 use std::fs::File;

modules/fdb/examples/fdb-to-sqlite.rs

@@ -1,6 +1,6 @@
 use std::{fs::File, path::PathBuf, time::Instant};
 
-use assembly_data::fdb::{mem::Database, sqlite::try_export_db};
+use assembly_fdb::{mem::Database, sqlite::try_export_db};
 use color_eyre::eyre::WrapErr;
 use mapr::Mmap;
 use rusqlite::Connection;

modules/fdb/examples/fdb-tree.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::mem::{Database, Tables};
+use assembly_fdb::mem::{Database, Tables};
 use mapr::Mmap;
 use std::{fs::File, path::PathBuf, time::Instant};
 use structopt::StructOpt;

modules/fdb/examples/lux-dest-factions.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::{
+use assembly_fdb::{
     common::Latin1Str,
     mem::{Database, Row, Table, Tables},
 };

modules/fdb/examples/lux-precond.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::{
+use assembly_fdb::{
     common::Latin1Str,
     mem::{Database, Row, Table, Tables},
 };

modules/fdb/examples/lux-rebuild-activity.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::mem::{Database, Row, Table, Tables};
+use assembly_fdb::mem::{Database, Row, Table, Tables};
 use color_eyre::eyre::{eyre, WrapErr};
 use mapr::Mmap;
 use std::{

modules/fdb/examples/lux-shaders.rs

@@ -1,4 +1,4 @@
-use assembly_data::fdb::{
+use assembly_fdb::{
     common::Latin1Str,
     mem::{Database, Row, Table, Tables},
 };
@@ -117,10 +117,13 @@ fn main() -> color_eyre::Result<()> {
         std::fs::create_dir(out)?;
     }
 
-    let all: Vec<_> = shaders.row_iter().map(|row| shad_loader.load(row)).collect();
-    let hal = HAL { _embedded: MapShaders {
-        mapShaders: all,
-    }};
+    let all: Vec<_> = shaders
+        .row_iter()
+        .map(|row| shad_loader.load(row))
+        .collect();
+    let hal = HAL {
+        _embedded: MapShaders { mapShaders: all },
+    };
 
     let string = serde_json::to_string(&hal)?;
     if let Some(out) = &opts.out {

modules/fdb/src/common.rs

@@ -151,6 +151,11 @@ impl Latin1Str {
     pub fn decode(&self) -> Cow<str> {
         WINDOWS_1252.decode(self.as_bytes()).0
     }
+
+    /// Hash the string using `SuperFashHash` / [`hsieh_hash`]
+    pub fn hash(&self) -> u32 {
+        hsieh_hash::digest(self.as_bytes())
+    }
 }
 
 /// Type-Parameters to [`Value`]

modules/fdb/src/lib.rs

@@ -29,7 +29,7 @@
 //! You can use the `mem` module to load a database from an in-memory buffer:
 //!
 //! ```
-//! use assembly_data::fdb::mem::Database;
+//! use assembly_fdb::mem::Database;
 //!
 //! let file: &[u8] = &[0,0,0,0,8,0,0,0];
 //! let db = Database::new(file);

modules/fdb/src/mem/c.rs

@@ -1,11 +1,11 @@
 #![allow(clippy::upper_case_acronyms)]
 use std::convert::TryFrom;
 
-use crate::fdb::file::{
+use crate::file::{
     FDBBucketHeader, FDBColumnHeader, FDBFieldValue, FDBHeader, FDBRowHeader,
     FDBRowHeaderListEntry, FDBTableDataHeader, FDBTableDefHeader, FDBTableHeader, IndirectValue,
 };
-use crate::fdb::{common::ValueType, file::ArrayHeader};
+use crate::{common::ValueType, file::ArrayHeader};
 use assembly_core::buffer::{MinimallyAligned, Repr, LEU32};
 
 /// An FDB header usable for unaligned reads

modules/fdb/src/mem/mod.rs

@@ -164,7 +164,9 @@ impl<'a> Tables<'a> {
 }
 
 #[allow(clippy::needless_lifetimes)] // <- clippy gets this wrong, presumably because of impl trait?
-fn map_column_header<'a>(buf: &'a [u8]) -> impl Fn(&'a FDBColumnHeaderC) -> Column<'a> + Copy + Clone {
+fn map_column_header<'a>(
+    buf: &'a [u8],
+) -> impl Fn(&'a FDBColumnHeaderC) -> Column<'a> + Copy + Clone {
     move |header: &FDBColumnHeaderC| {
         let column_header = header.extract();
         let name = get_latin1_str(buf, column_header.column_name_addr);

modules/fdb/src/reader/builder.rs

@@ -3,7 +3,7 @@ use assembly_core::displaydoc::Display;
 use thiserror::Error;
 
 use super::{DatabaseBufReader, DatabaseReader};
-use crate::fdb::{
+use crate::{
     common::{UnknownValueType, ValueType},
     core::Field,
     file::FDBFieldData,

modules/fdb/src/ro/buffer.rs

@@ -1,6 +1,6 @@
 //! # General methods for aligned access to a byte buffer
 
-use crate::fdb::{
+use crate::{
     common::Latin1Str,
     file::{
         FDBHeader, FDBRowHeader, FDBRowHeaderListEntry, FDBTableDataHeader, FDBTableDefHeader,
@@ -144,10 +144,7 @@ pub fn table_headers<'a>(buf: &'a [u8], header: &'a FDBHeader) -> Res<&'a [FDBTa
 }
 
 /// Get the table definition reference
-pub fn table_definition_ref(
-    buf: &[u8],
-    header: FDBTableHeader,
-) -> Res<&FDBTableDefHeader> {
+pub fn table_definition_ref(buf: &[u8], header: FDBTableHeader) -> Res<&FDBTableDefHeader> {
     get_at(buf, header.table_def_header_addr as usize)
 }
 

modules/fdb/src/ro/handle.rs

@@ -8,7 +8,7 @@ use super::{
     slice::{FDBBucketHeaderSlice, FDBColumnHeaderSlice, FDBFieldDataSlice, FDBTableHeaderSlice},
     BaseHandle, Handle,
 };
-use crate::fdb::{
+use crate::{
     common::{Latin1Str, UnknownValueType, ValueType},
     file::{
         FDBBucketHeader, FDBColumnHeader, FDBFieldData, FDBFieldValue, FDBHeader, FDBRowHeader,
@@ -163,7 +163,7 @@ where
 {
     /// Get the bucket for a particular id / hash
     pub fn get_bucket_for_hash(self, id: u32) -> BaseResult<P, FDBBucketHeader> {
-        self.map_into::<_,_,BufferError>(|buf, raw| {
+        self.map_into::<_, _, BufferError>(|buf, raw| {
             let bucket_count = raw.buckets.count as usize;
             let buckets_addr = raw.buckets.base_offset as usize;
             let slice: &[FDBBucketHeader] = buffer::get_slice_at(buf, buckets_addr, bucket_count)?;

modules/fdb/src/ro/slice.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::upper_case_acronyms)]
 //! # Handling of slice references into the in-memory DB file
 
-use crate::fdb::file::{FDBBucketHeader, FDBColumnHeader, FDBFieldData, FDBTableHeader};
+use crate::file::{FDBBucketHeader, FDBColumnHeader, FDBFieldData, FDBTableHeader};
 use std::convert::TryInto;
 
 /// Invariant: length must always be a multiple of 8 bytes

modules/fdb/src/store/mod.rs

@@ -6,7 +6,7 @@
 //! ## Usage
 //!
 //! ```
-//! use assembly_data::fdb::{
+//! use assembly_fdb::{
 //!     common::{ValueType, Latin1String},
 //!     core::{Field},
 //!     store::{Database, Table},

modules/fdb/src/store/tests.rs

@@ -1,4 +1,4 @@
-use crate::fdb::mem;
+use crate::mem;
 
 use super::super::core;
 use super::*;

modules/fdb/src/store/writer.rs

@@ -1,6 +1,6 @@
 use std::{hint::unreachable_unchecked, io};
 
-use crate::fdb::{
+use crate::{
     common::Latin1Str,
     file::{
         ArrayHeader, FDBBucketHeader, FDBColumnHeader, FDBFieldData, FDBRowHeader,

modules/sysdiagram/Cargo.toml

@@ -11,7 +11,7 @@ readme = "README.md"
 
 [dependencies]
 assembly-core = { path = "../core", version = "0.2.1" }
-ms-oforms = { path = "../../../ms-oforms", version = "0.3.0-alpha1" }
+ms-oforms = { version = "0.3.0-alpha1" }
 nom = "5.0"
 base64 = "0.10"
 cfb = "0.3.1"

modules/xml/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "assembly-xml"
+version = "0.1.0"
+edition = "2018"
+
+[features]
+default = ["serialize"]
+serialize = ["serde", "quick-xml/serialize"]
+
+[dependencies]
+thiserror = "1.0"
+displaydoc = "0.1.5"
+
+#[dependencies.assembly-core]
+#version = "0.2.1"
+#path = "../core"
+
+[dependencies.quick-xml]
+version = "0.20"
+features = ["encoding"]
+
+[dependencies.serde]
+version = "1"
+optional = true
+features = ["derive"]
+
+[dev-dependencies]
+structopt = "0.3"
+color-eyre = "0.5"
+serde_json = "1.0.61"

modules/xml/examples/dump-locale-prefix.rs

@@ -1,8 +1,8 @@
 use std::{collections::BTreeMap, fs::File, io::BufReader, path::PathBuf};
 
-use assembly_data::xml::common::exact::{expect_attribute, expect_end, expect_start, expect_text};
+use assembly_xml::common::exact::{expect_attribute, expect_end, expect_start, expect_text};
 use color_eyre::eyre::eyre;
-use quick_xml::{Reader as XmlReader, events::Event as XmlEvent};
+use quick_xml::{events::Event as XmlEvent, Reader as XmlReader};
 use structopt::StructOpt;
 
 #[derive(StructOpt)]

modules/xml/examples/xmldb-tree.rs

@@ -1,4 +1,4 @@
-use assembly_data::xml::{
+use assembly_xml::{
     common::{expect_decl, expect_end},
     database::{
         expect_column_or_end_columns, expect_columns, expect_database, expect_row_or_end_rows,

modules/xml/src/common/exact.rs

@@ -7,7 +7,7 @@ use {
     std::{error::Error as StdError, io::BufRead, str::FromStr},
 };
 
-use assembly_core::displaydoc::Display;
+use displaydoc::Display;
 use thiserror::Error;
 
 /// The errors for this module
@@ -77,10 +77,7 @@ pub fn expect_end<'a, 'b, 'c, B: BufRead>(
 }
 
 /// Expect some text and return it
-pub fn expect_text<B: BufRead>(
-    reader: &mut XmlReader<B>,
-    buf: &mut Vec<u8>,
-) -> Result<String> {
+pub fn expect_text<B: BufRead>(reader: &mut XmlReader<B>, buf: &mut Vec<u8>) -> Result<String> {
     if let Ok(XmlEvent::Text(e)) = reader.read_event(buf) {
         let text = e.unescape_and_decode(reader)?;
         Ok(text)

modules/xml/src/common/mod.rs

@@ -2,7 +2,7 @@
 
 use std::{io::BufRead, todo};
 
-use assembly_core::displaydoc::Display;
+use displaydoc::Display;
 use quick_xml::{events::Event, Reader};
 use thiserror::Error;
 

modules/xml/src/database/mod.rs

@@ -4,17 +4,19 @@
 //! file to store the client database. This was also used for LUPs to add their data
 //! independently of the rest of the game.
 
-use assembly_core::displaydoc::Display;
+use displaydoc::Display;
 use quick_xml::{events::Event, Reader};
-use std::{collections::HashMap, error::Error, fmt, io::BufRead, str::FromStr};
+use std::{collections::HashMap, error::Error, io::BufRead, str::FromStr};
 
 use super::common::{expect_elem, expect_named_elem, XmlError};
 
-#[cfg(feature = "serde-derives")]
+#[cfg(feature = "serialize")]
 use serde::{
     de::{self, Unexpected, Visitor},
     Deserialize, Deserializer,
 };
+#[cfg(feature = "serialize")]
+use std::fmt;
 
 /// The value types for the database
 ///
@@ -67,6 +69,7 @@ pub enum ValueType {
     DateTime,
 }
 
+#[cfg(feature = "serialize")]
 impl<'de> Deserialize<'de> for ValueType {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
@@ -117,10 +120,10 @@ impl FromStr for ValueType {
     }
 }
 
-#[cfg(feature = "serde-derives")]
+#[cfg(feature = "serialize")]
 struct ValueTypeVisitor;
 
-#[cfg(feature = "serde-derives")]
+#[cfg(feature = "serialize")]
 impl<'de> Visitor<'de> for ValueTypeVisitor {
     type Value = ValueType;
 
@@ -138,7 +141,7 @@ impl<'de> Visitor<'de> for ValueTypeVisitor {
 }
 
 /// A row of the database
-#[cfg_attr(feature = "serde-derives", derive(Deserialize))]
+#[cfg_attr(feature = "serialize", derive(Deserialize))]
 #[derive(Debug, Eq, PartialEq)]
 pub struct Row(HashMap<String, String>);
 
@@ -169,7 +172,7 @@ pub fn expect_rows<B: BufRead>(xml: &mut Reader<B>, buf: &mut Vec<u8>) -> Result
 }
 
 /// The information on a column
-#[cfg_attr(feature = "serde-derives", derive(Deserialize))]
+#[cfg_attr(feature = "serialize", derive(Deserialize))]
 pub struct Column {
     /// The name of the column
     pub name: String,
@@ -262,6 +265,7 @@ pub fn expect_row_or_end_rows<B: BufRead>(
 }
 
 #[cfg(test)]
+#[cfg(feature = "serialize")]
 mod tests {
     use quick_xml::{de::Deserializer, DeError};
 
@@ -272,6 +276,8 @@ mod tests {
 
     #[test]
     fn test_simple_deserialize() {
+        use serde::Deserialize;
+
         let st = br#"<row foo="bar"/></rows>"#;
         let c = BufReader::new(Cursor::new(st));
         let mut d = Deserializer::from_reader(c);

modules/xml/src/localization.rs

@@ -10,7 +10,7 @@ use std::{
     path::Path,
 };
 
-use assembly_core::displaydoc::Display;
+use displaydoc::Display;
 use quick_xml::{events::Event as XmlEvent, Error as XmlError, Reader as XmlReader};
 use thiserror::Error;