Merge pull request #18 from diceypotato/openlibrarytest

openlibrary: make slightly more robust and add some tests

src/handlers/scrapers/open_library.rs

@@ -1,58 +1,134 @@
+use crate::Result;
 use actix_web::{HttpResponse, web};
 use chrono::NaiveDate;
 use serde::Deserialize;
-use serde_json::Value;
 
-use crate::models::title_group::{ContentType, create_default_title_group};
+use crate::models::title_group::{ContentType, UserCreatedTitleGroup, create_default_title_group};
+
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+enum Description {
+    Typed {
+        #[allow(unused)]
+        #[serde(rename = "type")]
+        type_: String,
+        value: String,
+    },
+    Untyped(String),
+}
+
+impl AsRef<str> for Description {
+    fn as_ref(&self) -> &str {
+        match self {
+            Description::Typed { value, .. } => value,
+            Description::Untyped(value) => value,
+        }
+    }
+}
+
+impl From<Description> for String {
+    fn from(d: Description) -> Self {
+        match d {
+            Description::Typed { value, .. } => value,
+            Description::Untyped(value) => value,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct Work {
+    title: String,
+    description: Option<Description>,
+    first_publish_date: Option<String>,
+}
+
+fn parse_date(date: &str) -> Option<NaiveDate> {
+    date.parse::<i32>()
+        .ok()
+        .and_then(|y| NaiveDate::from_ymd_opt(y, 1, 1))
+        .or_else(|| NaiveDate::parse_from_str(date, "%B %d, %Y").ok())
+}
 
 #[derive(Debug, Deserialize)]
 pub struct GetOpenLibraryQuery {
     id: String,
 }
 
-pub async fn get_open_library_data(query: web::Query<GetOpenLibraryQuery>) -> HttpResponse {
-    let open_library_id = &query.id;
-    //TODO: check if there is an entry in the db with this open_library_id
-    let mut title_group = create_default_title_group();
-    title_group.external_links = vec![format!("https://openlibrary.org/works/{}", open_library_id)];
-    title_group.content_type = ContentType::Book;
+pub async fn get_open_library_data(query: web::Query<GetOpenLibraryQuery>) -> Result<HttpResponse> {
+    let url = format!("https://openlibrary.org/works/{}.json", query.id);
 
-    match reqwest::get(format!(
-        "https://openlibrary.org/works/{}.json",
-        open_library_id
-    ))
-    .await
-    {
-        Ok(response) => match response.json::<Value>().await {
-            Ok(json) => {
-                title_group.name = json["title"]
-                    .as_str()
-                    .unwrap_or("Title not found")
-                    .to_string();
-                title_group.description = json["description"]["value"]
-                    .as_str()
-                    .unwrap_or("Description not found")
-                    .to_string();
-                title_group.original_release_date = NaiveDate::from_ymd_opt(
-                    json["first_publish_date"]
-                        .as_str()
-                        .unwrap_or("0000")
-                        .parse()
-                        .unwrap_or(0),
-                    1,
-                    1,
-                )
-                .unwrap()
-                .and_hms_opt(0, 0, 0)
-                .unwrap();
-            }
-            Err(_) => {
-                return HttpResponse::InternalServerError().body("Failed to parse JSON");
-            }
-        },
-        Err(_) => {
-            return HttpResponse::InternalServerError().body("Failed to fetch data");
-        }
-    }
-    HttpResponse::Ok().json(serde_json::json!({"title_group": title_group}))
+    let work = reqwest::get(&url).await?.json::<Work>().await?;
+
+    // TODO: kill unwrap and make date nullable
+    let original_release_date = work
+        .first_publish_date
+        .as_ref()
+        .and_then(|d| parse_date(d))
+        .unwrap();
+
+    let description = work
+        .description
+        .map(String::from)
+        .unwrap_or_else(|| "Description not provided.".into());
+
+    let title_group = UserCreatedTitleGroup {
+        name: work.title,
+        description,
+        external_links: vec![format!("https://openlibrary.org/works/{}", query.id)],
+        original_release_date: original_release_date.into(),
+        content_type: ContentType::Book,
+        ..create_default_title_group()
+    };
+
+    Ok(HttpResponse::Ok().json(serde_json::json!({"title_group": title_group})))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_deserialization_moby_dick() {
+        let moby_dick = serde_json::from_str::<Work>(include_str!("testdata/OL102749W.json"));
+
+        assert!(moby_dick.is_ok());
+
+        let moby_dick = moby_dick.unwrap();
+
+        assert_eq!(moby_dick.title, "Moby Dick");
+        assert!(moby_dick.description.is_some());
+        assert_eq!(
+            moby_dick.description.unwrap().as_ref(),
+            "\"Command the murderous chalices! Drink ye harpooners! Drink and swear, ye men that man the deathful whaleboat's bow -- Death to Moby Dick!\" So Captain Ahab binds his crew to fulfil his obsession -- the destruction of the great white whale. Under his lordly but maniacal command the Pequod's commercial mission is perverted to one of vengeance. To Ahab, the monster that destroyed his body is not a creature, but the symbol of \"some unknown but still reasoning thing.\" Uncowed by natural disasters, ill omens, even death, Ahab urges his ship towards \"the undeliverable, nameless perils of the whale.\" Key letters from Melville to Nathaniel Hawthorne are printed at the end of this volume. - Back cover."
+        );
+        assert!(moby_dick.first_publish_date.is_none());
+    }
+
+    #[test]
+    fn test_deserialization_les_mis() {
+        let les_mis = serde_json::from_str::<Work>(include_str!("testdata/OL1063588W.json"));
+
+        assert!(les_mis.is_ok());
+
+        let les_mis = les_mis.unwrap();
+
+        assert_eq!(les_mis.title, "Les Misérables");
+        assert!(les_mis.description.is_some());
+        assert_eq!(
+            les_mis.description.unwrap().as_ref(),
+            "In this story of the trials of the peasant Jean Valjean--a man unjustly imprisoned, baffled by destiny, and hounded by his nemesis, the magnificently realized, ambiguously malevolent police detective Javert--Hugo achieves the sort of rare imaginative resonance that allows a work of art to transcend its genre."
+        );
+        assert_eq!(les_mis.first_publish_date, Some("1863".into()));
+    }
+
+    #[test]
+    fn test_parse_date() {
+        // OpenLibrary published date is not normalized, try a couple varieties.
+
+        let date = parse_date("1970");
+        assert_eq!(date, NaiveDate::from_ymd_opt(1970, 1, 1));
+
+        let date = parse_date("February 19, 1994");
+        assert_eq!(date, NaiveDate::from_ymd_opt(1994, 2, 19));
+    }
 }

src/handlers/scrapers/testdata/OL102749W.json

@@ -0,0 +1 @@
+{"description": "\"Command the murderous chalices! Drink ye harpooners! Drink and swear, ye men that man the deathful whaleboat's bow -- Death to Moby Dick!\" So Captain Ahab binds his crew to fulfil his obsession -- the destruction of the great white whale. Under his lordly but maniacal command the Pequod's commercial mission is perverted to one of vengeance. To Ahab, the monster that destroyed his body is not a creature, but the symbol of \"some unknown but still reasoning thing.\" Uncowed by natural disasters, ill omens, even death, Ahab urges his ship towards \"the undeliverable, nameless perils of the whale.\" Key letters from Melville to Nathaniel Hawthorne are printed at the end of this volume. - Back cover.", "title": "Moby Dick", "covers": [10544254, 8229204, 9341823, 2199528, 1007907, 10089516, 7726876, 10530031, 13195819, 10860984, 11098834, 11138515, 11358399, 11386586, 11420104, 12022194, 12807453, 12813499, 13022202], "subject_places": ["Massachusetts", "Nantucket", "New Bedford", "Cape Horn", "Cape of Good Hope", "Atlantic Ocean", "Pacific Ocean", "Indian Ocean"], "subjects": ["American Sea stories", "Mentally ill", "Whaling", "Science Fiction & Fantasy", "Whales", "great_books_of_the_western_world", "Translations into French", "Literature", "Captain Ahab (Fictitious character)", "American Adventure stories", "Sailors", "Sea stories", "Classic Literature", "Whaling in literature", "Young Adult Fiction", "Open Library Staff Picks", "open_syllabus_project", "Fiction", "Ship captains", "Whaling ships", "Juvenile fiction", "Chasse", "Whales in literature", "Shipwrecks", "Baleines", "Long Now Manual for Civilization", "General", "Children: Grades 4-6", "American fiction (fictional works by one author)", "Ahab, captain (fictitious character), fiction", "Whaling, fiction", "Children's fiction", "Whales, fiction", "Fiction, action & adventure", "Fiction, psychological", "Literature and fiction (general)", "Fiction, sea stories", "Fiction, fantasy, epic", "Poetry (poetic works by one author)", "Melville, herman, 1819-1891", "American literature, history and criticism", "Sea stories, history and criticism", "Literature and fiction, action and adventure", "Illustrations", "Pictorial works", "Drama (dramatic works by one author)", "Picture-writing in literature", "Readers (Primary)", "Readers for new literates", "Adventure and adventurers, fiction", "Ballenas", "Ficci\u00f3n", "Capitanes de barcos", "Enfermos mentales", "Naufragios", "Cuentos de mar", "Novela psicol\u00f3gica", "Romance literature", "Epic literature", "Adventure fiction", "Allegories", "Whalers (Persons)", "Drama", "Revenge", "Prohibition", "Achab (Personnage fictif)", "Romans, nouvelles", "Capitaines de navire", "Personnes vivant avec un trouble de sant\u00e9 mentale", "Action & Adventure", "Walfang", "Moby Dick (Melville, Herman)", "Fiction, historical, general", "Fiction, general", "Moby Dick", "Literatura infantil", "Ahab, captain (fictitious character)", "Ahab, captain (fictitious character)--fiction", "Whales--fiction", "Whaling--fiction", "Ps2384.m6 m45 1992", "813/.3", "Whaling ships--fiction", "Ship captains--fiction", "Mentally ill--fiction", "Ps2384 .m6 2001c", "Shipwrecks--fiction", "Sailors--fiction", "Ps2384 .m6 2003b", "Melville, herman , 1819-1891", "Ps2384 .m6 2002", "Comics & graphic novels, general", "Comic books, strips, etc.", "Fate and fatalism", "Symbolism", "R\u00e9cits de mer"], "subject_people": ["Herman Melville (1819-1891)", "Ishmael", "Ahab", "Starbuck", "Stubb", "Flask", "Queequeg", "Tashtego", "Daggoo", "Fedallah", "Pip"], "key": "/works/OL102749W", "authors": [{"author": {"key": "/authors/OL29497A"}, "type": {"key": "/type/author_role"}}], "excerpts": [{"pages": "1", "excerpt": "Call me Ishmael."}, {"excerpt": "All my means are sane, my motive and my object mad.", "comment": "Chapter XLI: Moby Dick", "author": {"key": "/people/huginn4126"}}, {"excerpt": "Consider the subtleness of the sea; how its most dreaded creatures glide under water, unapparent for the most part, and treacherously hidden beneath the loveliest tints of azure.", "comment": "Chapter LVIII: Brit", "author": {"key": "/people/huginn4126"}}], "type": {"key": "/type/work"}, "identifiers": {"wikidata": ["Q174596"], "bookbrainz": ["ea920175-d12c-40a4-8d40-135f9e2cc0c4"], "musicbrainz": ["fc85fbd0-18dc-4bf0-b3ff-6c77142379f7"]}, "subject_times": ["19th century"], "links": [{"title": "Moby-Dick - Wikipedia", "url": "https://en.wikipedia.org/wiki/Moby-Dick", "type": {"key": "/type/link"}}], "latest_revision": 119, "revision": 119, "created": {"type": "/type/datetime", "value": "2009-10-17T18:36:50.013545"}, "last_modified": {"type": "/type/datetime", "value": "2025-03-09T23:45:24.942698"}}

src/handlers/scrapers/testdata/OL1063588W.json

@@ -0,0 +1 @@
+{"authors": [{"author": {"key": "/authors/OL107571A"}, "type": {"key": "/type/author_role"}}], "cover_edition": {"key": "/books/OL11970756M"}, "covers": [12721865, 225607, 979889, 962622, 381914, 965195, 816172, 5080730, 105920, 106261, 645758, 3352368, 2913513, 6300619, 834670, 1257936, 7202179, 7328726, 8017278, 8357212, 10779617, 297595, 6524471, 2950202, 869658, 5785176, 759391, 405725, 5082382, 5734331, 794540, 2273606, 3344462, 2272149, 5015308, 5252976, 2269304, 3361222, 3333202, 419634, 8237834, 8866132, 8746787, 8787197, 10328975, 10450686, 10583450, 10583449, 12328148, 12341429, 12341814, 12374998, 6592758, 6615218, 6844264, 6844154, 6845448, 6854914, 6857079, 6857116, 6857137, 6857141, 6857930, 6873181, 6879761, 6879762, 6879779, 6879780, 6879781, 6883936, 6883937, 6883939, 6883940, 6883941, 6897133, 6988502, 6988503, 7296717, 7898885, 8050404, 8063799, 8648976, 8673384, 10054133, 12174652, 8752310, 10551221, 10693181, 11025756, 11299168, 11892668, 12712638, 10943098, 12584939, 10990799, 8775073, 8360722, 12588580, 8992357, 7533791], "description": {"type": "/type/text", "value": "In this story of the trials of the peasant Jean Valjean--a man unjustly imprisoned, baffled by destiny, and hounded by his nemesis, the magnificently realized, ambiguously malevolent police detective Javert--Hugo achieves the sort of rare imaginative resonance that allows a work of art to transcend its genre."}, "first_publish_date": "1863", "identifiers": {"wikidata": ["Q180736"]}, "key": "/works/OL1063588W", "subject_people": ["Jean Valjean", "Javert", "Javier Pe\u00f1alosa"], "subject_places": ["Toulon", "(France)", "Paris (France)", "Toulon Prison", "Francia", "Paris (Francia)", "Fa guo"], "subject_times": ["19th century", "19de eeu", "1815", "1832", "Siglo XIX", "July Revolution, 1830", "Jin dai"], "subjects": ["Poor", "Fiction", "Readers", "Fiction, historical", "France, fiction", "Ex-convicts", "History", "Historical fiction", "Manners and customs in fiction", "French fiction", "Continental european fiction (fictional works by one author)", "Ex-convicts in fiction", "Orphans in fiction", "Orphans", "Literature and fiction (general)", "Javert, inspector (fictitious character), fiction", "Classic Literature", "Fiction, political", "Social conditions", "Social life and customs", "Juvenile fiction", "Franse fiksie", "Leesboeke", "Franse taal", "French language", "Fiction, historical, general", "France in fiction", "Open Library Staff Picks", "Stuttering", "freedom", "redemption", "prison", "bread", "barricade", "Los", "Miserables", "Francia", "Paris (france), fiction", "Politics and government", "Graphic novels", "Comic books, strips", "Cartoons and comics", "Mis\u00e9rables (Hugo, Victor)", "Fiction, general", "Valjean, jean (fictitious character), fiction", "Police", "Vida social y costumbres", "Presos", "Polic\u00eda", "Ficci\u00f3n", "Hu\u00e9rfanos", "July Revolution (France : 1830) fast (OCoLC)fst01353472", "Literature", "FICTION / Classics", "FICTION / Historical", "Epic fiction", "Manners and customs", "Unterschicht", "French language materials", "French literature", "Electronic books", "Music, instruction and study", "Javert (Fictitious character)", "Fiction, fantasy, epic", "Suo xie", "Chang pian xiao shuo", "Children's fiction", "Adventure and adventurers, fiction", "Rescue work, fiction", "Police, fiction", "Romance literature", "Ex-d\u00e9tenus", "Romans, nouvelles", "M\u0153urs et coutumes", "Criminals", "Translations into English", "Prisoners", "Inspector Javert (Fictitious character)", "Long Now Manual for Civilization"], "title": "Les Mis\u00e9rables", "type": {"key": "/type/work"}, "latest_revision": 32, "revision": 32, "created": {"type": "/type/datetime", "value": "2009-12-09T20:20:25.168304"}, "last_modified": {"type": "/type/datetime", "value": "2025-03-25T05:29:26.658695"}}