|
|
@@ -1,45 +1,334 @@
|
|
|
use reqwest;
|
|
|
-use std::{fs::File, path::Path, io::Read};
|
|
|
|
|
|
-use scraper::{Html, Selector};
|
|
|
+// use std::io::Write;
|
|
|
+// use std::io::Read;
|
|
|
+
|
|
|
+use scraper;
|
|
|
+use std::{
|
|
|
+ collections::HashMap,
|
|
|
+ fs::File,
|
|
|
+ io::Write,
|
|
|
+ path::{Path, PathBuf},
|
|
|
+ sync::LazyLock,
|
|
|
+};
|
|
|
+
|
|
|
+use clap::{Parser, Subcommand};
|
|
|
+use std::{thread, time::Duration};
|
|
|
+
|
|
|
+#[derive(Parser)]
|
|
|
+#[command(about, long_about=None)]
|
|
|
+struct Cli {
|
|
|
+ /// Working directory
|
|
|
+ #[arg(short, long, default_value = "bible")]
|
|
|
+ work: PathBuf,
|
|
|
+
|
|
|
+ /// Bible Version
|
|
|
+ #[arg(short, long, default_value = "ESV")]
|
|
|
+ bible: String,
|
|
|
+
|
|
|
+ #[command(subcommand)]
|
|
|
+ command: Option<Commands>,
|
|
|
+}
|
|
|
+
|
|
|
+#[derive(Subcommand)]
|
|
|
+enum Commands {
|
|
|
+ /// Fetch from the web, using work directory for cache
|
|
|
+ Fetch {
|
|
|
+ /// Delay
|
|
|
+ #[arg(short, long, default_value = "10")]
|
|
|
+ delay: u32,
|
|
|
+ },
|
|
|
+ /// Extract information from cached files
|
|
|
+ Extract {},
|
|
|
+ /// Test something out
|
|
|
+ Test {},
|
|
|
+}
|
|
|
|
|
|
static APP_USER_AGENT: &str =
|
|
|
"Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0";
|
|
|
-static URL: &str = "https://www.bible.com/bible/59/GEN.1.ESV";
|
|
|
+
|
|
|
+static VERSION_URLS: LazyLock<HashMap<&str, &str>> = LazyLock::new(|| {
|
|
|
+ HashMap::from([
|
|
|
+ ("ESV", "https://www.bible.com/bible/59/GEN.1.ESV"),
|
|
|
+ ("KJV", "https://www.bible.com/bible/1/GEN.1.KJV"),
|
|
|
+ ("NIV", "https://www.bible.com/bible/111/GEN.1.NIV"),
|
|
|
+ ("YLT98", "https://www.bible.com/bible/821/GEN.1.YLT98"),
|
|
|
+ ])
|
|
|
+});
|
|
|
+
|
|
|
+// Start here
|
|
|
+// static URL: &str = "https://www.bible.com/bible/59/PSA.95.ESV";
|
|
|
+// "https://www.bible.com/bible/59/GEN.1.ESV";
|
|
|
+
|
|
|
+// And maybe:
|
|
|
+// https://www.bible.com/bible/2692/GEN.1.NASB2020
|
|
|
+// https://www.bible.com/bible/1/GEN.1.KJV
|
|
|
+// https://www.bible.com/bible/2692/GEN.1.NASB2020
|
|
|
+// https://www.bible.com/bible/111/GEN.1.NIV
|
|
|
+// https://www.bible.com/bible/821/GEN.1.YLT98
|
|
|
+// Catholic
|
|
|
+// https://www.bible.com/bible/42/GEN.1.CPDV
|
|
|
+
|
|
|
+// Audio
|
|
|
+// https://www.bible.com/audio-bible/59/GEN.1.ESV
|
|
|
+// <script type="application/ld+json">{"@context":"https://schema.org","@type":"AudioObject","mainEntityOfPage":{"@type":"WebPage","@id":"https://www.bible.com/audio-bible/59/GEN.1.ESV"},"headline":"Audio Bible: Listen to Genesis 1 English Standard Version 2016 ESV","contentUrl":"https://audio-bible-cdn.youversionapi.com/1/32k/GEN/1-9dcefc68c6f7244489f59475fc7a1866.mp3?version_id=59",
|
|
|
+
|
|
|
+// https://www.bible.com/verse-of-the-day
|
|
|
+
|
|
|
+struct FetchResult {
|
|
|
+ cached: bool,
|
|
|
+ html: String,
|
|
|
+}
|
|
|
+
|
|
|
+fn fetch_cache(client: &reqwest::blocking::Client, url: &str) -> FetchResult {
|
|
|
+ let (_, filename) = url.rsplit_once('/').unwrap();
|
|
|
+ let path = Path::new("bible").join(filename);
|
|
|
+
|
|
|
+ if path.exists() {
|
|
|
+ // File already exists -- use cached version.
|
|
|
+ let buffer = std::fs::read_to_string(path).unwrap();
|
|
|
+ return FetchResult {
|
|
|
+ cached: true,
|
|
|
+ html: buffer,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ println!("fetch_cache {} => {}", url, filename);
|
|
|
+ let res = client.get(url).send().unwrap();
|
|
|
+ let buffer = res.text().unwrap();
|
|
|
+
|
|
|
+ let mut file = File::create(path).unwrap();
|
|
|
+ let _ = file.write_all(buffer.as_bytes());
|
|
|
+ FetchResult {
|
|
|
+ cached: false,
|
|
|
+ html: buffer,
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
fn main() {
|
|
|
+ let cli = Cli::parse();
|
|
|
+ // println!("Work Dir: {:?}", cli.work);
|
|
|
+ // println!("Bible: {:?}", cli.bible);
|
|
|
+
|
|
|
+ if !VERSION_URLS.contains_key(cli.bible.as_str()) {
|
|
|
+ println!("Sorry, I don't know about Bible [{}].", cli.bible);
|
|
|
+ println!("I do know about the following:");
|
|
|
+
|
|
|
+ // Keys sorted in order.
|
|
|
+ for (name, _) in VERSION_URLS.iter() {
|
|
|
+ println!(" {}", name);
|
|
|
+ }
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ match &cli.command {
|
|
|
+ Some(Commands::Fetch { delay }) => {
|
|
|
+ let client = reqwest::blocking::Client::builder()
|
|
|
+ .user_agent(APP_USER_AGENT)
|
|
|
+ .build()
|
|
|
+ .unwrap();
|
|
|
+ let mut url = VERSION_URLS[cli.bible.as_str()].to_string();
|
|
|
+ println!("Fetch! [{}] with delay {} secs.", cli.bible, delay);
|
|
|
+ let mut more = true;
|
|
|
+
|
|
|
+ while more {
|
|
|
+ let result = fetch_cache(&client, url.as_str());
|
|
|
+ more = false;
|
|
|
+
|
|
|
+ let document = scraper::Html::parse_document(&result.html);
|
|
|
+
|
|
|
+ // TO FIX
|
|
|
+ // We want to upgrade this to use CSS selectors.
|
|
|
+ // For now, us the "working" code we have.
|
|
|
+ let a_selector = scraper::Selector::parse("div>a").unwrap();
|
|
|
+ for a in document.select(&a_selector) {
|
|
|
+
|
|
|
+ // Skip elements with a class attribute
|
|
|
+ if a.attr("class").is_some() {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ if let Some(href) = a.attr("href") {
|
|
|
+ if href.contains("/bible/") {
|
|
|
+ let text = a.text().collect::<Vec<_>>();
|
|
|
+
|
|
|
+ if text.len() != 1 {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if text[0] != "Next Chapter" {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Ok! We've found the Next Chapter a element!
|
|
|
+ if href.starts_with("/") {
|
|
|
+ url = String::from("https://www.bible.com") + href;
|
|
|
+ } else {
|
|
|
+ url = href.to_string();
|
|
|
+ }
|
|
|
+
|
|
|
+ // println!("Found HREF: {} => {}", href, url);
|
|
|
+ // panic!("Squirrel alert!");
|
|
|
+ more = true;
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if more {
|
|
|
+ if !result.cached {
|
|
|
+ thread::sleep(Duration::from_secs(*delay as u64));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ println!("I'm finished fetching!");
|
|
|
+ }
|
|
|
+ Some(Commands::Extract {}) => {
|
|
|
+ println!("Extract...");
|
|
|
+ }
|
|
|
+ Some(Commands::Test {}) => {
|
|
|
+ println!("Testing...");
|
|
|
+ }
|
|
|
+ None => {
|
|
|
+ println!("Looking for FETCH or EXTRACT");
|
|
|
+ println!("I've got nothing to do here...");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ return;
|
|
|
+
|
|
|
+
|
|
|
let client = reqwest::blocking::Client::builder()
|
|
|
.user_agent(APP_USER_AGENT)
|
|
|
.build()
|
|
|
.unwrap();
|
|
|
|
|
|
- // let res = client.get("https://httpbin.org/anything").send().unwrap();
|
|
|
- // println!("anything: {}", res.text().unwrap());
|
|
|
+ if true {
|
|
|
+ let mut url = String::from(URL);
|
|
|
+ let mut previous_url = url.clone();
|
|
|
+ let mut working = true;
|
|
|
|
|
|
- let mut file = File::open(Path::new("fetch1.html")).unwrap();
|
|
|
- let mut buffer = String::new();
|
|
|
- let _ = file.read_to_string(&mut buffer);
|
|
|
- drop(file);
|
|
|
+ while working {
|
|
|
+ // Begin the fetching process...
|
|
|
+ let result = fetch_cache(&client, url.as_str());
|
|
|
|
|
|
- /*
|
|
|
- let res = client.get(URL).send().unwrap();
|
|
|
- let buffer = res.text().unwrap();
|
|
|
- println!("{}", res.text().unwrap());
|
|
|
- */
|
|
|
+ working = false;
|
|
|
+
|
|
|
+ // Ok, HTML, get parsing!
|
|
|
+ let document = scraper::Html::parse_document(&result.html);
|
|
|
+
|
|
|
+ // For now, I don't care about parsing, just crawling/saving.
|
|
|
+ // Locate the a href.
|
|
|
+ let a_selector = scraper::Selector::parse("div>a").unwrap();
|
|
|
+ for a in document.select(&a_selector) {
|
|
|
+ let c = a.attr("class");
|
|
|
+ if c.is_some() {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ let href = a.attr("href");
|
|
|
|
|
|
- let document = scraper::Html::parse_document(&buffer);
|
|
|
- let h1_selector = scraper::Selector::parse("h1").unwrap();
|
|
|
+ if href.is_some() {
|
|
|
+ let href = href.unwrap();
|
|
|
+ if href.contains("/bible/") {
|
|
|
+ let text = a.text().collect::<Vec<_>>();
|
|
|
|
|
|
- let h1 = document.select(&h1_selector).next().unwrap();
|
|
|
- println!("h1 = {:?}", h1.text().collect::<Vec<_>>());
|
|
|
+ if text.len() != 1 {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if text[0] != "Next Chapter" {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
- let span_selector = scraper::Selector::parse("span").unwrap();
|
|
|
- for span in document.select(&span_selector) {
|
|
|
- if span.attr("data-usfm").is_some() {
|
|
|
- println!("span: {:?}", span.text().collect::<Vec<_>>());
|
|
|
+ // previous_url = url;
|
|
|
+ url = href.to_string();
|
|
|
+ if url.starts_with("/") {
|
|
|
+ url = String::from("https://www.bible.com") + &url;
|
|
|
+ }
|
|
|
+
|
|
|
+ working = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if working {
|
|
|
+ if !result.cached {
|
|
|
+ // Don't sleep if the results we just got were from the cache.
|
|
|
+ thread::sleep(Duration::from_secs(10));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ println!("I'm finished.");
|
|
|
+ } else {
|
|
|
+ // let res = client.get("https://httpbin.org/anything").send().unwrap();
|
|
|
+ // println!("anything: {}", res.text().unwrap());
|
|
|
+
|
|
|
+ let mut file = File::open(Path::new("fetch1.html")).unwrap();
|
|
|
+ let mut buffer = String::new();
|
|
|
+ let _ = file.read_to_string(&mut buffer);
|
|
|
+ drop(file);
|
|
|
+
|
|
|
+ /*
|
|
|
+ let res = client.get(URL).send().unwrap();
|
|
|
+ let buffer = res.text().unwrap();
|
|
|
+ println!("{}", res.text().unwrap());
|
|
|
+ */
|
|
|
+
|
|
|
+ let document = scraper::Html::parse_document(&buffer);
|
|
|
+ let h1_selector = scraper::Selector::parse("h1").unwrap();
|
|
|
+
|
|
|
+ let h1 = document.select(&h1_selector).next().unwrap();
|
|
|
+ println!("h1 = {:?}", h1.text().collect::<Vec<_>>());
|
|
|
+
|
|
|
+ // Selectors can't match on "if it exists" or
|
|
|
+ // return a matching value, type fields.
|
|
|
+
|
|
|
+ let span_selector = scraper::Selector::parse("span").unwrap();
|
|
|
+ for span in document.select(&span_selector) {
|
|
|
+ let data = span.attr("data-usfm");
|
|
|
+
|
|
|
+ if data.is_some() {
|
|
|
+ // Ok, iterate over these items...
|
|
|
+ for data_span in span.select(&span_selector) {
|
|
|
+ let d = data_span.attr("class");
|
|
|
+ // println!("data_span: {:?}", data_span);
|
|
|
+ if d.is_some() {
|
|
|
+ let dt = d.unwrap();
|
|
|
+ if dt.contains("content") {
|
|
|
+ let text = data_span.text().collect::<Vec<_>>();
|
|
|
+ println!("{} {:?}", data.unwrap(), text)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ GEN.1.25 ["And God made the beasts of the earth according to their kinds and the livestock according to their kinds, and everything that creeps on the ground according to its kind. And God saw that it was good."]
|
|
|
+ GEN.1.25 [" "]
|
|
|
+ GEN.1.26 ["Then God said, "]
|
|
|
+ GEN.1.26 ["“Let us make man"]
|
|
|
+ GEN.1.26 [" in our image, "]
|
|
|
+ GEN.1.26 ["after our likeness. And "]
|
|
|
+ GEN.1.26 ["let them have dominion over the fish of the sea and over the birds of the heavens and over the livestock and over all the earth and over every creeping thing that creeps on the earth.”"]
|
|
|
+ GEN.1.26 [" "]
|
|
|
+ GEN.1.27 ["So God created man in his own image,"]
|
|
|
+ GEN.1.27 ["in the image of God he created him;"]
|
|
|
+ GEN.1.27 [" "]
|
|
|
+ GEN.1.27 ["male and female he created them."]
|
|
|
+ GEN.1.27 [" "]
|
|
|
+ */
|
|
|
+
|
|
|
+ /*
|
|
|
+ let mut text = span.text().collect::<Vec<_>>();
|
|
|
+
|
|
|
+ println!("span: {} {:?}", data.unwrap(), text);
|
|
|
+ */
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
// let res = client.get("https://www.bible.com/bible/59/GEN.1.ESV").send().unwrap();
|
|
|
+ */
|
|
|
+
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
@@ -89,11 +378,11 @@ Book/chapter: <h1>
|
|
|
form and void, and darkness was over
|
|
|
the face of the deep. And the Spirit of
|
|
|
God was hovering over the face of the
|
|
|
- waters.</span></span>
|
|
|
+ waters.</span></span>
|
|
|
Next page link:
|
|
|
|
|
|
<div class="[pointer-events:all]">
|
|
|
<a href="/bible/59/GEN.2.ESV">
|
|
|
|
|
|
|
|
|
-*/
|
|
|
+*/
|
