Updated, added local testing.

Local testing uses httpbin image.

Cargo.toml

@@ -9,5 +9,8 @@ clap = { version = "4.5.34", features = ["derive"] }
 reqwest = { version = "0.12.15", features = ["blocking", "brotli", "deflate", "gzip"] }
 url = "2.5.4"
 
+[features]
+local-httpbin = []
+
 [dev-dependencies]
 testdir = "0.9.3"

docker-compose.yaml

@@ -0,0 +1,7 @@
+
+services:
+    httpbin:
+      image: kennethreitz/httpbin
+      ports:
+        - "80:80"
+

src/cache.rs

@@ -1,6 +1,6 @@
 // use sha256;
 use anyhow::{Context, Result, bail};
-use std::fs::{File, remove_file, read_dir, create_dir_all};
+use std::fs::{File, create_dir_all, read_dir, remove_file};
 use std::io::{BufRead, BufReader, Write};
 use std::path::PathBuf;
 use std::time::{Duration, SystemTime};
@@ -62,7 +62,7 @@ pub fn filename_from_url(url: &str) -> Result<String> {
 ///
 /// This also stores the url in the file, so I know what URL was called for
 /// this reqwest.
-/// 
+///
 /// It has each item on a single line:
 /// header: value
 /// The first line will be url: (Which is not part of original header.)
@@ -128,7 +128,7 @@ pub struct Cache {
 }
 
 // Should I also have std::io::Errors in here as well?
-// I can have File IO errors. 
+// I can have File IO errors.
 
 /// Status of fetch
 #[allow(dead_code)]
@@ -149,6 +149,18 @@ pub enum Status {
     ErrorStatus(u16),
 }
 
+impl Status {
+    /// Return pathbuf, if Status was success.
+    pub fn download_path(&self) -> Option<&PathBuf> {
+        match self {
+            Status::Fetched(path) | Status::Cached(path) => {
+                return Some(path);
+            }
+            _ => None,
+        }
+    }
+}
+
 /*
 Some possible content-type values:  We're only interested in a few of these...
 
@@ -285,7 +297,7 @@ impl Cache {
                                 }
                             }
                         }
-                    } 
+                    }
 
                     /*
                     if let Ok(access) = d.accessed() {
@@ -359,19 +371,22 @@ impl Cache {
     }
 
     /// Remove an extension from the filename.
-    /// 
+    ///
     /// Given something.tar.gz.header return something.tar.gz
     fn remove_from_filename(path: &mut PathBuf) {
         let filename = Self::pathbuf_filename(path);
         if let Some(parts) = filename.rsplit_once(".") {
             path.set_file_name(parts.0);
         } else {
-            panic!("Unable to locate the trailing extension . from: {}", path.display());
+            panic!(
+                "Unable to locate the trailing extension . from: {}",
+                path.display()
+            );
         }
     }
 
     /// Fetch, without using the cache.
-    /// 
+    ///
     /// This deletes the .header cache file, which forces a fetch.
     #[allow(dead_code)]
     pub fn fetch_nocache(&self, url: &str) -> Status {
@@ -386,7 +401,7 @@ impl Cache {
         }
         return self.fetch(url);
     }
-    
+
     // I'm not sure about using Result<Status> here...
     // It would allow for ? usage.
 
@@ -499,19 +514,81 @@ impl Cache {
             return Status::Fetched(fp);
             */
         } else {
-            // Status error?!
-            println!("Error {} {}", result.status(), url);
+            // Status error
+            // println!("Error {} {}", result.status(), url);
             return Status::ErrorStatus(u16::from(result.status()));
         }
     }
 }
 
+/*
+https://httpbin.org/anything
+/headers
+/ip
+/user-agent
+/status/404
+/status/200
+/cache/value for cache-control
+/cache (if-modified-since or if-none-match are present, returns 304)
+/etag/value for etag (if-none-match or if-match)
+/uuid
+
+/brotli
+/deflate
+/gzip
+
+^ I wonder what happens if I request one that isn't enabled in reqwest?
+
+ */
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::collections::HashMap;
     use testdir::testdir;
 
     #[test]
+    fn relative_test() {
+        let rel_abs: HashMap<(&str, &str), &str> = HashMap::from([
+            (
+                ("http://meow.org/rabbit", "/llama/index.html"),
+                "http://meow.org/llama/index.html",
+            ),
+            (
+                ("https://example.com/dir/index.html", "about.html"),
+                "https://example.com/dir/about.html",
+            ),
+            (
+                ("https://example.com/dir/index.html", "../and/about.html"),
+                "https://example.com/and/about.html",
+            ),
+            (
+                ("https://here.com/dir/index.html", "http://there.com/about.html"),
+                "http://there.com/about.html",
+            ),
+        ]);
+        for (base, url) in rel_abs {
+            if let Ok(abs) = relative_to_absolute(base.0, base.1) {
+                assert_eq!(abs, url);
+            } else {
+                panic!("Failed {} + {} = {}", base.0, base.1, url);
+            }
+        }
+    }
+
+    #[test]
+    fn url_to_filename_test() {
+        let mut dir = testdir!();
+        dir.push("cache");
+        let cache = Cache::new(dir, None).unwrap();
+        // url_to_basename
+        // filename_for_url
+        // append_to_filename
+        // remove_from_filename
+    }
+
+    #[test]
+    #[cfg(not(feature = "local-httpbin"))]
     fn cache_fetch() {
         let mut dir = testdir!();
         dir.push("cache");
@@ -540,4 +617,76 @@ mod tests {
 
         // println!("Dir: {:?}, Status: {:?}", t, r); // r has been partially moved.
     }
+
+    /*
+    Add to Config.toml:
+    [features]
+    local-httpbin = []
+
+    Use:
+    #[test]
+    #[cfg(feature = "local-httpbin")]
+
+    And then:
+    cargo test -F local-httpbin -- --show-output
+
+    This runs the local httpbin tests.
+     */
+
+    #[test]
+    #[cfg(feature = "local-httpbin")]
+    fn call_local() {
+        let mut dir = testdir!();
+        dir.push("cache");
+
+        // Make a copy of the cache directory PathBuf for verifying paths.
+        let mut t = dir.clone();
+
+        let cache = Cache::new(dir, None).unwrap();
+        let teapot_url = "http://127.0.0.1/status/418";
+
+        let r = cache.fetch(teapot_url);
+        if let Status::ErrorStatus(code) = r {
+            assert_eq!(code, 418);
+        } else {
+            panic!("Not an ErrorStatus");
+        }
+        println!("{:?}", r);
+
+        /*
+        I disabled brotli in the Client builder.
+        I get an error below about invalid UTF-8.  The httpbin server isn't smart
+        enough to see I don't support it, and sends it anyway.  :(
+
+        Probably because things like brotli (br) are usually only sent over https.
+        */
+
+        /*
+        let brot_url = "http://127.0.0.1/brotli";
+        let r = cache.fetch(brot_url);
+        println!("Brotli: {:?}", r);
+
+        if let Status::Fetched(path) = r {
+                let data = std::fs::read_to_string(path).unwrap();
+                println!("DATA:\n{}", data);
+        }
+        */
+    }
+
+    #[test]
+    #[cfg(feature = "local-httpbin")]
+    fn cache_local() {
+        let mut dir = testdir!();
+        dir.push("cache");
+
+        // Make a copy of the cache directory PathBuf for verifying paths.
+        let mut t = dir.clone();
+
+        let cache = Cache::new(dir, None).unwrap();
+        let etag_url = "http://127.0.0.1/etag/meow";
+
+        let r = cache.fetch(etag_url);
+        let r2 = cache.fetch(etag_url);
+        println!("{:?}\n{:?}", r, r2);
+    }
 }

src/main.rs

@@ -10,14 +10,6 @@ use std::process::Command;
 
 mod cache;
 
-// I'm not sure if reqwest::blocking's write_to is working right or not.
-// It seems like it might be storing the entire file in memory...
-// Which I don't want.  I might have to go back to tokio and chunk.
-
-// see reqwest/web-o/src/cache.rs for example cache
-// It restores reqwest::header::HeaderMap
-// (which allows duplicates... and ignores case on keys)
-
 #[derive(Parser)]
 #[command(
     about = "Go Updater",
@@ -33,10 +25,7 @@ struct Cli {
     #[arg(short, long, default_value = "cache")]
     cache: PathBuf,
 
-    // If I make this optional, I can't seem to get the display_help code
-    // to work now.  Not sure why.
     #[command(subcommand)]
-    // command: Commands,
     command: Option<Commands>,
 }
 
@@ -49,18 +38,26 @@ enum Commands {
 }
 
 /// Query `go env`
+///
+/// This is a good example of parsing output line by line.
 #[must_use]
 fn get_go_env() -> Result<HashMap<String, String>> {
     let output = Command::new("go").arg("env").output()?;
     if output.status.success() {
         let mut result = HashMap::<String, String>::new();
-        for mut line in String::from_utf8_lossy(output.stdout.as_slice()).split("\n") {
+
+        let output_str = std::str::from_utf8(output.stdout.as_slice())?;
+        for mut line in output_str.split("\n") {
             line = line.trim();
             if let Some(parts) = line.split_once("=") {
                 let value = parts.1.trim_matches('\'');
                 result.insert(parts.0.to_string(), value.to_string());
             }
         }
+        /*
+        for mut line in String::from_utf8_lossy(output.stdout.as_slice()).split("\n") {
+        }
+        */
         Ok(result)
     } else {
         bail!("Failed to query `go env`");
@@ -94,12 +91,12 @@ fn find_go() -> Result<String> {
 }
 
 const GO_URL: &str = "https://go.dev/dl/";
-// const GO_FILE: &str = "go-dl.html";
-
-// 2 MB download of html...
 
 // static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
 
+// I've changed the parser, so scrapy isn't needed.
+// I'll revisit this when I get further with html tokeparser.
+
 /*
 CSS Selector elements of interest:  (a href="" part).
 href are relative...
@@ -122,35 +119,11 @@ and grab the section of td's.  class=filename has a href, last has SHA256.
 <a class="download" href="/dl/go1.24.0.linux-arm64.tar.gz">go1.24.0.linux-arm64.tar.gz</a>
 <a class="download" href="/dl/go1.24.0.windows-amd64.zip">go1.24.0.windows-amd64.zip</a>
 <a class="download" href="/dl/go1.24.0.windows-amd64.msi">go1.24.0.windows-amd64.msi</a>
-
  */
 
-/*
-fn download_and_save(url: &str, filename: &str) -> Result<()> {
-    let client = reqwest::blocking::Client::builder()
-        .user_agent(APP_USER_AGENT)
-        .build()?;
-    print!("Downloading: {url} ");
-    let _ = stdout().flush();
-
-    let mut resp = client.get(url).send().context("Failed get")?;
-
-    if resp.status().is_success() {
-        let mut file =
-            File::create(filename).with_context(|| format!("Creating file {filename} failed."))?;
-        resp.copy_to(&mut file)?;
-    } else {
-        bail!("Status Code: {:?}", resp.status());
-    }
-    println!("OK");
-
-    Ok(())
-}
-*/
-
 // URL: https://go.dev/dl/go1.24.1.linux-amd64.tar.gz
 #[must_use]
-/// Get go version from download URL.
+/// Get go version from download link.
 fn version_from_url(url: &str, arch: &str) -> Option<String> {
     if let Some(parts) = url.split_once(arch) {
         if let Some(part) = parts.0.rsplit_once("/go") {
@@ -161,17 +134,7 @@ fn version_from_url(url: &str, arch: &str) -> Option<String> {
     None
 }
 
-#[must_use]
-/// Get go version from `go version` output.
-fn version_from_go(text: &str) -> Option<String> {
-    let parts: Vec<&str> = text.split(' ').collect();
-    if parts.len() == 4 {
-        return Some(parts[2].to_string().replace("go", ""));
-    }
-    None
-}
-
-/// Return just the href="<return this part>".
+/// Return just the href="<return just this part>".
 #[must_use]
 fn just_href(link: &str) -> Result<String> {
     let parts = link.split_once("href=\"").unwrap_or(("", "")).1;
@@ -182,26 +145,28 @@ fn just_href(link: &str) -> Result<String> {
     bail!("Unable to locate href");
 }
 
-/// Find a href link for given arch (architecture)
+/// Find a href link for given os and arch (architecture)
 ///
 /// Look for <a class="download" href="
 #[must_use]
-fn find_arch_link(arch: &str, fp: &File) -> Result<String> {
+fn find_arch_link(os_arch: &str, fp: &File) -> Result<String> {
     let reader = BufReader::new(fp);
     for line in reader.lines() {
         if let Ok(line) = line {
             if line.contains("a class=\"download\"") {
-                if line.contains(arch) {
+                if line.contains(os_arch) {
                     // Return just the href part.
                     return just_href(&line);
                 }
             }
         }
     }
-    bail!("Unable to locate architecture download link");
+    bail!("Unable to locate OS architecture download link");
 }
 
 /*
+// This is terrible, since it has all of the HTML in a string.
+
 /// find_link for given arch (architecture)
 ///
 /// Look for <a class="download" href=""
@@ -226,9 +191,9 @@ fn find_link(arch: &str) -> Result<String> {
 */
 
 /// Get value from given HashMap<String,String>
-/// 
+///
 /// This gives us a String, not a &String, and not an Option.
-fn hashget(hash: &HashMap<String,String>, key: &str) -> String {
+fn hashget(hash: &HashMap<String, String>, key: &str) -> String {
     if let Some(value) = hash.get(key) {
         return value.clone();
     }
@@ -243,7 +208,7 @@ fn main() -> Result<()> {
     This seems to be managed by go now.  So running ```go env``` would
     have/show GOROOT being set by that instance of go.
     GOPATH is another story.  It's where go stores things it downloads from
-    the web, and most certainly shouldn't be $HOME/go (!= GOROOT).
+    the web, and most certainly shouldn't be $HOME/go != GOROOT.
 
     I want to be a bit looser on requiring GOROOT later on.
     */
@@ -252,8 +217,7 @@ fn main() -> Result<()> {
     let go_path = env::var("GOPATH").unwrap_or(String::new());
     let go_root = env::var("GOROOT").unwrap_or(String::new());
 
-    let go_env = get_go_env()
-        .context("Calling `go env` failed.")?;
+    let go_env = get_go_env().context("Calling `go env` failed.")?;
     let mut go_version = hashget(&go_env, "GOVERSION");
     if go_version.starts_with("go") {
         go_version.remove(0);
@@ -264,10 +228,9 @@ fn main() -> Result<()> {
     let go_os = hashget(&go_env, "GOOS");
     let go_path = hashget(&go_env, "GOPATH");
     let go_root = hashget(&go_env, "GOROOT");
+    // Construct OS-ARCH value.
     let go_os_arch = format!("{go_os}-{go_arch}");
 
-    println!("{} - {}-{}", go_version, go_os, go_arch);
-
     // Initialize the cache.
     let cache = cache::Cache::new(cli.cache, None)?;
 
@@ -279,12 +242,53 @@ fn main() -> Result<()> {
             // Since the go.dev site doesn't allow caching or knowing when it changes,
             // we always end up in Status::Fetched arm.
 
+            if let Some(filename) = status.download_path() {
+                // Make this function the same whether or not we have a cache hit.
+
+                let fp = File::open(filename)?;
+                let link = find_arch_link(&go_os_arch, &fp);
+
+                if let Ok(relative) = link {
+                    // Download link for arch located.  Make absolute URL.
+                    let latest_version = version_from_url(&relative, &go_os_arch);
+                    if let Some(latest) = latest_version {
+                        println!("Version: {} [have {}]", latest, go_version);
+                        if go_version != latest {
+                            println!("Downloading newer version...");
+                            let abs = relative_to_absolute(GO_URL, &relative).unwrap();
+
+                            let latest_status = cache.fetch(&abs);
+                            if let Some(update) = latest_status.download_path() {
+                                // Ok, we have the update!  Now what?
+                                println!("Ready to install update.");
+
+                            } else {
+                                println!("Download failed: {:?}", latest_status);
+                            }
+                            // println!("Latest: {:?}", latest_status);
+                        } else {
+                            println!("You're already good to GO.");
+                        }
+                    } else {
+                        println!("Finding version failed: [{}]", relative);
+                    }
+                } else {
+                    bail!("Unable to locate download link");
+                }
+            } else {
+                println!("Status = {:?}", status);
+            }
+            /*
             match status {
-                cache::Status::Fetched(filename) => {
+                cache::Status::Fetched(filename) |
+                cache::Status::Cached(filename) => {
+                    // Make this function the same whether or not we have a cache hit.
+
                     let fp = File::open(filename)?;
                     let link = find_arch_link(&go_os_arch, &fp);
+
                     if let Ok(relative) = link {
-                        // Download link for arch located.  Make useable.
+                        // Download link for arch located.  Make absolute URL.
                         let abs = relative_to_absolute(GO_URL, &relative).unwrap();
                         println!("URL: {}", abs);
 
@@ -295,27 +299,21 @@ fn main() -> Result<()> {
                                 println!("Downloading newer version...");
                                 let latest_status = cache.fetch(&abs);
                                 println!("Latest: {:?}", latest_status);
+                            } else {
+                                println!("You're already good to GO.");
                             }
                         } else {
-                            println!("Finding version failed for string: [{}]", abs);
+                            println!("Finding version failed: [{}]", abs);
                         }
                     } else {
                         bail!("Unable to locate download link");
                     }
                 }
-                cache::Status::Cached(filename) => {
-                    println!("(from cache)"); // I wish I could see this.
-                    let fp = File::open(filename)?;
-                    let link = find_arch_link(&go_os_arch, &fp);
-                    if let Ok(relative) = link {
-                        let abs = relative_to_absolute(GO_URL, &relative).unwrap();
-                        println!("URL: {}", abs);
-                    }
-                }
                 _ => {
                     println!("Status = {:?}", status);
                 }
             }
+            */
         }
         Some(Commands::Info {}) => {
             println!("GOPATH      {}", go_path);