Release v0.23.0

Allow setting a different default user agent

.github/FUNDING.yml

@@ -1 +1,2 @@
 github: [kpcyrd]
+patreon: kpcyrd

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.22.0"
+version = "0.23.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -33,7 +33,7 @@ sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
 sn0int-common = { version="0.13.0", path="sn0int-common" }
-sn0int-std = { version="=0.22.0", path="sn0int-std" }
+sn0int-std = { version="=0.23.0", path="sn0int-std" }
 rustyline = "9.0"
 log = "0.4"
 env_logger = "0.9"
@@ -72,8 +72,8 @@ semver = "1"
 bytes = "0.4"
 bytesize = "1.0"
 ipnetwork = "0.18"
-strum = "0.21"
-strum_macros = "0.21"
+strum = "0.22"
+strum_macros = "0.22"
 embedded-triple = "0.1.0"
 humansize = "1.1.0"
 
@@ -93,7 +93,7 @@ os-version = "0.2"
 caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
 syscallz = "0.16"
-nix = "0.22"
+nix = "0.23"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"

README.md

@@ -14,33 +14,34 @@
 [registry]:     https://sn0int.com/
 
 sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
-package manager. It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
 
 [ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Attempt to bypass cloudflare with shodan
-- Harvest data and images from instagram profiles
-- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
@@ -226,6 +227,9 @@ For everything else please have a look at the [detailed list][1].
   - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)

docs/reference.rst

@@ -504,6 +504,39 @@ Extract exif metadata from an image.
     if last_err() then return end
     debug(exif)
 
+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------
 

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

sn0int-std/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.22.0"
+version = "0.23.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -27,13 +27,13 @@ ct-logs = "0.7"
 chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.8"
+pem = "1"
 url = "2.0"
 tungstenite = { version = "0.13", default-features = false }
 kuchiki = "0.8.0"
 maxminddb = "0.21"
-x509-parser = "0.10"
-der-parser = "5"
+x509-parser = "0.12"
+der-parser = "6"
 publicsuffix = { version="2", default-features=false }
 xml-rs = "0.8"
 geo = "0.18"
@@ -45,6 +45,7 @@ sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
 image = "0.23.0"
 kamadak-exif = "0.5.1"
+img_hash_median = "4.0.0"
 
 bs58 = "0.4"
 digest = "0.9"

sn0int-std/src/gfx/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use image::{self, DynamicImage, GenericImageView};
+pub use img_hash_median::HashAlg;
 
 pub mod exif;
 
@@ -79,6 +80,14 @@ impl Image {
     pub fn height(&self) -> u32 {
         self.image.height()
     }
+
+    pub fn perception_hash(&self, hash_alg: HashAlg) -> String {
+        let hasher = img_hash_median::HasherConfig::new()
+            .hash_alg(hash_alg)
+            .to_hasher();
+        let hash = hasher.hash_image(&self.image);
+        hash.to_base64()
+    }
 }
 
 impl AsRef<DynamicImage> for Image {
@@ -105,7 +114,6 @@ pub fn load(buf: &[u8]) -> Result<Image> {
     })
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;

sn0int-std/src/web.rs

@@ -65,21 +65,21 @@ impl HttpSession {
 
 #[derive(Debug, Default, Deserialize)]
 pub struct RequestOptions {
-    query: Option<HashMap<String, String>>,
-    headers: Option<HashMap<String, String>>,
-    basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
-    json: Option<serde_json::Value>,
-    form: Option<serde_json::Value>,
+    pub query: Option<HashMap<String, String>>,
+    pub headers: Option<HashMap<String, String>>,
+    pub basic_auth: Option<(String, String)>,
+    pub user_agent: Option<String>,
+    pub json: Option<serde_json::Value>,
+    pub form: Option<serde_json::Value>,
     #[serde(default)]
-    follow_redirects: usize,
-    body: Option<String>,
-    timeout: Option<u64>,
+    pub follow_redirects: usize,
+    pub body: Option<String>,
+    pub timeout: Option<u64>,
     #[serde(default)]
-    into_blob: bool,
-    proxy: Option<SocketAddr>,
+    pub into_blob: bool,
+    pub proxy: Option<SocketAddr>,
     #[serde(default)]
-    binary: bool,
+    pub binary: bool,
 }
 
 impl RequestOptions {
@@ -110,11 +110,8 @@ pub struct HttpRequest {
 }
 
 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, default_agent: fn() -> String) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, user_agent: String, options: RequestOptions) -> HttpRequest {
         let cookies = session.cookies.clone();
-
-        let user_agent = options.user_agent.unwrap_or_else(default_agent);
-
         let timeout = options.timeout.map(Duration::from_millis);
 
         let mut request = HttpRequest {

src/args.rs

@@ -97,18 +97,11 @@ pub enum SubCommand {
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
-    /// Execute a module that has been installed
-    pub module: String,
+    #[structopt(flatten)]
+    pub run: cmd::run_cmd::Args,
     /// Run a module from a path
     #[structopt(short="f", long="file")]
     pub file: bool,
-    /// Run modules concurrently
-    #[structopt(short="j", long="threads", default_value="1")]
-    pub threads: usize,
-    /// Verbose logging, once to print inserts even if they don't add new
-    /// data, twice to activate the debug() function
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    pub verbose: u64,
     /// Expose stdin to modules
     #[structopt(long="stdin")]
     pub stdin: bool,

src/cmd/run_cmd.rs

@@ -17,6 +17,7 @@ use serde::Serialize;
 use sn0int_common::metadata::Source;
 use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 
@@ -24,14 +25,26 @@ use structopt::clap::AppSettings;
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
-    #[structopt(short="j", long="threads", default_value="1")]
-    threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
+    /// Execute a module that has been installed
+    pub module: Option<String>,
+    /// Run investigations concurrently
+    #[structopt(short="j", default_value="1")]
+    pub threads: usize,
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    #[structopt(short="v", long, parse(from_occurrences))]
+    pub verbose: u64,
+    /// Set a specific socks5 proxy to use
+    #[structopt(short="X", long)]
+    pub proxy: Option<SocketAddr>,
+    /// Set a different default user agent
+    #[structopt(long)]
+    pub user_agent: Option<String>,
 }
 
 #[derive(Debug, Clone)]
 pub struct Params<'a> {
+    pub module: Option<&'a String>,
     pub threads: usize,
     pub verbose: u64,
     pub stdin: bool,
@@ -39,25 +52,61 @@ pub struct Params<'a> {
     pub grant_full_keyring: bool,
     pub deny_keyring: bool,
     pub exit_on_error: bool,
+    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<&'a String>,
+}
+
+impl<'a> Params<'a> {
+    pub fn get_module(&self, rl: &Shell) -> Result<Module> {
+        let module = if let Some(module) = self.module {
+            rl.library().get(module)?
+                .clone()
+        } else {
+            rl.module()
+                .map(|m| m.to_owned())
+                .ok_or_else(|| format_err!("No module selected"))?
+        };
+        Ok(module)
+    }
+
+    pub fn get_proxy(&self, rl: &Shell) -> Option<SocketAddr> {
+        if self.proxy.is_some() {
+            self.proxy.clone()
+        } else {
+            rl.config().network.proxy
+        }
+    }
+
+    pub fn get_user_agent(&self, rl: &Shell) -> Option<String> {
+        if let Some(user_agent) = self.user_agent {
+            Some(user_agent.to_string())
+        } else {
+            rl.config().network.user_agent.clone()
+        }
+    }
 }
 
 impl<'a> From<&'a args::Run> for Params<'a> {
     fn from(args: &args::Run) -> Params {
         Params {
-            threads: args.threads,
-            verbose: args.verbose,
+            module: args.run.module.as_ref(),
+            threads: args.run.threads,
+            verbose: args.run.verbose,
             stdin: args.stdin,
             grants: &args.grants,
             grant_full_keyring: args.grant_full_keyring,
             deny_keyring: args.deny_keyring,
             exit_on_error: args.exit_on_error,
+            proxy: args.run.proxy,
+            user_agent: args.run.user_agent.as_ref(),
         }
     }
 }
 
-impl From<Args> for Params<'static> {
-    fn from(args: Args) -> Params<'static> {
+impl<'a> From<&'a Args> for Params<'a> {
+    fn from(args: &Args) -> Params {
         Params {
+            module: args.module.as_ref(),
             threads: args.threads,
             verbose: args.verbose,
             stdin: false,
@@ -65,6 +114,8 @@ impl From<Args> for Params<'static> {
             grant_full_keyring: false,
             deny_keyring: false,
             exit_on_error: false,
+            proxy: args.proxy,
+            user_agent: args.user_agent.as_ref(),
         }
     }
 }
@@ -153,15 +204,14 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
 }
 
 pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let keyring = rl.keyring().request_keys(&module);
 
     let dns_config = Resolver::from_system_v4()?;
-    let proxy = rl.config().network.proxy;
 
     let args = get_args(rl, &module)?;
     for (arg, _pretty_arg, blobs) in args {
@@ -169,6 +219,7 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
                                           keyring.clone(),
                                           dns_config.clone(),
                                           proxy,
+                                          user_agent.clone(),
                                           options.clone(),
                                           module.clone(),
                                           arg,
@@ -181,15 +232,15 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
 }
 
 pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, proxy, user_agent, options);
     rl.signal_register().reset_ctrlc();
 
     if errors > 0 {
@@ -212,6 +263,6 @@ impl Cmd for Args {
             Some(options) => options.clone(),
             _ => HashMap::new(),
         };
-        execute(rl, self.into(), options)
+        execute(rl, Params::from(&self), options)
     }
 }

src/config.rs

@@ -75,4 +75,5 @@ fn default_registry() -> String {
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct NetworkConfig {
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
 }

src/engine/ctx.rs

@@ -215,6 +215,7 @@ pub struct LuaState {
     geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
     asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
     proxy: Option<SocketAddr>,
+    user_agent: Option<String>,
     options: HashMap<String, String>,
 }
 
@@ -372,7 +373,15 @@ impl State for LuaState {
         let mtx = self.http_sessions.lock().unwrap();
         let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
 
-        HttpRequest::new(session, method, url, options, || format!("sn0int/{}", env!("CARGO_PKG_VERSION")))
+        let user_agent = if let Some(user_agent) = &options.user_agent {
+            user_agent.to_string()
+        } else if let Some(user_agent) = &self.user_agent {
+            user_agent.to_string()
+        } else {
+            format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
+        };
+
+        HttpRequest::new(session, method, url, user_agent, options)
     }
 
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
@@ -472,6 +481,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
         geoip,
         asn,
         proxy: env.proxy,
+        user_agent: env.user_agent,
         options: env.options,
     });
 
@@ -517,6 +527,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
     runtime::http_fetch_json(&mut lua, state.clone());
     runtime::img_exif(&mut lua, state.clone());
     runtime::img_load(&mut lua, state.clone());
+    runtime::img_ahash(&mut lua, state.clone());
+    runtime::img_dhash(&mut lua, state.clone());
+    runtime::img_phash(&mut lua, state.clone());
     runtime::img_nudity(&mut lua, state.clone());
     runtime::info(&mut lua, state.clone());
     runtime::intval(&mut lua, state.clone());
@@ -662,6 +675,7 @@ impl Script {
         let keyring = Vec::new();
         let dns_config = Resolver::from_system_v4()?;
         let proxy = None;
+        let user_agent = None;
         let psl = PslReader::String(r#"
 // ===BEGIN ICANN DOMAINS===
 com
@@ -679,6 +693,7 @@ a.prod.fastly.net
             keyring,
             dns_config,
             proxy,
+            user_agent,
             options: HashMap::new(),
             blobs: Vec::new(),
             psl,

src/engine/mod.rs

@@ -34,6 +34,7 @@ pub struct Environment {
     pub keyring: Vec<KeyRingEntry>,
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
     pub options: HashMap<String, String>,
     pub blobs: Vec<Blob>,
     pub psl: PslReader,

src/ipc/child.rs

@@ -86,6 +86,7 @@ pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslRea
         keyring: start.keyring,
         dns_config: start.dns_config,
         proxy: start.proxy,
+        user_agent: start.user_agent,
         options: start.options,
         blobs: start.blobs,
         psl,

src/ipc/common.rs

@@ -12,6 +12,7 @@ pub struct StartCommand {
     pub keyring: Vec<KeyRingEntry>,
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
     pub options: HashMap<String, String>,
     pub module: Module,
     pub arg: serde_json::Value,
@@ -23,6 +24,7 @@ impl StartCommand {
                keyring: Vec<KeyRingEntry>,
                dns_config: Resolver,
                proxy: Option<SocketAddr>,
+               user_agent: Option<String>,
                options: HashMap<String, String>,
                module: Module,
                arg: serde_json::Value,
@@ -33,6 +35,7 @@ impl StartCommand {
             keyring,
             dns_config,
             proxy,
+            user_agent,
             options,
             module,
             arg,

src/ipc/parent.rs

@@ -105,6 +105,7 @@ pub fn run(module: Module,
            verbose: u64,
            has_stdin: bool,
            proxy: Option<SocketAddr>,
+           user_agent: Option<String>,
            options: HashMap<String, String>,
            blobs: Vec<Blob>,
 ) -> Result<ExitEvent> {
@@ -117,7 +118,7 @@ pub fn run(module: Module,
     };
 
     let mut ipc_parent = IpcParent::setup(&module)?;
-    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, user_agent, options, module, arg, blobs))?;
 
     let exit = loop {
         match ipc_parent.recv()? {

src/main.rs

@@ -2,6 +2,7 @@ use env_logger::Env;
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
 use sn0int::cmd::{self, LiteCmd};
+use sn0int::cmd::run_cmd::Params;
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
@@ -23,8 +24,11 @@ use std::path::Path;
 fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
 
+    let module = args.run.module.as_ref()
+        .ok_or_else(|| format_err!("Module is required"))?;
+
     let module = if args.file {
-        let path = Path::new(&args.module);
+        let path = Path::new(&module);
 
         let filename = path.file_stem()
             .ok_or_else(|| format_err!("Failed to decode filename"))?
@@ -34,7 +38,7 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         Module::load(&path.to_path_buf(), "anonymous", filename, true)
             .context(format!("Failed to parse {:?}", path))?
     } else {
-        rl.library().get(&args.module)?
+        rl.library().get(module)?
             .clone()
     };
 
@@ -47,10 +51,14 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         rl.set_target(Some(target));
     }
 
+    let mut params = Params::from(args);
+    // The module was already set and loaded
+    params.module = None;
+
     if args.dump_sandbox_init_msg {
-        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, params, Opt::collect(&args.options))
     } else {
-        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::execute(&mut rl, params, Opt::collect(&args.options))
     }
 }
 

src/notify/mod.rs

@@ -88,6 +88,7 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
     }
 
     let params = Params {
+        module: None,
         threads: 1,
         verbose,
         stdin: false,
@@ -95,13 +96,15 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
         grant_full_keyring: false,
         deny_keyring: false,
         exit_on_error: false,
+        proxy: None,
+        user_agent: None,
     };
 
     prepare_keyring(rl.keyring_mut(), module, &params)?;
     let args = vec![prepare_arg(notification)?];
 
     debug!("Executing notification module {:?}", module_name);
-    let errors = worker::spawn(rl, module, ratelimit, args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, module, ratelimit, args, &params, rl.config().network.proxy, None, options);
     debug!("Notification module {:?} exited with {:?} errors", module_name, errors);
 
     Ok(errors)

src/repl/mod.rs

@@ -82,6 +82,7 @@ pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
     let dns_config = Resolver::from_system_v4()?;
     let proxy = config.network.proxy;
+    let user_agent = config.network.user_agent.clone();
 
     let cache_dir = paths::cache_dir()?;
     let psl = PslReader::open(&cache_dir)?;
@@ -93,6 +94,7 @@ pub fn run(config: &Config) -> Result<()> {
         keyring,
         dns_config,
         proxy,
+        user_agent,
         options: HashMap::new(),
         blobs: Vec::new(),
         psl,

src/runtime/gfx.rs

@@ -36,6 +36,48 @@ pub fn img_load(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
+pub fn img_ahash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_ahash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Mean);
+
+        Ok(hash)
+    }))
+}
+
+pub fn img_dhash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_dhash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Gradient);
+
+        Ok(hash)
+    }))
+}
+
+pub fn img_phash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_phash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Median);
+
+        Ok(hash)
+    }))
+}
+
 #[derive(Debug, Serialize)]
 pub struct Nudity {
     nude: bool,

src/worker.rs

@@ -437,7 +437,15 @@ impl RatelimitEvent {
     }
 }
 
-pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args: Vec<(serde_json::Value, Option<String>, Vec<Blob>)>, params: &Params, proxy: Option<SocketAddr>, options: HashMap<String, String>) -> usize {
+pub fn spawn(rl: &mut Shell,
+             module: &Module,
+             ratelimit: &mut Ratelimiter,
+             args: Vec<(serde_json::Value, Option<String>, Vec<Blob>)>,
+             params: &Params,
+             proxy: Option<SocketAddr>,
+             user_agent: Option<String>,
+             options: HashMap<String, String>,
+) -> usize {
     // This function hangs if args is empty, so return early if that's the case
     if args.is_empty() {
         return 0;
@@ -463,6 +471,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args:
         let tx = tx.clone();
         let module = module.clone();
         let keyring = keyring.clone();
+        let user_agent = user_agent.clone();
         let options = options.clone();
         let signal_register = rl.signal_register().clone();
         pool.execute(move || {
@@ -476,7 +485,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args:
             }
 
             tx.send(Event2::Start);
-            let event = match ipc::parent::run(module, &tx, arg, keyring, verbose, has_stdin, proxy, options, blobs) {
+            let event = match ipc::parent::run(module, &tx, arg, keyring, verbose, has_stdin, proxy, user_agent, options, blobs) {
                 Ok(exit) => exit,
                 Err(err) => ExitEvent::SetupFailed(err.to_string()),
             };