Release v0.15.0

Add tab completion to sn0int repl

.travis.yml

@@ -21,10 +21,11 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
+  # https://github.com/rust-lang/rust/issues/68264
+  #- os: linux
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE=docker-registry
   - os: osx
     rust: stable
     env:
@@ -37,10 +38,10 @@ matrix:
   #  rust: stable
   #  env:
   #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows common"
 
 before_install:
 - ci/setup.sh "$TRAVIS_OS_NAME"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.14.0"
+version = "0.15.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -61,8 +61,6 @@ sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
 maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
 threadpool = "1.7"
 x509-parser = "0.6.0"
 der-parser = "3.0"
@@ -74,8 +72,8 @@ bytes = "0.4"
 xml-rs = "0.8"
 bytesize = "1.0"
 ipnetwork = "0.15"
-strum = "0.16"
-strum_macros = "0.16"
+strum = "0.17"
+strum_macros = "0.17"
 
 digest = "0.8.0"
 bs58 = "0.3"
@@ -87,15 +85,15 @@ sha3 = "0.8.0"
 hmac = "0.7"
 
 image = "0.22"
-kamadak-exif = "0.3.1"
+kamadak-exif = "0.4"
 walkdir = "2.2"
 nude = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.15"
+syscallz = "0.12"
+nix = "0.16"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"

README.md

@@ -116,10 +116,17 @@ For everything else please have a look at the [detailed list][1].
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
   - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
   - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
     - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
     - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
     - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -180,6 +187,7 @@ For everything else please have a look at the [detailed list][1].
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
   - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)

ci/run.sh

@@ -7,10 +7,6 @@ case "$1" in
         ;;
     test)
         ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/index.rst

@@ -41,6 +41,7 @@ Getting Started
    scripting
    database
    structs
+   activity
    keyring
    config
    sandbox

docs/install.rst

@@ -47,8 +47,6 @@ Docker
 Alpine
 ------
 
-On alpine edge, with enabled testing repositories:
-
 .. code-block:: bash
 
     $ apk add sn0int
@@ -56,8 +54,6 @@ On alpine edge, with enabled testing repositories:
 OpenBSD
 -------
 
-On -current:
-
 .. code-block:: bash
 
     $ pkg_add sn0int

docs/keyring.rst

@@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/reference.rst

@@ -643,6 +643,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 

docs/structs.rst

@@ -259,6 +259,35 @@ is ``140.82.112.0/20``.
     This field isn't strictly defined and meant to be used as a human
     meaningful name if available.
 
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

examples/maxmind.rs

@@ -1,7 +1,6 @@
 extern crate sn0int;
 extern crate env_logger;
 extern crate chrootable_https;
-#[macro_use] extern crate log;
 
 // workaround for rustc 1.29.2 support
 #[cfg(not(target_os = "openbsd"))]
@@ -11,43 +10,17 @@ extern crate structopt;
 
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
-use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
     #[structopt(name="asn")]
     Asn(AsnArgs),
     #[structopt(name="geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Debug, StructOpt)]
 pub struct AsnArgs {
     ip: IpAddr,
@@ -55,7 +28,8 @@ pub struct AsnArgs {
 
 impl AsnArgs {
     fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+        let path = AsnDB::cache_path()?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -71,7 +45,8 @@ pub struct GeoIPArgs {
 
 impl GeoIPArgs {
     fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+        let path = GeoIP::cache_path()?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -85,7 +60,6 @@ fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
     match args {
-        Args::Download(args) => args.run(),
         Args::Asn(args) => args.run(),
         Args::GeoIP(args) => args.run(),
     }

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/ratelimit.lua

@@ -0,0 +1,11 @@
+-- Description: Run script with a global ratelimit
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- this shouldn't complete in less than 5 seconds
+    for i=1, 20 do
+        ratelimit_throttle('foo', 4, 1000)
+        info(sn0int_time())
+    end
+end

src/archive.rs

@@ -1,40 +0,0 @@
-use crate::errors::*;
-use tar;
-use libflate::gzip;
-use std::io;
-use std::fs::File;
-use std::path::Path;
-
-
-pub fn extract<R: io::Read, P: AsRef<Path>>(read: &mut R, filter: &str, target: P) -> Result<()> {
-    let file = gzip::Decoder::new(read)?;
-    let mut ar = tar::Archive::new(file);
-
-    for entry in ar.entries()? {
-        let mut entry = entry?;
-        let file_name = {
-            let path = entry.path()?;
-            path.file_name()
-                .ok_or_else(|| format_err!("Invalid path in archive"))?
-                .to_str()
-                .ok_or_else(|| format_err!("Filename is invalid utf8"))?
-                .to_owned()
-        };
-
-        debug!("Found in archive: {:?}", file_name);
-
-        if filter == file_name {
-            debug!("Extracting to {:?}", target.as_ref());
-
-            let mut target = File::create(target)?;
-            let n = io::copy(&mut entry, &mut target)?;
-            debug!("Wrote {:?} bytes", n);
-
-            return Ok(());
-        }
-
-        debug!("Skipping file");
-    }
-
-    bail!("Nothing in archive matched filter")
-}

src/args.rs

@@ -57,6 +57,9 @@ pub enum SubCommand {
     /// Delete from the database
     #[structopt(name="delete")]
     Delete(cmd::delete_cmd::Args),
+    /// Query logged activity
+    #[structopt(name="activity")]
+    Activity(cmd::activity_cmd::Args),
     /// Include entities in the scope
     #[structopt(name="scope")]
     Scope(cmd::scope_cmd::Args),

src/cmd/activity_cmd.rs

@@ -0,0 +1,88 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::shell::Shell;
+use crate::models::*;
+use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use std::convert::TryFrom;
+use std::io;
+use std::str::FromStr;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug)]
+pub struct TimeSpec {
+    datetime: NaiveDateTime,
+}
+
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+
+        let datetime = match s {
+            "today" => today,
+            "yesterday" => today - Duration::days(1),
+            // x {second,minute,hour,day,week,month,year}s? ago
+            s => NaiveDateTime::from_str(s)?,
+        };
+
+        Ok(TimeSpec {
+            datetime,
+        })
+    }
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Only query events for a given topic
+    #[structopt(short="t", long="topic")]
+    topic: Option<String>,
+    /// Only query events starting from that datetime
+    #[structopt(long="since")]
+    since: Option<TimeSpec>,
+    /// Only query events until this datetime
+    #[structopt(long="until")]
+    until: Option<TimeSpec>,
+    /// Try to select the previous event before --since as an initial state
+    #[structopt(short="i", long="initial")]
+    initial: bool,
+    /// Only query events that are tied to a location
+    #[structopt(short="l", long="location")]
+    location: bool,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let filter = ActivityFilter {
+            topic: self.topic,
+            since: self.since.map(|t| t.datetime),
+            until: self.until.map(|t| t.datetime),
+            location: self.location,
+        };
+
+        let mut stdout = io::stdout();
+        let events = Activity::query(rl.db(), &filter)?;
+
+        if self.initial {
+            if let Some(first) = events.get(0) {
+                let previous = Activity::previous(rl.db(), first, &filter)?;
+                if let Some(previous) = previous {
+                    let mut previous = JsonActivity::try_from(previous)?;
+                    previous.initial = true;
+                    previous.write_to(&mut stdout)?;
+                }
+            }
+        }
+
+        for activity in events {
+            let activity = JsonActivity::try_from(activity)?;
+            activity.write_to(&mut stdout)?;
+        }
+
+        Ok(())
+    }
+}

src/cmd/add_cmd.rs

@@ -11,6 +11,7 @@ use crate::utils;
 use crate::term;
 use std::fs;
 use std::net;
+use std::net::SocketAddr;
 use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
@@ -37,7 +38,7 @@ pub enum Target {
     #[structopt(name="ipaddr")]
     IpAddr(AddIpAddr),
     /// Insert url into the database
-    #[structopt(name="subdomain")]
+    #[structopt(name="url")]
     Url(AddUrl),
     /// Insert email into the database
     #[structopt(name="email")]
@@ -63,6 +64,9 @@ pub enum Target {
     /// Insert ip network into the database
     #[structopt(name="netblock")]
     Netblock(AddNetblock),
+    /// Insert port into the database
+    #[structopt(name="port")]
+    Port(AddPort),
     /// Insert a crypto currency address into the database
     #[structopt(name="cryptoaddr")]
     CryptoAddr(AddCryptoAddr),
@@ -83,6 +87,7 @@ impl Cmd for Args {
             Target::Breach(args) => args.insert(rl, self.dry_run),
             Target::Image(args) => args.insert(rl, self.dry_run),
             Target::Netblock(args) => args.insert(rl, self.dry_run),
+            Target::Port(args) => args.insert(rl, self.dry_run),
             Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
         }
     }
@@ -555,6 +560,71 @@ impl IntoInsert for AddNetblock {
     }
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddPort {
+    protocol: Option<String>,
+    addr: Option<SocketAddr>,
+}
+
+impl IntoInsert for AddPort {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        let protocol = if let Some(protocol) = self.protocol {
+            protocol
+        } else {
+            utils::question("Protocol (tcp/udp)")?
+        };
+
+        let addr = if let Some(addr) = self.addr {
+            addr
+        } else {
+            let addr = utils::question("IP:Port")?;
+            addr.parse()?
+        };
+
+        let family = match addr.ip() {
+            net::IpAddr::V4(_) => "4",
+            net::IpAddr::V6(_) => "6",
+        };
+
+        let ip_addr_id = match rl.db().insert_struct(NewIpAddr {
+            family: family.to_string(),
+            value: addr.ip().to_string(),
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            latitude: None,
+            longitude: None,
+            asn: None,
+            as_org: None,
+            description: None,
+            reverse_dns: None,
+            unscoped: false,
+        }, true)? {
+            Some((_, ip_addr_id)) => ip_addr_id,
+            _ => bail!("IpAddr is out out of scope"),
+        };
+
+        let value = format!("{}/{}", protocol, addr);
+
+        Ok(Insert::Port(NewPort {
+            ip_addr_id,
+            value,
+            ip_addr: addr.ip().to_string(),
+            port: addr.port() as i32,
+            protocol,
+            status: None,
+
+            banner: None,
+            service: None,
+            version: None,
+
+            unscoped: false,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddCryptoAddr {
     address: Option<String>,

src/cmd/mod.rs

@@ -11,6 +11,7 @@ pub trait Cmd: structopt::StructOpt + Sized {
     }
 }
 
+pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
 pub mod autoscope_cmd;

src/crt.rs

@@ -11,6 +11,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 #[derive(Debug, PartialEq)]
 pub enum AlternativeName {
     DnsName(String),
+    Email(String),
     IpAddr(IpAddr),
 }
 
@@ -37,6 +38,7 @@ pub fn san_value(o: DerObject) -> Result<AlternativeName> {
 
     match (o.class, o.tag, &o.content) {
         (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
+        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
         (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
         _ => bail!("Unexpected object: {:?}", o),
     }
@@ -49,6 +51,13 @@ pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
         .map_err(Error::from)
 }
 
+pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
+    debug!("Reading as email: {:?}", v);
+    String::from_utf8(v.to_vec())
+        .map(AlternativeName::Email)
+        .map_err(Error::from)
+}
+
 pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
     debug!("Reading as ipaddr: {:?}", v);
     match v.len() {
@@ -68,7 +77,8 @@ pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
     pub valid_names: Vec<String>,
-    pub valid_ipaddrs: Vec<IpAddr>
+    pub valid_emails: Vec<String>,
+    pub valid_ipaddrs: Vec<IpAddr>,
 }
 
 impl Certificate {
@@ -103,6 +113,7 @@ impl Certificate {
         };
 
         let mut valid_names = HashSet::new();
+        let mut valid_emails = HashSet::new();
         let mut valid_ipaddrs = HashSet::new();
 
         for x in crt.tbs_certificate.subject.rdn_seq {
@@ -131,15 +142,18 @@ impl Certificate {
             for v in values {
                 match v {
                     AlternativeName::DnsName(v) => valid_names.insert(v),
+                    AlternativeName::Email(v) => valid_emails.insert(v),
                     AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
                 };
             }
         }
 
         let valid_names = valid_names.into_iter().collect();
+        let valid_emails = valid_emails.into_iter().collect();
         let valid_ipaddrs = valid_ipaddrs.into_iter().collect();
         Ok(Certificate {
             valid_names,
+            valid_emails,
             valid_ipaddrs,
         })
     }
@@ -197,6 +211,7 @@ WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
         x.valid_names.sort();
         assert_eq!(x, Certificate {
             valid_names: vec!["github.com".into(), "www.github.com".into()],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -235,6 +250,7 @@ ZkZZmqNn2Q8=
                 "*.cloudflare-dns.com".into(),
                 "cloudflare-dns.com".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![
                 "1.0.0.1".parse().unwrap(),
                 "1.1.1.1".parse().unwrap(),
@@ -351,6 +367,7 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
                 "witt-weiden.dam.staging.aboutyou.cloud".into(),
                 "www.aboutyou.de".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -396,4 +413,50 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
             .expect("Failed to process san value");
         assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
     }
+
+    #[test]
+    fn test_san_email() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIIE5zCCA8+gAwIBAgIQBvsKfZ5AGSW3Vc8Ldto1hTANBgkqhkiG9w0BAQUFADBp
+MSQwIgYJKoZIhvcNAQkBFhVwa2lfYWRtaW5Ac3VuZ2FyZC5jb20xJjAkBgNVBAoT
+HVN1bkdhcmQgQXZhaWxhYmlsaXR5IFNlcnZpY2VzMRkwFwYDVQQDExBTQVMgUHVi
+bGljIENBIHYxMB4XDTEwMDkwMjE2MzY0OVoXDTExMTAwMTE2MzEwMFowgbQxCzAJ
+BgNVBAYTAlVTMQswCQYDVQQIEwJPUjERMA8GA1UEBxMIUG9ydGxhbmQxEzARBgoJ
+kiaJk/IsZAEZFgNjb20xHDAaBgoJkiaJk/IsZAEZFgxqaXZlc29mdHdhcmUxHDAa
+BgNVBAoTE0ppdmUgU29mdHdhcmUsIEluYy4xEDAOBgNVBAsTB0hvc3RpbmcxIjAg
+BgNVBAMTGSouaG9zdGVkLmppdmVzb2Z0d2FyZS5jb20wggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC0oornTIyL5YjZMpNwy+V2YJbLqaLrbPrbWFCsNJDx
+dnubjfR71aW+YYlUZF8zoq4jFetkblCehyvPEb5tD/l3/WZhiXYOziPDrsEVCngF
+3/b0H3Dyk6mNWBZcNpJkdpOx1YB6Zer8eKzFOr7Qj3aevOR/bEe2NARJIaO0Rjwe
+YIWY0arKRm6z4nJD8fYAvFV6wRWmHsZO9ci7hiGeW3YL6jQYJqLeuwXm64l0jptb
+Qg8r8c1V5BXETlvQJL34gUozEl9jDpzR7KoXtErhlU2ytl9Wg+fOxYuWgx8vER0/
+7Hqc/qD5e7B+NtwgfEio7SvNGA/HhjNxW2Wbrx4qooJRAgMBAAGjggE9MIIBOTAO
+BgNVHQ8BAf8EBAMCA6gwEQYJYIZIAYb4QgEBBAQDAgbAMCIGA1UdEQQbMBmBF3N1
+YmplY3RuYW1lQGV4YW1wbGUuY29tMB8GA1UdIwQYMBaAFDhBxvKFgYP96+IaNpI7
+JmEWgRESMFQGA1UdIARNMEswSQYJKoZIhvcNBQYBMDwwOgYIKwYBBQUHAgEWLmh0
+dHBzOi8vY2VydGlmaWNhdGUuc3VuZ2FyZC5jb20vU0FTX0NBX0NQUy5wZGYwRQYD
+VR0fBD4wPDA6oDigNoY0aHR0cHM6Ly9jZXJ0aWZpY2F0ZS5zdW5nYXJkLmNvbS9T
+QVNfUHVibGljX0NBX3YxLmNybDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4E
+FgQUhJ99py6oeCYBzcePPjhOxHVDBvwwDQYJKoZIhvcNAQEFBQADggEBAF/DAJgX
+f50x8t8Im96AUn4DqC+T0QZIYHihpj2uCwWDbdp5efppqTrk6FrpFOzQy0TRkstb
+Q3zKSgduedQiwii9qh88O1h2gbSTqfi55ApOIGoCiiRCqio2p4tbKyqPV3Q0eyYw
+K4f9GAOcawvNsI//mx99ol/ZGEamydeL9G0qiKrhqSxd2TGmFaIVJdu9fh59hos4
+6t9c4FVyYygdsIeGHkHjpB2bKjZhnJpKRh9dGWctcjdHMITBBqgiRH9OZa/w6SPE
+UT+11L6q7MSIXSIMV8kJSUUYE92P7bnAqViTIuu/hHnfmIhiy6t7AuT2QHEhqDab
+EF4l5MwdUqs8FvM=
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        x.valid_emails.sort();
+        x.valid_ipaddrs.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec![
+                "*.hosted.jivesoftware.com".into(),
+            ],
+            valid_emails: vec![
+                "subjectname@example.com".into(),
+            ],
+            valid_ipaddrs: vec![],
+        });
+    }
 }

src/db/mod.rs

@@ -208,6 +208,17 @@ impl Database {
         }
     }
 
+    pub fn insert_activity(&self, obj: NewActivity) -> Result<bool> {
+        if let Some(uniq) = &obj.uniq {
+            if Activity::uniq(self, uniq)?.is_some() {
+                // unique tag set and event already logged
+                return Ok(false);
+            }
+        }
+        obj.insert(&self)?;
+        Ok(true)
+    }
+
     pub fn insert_subdomain_ipaddr_struct(&self, subdomain_ipaddr: &NewSubdomainIpAddr) -> Result<Option<(DbChange, i32)>> {
         if let Some(subdomain_ipaddr_id) = SubdomainIpAddr::get_id_opt(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))? {
             Ok(Some((DbChange::None, subdomain_ipaddr_id)))

src/engine/ctx.rs

@@ -6,19 +6,21 @@ use crate::engine::{Environment, Reporter};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
-use crate::models::{Insert, Update};
+use crate::models::*;
 use crate::psl::{Psl, PslReader};
 use crate::lazy::Lazy;
 use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
-use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
+use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent, RatelimitEvent};
+use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
 use std::collections::HashMap;
 use std::result;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
+use std::thread;
 use rand::prelude::*;
 use rand::distributions::Alphanumeric;
 
@@ -84,6 +86,18 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
     }
 
+    fn db_activity(&self, activity: InsertActivity) -> Result<()> {
+        let activity = activity.try_into_new()?;
+
+        self.send(&Event::Database(DatabaseEvent::Activity(activity)));
+        let reply = self.recv()?;
+        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
+
+        reply
+            .map(|_| ())
+            .map_err(|err| format_err!("Failed to add to database: {:?}", err))
+    }
+
     fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
         self.send(&Event::Database(DatabaseEvent::Select((family, value))));
         let reply = self.recv()?;
@@ -114,6 +128,21 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
     }
 
+    fn ratelimit(&self, key: String, passes: u32, time: u32) -> Result<()> {
+        let ratelimit = Event::Ratelimit(RatelimitEvent::new(key, passes, time));
+        loop {
+            self.send(&ratelimit);
+            let reply = self.recv()?;
+            let reply: result::Result<RatelimitResponse, String> = serde_json::from_value(reply)?;
+            match reply {
+                Ok(RatelimitResponse::Retry(delay)) => thread::sleep(delay),
+                Ok(RatelimitResponse::Pass) => break,
+                Err(err) => bail!("Unexpected error case for ratelimit: {}", err),
+            }
+        }
+        Ok(())
+    }
+
     #[inline]
     fn random_id(&self) -> String {
         thread_rng().sample_iter(&Alphanumeric).take(16).collect()
@@ -173,8 +202,8 @@ pub struct LuaState {
     keyring: Vec<KeyRingEntry>, // TODO: maybe hashmap
     dns_config: Resolver,
     psl: Mutex<Lazy<PslReader, Arc<Psl>>>,
-    geoip: Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>,
-    asn: Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>,
+    geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
+    asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
     proxy: Option<SocketAddr>,
     options: HashMap<String, String>,
 }
@@ -236,15 +265,23 @@ impl State for LuaState {
     }
 
     fn geoip(&self) -> Result<Arc<GeoIP>> {
-        let mut geoip = self.geoip.lock().unwrap();
-        let geoip = geoip.get()?;
-        Ok(geoip.clone())
+        if let Some(mtx) = &self.geoip {
+            let mut geoip = mtx.lock().unwrap();
+            let geoip = geoip.get()?;
+            Ok(geoip.clone())
+        } else {
+            bail!("No geoip database loaded")
+        }
     }
 
     fn asn(&self) -> Result<Arc<AsnDB>> {
-        let mut asn = self.asn.lock().unwrap();
-        let asn = asn.get()?;
-        Ok(asn.clone())
+        if let Some(mtx) = &self.asn {
+            let mut asn = mtx.lock().unwrap();
+            let asn = asn.get()?;
+            Ok(asn.clone())
+        } else {
+            bail!("No asn database loaded")
+        }
     }
 
     fn sock_connect(&self, host: &str, port: u16, options: &SocketOptions) -> Result<String> {
@@ -369,6 +406,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     let mut lua = hlua::Lua::new();
     lua.open_string();
 
+    let geoip = env.geoip.map(|db| Mutex::new(Lazy::from(db)));
+    let asn = env.asn.map(|db| Mutex::new(Lazy::from(db)));
+
     let state = Arc::new(LuaState {
         error: Mutex::new(None),
         logger,
@@ -381,8 +421,8 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
         keyring: env.keyring,
         dns_config: env.dns_config,
         psl: Mutex::new(Lazy::from(env.psl)),
-        geoip: Mutex::new(Lazy::from(env.geoip)),
-        asn: Mutex::new(Lazy::from(env.asn)),
+        geoip,
+        asn,
         proxy: env.proxy,
         options: env.options,
     });
@@ -404,6 +444,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::datetime(&mut lua, state.clone());
     runtime::db_add(&mut lua, state.clone());
     runtime::db_add_ttl(&mut lua, state.clone());
+    runtime::db_activity(&mut lua, state.clone());
     runtime::db_select(&mut lua, state.clone());
     runtime::db_update(&mut lua, state.clone());
     runtime::debug(&mut lua, state.clone());
@@ -440,6 +481,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
     runtime::psl_domain_from_dns_name(&mut lua, state.clone());
+    runtime::ratelimit_throttle(&mut lua, state.clone());
     runtime::regex_find(&mut lua, state.clone());
     runtime::regex_find_all(&mut lua, state.clone());
     runtime::semver_match(&mut lua, state.clone());
@@ -555,8 +597,8 @@ com
 a.prod.fastly.net
 // ===END PRIVATE DOMAINS===
 "#.into());
-        let geoip = GeoIP::open_reader()?;
-        let asn = AsnDB::open_reader()?;
+        let geoip = GeoIP::try_open_reader()?;
+        let asn = AsnDB::try_open_reader()?;
 
         let env = Environment {
             verbose: 0,

src/engine/isolation.rs

@@ -206,6 +206,7 @@ pub fn spawn_module(module: Module,
             Event::Log(event) => tx.send(Event2::Log(event)),
             Event::Database(object) => supervisor.send_event_callback(object, &tx),
             Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
+            Event::Ratelimit(req) => supervisor.send_event_callback(req, &tx),
             Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
             Event::Exit(event) => {
                 if let ExitEvent::Err(err) = &event {
@@ -221,7 +222,7 @@ pub fn spawn_module(module: Module,
     Ok(exit)
 }
 
-pub fn run_worker(geoip: MaxmindReader, asn: MaxmindReader, psl: PslReader) -> Result<()> {
+pub fn run_worker(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
     let mut reporter = StdioReporter::setup();
     let start = reporter.recv_start()?;
 

src/engine/mod.rs

@@ -38,8 +38,8 @@ pub struct Environment {
     pub options: HashMap<String, String>,
     pub blobs: Vec<Blob>,
     pub psl: PslReader,
-    pub geoip: MaxmindReader,
-    pub asn: MaxmindReader,
+    pub geoip: Option<MaxmindReader>,
+    pub asn: Option<MaxmindReader>,
 }
 
 #[derive(Debug)]

src/errors.rs

@@ -1,2 +1,3 @@
-pub use failure::{Error, ResultExt};
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
 pub type Result<T> = ::std::result::Result<T, Error>;

src/geoip/mod.rs

@@ -1,34 +1,25 @@
-use chrootable_https::Client;
-use crate::archive;
 use crate::errors::*;
 use crate::lazy::LazyInit;
 use crate::paths;
-use crate::worker;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
 use std::net::IpAddr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::io::Read;
 use std::sync::Arc;
 
-pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
-pub static GEOIP_ASN_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz";
-
 pub mod models;
 use self::models::GeoLookup;
 use self::models::AsnLookup;
 
 
 pub trait Maxmind: Sized {
-    fn archive_filename() -> &'static str;
-
-    fn archive_url() -> &'static str;
+    fn filename() -> &'static str;
 
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
-    // TODO: refactor this to return Path
-    fn cache_path() -> Result<String> {
+    fn cache_path() -> Result<PathBuf> {
         // use system path if exists
         for path in &[
             // Archlinux
@@ -37,21 +28,17 @@ pub trait Maxmind: Sized {
             "/usr/local/share/examples/libmaxminddb/",
         ] {
             let path = Path::new(path);
-            let path = path.join(Self::archive_filename());
+            let path = path.join(Self::filename());
 
             if path.exists() {
-                let path = path.to_str()
-                    .ok_or_else(|| format_err!("Failed to decode path"))?;
-                return Ok(path.to_string());
+                return Ok(path);
             }
         }
 
         // use cache path
         let path = paths::cache_dir()?
-            .join(Self::archive_filename());
-        let path = path.to_str()
-            .ok_or_else(|| format_err!("Failed to decode path"))?;
-        Ok(path.to_string())
+            .join(Self::filename());
+        Ok(path)
     }
 
     fn from_buf(buf: Vec<u8>) -> Result<Self> {
@@ -60,37 +47,20 @@ pub trait Maxmind: Sized {
         Ok(Self::new(reader))
     }
 
-    fn open(path: &str) -> Result<Self> {
+    fn open(path: &Path) -> Result<Self> {
         let buf = fs::read(path)?;
         Self::from_buf(buf)
     }
 
-    fn open_reader() -> Result<MaxmindReader> {
-        let path = Self::cache_path()?;
-        MaxmindReader::open_path(path)
-    }
-
-    fn open_or_download() -> Result<Self> {
+    fn try_open_reader() -> Result<Option<MaxmindReader>> {
         let path = Self::cache_path()?;
 
-        if File::open(&path).is_err() {
-            worker::spawn_fn(&format!("Downloading {:?}", Self::archive_filename()), || {
-                Self::download(&path, Self::archive_filename(), Self::archive_url())
-            }, false)?;
-        };
-
-        Self::open(&path)
-    }
-
-    fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
-        debug!("Downloading {:?}...", url);
-        let client = Client::with_system_resolver()?;
-        let resp = client.get(url)
-            .wait_for_response()
-            .context("http request failed")?;
-        debug!("Downloaded {} bytes", resp.body.len());
-        archive::extract(&mut &resp.body[..], filter, path)?;
-        Ok(())
+        if path.exists() {
+            let db = MaxmindReader::open_path(path)?;
+            Ok(Some(db))
+        } else {
+            Ok(None)
+        }
     }
 }
 
@@ -141,15 +111,10 @@ impl fmt::Debug for GeoIP {
 
 impl Maxmind for GeoIP {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-City.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_CITY_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         GeoIP {
@@ -178,15 +143,10 @@ impl fmt::Debug for AsnDB {
 
 impl Maxmind for AsnDB {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-ASN.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_ASN_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         AsnDB {
@@ -205,22 +165,29 @@ impl AsnDB {
 
 #[cfg(test)]
 mod tests {
+    // You need geoip setup on your system to run this
+    /*
     use super::*;
 
     #[test]
+    #[ignore]
     fn test_geoip_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let geoip = GeoIP::open_or_download().expect("Failed to load geoip");
+        let path = GeoIP::cache_path().unwrap();
+        let geoip = GeoIP::open(&path).unwrap();
         let lookup = geoip.lookup(ip).expect("GeoIP lookup failed");
         println!("{:#?}", lookup);
         assert_eq!(lookup.city, None);
     }
 
     #[test]
+    #[ignore]
     fn test_asn_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let asndb = AsnDB::open_or_download().expect("Failed to load asndb");
+        let path = AsnDB::cache_path().unwrap();
+        let asndb = AsnDB::open(&path).unwrap();
         let lookup = asndb.lookup(ip).expect("ASN lookup failed");
         println!("{:#?}", lookup);
     }
+    */
 }

src/gfx/exif.rs

@@ -11,11 +11,12 @@ pub struct Location {
 }
 
 impl Location {
-    pub fn try_from(fields: &[exif::Field]) -> Result<Location> {
+    fn try_from_iter<'a, I: IntoIterator<Item=&'a exif::Field>>(iter: I) -> Result<Self> {
         let mut builder = LocationBuilder::default();
-        fields.iter()
-            .map(|f| builder.add_one(f))
-            .collect::<Result<()>>()?;
+        for f in iter {
+            debug!("Exif field: {:?}", f.display_value().to_string());
+            builder.add_one(f)?;
+        }
         builder.build()
     }
 }
@@ -63,9 +64,8 @@ pub fn gps(img: &[u8]) -> Result<Option<Location>> {
     let mut buf = io::BufReader::new(img);
     let reader = exif::Reader::new(&mut buf)?;
     let fields = reader.fields();
-    debug!("Exif fields: {:?}", fields);
 
-    let location = Location::try_from(fields).ok();
+    let location = Location::try_from_iter(fields).ok();
     Ok(location)
 }
 
@@ -115,18 +115,18 @@ mod tests {
     fn verify_exif_location() {
         test_init();
 
-        let location = Location::try_from(&[
+        let location = Location::try_from_iter(&[
             exif::Field {
                 tag: exif::Tag::GPSLatitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'N']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'N']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'E']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'E']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLatitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 43,
                     denom: 1,
@@ -139,7 +139,7 @@ mod tests {
                 }]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 11,
                     denom: 1,

src/html.rs

@@ -1,4 +1,4 @@
-use crate::errors::Result;
+use crate::errors::*;
 
 use kuchiki;
 use kuchiki::traits::TendrilSink;

src/lib.rs

@@ -1,10 +1,8 @@
 #![warn(unused_extern_crates)]
-#[macro_use] extern crate failure;
 #[macro_use] extern crate maplit;
 use url;
 use hlua_badtouch as hlua;
 #[macro_use] extern crate serde_derive;
-#[macro_use] extern crate log;
 #[macro_use] extern crate structopt;
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate diesel;
@@ -12,7 +10,6 @@ use hlua_badtouch as hlua;
 #[macro_use] extern crate crossbeam_channel as channel;
 
 pub mod api;
-pub mod archive;
 pub mod args;
 pub mod auth;
 pub mod autonoscope;
@@ -36,6 +33,7 @@ pub mod models;
 pub mod paths;
 pub mod psl;
 pub mod options;
+pub mod ratelimits;
 pub mod registry;
 pub mod repl;
 pub mod runtime;

src/main.rs

@@ -1,7 +1,3 @@
-#![warn(unused_extern_crates)]
-#[macro_use] extern crate failure;
-#[macro_use] extern crate log;
-
 use env_logger::{self, Env};
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
@@ -22,7 +18,6 @@ use std::fs::OpenOptions;
 use std::io::Write;
 use std::path::Path;
 
-
 fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
 
@@ -54,8 +49,8 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
 }
 
 fn run_sandbox() -> Result<()> {
-    let geoip = GeoIP::open_reader()?;
-    let asn = AsnDB::open_reader()?;
+    let geoip = GeoIP::try_open_reader()?;
+    let asn = AsnDB::try_open_reader()?;
     let psl = PslReader::open()?;
 
     sandbox::init()
@@ -113,6 +108,7 @@ fn run() -> Result<()> {
         Some(SubCommand::Add(add)) => run_cmd(&args, add, &config),
         Some(SubCommand::Select(select)) => run_cmd(&args, select, &config),
         Some(SubCommand::Delete(delete)) => run_cmd(&args, delete, &config),
+        Some(SubCommand::Activity(activity)) => run_cmd(&args, activity, &config),
         Some(SubCommand::Scope(scope)) => run_cmd(&args, scope, &config),
         Some(SubCommand::Noscope(noscope)) => run_cmd(&args, noscope, &config),
         Some(SubCommand::Workspace(workspace)) => run_cmd(&args, workspace, &config),

src/models/activity.rs

@@ -0,0 +1,185 @@
+use crate::errors::*;
+use crate::schema::activity;
+use diesel;
+use diesel::prelude::*;
+use diesel::query_builder::BoxedSelectStatement;
+use crate::models::*;
+use chrono::NaiveDateTime;
+use std::convert::TryFrom;
+use std::io::Write;
+
+
+#[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
+#[table_name="activity"]
+pub struct Activity {
+    pub id: i32,
+    pub topic: String,
+    pub time: NaiveDateTime,
+    pub uniq: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub radius: Option<i32>,
+    pub content: String,
+}
+
+impl Activity {
+    pub fn uniq(db: &Database, my_uniq: &str) -> Result<Option<Self>> {
+        use crate::schema::activity::dsl::*;
+        activity.filter(uniq.eq(my_uniq))
+            .first::<Self>(db.db())
+            .optional()
+            .map_err(Error::from)
+    }
+
+    fn build_query_except_since(filter: &ActivityFilter) -> BoxedSelectStatement<(diesel::sql_types::Integer, diesel::sql_types::Text, diesel::sql_types::Timestamp, diesel::sql_types::Nullable<diesel::sql_types::Text>, diesel::sql_types::Nullable<diesel::sql_types::Float>, diesel::sql_types::Nullable<diesel::sql_types::Float>, diesel::sql_types::Nullable<diesel::sql_types::Integer>, diesel::sql_types::Text), activity::table, diesel::sqlite::Sqlite> {
+        use crate::schema::activity::dsl::*;
+
+        let mut query = activity.into_boxed();
+
+        if let Some(my_topic) = &filter.topic {
+            query = query.filter(topic.eq(my_topic));
+        }
+
+        // "since" filter is not applied
+
+        if let Some(until) = &filter.until {
+            query = query.filter(time.le(until));
+        }
+
+        if filter.location {
+            query = query
+                .filter(latitude.is_not_null())
+                .filter(longitude.is_not_null());
+        }
+
+        query
+    }
+
+    pub fn query(db: &Database, filter: &ActivityFilter) -> Result<Vec<Self>> {
+        use crate::schema::activity::dsl::*;
+
+        let mut query = Self::build_query_except_since(filter);
+
+        if let Some(since) = &filter.since {
+            query = query.filter(time.ge(since));
+        }
+
+        query
+            .order_by((time.asc(), id.asc()))
+            .load::<Self>(db.db())
+            .map_err(Error::from)
+    }
+
+    pub fn previous(db: &Database, previous: &Activity, filter: &ActivityFilter) -> Result<Option<Self>> {
+        use crate::schema::activity::dsl::*;
+
+        Self::build_query_except_since(filter)
+            .filter(time.lt(previous.time))
+            .order_by((time.desc(), id.desc()))
+            .first::<Self>(db.db())
+            .optional()
+            .map_err(Error::from)
+    }
+}
+
+pub struct ActivityFilter {
+    pub topic: Option<String>,
+    pub since: Option<NaiveDateTime>,
+    pub until: Option<NaiveDateTime>,
+    pub location: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct JsonActivity {
+    pub id: i32,
+    pub initial: bool,
+    pub topic: String,
+    pub time: NaiveDateTime,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub uniq: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub latitude: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub longitude: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub radius: Option<i32>,
+    pub content: serde_json::Value,
+}
+
+impl JsonActivity {
+    pub fn write_to<W: Write>(self, mut w: W) -> Result<()> {
+        let s = serde_json::to_string(&self)?;
+        writeln!(w, "{}", s)?;
+        Ok(())
+    }
+}
+
+impl TryFrom<Activity> for JsonActivity {
+    type Error = Error;
+
+    fn try_from(a: Activity) -> Result<Self> {
+        let content = serde_json::from_str(&a.content)?;
+
+        Ok(JsonActivity {
+            id: a.id,
+            initial: false,
+            topic: a.topic,
+            time: a.time,
+            uniq: a.uniq,
+            latitude: a.latitude,
+            longitude: a.longitude,
+            radius: a.radius,
+            content,
+        })
+    }
+}
+
+#[derive(Debug, Clone, Insertable, Serialize, Deserialize)]
+#[table_name="activity"]
+pub struct NewActivity {
+    pub topic: String,
+    pub time: NaiveDateTime,
+    pub uniq: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub radius: Option<i32>,
+    pub content: String,
+}
+
+impl NewActivity {
+    pub fn insert(&self, db: &Database) -> Result<()> {
+        diesel::insert_into(activity::table)
+            .values(self)
+            .execute(db.db())?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InsertActivity {
+    pub topic: String,
+    pub time: NaiveDateTime,
+    pub uniq: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub radius: Option<i32>,
+    pub content: serde_json::Value,
+}
+
+impl InsertToNew for InsertActivity {
+    type Target = NewActivity;
+
+    #[inline]
+    fn try_into_new(self) -> Result<NewActivity> {
+        let content = serde_json::to_string(&self.content)?;
+        Ok(NewActivity {
+            topic: self.topic,
+            time: self.time,
+            uniq: self.uniq,
+            latitude: self.latitude,
+            longitude: self.longitude,
+            radius: self.radius,
+            content,
+        })
+    }
+}

src/models/cryptoaddr.rs

@@ -92,20 +92,20 @@ impl Model for CryptoAddr {
     fn get(db: &Database, query: &Self::ID) -> Result<Self> {
         use crate::schema::cryptoaddrs::dsl::*;
 
-        let netblock = cryptoaddrs.filter(value.eq(query))
+        let cryptoaddr = cryptoaddrs.filter(value.eq(query))
             .first::<Self>(db.db())?;
 
-        Ok(netblock)
+        Ok(cryptoaddr)
     }
 
     fn get_opt(db: &Database, query: &Self::ID) -> Result<Option<Self>> {
         use crate::schema::cryptoaddrs::dsl::*;
 
-        let netblock = cryptoaddrs.filter(value.eq(query))
+        let cryptoaddr = cryptoaddrs.filter(value.eq(query))
             .first::<Self>(db.db())
             .optional()?;
 
-        Ok(netblock)
+        Ok(cryptoaddr)
     }
 }
 

src/models/mod.rs

@@ -424,3 +424,6 @@ pub use self::port::*;
 
 mod cryptoaddr;
 pub use self::cryptoaddr::*;
+
+mod activity;
+pub use self::activity::*;

src/models/netblock.rs

@@ -129,7 +129,7 @@ pub struct PrintableNetblock {
 
 impl fmt::Display for PrintableNetblock {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "{:?}", self.value) // TODO: cidr type
+        write!(w, "{}", self.value)
     }
 }
 

src/models/port.rs

@@ -17,7 +17,7 @@ pub struct Port {
     pub ip_addr: String,
     pub port: i32,
     pub protocol: String,
-    pub status: String,
+    pub status: Option<String>,
     pub unscoped: bool,
 
     pub banner: Option<String>,
@@ -147,7 +147,7 @@ impl Printable<PrintablePort> for Port {
 pub struct DetailedPort {
     id: i32,
     value: String,
-    status: String,
+    status: Option<String>,
     unscoped: bool,
 
     banner: Option<String>,
@@ -210,7 +210,7 @@ pub struct NewPort {
     pub ip_addr: String,
     pub port: i32,
     pub protocol: String,
-    pub status: String,
+    pub status: Option<String>,
 
     pub banner: Option<String>,
     pub service: Option<String>,
@@ -242,7 +242,7 @@ impl Upsertable<Port> for NewPort {
     fn upsert(self, existing: &Port) -> Self::Update {
         Self::Update {
             id: existing.id,
-            status: Self::upsert_opt(Some(self.status), &Some(existing.status.clone())),
+            status: Self::upsert_opt(self.status, &existing.status),
             banner: Self::upsert_opt(self.banner, &existing.banner),
             service: Self::upsert_opt(self.service, &existing.service),
             version: Self::upsert_opt(self.version, &existing.version),
@@ -264,7 +264,7 @@ pub struct InsertPort {
     pub ip_addr: net::IpAddr,
     pub port: i32,
     pub protocol: String,
-    pub status: String,
+    pub status: Option<String>,
 
     pub banner: Option<String>,
     pub service: Option<String>,
@@ -278,10 +278,11 @@ impl InsertToNew for InsertPort {
         let addr = SocketAddr::new(self.ip_addr, self.port as u16);
         let value = format!("{}/{}", self.protocol, addr);
 
-        match self.status.as_str() {
-            "open" => (),
-            "closed" => (),
-            s => bail!("unsupported port status: {:?}", s),
+        match self.status.as_deref() {
+            Some("open") => (),
+            Some("closed") => (),
+            Some(s) => bail!("unsupported port status: {:?}", s),
+            None => (),
         }
 
         Ok(NewPort {
@@ -330,7 +331,7 @@ impl Upsert for PortUpdate {
 
 impl Updateable<Port> for PortUpdate {
     fn changeset(&mut self, existing: &Port) {
-        Self::clear_if_equal(&mut self.status, &Some(existing.status.clone()));
+        Self::clear_if_equal(&mut self.status, &existing.status);
         Self::clear_if_equal(&mut self.banner, &existing.banner);
         Self::clear_if_equal(&mut self.service, &existing.service);
         Self::clear_if_equal(&mut self.version, &existing.version);

src/ratelimits.rs

@@ -0,0 +1,67 @@
+use chrono::prelude::*;
+use crate::worker::RatelimitSender;
+use std::collections::HashMap;
+use std::time::Duration;
+
+pub struct Ratelimiter {
+    buckets: HashMap<String, Bucket>,
+}
+
+impl Ratelimiter {
+    pub fn new() -> Ratelimiter {
+        Ratelimiter {
+            buckets: HashMap::new(),
+        }
+    }
+
+    pub fn pass(&mut self, tx: RatelimitSender, key: &str, passes: u32, time: u32) {
+        let bucket = if let Some(bucket) = self.buckets.get_mut(key) {
+            bucket
+        } else {
+            let bucket = Bucket::new();
+            self.buckets.insert(key.to_string(), bucket);
+            self.buckets.get_mut(key).unwrap()
+        };
+        let reply = bucket.pass(passes as usize, time);
+        tx.send(Ok(reply)).unwrap();
+    }
+}
+
+struct Bucket {
+    passes: Vec<DateTime<Utc>>,
+}
+
+impl Bucket {
+    pub fn new() -> Bucket {
+        Bucket {
+            passes: Vec::new(),
+        }
+    }
+
+    pub fn pass(&mut self, passes: usize, time: u32) -> RatelimitResponse {
+        let now = Utc::now();
+        let time = chrono::Duration::milliseconds(time as i64);
+        let retain = now - time;
+        self.passes.retain(|x| *x >= retain);
+
+        if self.passes.len() >= passes {
+            if let Some(min) = self.passes.iter().min() {
+                let delay = time - (now - *min);
+                RatelimitResponse::Retry(delay.to_std().unwrap())
+            } else {
+                // This should never happen unless passes is zero
+                RatelimitResponse::Retry(Duration::from_millis(100))
+            }
+        } else {
+            let now = Utc::now();
+            self.passes.push(now);
+            RatelimitResponse::Pass
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum RatelimitResponse {
+    Retry(Duration),
+    Pass,
+}

src/repl/complete.rs

@@ -1,3 +1,4 @@
+use crate::repl::tokenize::{self, Token};
 use rustyline::Context;
 use rustyline::completion::Completer;
 use rustyline::highlight::Highlighter;
@@ -5,7 +6,16 @@ use rustyline::hint::Hinter;
 use std::borrow::Cow;
 
 
-pub struct ReplCompleter;
+#[derive(Default)]
+pub struct ReplCompleter {
+    globals: Vec<String>,
+}
+
+impl ReplCompleter {
+    pub fn set(&mut self, globals: Vec<String>) {
+        self.globals = globals;
+    }
+}
 
 impl rustyline::Helper for ReplCompleter {}
 
@@ -13,13 +23,25 @@ impl Completer for ReplCompleter {
     type Candidate = String;
 
     #[inline]
-    fn complete(&self, _line: &str, pos: usize, _ctx: &Context<'_>) -> rustyline::Result<(usize, Vec<String>)> {
+    fn complete(&self, line: &str, pos: usize, _ctx: &Context<'_>) -> rustyline::Result<(usize, Vec<String>)> {
         if pos == 0 {
             Ok((0, vec![
                 String::from("return "),
             ]))
         } else {
-            Ok((0, vec![]))
+            let filter = match tokenize::parse_last(&line[..pos]) {
+                Token::Name(name) => name,
+                Token::Empty => String::new(),
+                _ => return Ok((0, vec![])),
+            };
+
+            let mut options = Vec::new();
+            for g in &self.globals {
+                if g.starts_with(&filter) {
+                    options.push(g.to_string());
+                }
+            }
+            Ok((pos - filter.len(), options))
         }
     }
 }

src/repl/mod.rs

@@ -1,26 +1,90 @@
 use crate::config::Config;
 use crate::errors::*;
 use crate::engine::{ctx, Environment, DummyReporter};
-use crate::engine::ctx::State;
+use crate::engine::ctx::{State, LuaState};
 use crate::geoip::{Maxmind, AsnDB, GeoIP};
-use crate::hlua::AnyLuaValue;
+use crate::hlua::{Lua, AnyLuaValue};
 use crate::psl::PslReader;
 use crate::shell::readline::Readline;
 use crate::runtime::format_lua;
 use chrootable_https::Resolver;
 use std::collections::HashMap;
+use std::sync::Arc;
 
 mod complete;
 use self::complete::ReplCompleter;
+mod tokenize;
 
+pub struct Repl<'a> {
+    rl: Readline<ReplCompleter>,
+    lua: Lua<'a>,
+    state: Arc<LuaState>,
+}
+
+impl<'a> Repl<'a> {
+    pub fn new(lua: Lua<'a>, state: Arc<LuaState>) -> Repl<'a> {
+        let rl = Readline::with(ReplCompleter::default());
+        Repl {
+            rl,
+            lua,
+            state,
+        }
+    }
+
+    fn update_globals(&mut self) {
+        let mut globals = Vec::new();
+        for item in self.lua.globals_table().iter::<String, AnyLuaValue>() {
+            if let Some((k, _)) = item {
+                globals.push(k);
+            }
+        }
+        if let Some(helper) = self.rl.helper_mut() {
+            debug!("updating globals: {:?}", globals);
+            helper.set(globals);
+        }
+    }
+
+    pub fn run(&mut self) {
+        loop {
+            self.update_globals();
+
+            match self.rl.readline("> ") {
+                Ok(line) => {
+                    self.rl.add_history_entry(line.as_str());
+                    self.exec(&line);
+                },
+                Err(_) => break,
+            }
+        }
+    }
+
+    pub fn exec(&mut self, line: &str) {
+        match self.lua.execute::<AnyLuaValue>(line) {
+            Ok(val) => {
+                if val != AnyLuaValue::LuaNil {
+                    let mut out = String::new();
+                    format_lua(&mut out, &val);
+                    println!("{}", out);
+                }
+                if let Some(err) = self.state.last_error() {
+                    println!("Error: {}", err);
+                    self.state.clear_error();
+                }
+            },
+            Err(err) => {
+                println!("Fatal: {}", err);
+            }
+        }
+    }
+}
 
 pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
     let dns_config = Resolver::from_system()?;
     let proxy = config.network.proxy;
     let psl = PslReader::open()?;
-    let geoip = GeoIP::open_reader()?;
-    let asn = AsnDB::open_reader()?;
+    let geoip = GeoIP::try_open_reader()?;
+    let asn = AsnDB::try_open_reader()?;
 
     let env = Environment {
         verbose: 0, // this doesn't do anything since we use a dummy reporter
@@ -35,39 +99,15 @@ pub fn run(config: &Config) -> Result<()> {
     };
 
     let tx = DummyReporter::new();
-    let (mut lua, state) = ctx::ctx(env, tx);
+    let (lua, state) = ctx::ctx(env, tx);
+    let mut repl = Repl::new(lua, state);
 
     println!(r#":: sn0int v{} lua repl
 Assign variables with `a = sn0int_version()` and `return a` to print
 Read the docs at https://sn0int.readthedocs.io/en/stable/reference.html
 "#, env!("CARGO_PKG_VERSION"));
 
-    let mut rl = Readline::with(ReplCompleter);
-    loop {
-        match rl.readline("> ") {
-            Ok(line) => {
-                rl.add_history_entry(line.as_str());
-
-                match lua.execute::<AnyLuaValue>(&line) {
-                    Ok(val) => {
-                        if val != AnyLuaValue::LuaNil {
-                            let mut out = String::new();
-                            format_lua(&mut out, &val);
-                            println!("{}", out);
-                        }
-                        if let Some(err) = state.last_error() {
-                            println!("Error: {}", err);
-                            state.clear_error();
-                        }
-                    },
-                    Err(err) => {
-                        println!("Fatal: {}", err);
-                    }
-                }
-            },
-            Err(_) => break,
-        }
-    }
+    repl.run();
 
     Ok(())
 }

src/repl/tokenize.rs

@@ -0,0 +1,119 @@
+#[derive(Debug, PartialEq)]
+pub enum Token {
+    Empty,
+    Name(String),
+    InString(StringState),
+    Value(String),
+}
+
+#[derive(Debug, PartialEq)]
+pub struct StringState {
+    quote: char,
+    escape: bool,
+    buf: String,
+}
+
+impl StringState {
+    pub fn new(quote: char) -> StringState {
+        StringState {
+            quote,
+            escape: false,
+            buf: String::new(),
+        }
+    }
+}
+
+pub fn parse_last(line: &str) -> Token {
+    let mut token = Token::Empty;
+
+    for c in line.chars() {
+        match &mut token {
+            Token::Empty => {
+                match c {
+                    'a'..='z' | 'A'..='Z' | '_' => {
+                        token = Token::Name(c.to_string());
+                    },
+                    '0'..='9' => {
+                        token = Token::Value(c.to_string());
+                    },
+                    '\'' | '"' => {
+                        token = Token::InString(StringState::new(c));
+                    },
+                    // ignore operators
+                    _ => (),
+                }
+            },
+            Token::Name(s) => {
+                match c {
+                    'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => {
+                        s.push(c);
+                    },
+                    // ignore operators
+                    _ => {
+                        token = Token::Empty;
+                    },
+                }
+            },
+            Token::InString(s) => {
+                if s.escape {
+                    s.buf.push(c);
+                    s.escape = false;
+                } else if c == '\\' {
+                    s.escape = true;
+                } else if c == s.quote {
+                    // done
+                    token = Token::Empty;
+                } else {
+                    s.buf.push(c);
+                }
+            },
+            Token::Value(s) => {
+                match c {
+                    '0'..='9' => {
+                        s.push(c);
+                    },
+                    // ignore operators
+                    // a name can't directly follow a value
+                    _ => {
+                        token = Token::Empty;
+                    },
+                }
+            },
+        }
+    }
+
+    token
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    pub fn test_empty() {
+        let token = parse_last("");
+        assert_eq!(token, Token::Empty);
+    }
+
+    #[test]
+    pub fn test_abc() {
+        let token = parse_last("abc");
+        assert_eq!(token, Token::Name("abc".into()));
+    }
+
+    #[test]
+    pub fn test_in_string() {
+        let token = parse_last("return url_encode(\"asdf");
+        assert_eq!(token, Token::InString(StringState {
+            quote: '"',
+            escape: false,
+            buf: "asdf".into(),
+        }));
+    }
+
+    #[test]
+    pub fn test_in_func() {
+        let token = parse_last("return url_encode(\"asdf\") .. url_deco");
+        assert_eq!(token, Token::Name("url_deco".into()));
+    }
+}

src/runtime/db.rs

@@ -104,6 +104,18 @@ pub fn db_add_ttl(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
+pub fn db_activity(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<()> {
+        let v: LuaJsonValue = v.into();
+        let v: serde_json::Value = v.into();
+        let activity: InsertActivity = serde_json::from_value(v)
+            .map_err(|e| state.set_error(e.into()))?;
+
+        state.db_activity(activity)
+            .map_err(|e| state.set_error(e))
+    }))
+}
+
 pub fn db_select(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("db_select", hlua::function2(move |family: String, value: String| -> Result<Option<i32>> {
         let family = Family::from_str(&family)

src/runtime/geoip.rs

@@ -49,6 +49,8 @@ pub fn geoip_lookup(lua: &mut hlua::Lua, state: Arc<dyn State>) {
 
 #[cfg(test)]
 mod tests {
+    // You need geoip setup on your system to run this
+    /*
     use crate::engine::ctx::Script;
 
     #[test]
@@ -72,4 +74,5 @@ mod tests {
         "#).expect("Failed to load script");
         script.test().expect("Script failed");
     }
+    */
 }

src/runtime/mod.rs

@@ -24,6 +24,7 @@ import_fns!(logger);
 import_fns!(options);
 import_fns!(pgp);
 import_fns!(psl);
+import_fns!(ratelimits);
 import_fns!(regex);
 import_fns!(semver);
 import_fns!(sleep);

src/runtime/ratelimits.rs

@@ -0,0 +1,12 @@
+use crate::errors::*;
+use crate::engine::ctx::State;
+use crate::hlua;
+use std::sync::Arc;
+
+
+pub fn ratelimit_throttle(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("ratelimit_throttle", hlua::function3(move |key: String, passes: u32, time: u32| -> Result<()> {
+        state.ratelimit(key, passes, time)
+            .map_err(|e| state.set_error(e))
+    }))
+}

src/schema.rs

@@ -15,6 +15,19 @@ table! {
     }
 }
 
+table! {
+    activity (id) {
+        id -> Integer,
+        topic -> Text,
+        time -> Timestamp,
+        uniq -> Nullable<Text>,
+        latitude -> Nullable<Float>,
+        longitude -> Nullable<Float>,
+        radius -> Nullable<Integer>,
+        content -> Text,
+    }
+}
+
 table! {
     autonoscope (id) {
         id -> Integer,
@@ -184,7 +197,7 @@ table! {
         ip_addr -> Text,
         port -> Integer,
         protocol -> Text,
-        status -> Text,
+        status -> Nullable<Text>,
         unscoped -> Bool,
         banner -> Nullable<Text>,
         service -> Nullable<Text>,
@@ -246,6 +259,7 @@ joinable!(urls -> subdomains (subdomain_id));
 
 allow_tables_to_appear_in_same_query!(
     accounts,
+    activity,
     autonoscope,
     breach_emails,
     breaches,

src/shell/complete.rs

@@ -102,6 +102,7 @@ impl Completer for CmdCompleter {
                                         "subdomain",
                                         "ipnet",
                                         "ipaddr",
+                                        "url",
                                         "email",
                                         "phonenumber",
                                         "device",
@@ -110,6 +111,7 @@ impl Completer for CmdCompleter {
                                         "breach",
                                         "image",
                                         "netblock",
+                                        "port",
                                         "cryptoaddr"];
 
                         let results: Vec<String> = options.iter()

src/shell/mod.rs

@@ -10,7 +10,6 @@ use crate::worker::VoidSender;
 use colored::Colorize;
 use crate::db::{self, Database};
 use crate::engine::{Engine, Module};
-use crate::geoip::{GeoIP, AsnDB, Maxmind};
 use crate::update::AutoUpdater;
 use std::collections::HashMap;
 use std::str::FromStr;
@@ -30,6 +29,7 @@ use self::readline::{Readline, ReadlineError};
 
 #[derive(Debug)]
 pub enum Command {
+    Activity,
     Add,
     Autonoscope,
     Autoscope,
@@ -54,6 +54,7 @@ pub enum Command {
 impl Command {
     pub fn as_str(&self) -> &'static str {
         match *self {
+            Command::Activity => "activity",
             Command::Add => "add",
             Command::Autonoscope => "autonoscope",
             Command::Autoscope => "autoscope",
@@ -78,6 +79,7 @@ impl Command {
     pub fn list_all() -> &'static [&'static str] {
         lazy_static! {
             static ref COMMANDS: Vec<&'static str> = vec![
+                Command::Activity.as_str(),
                 Command::Add.as_str(),
                 Command::Autonoscope.as_str(),
                 Command::Autoscope.as_str(),
@@ -107,6 +109,7 @@ impl FromStr for Command {
 
     fn from_str(s: &str) -> Result<Self> {
         match s {
+            "activity" => Ok(Command::Activity),
             "add" => Ok(Command::Add),
             "autonoscope" => Ok(Command::Autonoscope),
             "autoscope" => Ok(Command::Autoscope),
@@ -420,6 +423,7 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
     let line = rl.readline();
     debug!("Received line: {:?}", line);
     match line {
+        Some((Command::Activity, args)) => cmd::<activity_cmd::Args>(rl, &args)?,
         Some((Command::Add, args)) => cmd::<add_cmd::Args>(rl, &args)?,
         Some((Command::Autonoscope, args)) => autonoscope_cmd::run(rl, &args)?,
         Some((Command::Autoscope, args)) => autoscope_cmd::run(rl, &args)?,
@@ -462,10 +466,6 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
 
     let psl = PslReader::open_or_download()
         .context("Failed to download public suffix list")?;
-    let _geoip = GeoIP::open_or_download()
-        .context("Failed to download GeoIP database")?;
-    let _asndb = AsnDB::open_or_download()
-        .context("Failed to download ASN database")?;
     let engine = Engine::new(verbose_init, &config)?;
     let keyring = KeyRing::init()?;
 

src/utils.rs

@@ -1,21 +1,26 @@
 use crate::errors::*;
-
-use std::io::{self, Write};
+use rustyline::error::ReadlineError;
 use std::str::FromStr;
 
 
-pub fn read_line() -> Result<String> {
-    let mut buf = String::new();
-    io::stdin().read_line(&mut buf)?;
-    let buf = buf.trim().to_string();
-    Ok(buf)
+pub fn read_line(prompt: &str) -> Result<String> {
+    let mut rl = rustyline::Editor::<()>::new();
+    let mut line = rl.readline(prompt)
+        .map_err(|err| match err {
+            ReadlineError::Eof => format_err!("Failed to read line from input"),
+            ReadlineError::Interrupted => format_err!("Prompt has been canceled"),
+            err => err.into(),
+        })?;
+    if let Some(idx) = line.find('\n') {
+        line.truncate(idx);
+    }
+    info!("Read from prompt: {:?}", line);
+    Ok(line)
 }
 
 pub fn question(text: &str) -> Result<String> {
-    print!("\x1b[1m[\x1b[34m?\x1b[0;1m]\x1b[0m {}: ", text);
-    io::stdout().flush()?;
-
-    read_line()
+    let prompt = format!("\x1b[1m[\x1b[34m?\x1b[0;1m]\x1b[0m {}: ", text);
+    read_line(&prompt)
 }
 
 pub fn question_opt(text: &str) -> Result<Option<String>> {

src/worker.rs

@@ -9,6 +9,7 @@ use crate::engine::{self, Module};
 use crate::engine::isolation::Supervisor;
 use crate::models::*;
 use serde_json;
+use crate::ratelimits::{Ratelimiter, RatelimitResponse};
 use crate::shell::Shell;
 use std::collections::HashMap;
 use std::result;
@@ -23,12 +24,14 @@ use threadpool::ThreadPool;
 
 type DbSender = mpsc::Sender<result::Result<Option<i32>, String>>;
 pub type VoidSender = mpsc::Sender<result::Result<(), String>>;
+pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
 
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Event {
     Log(LogEvent),
     Database(DatabaseEvent),
     Stdio(StdioEvent),
+    Ratelimit(RatelimitEvent),
     Blob(Blob),
     Exit(ExitEvent),
 }
@@ -38,6 +41,7 @@ pub enum Event2 {
     Start,
     Log(LogEvent),
     Database((DatabaseEvent, DbSender)),
+    Ratelimit((RatelimitEvent, RatelimitSender)),
     Blob((Blob, VoidSender)),
     Exit(ExitEvent),
 }
@@ -125,6 +129,7 @@ impl LogEvent {
 pub enum DatabaseEvent {
     Insert(Insert),
     InsertTtl((Insert, i32)),
+    Activity(NewActivity),
     Select((Family, String)),
     Update((String, Update)),
 }
@@ -201,10 +206,45 @@ impl DatabaseEvent {
         tx.send(result).expect("Failed to send db result to channel");
     }
 
+    pub fn activity<T: SpinLogger>(object: NewActivity, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
+        let result = db.insert_activity(object.clone());
+        debug!("{:?} => {:?}", object, result);
+
+        let result = match result {
+            Ok(true) => {
+                let mut log = format!("{:?} ", object.topic);
+                if let Some(uniq) = &object.uniq {
+                    log.push_str(&format!("({:?}) ", uniq));
+                }
+                log.push_str(&format!("@ {}", object.time));
+
+                if let (Some(ref lat), Some(ref lon)) = (object.latitude, object.longitude) {
+                    log.push_str(&format!(" ({}, {})", lat, lon));
+                }
+
+                if verbose > 0 {
+                    log.push_str(&format!(": {}", object.content));
+                }
+
+                spinner.log(&log);
+                Ok(None)
+            },
+            Ok(false) => Ok(None),
+            Err(err) => {
+                let err = err.to_string();
+                spinner.error(&err);
+                Err(err)
+            },
+        };
+
+        tx.send(result).expect("Failed to send db result to channel");
+    }
+
     pub fn apply<T: SpinLogger>(self, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
         match self {
             DatabaseEvent::Insert(object) => Self::insert(object, None, tx, spinner, db, verbose),
             DatabaseEvent::InsertTtl((object, ttl)) => Self::insert(object, Some(ttl), tx, spinner, db, verbose),
+            DatabaseEvent::Activity(object) => Self::activity(object, tx, spinner, db, verbose),
             DatabaseEvent::Select((family, value)) => {
                 let result = db.get_opt(&family, &value)
                     .map_err(|e| e.to_string());
@@ -283,6 +323,32 @@ impl StdioEvent {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct RatelimitEvent {
+    key: String,
+    passes: u32,
+    time: u32,
+}
+
+impl EventWithCallback for RatelimitEvent {
+    type Payload = RatelimitResponse;
+
+    fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
+        Event2::Ratelimit((self, tx))
+    }
+}
+
+impl RatelimitEvent {
+    #[inline]
+    pub fn new(key: String, passes: u32, time: u32) -> RatelimitEvent {
+        RatelimitEvent {
+            key,
+            passes,
+            time,
+        }
+    }
+}
+
 pub fn spawn(rl: &mut Shell, module: &Module, args: Vec<(serde_json::Value, Option<String>, Vec<Blob>)>, params: &Params, proxy: Option<SocketAddr>, options: HashMap<String, String>) -> usize {
     // This function hangs if args is empty, so return early if that's the case
     if args.is_empty() {
@@ -328,6 +394,8 @@ pub fn spawn(rl: &mut Shell, module: &Module, args: Vec<(serde_json::Value, Opti
         expected += 1;
     }
 
+    let mut ratelimit = Ratelimiter::new();
+
     let mut errors = 0;
     let mut failed = Vec::new();
     let timeout = Duration::from_millis(100);
@@ -344,6 +412,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, args: Vec<(serde_json::Value, Opti
                         },
                         Event2::Log(log) => log.apply(&mut stack.prefixed(name)),
                         Event2::Database((db, tx)) => db.apply(tx, &mut stack.prefixed(name), rl.db(), verbose),
+                        Event2::Ratelimit((req, tx)) => ratelimit.pass(tx, &req.key, req.passes, req.time),
                         Event2::Blob((blob, tx)) => rl.store_blob(tx, &blob),
                         Event2::Exit(event) => {
                             debug!("Received exit: {:?} -> {:?}", name, event);
@@ -399,6 +468,7 @@ pub fn spawn_fn<F, T>(label: &str, f: F, clear: bool) -> Result<T>
                     Some(Event::Log(log)) => log.apply(&mut *spinner),
                     Some(Event::Database(_)) => (),
                     Some(Event::Stdio(_)) => (),
+                    Some(Event::Ratelimit(_)) => (),
                     Some(Event::Blob(_)) => (),
                     // TODO: refactor
                     Some(Event::Exit(ExitEvent::Ok)) => break,
@@ -491,6 +561,7 @@ pub fn spawn_multi<T: Task, F>(tasks: Vec<T>, mut done_fn: F, threads: usize) ->
                         },
                         Event2::Log(log) => log.apply(&mut stack.prefixed(&name)),
                         Event2::Database(_) => (),
+                        Event2::Ratelimit(_) => (),
                         Event2::Blob(_) => (),
                         Event2::Exit(event) => {
                             debug!("Received exit: {:?} -> {:?}", name, event);