Release v0.19.0

Make sn0int more forgiving with accidential ^C

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.18.1"
+version = "0.19.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -17,12 +17,27 @@ members = ["sn0int-common",
            "sn0int-registry",
            "sn0int-std"]
 
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
+
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.10.0", path="sn0int-common" }
-sn0int-std = { version="=0.18.1", path="sn0int-std" }
+sn0int-common = { version="0.11.0", path="sn0int-common" }
+sn0int-std = { version="=0.19.0", path="sn0int-std" }
 rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
@@ -35,13 +50,13 @@ lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = "0.16.0"
+libsqlite3-sys = { version = "0.17.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
 dirs = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.15"
+chrootable-https = "0.15.2"
 base64 = "0.12"
 data-encoding = "2.1.2"
 serde_urlencoded = "0.6"
@@ -76,11 +91,12 @@ hmac = "0.7"
 walkdir = "2.2"
 nude = "0.3"
 glob = "0.3.0"
+os-version = "0.1.1"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.12"
+syscallz = "0.14.0"
 nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]

README.md

@@ -58,6 +58,39 @@ Mac OSX
 
     brew install sn0int
 
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+    apt install debian-keyring
+    gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    apt update
+    apt install sn0int
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
 For everything else please have a look at the [detailed list][1].
 
 [1]: https://sn0int.readthedocs.io/en/latest/install.html
@@ -68,10 +101,12 @@ For everything else please have a look at the [detailed list][1].
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
   - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
   - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
@@ -129,6 +164,18 @@ For everything else please have a look at the [detailed list][1].
   - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
   - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
   - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)

ci/run.sh

@@ -19,8 +19,8 @@ case "$1" in
         cargo test --verbose -- --ignored
         ;;
     windows)
-        cargo build --verbose --features=sqlite-bundled
-        cargo build --verbose --examples --features=sqlite-bundled
+        cargo build --verbose
+        cargo build --verbose --examples
         ;;
     boxxy)
         cargo build --verbose --examples

docs/build.rst

@@ -73,10 +73,4 @@ After all dependencies have been installed, simply build the binary:
 
     $ cargo build --release
 
-Note that you need a different command on windows:
-
-.. code-block:: bash
-
-    $ cargo build --release --features=sqlite-bundled
-
 After the build finished the binary is located at ``target/release/sn0int``.

docs/index.rst

@@ -43,6 +43,7 @@ Getting Started
    database
    structs
    activity
+   notifications
    keyring
    config
    sandbox

docs/install.rst

@@ -22,9 +22,20 @@ Mac OSX
 Debian/Ubuntu/Kali
 ------------------
 
-Note that debian `doesn't ship the geoip2-database
-<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
-download them automatically during the first run.
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    $ apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ apt update
+    $ apt install sn0int
+
+Fedora/CentOS/Redhat
+--------------------
 
 Using rust+cargo from the repos might work for you, but we only officially
 support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
@@ -32,7 +43,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -63,8 +74,15 @@ Gentoo
 
 .. code-block:: bash
 
-    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
-    emerge --ask net-analyzer/sn0int
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
@@ -78,4 +96,4 @@ Make sure rust is installed and setup.
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f --path . --features=sqlite-bundled
+    $ cargo install -f --path .

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/sandbox.rst

@@ -6,7 +6,7 @@ child process. It's important to note that there's a basic sandbox that's
 active on every operating system, and there's a second line of defense on
 supported operating systems.
 
-The first line of defense is the restrictive stdlib. It's assumed that and
+The first line of defense is the restrictive stdlib. It's assumed that an
 attacker gains full control over the lua code and is able to call any function
 with arbitrary arguments. The stdlib only provides functions that are
 considered safe, so for example it's not possible to start a process or open a

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/notify-ohai.lua

@@ -0,0 +1,8 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-slow.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    -- TODO your code here
+    sleep(1)
+    info('notication!: ' .. json_encode(arg))
+end

sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.10.0"
+version = "0.11.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"

sn0int-common/src/metadata.rs

@@ -44,6 +44,7 @@ pub enum Source {
     Netblocks,
     CryptoAddrs(Option<String>),
     KeyRing(String),
+    Notifications,
 }
 
 impl Source {
@@ -63,6 +64,7 @@ impl Source {
             Source::Ports => "ports",
             Source::Netblocks => "netblocks",
             Source::CryptoAddrs(_) => "cryptoaddrs",
+            Source::Notifications => "notifications",
             Source::KeyRing(_) => "keyring",
         }
     }
@@ -94,6 +96,7 @@ impl FromStr for Source {
             ("ports", None) => Ok(Source::Ports),
             ("netblocks", None) => Ok(Source::Netblocks),
             ("cryptoaddrs", param) => Ok(Source::CryptoAddrs(param.map(String::from))),
+            ("notifications", None) => Ok(Source::Notifications),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.10.0", path="../sn0int-common" }
+sn0int-common = { version="0.11.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-std/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.18.1"
+version = "0.19.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -24,19 +24,20 @@ serde_urlencoded = "0.6"
 webpki = "0.21"
 webpki-roots = "0.19.0"
 ct-logs = "0.6"
-chrootable-https = "0.15"
+#chrootable-https = { path = "../../chrootable-https" }
+chrootable-https = "0.15.2"
 http = "0.2"
 bufstream = "0.1.4"
 pem = "0.7"
 url = "2.0"
 tungstenite = { version = "0.10.1", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.13"
-x509-parser = "0.6.2"
+maxminddb = "0.14"
+x509-parser = "0.7"
 der-parser = "3.0"
 publicsuffix = { version="1.5", default-features=false }
 xml-rs = "0.8"
-geo = "0.12"
+geo = "0.13"
 bytes = "0.4"
 base64 = "0.12"
 chrono = { version = "0.4", features = ["serde"] }

sn0int-std/src/geoip/models.rs

@@ -3,14 +3,8 @@ use maxminddb::geoip2;
 use std::collections::BTreeMap;
 
 
-fn from_geoip_model_names(names: Option<BTreeMap<String, String>>) -> Option<String> {
-    let names = match names {
-        Some(names) => names,
-        _ => return None,
-    };
-
-    names.get("en")
-        .map(|x| x.to_owned())
+fn from_geoip_model_names(names: Option<BTreeMap<&str, &str>>) -> Option<String> {
+    names?.get("en").map(|x| x.to_string())
 }
 
 #[derive(Debug, Serialize)]
@@ -24,7 +18,7 @@ pub struct GeoLookup {
     pub longitude: Option<f64>,
 }
 
-impl From<geoip2::City> for GeoLookup {
+impl<'a> From<geoip2::City<'a>> for GeoLookup {
     fn from(lookup: geoip2::City) -> GeoLookup {
         // parse maxminddb lookup
         let continent = match lookup.continent {
@@ -79,17 +73,11 @@ pub struct Continent {
 
 impl Continent {
     fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
-        let code = match continent.code {
-            Some(code) => code,
-            _ => return None,
-        };
-        let name = match from_geoip_model_names(continent.names) {
-            Some(name) => name,
-            _ => return None,
-        };
+        let code = continent.code?;
+        let name = from_geoip_model_names(continent.names)?;
 
         Some(Continent {
-            code,
+            code: code.to_string(),
             name,
         })
     }
@@ -103,17 +91,11 @@ pub struct Country {
 
 impl Country {
     fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
-        let code = match country.iso_code {
-            Some(code) => code,
-            _ => return None,
-        };
-        let name = match from_geoip_model_names(country.names) {
-            Some(name) => name,
-            _ => return None,
-        };
+        let code = country.iso_code?;
+        let name = from_geoip_model_names(country.names)?;
 
         Some(Country {
-            code,
+            code: code.to_string(),
             name,
         })
     }
@@ -152,19 +134,14 @@ pub struct AsnLookup {
 impl AsnLookup {
     pub fn try_from(lookup: geoip2::Isp) -> Result<AsnLookup> {
         // parse maxminddb lookup
-        let asn = match lookup.autonomous_system_number {
-            Some(asn) => asn,
-            _ => bail!("autonomous_system_number not set"),
-        };
-        let as_org = match lookup.autonomous_system_organization {
-            Some(org) => org,
-            _ => bail!("autonomous_system_organization not set"),
-        };
+        let asn = lookup.autonomous_system_number
+            .ok_or_else(|| format_err!("autonomous_system_number not set"))?;
+        let as_org = lookup.autonomous_system_organization
+            .ok_or_else(|| format_err!("autonomous_system_organization not set"))?;
 
-        // return result
         Ok(AsnLookup {
             asn,
-            as_org,
+            as_org: as_org.to_string(),
         })
     }
 }

sn0int-std/src/psl.rs

@@ -60,7 +60,7 @@ impl PslReader {
     }
 
     pub fn download(path: &Path, url: &str) -> Result<()> {
-        let client = Client::with_system_resolver()?;
+        let client = Client::with_system_resolver_v4()?;
         let resp = client.get(url)
             .wait_for_response()
             .context("http request failed")?;

src/api.rs

@@ -18,6 +18,7 @@ pub struct Client {
     server: String,
     client: chrootable_https::Client<chrootable_https::Resolver>,
     session: Option<String>,
+    os_version: String,
 }
 
 impl Client {
@@ -25,14 +26,19 @@ impl Client {
         let client = if let Some(proxy) = config.network.proxy {
             chrootable_https::Client::with_socks5(proxy)
         } else {
-            chrootable_https::Client::with_system_resolver()
-                .context("Failed to load dns configuration")?
+            chrootable_https::Client::with_system_resolver_v4()?
+        };
+
+        let os_version = match os_version::detect() {
+            Ok(v) => v.to_string(),
+            Err(_) => String::from("unknown"),
         };
 
         Ok(Client {
             server: config.core.registry.clone(),
             client,
             session: None,
+            os_version,
         })
     }
 
@@ -45,10 +51,11 @@ impl Client {
     }
 
     #[inline]
-    fn user_agent() -> String {
-        format!("{}, {}",
+    fn user_agent(&self) -> String {
+        format!("{}, {}, {}",
             web::default_user_agent(),
-            embedded_triple::get()
+            embedded_triple::get(),
+            self.os_version,
         )
     }
 
@@ -57,7 +64,7 @@ impl Client {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
-        request.header("User-Agent", Self::user_agent());
+        request.header("User-Agent", self.user_agent());
 
         let request = request.body(body)?;
 

src/args.rs

@@ -72,6 +72,12 @@ pub enum SubCommand {
     /// Manage workspaces
     #[structopt(name="workspace")]
     Workspace(cmd::workspace_cmd::Args),
+    /// Calendar
+    #[structopt(name="cal")]
+    Cal(cmd::cal_cmd::Args),
+    /// Notify
+    #[structopt(name="notify")]
+    Notify(cmd::notify_cmd::Args),
     /// Verify blob storage for corrupt and dangling blobs
     #[structopt(name="fsck")]
     Fsck(cmd::fsck_cmd::Args),

src/cal/date.rs

@@ -0,0 +1,397 @@
+use crate::errors::*;
+
+use chrono::Utc;
+use chrono::prelude::*;
+use crate::cal::{ActivityGrade, DateArg};
+use crate::models::*;
+use std::collections::HashMap;
+use std::collections::VecDeque;
+
+const MONTH_LINES: i32 = 7;
+
+fn merge_months(ctx: &DateContext, months: &[DateSpec]) -> String {
+    let mut months = months.iter()
+        .map(|ds| {
+            let month = ds.to_term_string(ctx);
+            month.lines()
+                .map(String::from)
+                .collect::<VecDeque<_>>()
+        })
+        .collect::<Vec<_>>();
+
+    let mut out = String::new();
+    for i in 0..=MONTH_LINES {
+        let mut first = true;
+        for m in &mut months {
+            if !first {
+                out.push_str("   ");
+            }
+            if let Some(line) = m.pop_front() {
+                out.push_str(&line);
+            } else {
+                out.push_str(&" ".repeat(21));
+            }
+            first = false;
+        }
+        if i < MONTH_LINES {
+            out.push('\n');
+        }
+    }
+    out
+}
+
+fn chunk_months(ctx: &DateContext, months: &[DateSpec]) -> String {
+    months
+        .chunks(3)
+        .map(|m| merge_months(ctx, m))
+        .fold(String::new(), |a, b| {
+            if a.is_empty() {
+                a + &b
+            } else {
+                a + "\n" + &b
+            }
+        })
+}
+
+fn days_in_month(year: i32, month: u32) -> i64 {
+    let start = Utc.ymd(year, month, 1);
+    let end = if month == 12 {
+        Utc.ymd(year + 1, 1, 1)
+    } else {
+        Utc.ymd(year, month + 1, 1)
+    };
+    end.signed_duration_since(start).num_days()
+}
+
+fn setup_graph_map(events: &[Activity]) -> (HashMap<NaiveDate, u64>, u64) {
+    debug!("Found {} events in selected range", events.len());
+
+    let mut cur = None;
+    let mut ctr = 0;
+    let mut max = 0;
+
+    let mut map = HashMap::new();
+    for event in events {
+        let date = event.time.date();
+        if let Some(cur) = cur.as_mut() {
+            if date == *cur {
+                ctr += 1;
+            } else {
+                if ctr > max {
+                    max = ctr;
+                }
+                map.insert(cur.clone(), ctr);
+                *cur = date;
+                ctr = 1;
+            }
+        } else {
+            cur = Some(date);
+            ctr = 1;
+        }
+    }
+
+    if ctr > 0 {
+        if let Some(cur) = cur.take() {
+            if ctr > max {
+                max = ctr;
+            }
+            map.insert(cur, ctr);
+        }
+    }
+
+    debug!("Maximum events per day is {}", max);
+
+    (map, max)
+}
+
+pub struct DateContext {
+    events: HashMap<NaiveDate, u64>,
+    max: u64,
+    today: NaiveDate,
+}
+
+impl DateContext {
+    pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
+        let (events, max) = setup_graph_map(&events);
+        DateContext {
+            events,
+            max,
+            today,
+        }
+    }
+
+    #[inline]
+    fn is_today(&self, date: &NaiveDate) -> bool {
+        self.today == *date
+    }
+
+    #[inline]
+    fn is_future(&self, date: &NaiveDate) -> bool {
+        self.today < *date
+    }
+
+    fn activity_for_day(&self, date: &NaiveDate) -> ActivityGrade {
+        if let Some(events) = self.events.get(date) {
+            ActivityGrade::from_ratio(*events, self.max)
+        } else {
+            ActivityGrade::None
+        }
+    }
+}
+
+pub enum DateSpec {
+    Year(i32),
+    YearMonth((i32, u32)),
+    YearMonthContext((i32, u32, u32)),
+}
+
+impl DateSpec {
+    pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateSpec> {
+        if args.len() > 2 {
+            bail!("Too many datespec args");
+        }
+
+        let today = Utc::today();
+        let ds = match (args.get(0), args.get(1), context) {
+            (None, _, None) => DateSpec::YearMonth((today.year(), today.month())),
+            (None, _, Some(context)) => DateSpec::YearMonthContext((today.year(), today.month(), context)),
+
+            (Some(DateArg::Month(month)), None, None) => DateSpec::YearMonth((today.year(), *month)),
+            (Some(DateArg::Num(year)), None, None) => DateSpec::Year(*year),
+            (Some(DateArg::Month(month)), Some(DateArg::Num(year)), None) => DateSpec::YearMonth((*year, *month)),
+            (Some(DateArg::Num(year)), Some(DateArg::Month(month)), None) => DateSpec::YearMonth((*year, *month)),
+
+            (Some(DateArg::Month(month)), None, Some(context)) => DateSpec::YearMonthContext((today.year(), *month, context)),
+            (Some(DateArg::Month(month)), Some(DateArg::Num(year)), Some(context)) => DateSpec::YearMonthContext((*year, *month, context)),
+            (Some(DateArg::Num(year)), Some(DateArg::Month(month)), Some(context)) => DateSpec::YearMonthContext((*year, *month, context)),
+            _ => bail!("Combination of datespec args is invalid"),
+        };
+        Ok(ds)
+    }
+
+    pub fn start(&self) -> NaiveDate {
+        match self {
+            DateSpec::Year(year) => NaiveDate::from_ymd(*year, 1, 1),
+            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd(*year, *month, 1),
+            DateSpec::YearMonthContext((year, month, context)) => {
+                let mut year = *year - (*context / 12) as i32;
+                let context = context % 12;
+                let month = if context >= *month {
+                    year -= 1;
+                    12 - context + month
+                } else {
+                    month - context
+                };
+                NaiveDate::from_ymd(year, month, 1)
+            },
+        }
+    }
+
+    pub fn end(&self) -> NaiveDate {
+        match self {
+            DateSpec::Year(year) => NaiveDate::from_ymd(year + 1, 1, 1),
+            DateSpec::YearMonth((year, month)) => {
+                let (year, month) = if *month == 12 {
+                    (*year + 1, 1)
+                } else {
+                    (*year, *month + 1)
+                };
+                NaiveDate::from_ymd(year, month, 1)
+            },
+            DateSpec::YearMonthContext((year, month, _context)) => {
+                let (year, month) = if *month == 12 {
+                    (*year + 1, 1)
+                } else {
+                    (*year, *month + 1)
+                };
+                NaiveDate::from_ymd(year, month, 1)
+            },
+        }
+    }
+
+    pub fn to_term_string(&self, ctx: &DateContext) -> String {
+        match self {
+            DateSpec::Year(year) => {
+                let months = (1..=12)
+                    .map(|month| DateSpec::YearMonth((*year, month)))
+                    .collect::<Vec<_>>();
+                chunk_months(ctx, &months)
+            },
+            DateSpec::YearMonth((year, month)) => {
+                let mut w = String::new();
+
+                let start = Utc.ymd(*year, *month, 1);
+                let days = days_in_month(*year, *month) as u32;
+
+                w.push_str(&format!("{:^21}\n", start.format("%B %Y")));
+                w.push_str(" Su Mo Tu We Th Fr Sa\n");
+
+                let mut cur_week_day = start.weekday();
+                let week_progress = cur_week_day.num_days_from_sunday() as usize;
+                w.push_str(&"   ".repeat(week_progress));
+
+                let mut week_written = week_progress * 3;
+                for cur_day in 1..=days {
+                    let date = NaiveDate::from_ymd(*year, *month, cur_day);
+
+                    if !ctx.is_future(&date) {
+                        let activity = ctx.activity_for_day(&date);
+                        w.push_str(activity.as_term_str());
+                    }
+
+                    if ctx.is_today(&date) {
+                        w.push_str("\x1b[1m#");
+                    } else {
+                        w.push(' ');
+                    }
+                    w.push_str(&format!("{:2}", cur_day));
+                    week_written += 3;
+                    w.push_str("\x1b[0m");
+
+                    // detect end of the week
+                    if cur_week_day == Weekday::Sat {
+                        if cur_day != days {
+                            w.push('\n');
+                        }
+                        week_written = 0;
+                    }
+
+                    cur_week_day = cur_week_day.succ();
+                }
+                if week_written != 0 {
+                    w.push_str(&" ".repeat(21 - week_written));
+                }
+
+                w
+            }
+            DateSpec::YearMonthContext((_year, _month, context)) => {
+                let start = self.start();
+                let mut year = start.year();
+                let mut month = start.month();
+
+                let mut months = Vec::new();
+
+                for _ in 0..=*context {
+                    months.push(DateSpec::YearMonth((year, month)));
+
+                    if month == 12 {
+                        year += 1;
+                        month = 1;
+                    } else {
+                        month += 1;
+                    }
+                }
+
+                chunk_months(ctx, &months)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn context() -> DateContext {
+        DateContext {
+            events: HashMap::new(),
+            max: 0,
+            today: NaiveDate::from_ymd(2020, 05, 30),
+        }
+    }
+
+    #[test]
+    fn test_days_in_month_2020_05() {
+        let days = days_in_month(2020, 05);
+        assert_eq!(days, 31);
+    }
+
+    #[test]
+    fn test_days_in_month_2020_04() {
+        let days = days_in_month(2020, 04);
+        assert_eq!(days, 30);
+    }
+
+    #[test]
+    fn test_days_in_month_2020_02() {
+        let days = days_in_month(2020, 02);
+        assert_eq!(days, 29);
+    }
+
+    #[test]
+    fn small_max_activity_0() {
+        let events = HashMap::new();
+        let ctx = DateContext {
+            events,
+            max: 0,
+            today: NaiveDate::from_ymd(2020, 06, 06),
+        };
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 06, 06));
+        assert_eq!(grade, ActivityGrade::None);
+    }
+
+    #[test]
+    fn test_datespec_year_month() {
+        let ds = DateSpec::YearMonth((2020, 05));
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "      May 2020       
+ Su Mo Tu We Th Fr Sa
+               \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m\u{1b}[1m#30\u{1b}[0m
+ 31\u{1b}[0m                  ");
+    }
+
+    #[test]
+    fn test_datespec_year_month_ends_on_sat() {
+        let ds = DateSpec::YearMonth((2020, 10));
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "    October 2020     
+ Su Mo Tu We Th Fr Sa
+              1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m
+  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m
+ 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m
+ 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m
+ 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m");
+    }
+
+    #[test]
+    fn test_datespec_year() {
+        let ds = DateSpec::Year(2020);
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "    January 2020            February 2020            March 2020      
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+         \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m                     \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 31\u{1b}[0m      \u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 31\u{1b}[0m            
+                                                                     
+     April 2020               May 2020                June 2020      
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+         \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m                  \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m        1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m     7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m    14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m    21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m         \u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m\u{1b}[1m#30\u{1b}[0m    28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m            
+                         31\u{1b}[0m                                          
+      July 2020              August 2020           September 2020    
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m                       1\u{1b}[0m           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m
+  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m     2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m     6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m
+ 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m     9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m    13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m
+ 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m    16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m    20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m
+ 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m       23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m    27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m         
+                         30\u{1b}[0m 31\u{1b}[0m                                       
+    October 2020            November 2020           December 2020    
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+              1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m     1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m
+  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m     8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m     6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m
+ 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m    15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m    13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m
+ 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m    22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m    20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m
+ 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m    29\u{1b}[0m 30\u{1b}[0m                   27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m      
+                                                                     ");
+    }
+}

src/cal/mod.rs

@@ -0,0 +1,166 @@
+use crate::errors::*;
+use std::str::FromStr;
+
+pub mod date;
+pub mod time;
+
+#[derive(Debug)]
+pub enum DateArg {
+    Month(u32),
+    Num(i32),
+}
+
+impl FromStr for DateArg {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<DateArg> {
+        let ds = match s.to_lowercase().as_str() {
+            "jan" | "january"   => DateArg::Month(1),
+            "feb" | "february"  => DateArg::Month(2),
+            "mar" | "march"     => DateArg::Month(3),
+            "apr" | "april"     => DateArg::Month(4),
+            "may"               => DateArg::Month(5),
+            "jun" | "june"      => DateArg::Month(6),
+            "jul" | "july"      => DateArg::Month(7),
+            "aug" | "august"    => DateArg::Month(8),
+            "sep" | "september" => DateArg::Month(9),
+            "oct" | "october"   => DateArg::Month(10),
+            "nov" | "november"  => DateArg::Month(11),
+            "dec" | "december"  => DateArg::Month(12),
+            _ => {
+                let num = s.parse::<i32>()
+                    .context("Input is not a month and not a number")?;
+                DateArg::Num(num)
+            },
+        };
+        Ok(ds)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum ActivityGrade {
+    None,
+    One,
+    Two,
+    Three,
+    Four,
+}
+
+impl ActivityGrade {
+    fn as_term_str(&self) -> &'static str {
+        match self {
+            ActivityGrade::None => "\x1b[97m\x1b[48;5;238m",
+            ActivityGrade::One => "\x1b[30m\x1b[48;5;148m",
+            ActivityGrade::Two => "\x1b[30m\x1b[48;5;71m",
+            ActivityGrade::Three => "\x1b[97m\x1b[48;5;34m",
+            ActivityGrade::Four => "\x1b[97m\x1b[48;5;22m",
+        }
+    }
+
+    pub fn from_ratio(num: u64, max: u64) -> ActivityGrade {
+        let max = max as f64;
+        let num = num as f64;
+        let step = max / 4.0;
+
+        let x = num / step;
+
+        if x <= 1.0 {
+            ActivityGrade::One
+        } else if x <= 2.0 {
+            ActivityGrade::Two
+        } else if x <= 3.0 {
+            ActivityGrade::Three
+        } else {
+            ActivityGrade::Four
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn small_max_activity_1() {
+        let grade = ActivityGrade::from_ratio(1, 1);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_2() {
+        let grade = ActivityGrade::from_ratio(2, 2);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_3() {
+        let grade = ActivityGrade::from_ratio(3, 3);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_4() {
+        let grade = ActivityGrade::from_ratio(4, 4);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_2_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 2);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_3_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 3);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_3_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 3);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_3() {
+        let grade = ActivityGrade::from_ratio(3, 4);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 4);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 4);
+        assert_eq!(grade, ActivityGrade::One);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_4() {
+        let grade = ActivityGrade::from_ratio(4, 5);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_3() {
+        let grade = ActivityGrade::from_ratio(3, 5);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 5);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 5);
+        assert_eq!(grade, ActivityGrade::One);
+    }
+}

src/cal/time.rs

@@ -0,0 +1,194 @@
+use crate::errors::*;
+
+use chrono::Duration;
+use chrono::prelude::*;
+use crate::cal::{ActivityGrade, DateArg};
+use crate::models::*;
+use std::collections::HashMap;
+
+const MIN_PER_DAY: u32 = 1440;
+
+fn round_to_slice(time: &NaiveDateTime, slice_duration: u32) -> NaiveDateTime {
+    let date = time.date();
+    let hour = time.hour();
+    let mins = time.minute();
+    let slice = mins - (mins % slice_duration);
+    date.and_hms(hour, slice, 0)
+}
+
+fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDateTime, u64>, u64) {
+    debug!("Found {} events in selected range", events.len());
+
+    let mut cur = None;
+    let mut ctr = 0;
+    let mut max = 0;
+
+    let mut map = HashMap::new();
+    for event in events {
+        let time = round_to_slice(&event.time, slice_duration);
+
+        if let Some(cur) = cur.as_mut() {
+            if time == *cur {
+                ctr += 1;
+            } else {
+                if ctr > max {
+                    max = ctr;
+                }
+                map.insert(cur.clone(), ctr);
+                *cur = time;
+                ctr = 1;
+            }
+        } else {
+            cur = Some(time);
+            ctr = 1;
+        }
+    }
+
+    if ctr > 0 {
+        if let Some(cur) = cur.take() {
+            if ctr > max {
+                max = ctr;
+            }
+            map.insert(cur, ctr);
+        }
+    }
+
+    debug!("Maximum events per slice is {}", max);
+
+    (map, max)
+}
+
+pub struct DateTimeContext {
+    events: HashMap<NaiveDateTime, u64>,
+    max: u64,
+    now: NaiveDateTime,
+    pub slice_width: u32,
+    pub slice_duration: u32,
+}
+
+impl DateTimeContext {
+    pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
+        let (events, max) = setup_graph_map(&events, slice_duration);
+        DateTimeContext {
+            events,
+            max,
+            now: round_to_slice(&now, slice_duration),
+            slice_width,
+            slice_duration,
+        }
+    }
+
+    #[inline]
+    fn is_future(&self, time: &NaiveDateTime) -> bool {
+        self.now < *time
+    }
+
+    #[inline]
+    fn slices_per_hour(&self) -> u32 {
+        60 / self.slice_duration
+    }
+
+    #[inline]
+    fn hour_width(&self) -> u32 {
+        self.slices_per_hour() * self.slice_width
+    }
+
+    fn activity_for_slice(&self, time: &NaiveDateTime) -> ActivityGrade {
+        if let Some(events) = self.events.get(time) {
+            ActivityGrade::from_ratio(*events, self.max)
+        } else {
+            ActivityGrade::None
+        }
+    }
+}
+
+pub struct DateTimeSpec {
+    start: NaiveDate,
+    end: NaiveDate,
+}
+
+impl DateTimeSpec {
+    pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateTimeSpec> {
+        let today = Utc::today().naive_utc();
+        if args.is_empty() {
+            let mut start = today;
+
+            if let Some(context) = context {
+                start = start.checked_sub_signed(Duration::days(context as i64))
+                    .ok_or_else(|| format_err!("Failed travel back in time"))?;
+            }
+
+            Ok(DateTimeSpec {
+                start,
+                end: today,
+            })
+        } else {
+            todo!()
+        }
+    }
+
+    pub fn start(&self) -> &NaiveDate {
+        &self.start
+    }
+
+    pub fn end(&self) -> &NaiveDate {
+        &self.end
+    }
+
+    pub fn to_term_string(&self, ctx: &DateTimeContext) -> String {
+        let mut w = String::new();
+
+        // add legend
+        w.push_str(&" ".repeat(11));
+        for x in 0..24 {
+            w.push_str(&format!("{:02}", x));
+
+            for i in 0..ctx.hour_width() {
+                if i >= 2 {
+                    w.push(' ');
+                }
+            }
+        }
+        w.push('\n');
+
+        // add days
+        let mut date = self.start;
+        while date <= self.end {
+            w.push_str(&date.format("%Y-%m-%d ").to_string());
+
+            let mut hours = 0;
+            let mut mins = 0;
+
+            for _ in 0..(MIN_PER_DAY / ctx.slice_duration) {
+                let time = date.and_hms(hours, mins, 0);
+
+                if !ctx.is_future(&time) {
+                    let activity = ctx.activity_for_slice(&time);
+                    w.push_str(activity.as_term_str());
+                } else {
+                    w.push_str("\x1b[0m");
+                }
+
+                for _ in 0..ctx.slice_width {
+                    w.push(' ');
+                }
+
+                mins += ctx.slice_duration;
+                if mins >= 60 {
+                    hours += 1;
+                    mins = 0;
+                }
+            }
+            w.push_str("\x1b[0m");
+
+            if date < self.end {
+                w.push('\n');
+            }
+
+            date = date.checked_add_signed(Duration::days(1))
+                .expect("Failed to get next day");
+        }
+
+        w
+    }
+}

src/cmd/cal_cmd.rs

@@ -0,0 +1,66 @@
+use crate::errors::*;
+
+use chrono::Utc;
+use crate::cal::DateArg;
+use crate::cal::date::{DateContext, DateSpec};
+use crate::cal::time::{DateTimeContext, DateTimeSpec};
+use crate::cmd::Cmd;
+use crate::models::*;
+use crate::shell::Shell;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Show additional months for context
+    #[structopt(short="C", long)]
+    context: Option<u32>,
+    /// Group events in 12 min slices
+    #[structopt(short="T", long, group = "view")]
+    time: bool,
+    /// Group events by hour
+    #[structopt(short="H", long, group = "view")]
+    hourly: bool,
+    args: Vec<DateArg>,
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        if self.time || self.hourly {
+            let dts = DateTimeSpec::from_args(&self.args, self.context)
+                .context("Failed to parse date spec")?;
+            let filter = ActivityFilter {
+                topic: None,
+                since: Some(dts.start().and_hms(0, 0, 0)),
+                until: Some(dts.end().and_hms(23, 59, 59)),
+                location: false,
+            };
+            let events = Activity::query(rl.db(), &filter)?;
+
+            let (slice_width, slice_duration) = if self.hourly {
+                (3, 60)
+            } else {
+                (1, 12)
+            };
+
+            let ctx = DateTimeContext::new(&events, Utc::now().naive_utc(), slice_width, slice_duration);
+            println!("{}", dts.to_term_string(&ctx));
+        } else {
+            let ds = DateSpec::from_args(&self.args, self.context)
+                .context("Failed to parse date spec")?;
+            let filter = ActivityFilter {
+                topic: None,
+                since: Some(ds.start().and_hms(0, 0, 0)),
+                until: Some(ds.end().and_hms(23, 59, 59)),
+                location: false,
+            };
+            let events = Activity::query(rl.db(), &filter)?;
+            let ctx = DateContext::new(&events, Utc::today().naive_utc());
+            println!("{}", ds.to_term_string(&ctx));
+        }
+        Ok(())
+    }
+}

src/cmd/export_cmd.rs

@@ -3,6 +3,7 @@ use crate::errors::*;
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
 use crate::db::ttl;
+use crate::models::*;
 use crate::shell::Shell;
 use serde_json;
 use serde::Serialize;
@@ -10,7 +11,6 @@ use std::io::{self, Write};
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use strum_macros::{EnumString, IntoStaticStr};
-use crate::models::*;
 
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
@@ -22,7 +22,7 @@ pub struct Args {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         match self.format {
             Format::Json => export::<JsonFormat>(rl),
             Format::JsonBlobs => export::<JsonBlobsFormat>(rl),

src/cmd/mod.rs

@@ -20,6 +20,7 @@ pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
 pub mod autoscope_cmd;
+pub mod cal_cmd;
 pub mod delete_cmd;
 pub mod export_cmd;
 pub mod fsck_cmd;
@@ -29,6 +30,7 @@ pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
 pub mod noscope_cmd;
+pub mod notify_cmd;
 pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;

src/cmd/notify_cmd.rs

@@ -0,0 +1,92 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::engine::Module;
+// use crate::models::*;
+use crate::notify::{self, Notification};
+use crate::options::{self, Opt};
+use crate::shell::Shell;
+use crate::term;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Subcommand {
+    /// Manually add a notification to the outbox
+    Send(SendArgs),
+    /// Show the current outbox
+    Outbox,
+    /// Execute a module directly instead of sending a message
+    Exec(ExecArgs),
+    /// Try to deliver all messages in our outbox
+    Deliver,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct SendArgs {
+    /// Evaluate the routing rules, but do not actually send a notification
+    #[structopt(short="n", long)]
+    pub dry_run: bool,
+    pub topic: String,
+    #[structopt(flatten)]
+    pub notification: Notification,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct ExecArgs {
+    pub module: String,
+    #[structopt(short="o", long="option")]
+    pub options: Vec<options::Opt>,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
+    #[structopt(flatten)]
+    pub notification: Notification,
+}
+
+fn print_summary(module: &Module, sent: usize, errors: usize) {
+        let mut out = if sent == 1 {
+            String::from("Sent 1 notification")
+        } else {
+            format!("Sent {} notifications", sent)
+        };
+
+        out.push_str(&format!(" with {}", module.canonical()));
+
+        if errors > 0 {
+            out.push_str(&format!(" ({} errors)", errors));
+        }
+
+        term::info(&out);
+}
+
+fn send(args: SendArgs, rl: &mut Shell) -> Result<()> {
+    notify::run_router(rl, &mut term::Term, args.dry_run, &args.topic, &args.notification)?;
+    Ok(())
+}
+
+fn exec(args: ExecArgs, rl: &mut Shell) -> Result<()> {
+    let module = rl.library().get(&args.module)?.clone();
+    let options = Opt::collect(&args.options);
+    let errors = notify::exec(rl, &module, options, args.verbose, &args.notification)?;
+    print_summary(&module, 1, errors);
+    Ok(())
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        match self.subcommand {
+            Subcommand::Send(args) => send(args, rl),
+            Subcommand::Outbox => todo!(),
+            Subcommand::Exec(args) => exec(args, rl),
+            Subcommand::Deliver => todo!(),
+        }
+    }
+}

src/cmd/pkg_cmd.rs

@@ -12,6 +12,7 @@ use crate::update::AutoUpdater;
 use crate::worker;
 use colored::Colorize;
 use sn0int_common::ModuleID;
+use std::collections::HashSet;
 use std::fmt::Write;
 use std::sync::Arc;
 use structopt::StructOpt;
@@ -174,17 +175,30 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             let updater = Arc::new(Updater::new(&config)?);
             let mut autoupdate = AutoUpdater::load()?;
 
+            let installed = library.list()
+                .into_iter()
+                .map(|module| module.id())
+                .collect::<HashSet<_>>();
+
             let modules = client.quickstart()?
                 .into_iter()
-                .map(|module| {
-                    InstallTask::new(Install {
-                        module: ModuleID {
-                            author: module.author,
-                            name: module.name,
-                        },
-                        version: None,
-                        force: false,
-                    }, updater.clone())
+                .filter_map(|module| {
+                    let id = ModuleID {
+                        author: module.author,
+                        name: module.name,
+                    };
+
+                    if !installed.contains(&id) {
+                        info!("Queueing for install: {}", id);
+                        Some(InstallTask::new(Install {
+                            module: id,
+                            version: None,
+                            force: false,
+                        }, updater.clone()))
+                    } else {
+                        info!("Skipping already installed module: {}", id);
+                        None
+                    }
                 })
                 .collect::<Vec<_>>();
 

src/cmd/run_cmd.rs

@@ -92,7 +92,7 @@ fn prepare_args<T: Scopable + Serialize + Model>(rl: &Shell, filter: &Filter, pa
         .collect()
 }
 
-fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
+pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
     for namespace in keyring.unauthorized_namespaces(&module) {
         let grant_access = if params.deny_keyring {
             false
@@ -133,6 +133,7 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
         Some(Source::Ports) => prepare_args::<Port>(rl, &filter, None),
         Some(Source::Netblocks) => prepare_args::<Netblock>(rl, &filter, None),
         Some(Source::CryptoAddrs(currency)) => prepare_args::<CryptoAddr>(rl, &filter, currency.as_ref()),
+        Some(Source::Notifications) => bail!("Notification modules can't be executed like this"),
         Some(Source::KeyRing(namespace)) => {
             let keyring = rl.keyring();
             if keyring.is_access_granted(&module, &namespace) {
@@ -206,7 +207,7 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         let options = match rl.options_mut() {
             Some(options) => options.clone(),
             _ => HashMap::new(),

src/cmd/select_cmd.rs

@@ -94,7 +94,7 @@ impl<'a, 'b> Printer<'a, 'b> {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         let printer = Printer::new(rl, &self);
 
         match &self.subcommand {

src/cmd/target_cmd.rs

@@ -44,6 +44,7 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
             Source::Ports => select::<Port>(rl, None)?,
             Source::Netblocks => select::<Netblock>(rl, None)?,
             Source::CryptoAddrs(currency) => select::<CryptoAddr>(rl, currency.as_ref())?,
+            Source::Notifications => bail!("Notifications can't be set as target"),
             Source::KeyRing(namespace) => {
                 for key in rl.keyring().list_for(&namespace) {
                     println!("{}:{}", key.namespace, key.name);
@@ -79,6 +80,7 @@ fn count_selected(rl: &mut Shell, source: &Source) -> Result<usize> {
         Source::Ports => db.filter::<Port>(&filter)?.len(),
         Source::Netblocks => db.filter::<Netblock>(&filter)?.len(),
         Source::CryptoAddrs(currency) => db.filter_with_param::<CryptoAddr>(&filter, currency.as_ref())?.len(),
+        Source::Notifications => bail!("Notifications can't be set as target"),
         Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
     };
     Ok(num)

src/config.rs

@@ -1,5 +1,6 @@
 use dirs;
 use crate::errors::*;
+use crate::notify::NotificationConfig;
 use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -15,6 +16,8 @@ pub struct Config {
     pub namespaces: HashMap<String, PathBuf>,
     #[serde(default)]
     pub network: NetworkConfig,
+    #[serde(default)]
+    pub notifications: HashMap<String, NotificationConfig>,
 }
 
 impl Config {

src/db/ttl.rs

@@ -1,7 +1,10 @@
-use crate::errors::*;
 use crate::db::{Database, Table};
-use crate::schema::*;
+use crate::errors::*;
 use crate::models::*;
+use crate::notify::{self, Notification};
+use crate::schema::*;
+use crate::shell::Shell;
+use crate::term::{self, Term};
 use chrono::{NaiveDateTime, Duration, Utc};
 use diesel;
 use diesel::prelude::*;
@@ -13,6 +16,7 @@ pub struct Ttl {
     pub id: i32,
     pub family: String,
     pub key: i32,
+    pub value: String,
     pub expire: NaiveDateTime,
 }
 
@@ -21,14 +25,16 @@ pub struct Ttl {
 pub struct NewTtl<'a> {
     pub family: &'a str,
     pub key: i32,
+    pub value: String,
     pub expire: NaiveDateTime,
 }
 
 impl Ttl {
-    pub fn new(obj: &Insert, key: i32, expire: NaiveDateTime) -> NewTtl {
+    pub fn new(obj: &Insert, key: i32, value: String, expire: NaiveDateTime) -> NewTtl {
         NewTtl {
             family: obj.table(),
             key,
+            value,
             expire,
         }
     }
@@ -59,7 +65,7 @@ impl Ttl {
         expire_at.naive_utc()
     }
 
-    pub fn create(obj: &Insert, key: i32, ttl: i32, db: &Database) -> Result<()> {
+    pub fn create(obj: &Insert, key: i32, value: String, ttl: i32, db: &Database) -> Result<()> {
         debug!("Creating ttl on record");
         let expire = Self::ttl_to_datetime(ttl);
 
@@ -67,6 +73,7 @@ impl Ttl {
             .values(NewTtl {
                 family: obj.table(),
                 key,
+                value,
                 expire,
             })
             .execute(db.db())?;
@@ -127,12 +134,21 @@ impl Ttl {
     }
 }
 
-pub fn reap_expired(db: &Database) -> Result<()> {
+pub fn reap_expired(rl: &mut Shell) -> Result<()> {
     debug!("Reaping expired entities");
 
-    for expired in Ttl::expired(db)? {
+    for expired in Ttl::expired(rl.db())? {
         debug!("Expired: {:?}", expired);
-        expired.delete(db)?;
+        expired.delete(rl.db())?;
+
+        let subject = format!("Deleted {} {:?}", &expired.family, &expired.value);
+        let topic = &format!("db:{}:{}:delete", &expired.family, &expired.value);
+        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &topic, &Notification {
+            subject,
+            body: None,
+        }) {
+            term::error(&format!("Failed to send notifications: {}", err));
+        }
     }
 
     debug!("Finished reaping expired entities");

src/engine/ctx.rs

@@ -114,8 +114,8 @@ pub trait State {
             .map_err(Error::from)
     }
 
-    fn db_update(&self, object: String, update: Update) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Update((object, update))));
+    fn db_update(&self, family: Family, value: String, update: Update) -> Result<DatabaseResponse> {
+        self.send(&Event::Database(DatabaseEvent::Update((family, value, update))));
         self.db_recv()
             .context("Failed to update database")
             .map_err(Error::from)

src/lib.rs

@@ -14,6 +14,7 @@ pub mod args;
 pub mod auth;
 pub mod autonoscope;
 pub mod blobs;
+pub mod cal;
 pub mod cmd;
 pub mod config;
 use sn0int_std::crt;
@@ -33,6 +34,7 @@ use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
 use sn0int_std::mqtt;
+pub mod notify;
 pub mod paths;
 pub use sn0int_std::psl;
 pub mod options;

src/main.rs

@@ -125,6 +125,8 @@ fn run() -> Result<()> {
         Some(SubCommand::Workspace(workspace)) => workspace.run(&config),
         Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
         Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),
+        Some(SubCommand::Cal(cal)) => run_cmd(&args, cal, &config),
+        Some(SubCommand::Notify(notify)) => run_cmd(&args, notify, &config),
         Some(SubCommand::Repl) => repl::run(&config),
         Some(SubCommand::Paths) => paths::run(&config),
         Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),

src/models/account.rs

@@ -358,13 +358,13 @@ impl Updateable<Account> for AccountUpdate {
         Self::clear_if_equal(&mut self.profile_pic, &existing.profile_pic);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "displayname", &self.displayname);
-        Self::push_value(updates, "email", &self.email);
-        Self::push_value(updates, "url", &self.url);
-        Self::push_value(updates, "last_seen", &self.last_seen);
-        Self::push_value(updates, "birthday", &self.birthday);
-        Self::push_value(updates, "phonenumber", &self.phonenumber);
-        Self::push_value(updates, "profile_pic", &self.profile_pic);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "displayname", &self.displayname, colors);
+        Self::push_value(updates, "email", &self.email, colors);
+        Self::push_value(updates, "url", &self.url, colors);
+        Self::push_value(updates, "last_seen", &self.last_seen, colors);
+        Self::push_value(updates, "birthday", &self.birthday, colors);
+        Self::push_value(updates, "phonenumber", &self.phonenumber, colors);
+        Self::push_value(updates, "profile_pic", &self.profile_pic, colors);
     }
 }

src/models/activity.rs

@@ -37,16 +37,19 @@ impl Activity {
         let mut query = activity.into_boxed();
 
         if let Some(my_topic) = &filter.topic {
+            debug!("Filtering topic to be {:?}", my_topic);
             query = query.filter(topic.eq(my_topic));
         }
 
         // "since" filter is not applied
 
         if let Some(until) = &filter.until {
+            debug!("Filtering until <= {}", until);
             query = query.filter(time.le(until));
         }
 
         if filter.location {
+            debug!("Filtering latitude and longitude != null");
             query = query
                 .filter(latitude.is_not_null())
                 .filter(longitude.is_not_null());
@@ -61,6 +64,7 @@ impl Activity {
         let mut query = Self::build_query_except_since(filter);
 
         if let Some(since) = &filter.since {
+            debug!("Filtering since >= {}", since);
             query = query.filter(time.ge(since));
         }
 

src/models/breach_email.rs

@@ -211,7 +211,7 @@ impl Updateable<BreachEmail> for BreachEmailUpdate {
         Self::clear_if_equal(&mut self.password, &existing.password);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "password", &self.password);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "password", &self.password, colors);
     }
 }

src/models/cryptoaddr.rs

@@ -342,14 +342,14 @@ impl Updateable<CryptoAddr> for CryptoAddrUpdate {
         Self::clear_if_equal(&mut self.description, &existing.description);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "currency", &self.currency);
-        Self::push_value(updates, "denominator", &self.denominator);
-        Self::push_value(updates, "balance", &self.balance);
-        Self::push_value(updates, "received", &self.received);
-        Self::push_value(updates, "first_seen", &self.first_seen);
-        Self::push_value(updates, "last_withdrawal", &self.last_withdrawal);
-        Self::push_value(updates, "description", &self.description);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "currency", &self.currency, colors);
+        Self::push_value(updates, "denominator", &self.denominator, colors);
+        Self::push_value(updates, "balance", &self.balance, colors);
+        Self::push_value(updates, "received", &self.received, colors);
+        Self::push_value(updates, "first_seen", &self.first_seen, colors);
+        Self::push_value(updates, "last_withdrawal", &self.last_withdrawal, colors);
+        Self::push_value(updates, "description", &self.description, colors);
     }
 }
 

src/models/device.rs

@@ -332,10 +332,10 @@ impl Updateable<Device> for DeviceUpdate {
         Self::clear_if_equal(&mut self.last_seen, &existing.last_seen);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "name", &self.name);
-        Self::push_value(updates, "hostname", &self.hostname);
-        Self::push_value(updates, "vendor", &self.vendor);
-        Self::push_value(updates, "last_seen", &self.last_seen);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "name", &self.name, colors);
+        Self::push_value(updates, "hostname", &self.hostname, colors);
+        Self::push_value(updates, "vendor", &self.vendor, colors);
+        Self::push_value(updates, "last_seen", &self.last_seen, colors);
     }
 }

src/models/domain.rs

@@ -122,24 +122,6 @@ impl Domain {
     }
 }
 
-pub struct PrintableDomain {
-    value: String,
-}
-
-impl fmt::Display for PrintableDomain {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "{:?}", self.value)
-    }
-}
-
-impl Printable<PrintableDomain> for Domain {
-    fn printable(&self, _db: &Database) -> Result<PrintableDomain> {
-        Ok(PrintableDomain {
-            value: self.value.to_string(),
-        })
-    }
-}
-
 pub struct DetailedDomain {
     id: i32,
     value: String,
@@ -222,14 +204,6 @@ impl Upsertable<Domain> for NewDomain {
     }
 }
 
-impl Printable<PrintableDomain> for NewDomain {
-    fn printable(&self, _db: &Database) -> Result<PrintableDomain> {
-        Ok(PrintableDomain {
-            value: self.value.to_string(),
-        })
-    }
-}
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct InsertDomain {
     pub value: String,
@@ -240,6 +214,9 @@ impl InsertToNew for InsertDomain {
 
     fn try_into_new(self) -> Result<NewDomain> {
         let value = self.value.to_lowercase();
+        if value.contains('*') {
+            bail!("Asterisks inside domains are not valid");
+        }
         Ok(NewDomain {
             value,
             unscoped: false,

src/models/email.rs

@@ -335,9 +335,9 @@ impl Updateable<Email> for EmailUpdate {
         Self::clear_if_equal(&mut self.valid, &existing.valid);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "displayname", &self.displayname);
-        Self::push_value(updates, "valid", &self.valid);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "displayname", &self.displayname, colors);
+        Self::push_value(updates, "valid", &self.valid, colors);
     }
 }
 

src/models/image.rs

@@ -464,19 +464,19 @@ impl Updateable<Image> for ImageUpdate {
         Self::clear_if_equal(&mut self.phash, &existing.phash);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "filename", &self.filename);
-        Self::push_value(updates, "mime", &self.mime);
-        Self::push_value(updates, "width", &self.width);
-        Self::push_value(updates, "height", &self.height);
-        Self::push_value(updates, "created", &self.created);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "filename", &self.filename, colors);
+        Self::push_value(updates, "mime", &self.mime, colors);
+        Self::push_value(updates, "width", &self.width, colors);
+        Self::push_value(updates, "height", &self.height, colors);
+        Self::push_value(updates, "created", &self.created, colors);
 
-        Self::push_value(updates, "latitude", &self.latitude);
-        Self::push_value(updates, "longitude", &self.longitude);
+        Self::push_value(updates, "latitude", &self.latitude, colors);
+        Self::push_value(updates, "longitude", &self.longitude, colors);
 
-        Self::push_value(updates, "nudity", &self.nudity);
-        Self::push_value(updates, "ahash", &self.ahash);
-        Self::push_value(updates, "dhash", &self.dhash);
-        Self::push_value(updates, "phash", &self.phash);
+        Self::push_value(updates, "nudity", &self.nudity, colors);
+        Self::push_value(updates, "ahash", &self.ahash, colors);
+        Self::push_value(updates, "dhash", &self.dhash, colors);
+        Self::push_value(updates, "phash", &self.phash, colors);
     }
 }

src/models/ipaddr.rs

@@ -444,17 +444,17 @@ impl Updateable<IpAddr> for IpAddrUpdate {
         Self::clear_if_equal(&mut self.reverse_dns, &existing.reverse_dns);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "continent", &self.continent);
-        Self::push_value(updates, "continent_code", &self.continent_code);
-        Self::push_value(updates, "country", &self.country);
-        Self::push_value(updates, "country_code", &self.country_code);
-        Self::push_value(updates, "city", &self.city);
-        Self::push_value(updates, "latitude", &self.latitude);
-        Self::push_value(updates, "longitude", &self.longitude);
-        Self::push_value(updates, "asn", &self.asn);
-        Self::push_value(updates, "as_org", &self.as_org);
-        Self::push_value(updates, "description", &self.description);
-        Self::push_value(updates, "reverse_dns", &self.reverse_dns);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "continent", &self.continent, colors);
+        Self::push_value(updates, "continent_code", &self.continent_code, colors);
+        Self::push_value(updates, "country", &self.country, colors);
+        Self::push_value(updates, "country_code", &self.country_code, colors);
+        Self::push_value(updates, "city", &self.city, colors);
+        Self::push_value(updates, "latitude", &self.latitude, colors);
+        Self::push_value(updates, "longitude", &self.longitude, colors);
+        Self::push_value(updates, "asn", &self.asn, colors);
+        Self::push_value(updates, "as_org", &self.as_org, colors);
+        Self::push_value(updates, "description", &self.description, colors);
+        Self::push_value(updates, "reverse_dns", &self.reverse_dns, colors);
     }
 }

src/models/mod.rs

@@ -1,9 +1,10 @@
+use crate::db::{Database, Table, Filter, Family};
+use crate::engine::ctx::State;
 use crate::errors::*;
-use crate::db::{Database, Table, Filter};
 use crate::fmt;
 use crate::schema::*;
+use std::borrow::Cow;
 use std::sync::Arc;
-use crate::engine::ctx::State;
 
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -28,39 +29,39 @@ pub enum Insert {
 }
 
 impl Insert {
-    pub fn label(&self, db: &Database) -> Result<String> {
-        let label = match self {
-            Insert::Domain(x) => format!("{:?}", x.value),
-            Insert::Subdomain(x) => format!("{:?}", x.value),
-            Insert::IpAddr(x) => format!("{:?}", x.value),
+    pub fn value(&self, db: &Database) -> Result<Cow<String>> {
+        let value = match self {
+            Insert::Domain(x) => Cow::Borrowed(&x.value),
+            Insert::Subdomain(x) => Cow::Borrowed(&x.value),
+            Insert::IpAddr(x) => Cow::Borrowed(&x.value),
             Insert::SubdomainIpAddr(x) => {
                 let subdomain = Subdomain::by_id(db, x.subdomain_id)?;
                 let ipaddr = IpAddr::by_id(db, x.ip_addr_id)?;
-                format!("{:?}+{:?}", subdomain.value, ipaddr.value)
+                Cow::Owned(format!("{}+{}", subdomain.value, ipaddr.value))
             },
-            Insert::Url(x) => format!("{:?}", x.value),
-            Insert::Email(x) => format!("{:?}", x.value),
-            Insert::PhoneNumber(x) => format!("{:?}", x.value),
-            Insert::Device(x) => format!("{:?}", x.value),
-            Insert::Network(x) => format!("{:?}", x.value),
+            Insert::Url(x) => Cow::Borrowed(&x.value),
+            Insert::Email(x) => Cow::Borrowed(&x.value),
+            Insert::PhoneNumber(x) => Cow::Borrowed(&x.value),
+            Insert::Device(x) => Cow::Borrowed(&x.value),
+            Insert::Network(x) => Cow::Borrowed(&x.value),
             Insert::NetworkDevice(x) => {
                 let network = Network::by_id(db, x.network_id)?;
                 let device = Device::by_id(db, x.device_id)?;
-                format!("{:?}+{:?}", network.value, device.value)
+                Cow::Owned(format!("{}+{}", network.value, device.value))
             },
-            Insert::Account(x) => format!("{:?}", x.value),
-            Insert::Breach(x) => format!("{:?}", x.value),
+            Insert::Account(x) => Cow::Borrowed(&x.value),
+            Insert::Breach(x) => Cow::Borrowed(&x.value),
             Insert::BreachEmail(x) => {
                 let breach = Breach::by_id(db, x.breach_id)?;
                 let email = Email::by_id(db, x.email_id)?;
-                format!("{:?}+{:?}", breach.value, email.value)
+                Cow::Owned(format!("{}+{}", breach.value, email.value))
             }
-            Insert::Image(x) => format!("{:?}", x.value),
-            Insert::Port(x) => format!("{:?}", x.value),
-            Insert::Netblock(x) => format!("{:?}", x.value),
-            Insert::CryptoAddr(x) => format!("{:?}", x.value),
+            Insert::Image(x) => Cow::Borrowed(&x.value),
+            Insert::Port(x) => Cow::Borrowed(&x.value),
+            Insert::Netblock(x) => Cow::Borrowed(&x.value),
+            Insert::CryptoAddr(x) => Cow::Borrowed(&x.value),
         };
-        Ok(label)
+        Ok(value)
     }
 
     #[inline]
@@ -68,26 +69,27 @@ impl Insert {
         Table::from(self).into()
     }
 
-    pub fn printable(&self, db: &Database) -> Result<String> {
-        Ok(match self {
-            Insert::Domain(x) => format!("Domain: {}", x.printable(db)?),
-            Insert::Subdomain(x) => format!("Subdomain: {}", x.printable(db)?),
-            Insert::IpAddr(x) => format!("IpAddr: {}", x.printable(db)?),
-            Insert::SubdomainIpAddr(x) => x.printable(db)?.to_string(),
-            Insert::Url(x) => format!("Url: {}", x.printable(db)?),
-            Insert::Email(x) => format!("Email: {}", x.printable(db)?),
-            Insert::PhoneNumber(x) => format!("PhoneNumber: {}", x.printable(db)?),
-            Insert::Device(x) => format!("Device: {}", x.printable(db)?),
-            Insert::Network(x) => format!("Network: {}", x.printable(db)?),
-            Insert::NetworkDevice(x) => x.printable(db)?.to_string(),
-            Insert::Account(x) => format!("Account: {}", x.printable(db)?),
-            Insert::Breach(x) => format!("Breach: {}", x.printable(db)?),
-            Insert::BreachEmail(x) => x.printable(db)?.to_string(),
-            Insert::Image(x) => format!("Image: {}", x.printable(db)?),
-            Insert::Port(x) => format!("Port: {}", x.printable(db)?),
-            Insert::Netblock(x) => format!("Netblock: {}", x.printable(db)?),
-            Insert::CryptoAddr(x) => format!("CryptoAddr: {}", x.printable(db)?),
-        })
+    #[inline]
+    pub fn family(&self) -> &str {
+        match self {
+            Insert::Domain(_) => Family::Domain.as_str(),
+            Insert::Subdomain(_) => Family::Subdomain.as_str(),
+            Insert::IpAddr(_) => Family::Ipaddr.as_str(),
+            Insert::SubdomainIpAddr(_) => Family::SubdomainIpaddr.as_str(),
+            Insert::Url(_) => Family::Url.as_str(),
+            Insert::Email(_) => Family::Email.as_str(),
+            Insert::PhoneNumber(_) => Family::Phonenumber.as_str(),
+            Insert::Device(_) => Family::Device.as_str(),
+            Insert::Network(_) => Family::Network.as_str(),
+            Insert::NetworkDevice(_) => Family::NetworkDevice.as_str(),
+            Insert::Account(_) => Family::Account.as_str(),
+            Insert::Breach(_) => Family::Breach.as_str(),
+            Insert::BreachEmail(_) => Family::BreachEmail.as_str(),
+            Insert::Image(_) => Family::Image.as_str(),
+            Insert::Port(_) => Family::Port.as_str(),
+            Insert::Netblock(_) => Family::Netblock.as_str(),
+            Insert::CryptoAddr(_) => Family::Cryptoaddr.as_str(),
+        }
     }
 }
 
@@ -152,25 +154,42 @@ impl Update {
             Update::CryptoAddr(update)    => update.is_dirty(),
         }
     }
-}
 
-impl fmt::Display for Update {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+    pub fn to_plain_str(&self) -> String {
         match self {
-            Update::Subdomain(update)     => write!(w, "{}", update.to_string()),
-            Update::IpAddr(update)        => write!(w, "{}", update.to_string()),
-            Update::Url(update)           => write!(w, "{}", update.to_string()),
-            Update::Email(update)         => write!(w, "{}", update.to_string()),
-            Update::PhoneNumber(update)   => write!(w, "{}", update.to_string()),
-            Update::Device(update)        => write!(w, "{}", update.to_string()),
-            Update::Network(update)       => write!(w, "{}", update.to_string()),
-            Update::NetworkDevice(update) => write!(w, "{}", update.to_string()),
-            Update::Account(update)       => write!(w, "{}", update.to_string()),
-            Update::BreachEmail(update)   => write!(w, "{}", update.to_string()),
-            Update::Image(update)         => write!(w, "{}", update.to_string()),
-            Update::Port(update)          => write!(w, "{}", update.to_string()),
-            Update::Netblock(update)      => write!(w, "{}", update.to_string()),
-            Update::CryptoAddr(update)    => write!(w, "{}", update.to_string()),
+            Update::Subdomain(update)       => update.to_plain_str(),
+            Update::IpAddr(update)          => update.to_plain_str(),
+            Update::Url(update)             => update.to_plain_str(),
+            Update::Email(update)           => update.to_plain_str(),
+            Update::PhoneNumber(update)     => update.to_plain_str(),
+            Update::Device(update)          => update.to_plain_str(),
+            Update::Network(update)         => update.to_plain_str(),
+            Update::NetworkDevice(update)   => update.to_plain_str(),
+            Update::Account(update)         => update.to_plain_str(),
+            Update::BreachEmail(update)     => update.to_plain_str(),
+            Update::Image(update)           => update.to_plain_str(),
+            Update::Port(update)            => update.to_plain_str(),
+            Update::Netblock(update)        => update.to_plain_str(),
+            Update::CryptoAddr(update)      => update.to_plain_str(),
+        }
+    }
+
+    pub fn to_term_str(&self) -> String {
+        match self {
+            Update::Subdomain(update)       => update.to_term_str(),
+            Update::IpAddr(update)          => update.to_term_str(),
+            Update::Url(update)             => update.to_term_str(),
+            Update::Email(update)           => update.to_term_str(),
+            Update::PhoneNumber(update)     => update.to_term_str(),
+            Update::Device(update)          => update.to_term_str(),
+            Update::Network(update)         => update.to_term_str(),
+            Update::NetworkDevice(update)   => update.to_term_str(),
+            Update::Account(update)         => update.to_term_str(),
+            Update::BreachEmail(update)     => update.to_term_str(),
+            Update::Image(update)           => update.to_term_str(),
+            Update::Port(update)            => update.to_term_str(),
+            Update::Netblock(update)        => update.to_term_str(),
+            Update::CryptoAddr(update)      => update.to_term_str(),
         }
     }
 }
@@ -274,9 +293,15 @@ impl Upsert for NullUpdate {
 }
 
 pub trait Updateable<M> {
-    fn to_string(&self) -> String {
+    fn to_plain_str(&self) -> String {
         let mut updates = Vec::new();
-        self.fmt(&mut updates);
+        self.fmt(&mut updates, false);
+        updates.join(", ")
+    }
+
+    fn to_term_str(&self) -> String {
+        let mut updates = Vec::new();
+        self.fmt(&mut updates, true);
         updates.join(", ")
     }
 
@@ -304,22 +329,31 @@ pub trait Updateable<M> {
     fn changeset(&mut self, existing: &M);
 
     #[inline]
-    fn push_value<D: fmt::Debug>(updates: &mut Vec<String>, name: &str, value: &Option<D>) {
+    fn push_value<D: fmt::Debug>(updates: &mut Vec<String>, name: &str, value: &Option<D>, colors: bool) {
         if let Some(v) = value {
-            updates.push(format!("{} => \x1b[33m{:?}\x1b[0m", name, v));
+            if colors {
+                updates.push(format!("{} => \x1b[33m{:?}\x1b[0m", name, v));
+            } else {
+                updates.push(format!("{} => {:?}", name, v));
+            }
         }
     }
 
     #[inline]
-    fn push_raw<T: AsRef<str>>(updates: &mut Vec<String>, name: &str, value: Option<T>) {
+    fn push_raw<T: AsRef<str>>(updates: &mut Vec<String>, name: &str, value: Option<T>, colors: bool) {
         if let Some(v) = value {
-            updates.push(format!("{} => \x1b[33m{}\x1b[0m", name, v.as_ref()));
+            if colors {
+                updates.push(format!("{} => \x1b[33m{}\x1b[0m", name, v.as_ref()));
+            } else {
+                updates.push(format!("{} => {}", name, v.as_ref()));
+            }
         }
     }
 
-    fn fmt(&self, updates: &mut Vec<String>);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool);
 }
 
+// TODO: Printable could probably be dropped
 pub trait Printable<T: Sized> {
     fn printable(&self, db: &Database) -> Result<T>;
 }

src/models/netblock.rs

@@ -307,9 +307,9 @@ impl Updateable<Netblock> for NetblockUpdate {
         Self::clear_if_equal(&mut self.description, &existing.description);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "asn", &self.asn);
-        Self::push_value(updates, "as_org", &self.as_org);
-        Self::push_value(updates, "description", &self.description);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "asn", &self.asn, colors);
+        Self::push_value(updates, "as_org", &self.as_org, colors);
+        Self::push_value(updates, "description", &self.description, colors);
     }
 }

src/models/network.rs

@@ -310,9 +310,9 @@ impl Updateable<Network> for NetworkUpdate {
         Self::clear_if_equal(&mut self.description, &existing.description);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "latitude", &self.latitude);
-        Self::push_value(updates, "longitude", &self.longitude);
-        Self::push_value(updates, "description", &self.description);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "latitude", &self.latitude, colors);
+        Self::push_value(updates, "longitude", &self.longitude, colors);
+        Self::push_value(updates, "description", &self.description, colors);
     }
 }

src/models/network_device.rs

@@ -194,8 +194,8 @@ impl Updateable<NetworkDevice> for NetworkDeviceUpdate {
         Self::clear_if_equal(&mut self.last_seen, &existing.last_seen);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "ipaddr", &self.ipaddr);
-        Self::push_value(updates, "last_seen", &self.last_seen);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "ipaddr", &self.ipaddr, colors);
+        Self::push_value(updates, "last_seen", &self.last_seen, colors);
     }
 }

src/models/phonenumber.rs

@@ -377,16 +377,16 @@ impl Updateable<PhoneNumber> for PhoneNumberUpdate {
         Self::clear_if_equal(&mut self.caller_type, &existing.caller_type);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "name", &self.name);
-        Self::push_value(updates, "valid", &self.valid);
-        Self::push_value(updates, "last_online", &self.last_online);
-        Self::push_value(updates, "country", &self.country);
-        Self::push_value(updates, "carrier", &self.carrier);
-        Self::push_value(updates, "line", &self.line);
-        Self::push_value(updates, "is_ported", &self.is_ported);
-        Self::push_value(updates, "last_ported", &self.last_ported);
-        Self::push_value(updates, "caller_name", &self.caller_name);
-        Self::push_value(updates, "caller_type", &self.caller_type);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "name", &self.name, colors);
+        Self::push_value(updates, "valid", &self.valid, colors);
+        Self::push_value(updates, "last_online", &self.last_online, colors);
+        Self::push_value(updates, "country", &self.country, colors);
+        Self::push_value(updates, "carrier", &self.carrier, colors);
+        Self::push_value(updates, "line", &self.line, colors);
+        Self::push_value(updates, "is_ported", &self.is_ported, colors);
+        Self::push_value(updates, "last_ported", &self.last_ported, colors);
+        Self::push_value(updates, "caller_name", &self.caller_name, colors);
+        Self::push_value(updates, "caller_type", &self.caller_type, colors);
     }
 }

src/models/port.rs

@@ -337,10 +337,10 @@ impl Updateable<Port> for PortUpdate {
         Self::clear_if_equal(&mut self.version, &existing.version);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "status", &self.status);
-        Self::push_value(updates, "banner", &self.banner);
-        Self::push_value(updates, "service", &self.service);
-        Self::push_value(updates, "version", &self.version);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "status", &self.status, colors);
+        Self::push_value(updates, "banner", &self.banner, colors);
+        Self::push_value(updates, "service", &self.service, colors);
+        Self::push_value(updates, "version", &self.version, colors);
     }
 }

src/models/subdomain.rs

@@ -258,6 +258,9 @@ impl InsertToNew for InsertSubdomain {
 
     fn try_into_new(self) -> Result<NewSubdomain> {
         let value = self.value.to_lowercase();
+        if value.contains('*') {
+            bail!("Asterisks inside domains are not valid");
+        }
         Ok(NewSubdomain {
             domain_id: self.domain_id,
             value,
@@ -294,7 +297,7 @@ impl Updateable<Subdomain> for SubdomainUpdate {
         Self::clear_if_equal(&mut self.resolvable, &existing.resolvable);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "resolvable", &self.resolvable);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "resolvable", &self.resolvable, colors);
     }
 }

src/models/url.rs

@@ -354,12 +354,12 @@ impl Updateable<Url> for UrlChangeset {
         Self::clear_if_equal(&mut self.redirect, &existing.redirect);
     }
 
-    fn fmt(&self, updates: &mut Vec<String>) {
-        Self::push_value(updates, "online", &self.online);
-        Self::push_value(updates, "status", &self.status);
-        Self::push_raw(updates, "body", self.body.as_ref().map(|x| format!("[{} bytes]", x.len())));
-        Self::push_value(updates, "title", &self.title);
-        Self::push_value(updates, "redirect", &self.redirect);
+    fn fmt(&self, updates: &mut Vec<String>, colors: bool) {
+        Self::push_value(updates, "online", &self.online, colors);
+        Self::push_value(updates, "status", &self.status, colors);
+        Self::push_raw(updates, "body", self.body.as_ref().map(|x| format!("[{} bytes]", x.len())), colors);
+        Self::push_value(updates, "title", &self.title, colors);
+        Self::push_value(updates, "redirect", &self.redirect, colors);
     }
 }
 

src/notify/mod.rs

@@ -0,0 +1,235 @@
+use crate::errors::*;
+use crate::cmd::run_cmd::prepare_keyring;
+use crate::cmd::run_cmd::Params;
+use crate::engine::Module;
+use crate::options;
+use crate::shell::Shell;
+use crate::term::SpinLogger;
+use crate::worker;
+use serde::de::{self, Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
+use sn0int_common::metadata::Source;
+use sn0int_std::blobs::Blob;
+use std::collections::HashMap;
+use std::result;
+use std::str::FromStr;
+
+#[derive(Debug, StructOpt, Serialize)]
+pub struct Notification {
+    pub subject: String,
+    pub body: Option<String>,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct NotificationConfig {
+    #[serde(default)]
+    pub workspaces: Vec<String>,
+    #[serde(default)]
+    pub topics: Vec<Glob>,
+    pub script: String,
+    #[serde(default)]
+    pub options: Vec<options::Opt>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Glob {
+    patterns: Vec<glob::Pattern>,
+    src: String,
+}
+
+impl Glob {
+    fn matches(&self, topic: &str) -> bool {
+        let mut filter = self.patterns.iter();
+        let mut topic = topic.split(':');
+
+        loop {
+            match (filter.next(), topic.next()) {
+                (Some(filter), Some(topic)) => if !filter.matches(&topic) {
+                    return false;
+                },
+                (None, None) => return true,
+                (_, _) => return false,
+            }
+        }
+    }
+}
+
+impl FromStr for Glob {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Glob> {
+        let patterns = s.split(':')
+            .map(|s| glob::Pattern::new(s).map_err(Error::from))
+            .collect::<Result<Vec<_>>>()?;
+        Ok(Glob {
+            patterns,
+            src: s.to_string(),
+        })
+    }
+}
+
+impl Serialize for Glob {
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+        where S: Serializer
+    {
+        serializer.serialize_str(&self.src)
+    }
+}
+
+impl<'de> Deserialize<'de> for Glob {
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+        where D: Deserializer<'de>
+    {
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}
+
+fn apply_rule<T>(name: &str, filter: &[T], value: &str, cmp: fn(&T, &str) -> bool) -> bool {
+    if !filter.is_empty() {
+        debug!("{} filter is active", name);
+        if !filter.iter().any(|filter| cmp(filter, value)) {
+            debug!("{} isn't allow-listed, aborting", name);
+            return false;
+        }
+        debug!("{} was allow-listed", name);
+    }
+    true
+}
+
+impl NotificationConfig {
+    fn matches(&self, workspace: &str, topic: &str) -> bool {
+        debug!("testing notification with rules");
+        if !apply_rule("workspace", &self.workspaces, workspace, |filter, value| filter == value) {
+            return false;
+        }
+        if !apply_rule("topic", &self.topics, topic, |filter, value| filter.matches(value)) {
+            return false;
+        }
+        debug!("notification matches this config");
+        true
+    }
+}
+
+pub fn trigger_notify_event<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, topic: &str, notification: &Notification) -> Result<()> {
+    run_router(rl, spinner, false, topic, notification)
+}
+
+fn prepare_arg(notification: &Notification) -> Result<(serde_json::Value, Option<String>, Vec<Blob>)> {
+    let arg = serde_json::to_value(notification)?;
+    Ok((arg, None, vec![]))
+}
+
+pub fn exec(rl: &mut Shell, module: &Module, options: HashMap<String, String>, verbose: u64, notification: &Notification) -> Result<usize> {
+    if *module.source() != Some(Source::Notifications) {
+        bail!("Module doesn't take notifications as source");
+    }
+
+    let params = Params {
+        threads: 1,
+        verbose,
+        stdin: false,
+        grants: &[],
+        grant_full_keyring: false,
+        deny_keyring: false,
+        exit_on_error: false,
+    };
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let args = vec![prepare_arg(&notification)?];
+
+    rl.signal_register().catch_ctrl();
+    let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);
+    rl.signal_register().reset_ctrlc();
+
+    Ok(errors)
+}
+
+pub fn run_router<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, dry_run: bool, topic: &str, notification: &Notification) -> Result<()> {
+    let configs = rl.config().notifications.clone();
+
+    for (name, config) in configs {
+        if config.matches(rl.workspace(), topic) {
+            let module = rl.library().get(&config.script)?.clone();
+            if dry_run {
+                spinner.success(&format!("Executed {} {:?} (dry-run)", module.canonical(), name));
+            } else {
+                let options = options::Opt::collect(&config.options);
+                match exec(rl, &module, options, 0, notification) {
+                    Ok(0) => {
+                        let msg = format!("Executed {} {:?}", module.canonical(), name);
+                        spinner.success(&msg);
+                    },
+                    Ok(errors) => {
+                        let msg = format!("Executed {} {:?} ({} errors)", module.canonical(), name, errors);
+                        spinner.error(&msg);
+                    },
+                    Err(err) => {
+                        spinner.error(&format!("Fatal {} {:?}: {}", module.canonical(), name, err));
+                    },
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn match_topic_str(filter: &str, value: &str) -> bool {
+        let filter: Glob = filter.parse().unwrap();
+        filter.matches(value)
+    }
+
+    #[test]
+    fn test_match_topic_exact() {
+        assert!(match_topic_str("topic:hello-world", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_match_topic_starts_with() {
+        assert!(match_topic_str("topic:*", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_match_topic_ends_with() {
+        assert!(match_topic_str("*:hello-world", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_one_section() {
+        assert!(match_topic_str("a:*:z", "a:b:z"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_not_two_sections() {
+        assert!(!match_topic_str("a:*:z", "a:b:c:z"));
+    }
+
+    #[test]
+    fn test_match_topic_two_wildcards_two_sections() {
+        assert!(match_topic_str("a:*:*:z", "a:b:c:z"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_not_two_sections_start() {
+        assert!(!match_topic_str("a:*", "a:b:c"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_not_two_sections_end() {
+        assert!(!match_topic_str("*:z", "b:c:z"));
+    }
+
+    #[test]
+    fn test_match_topic_many_wildcards() {
+        assert!(match_topic_str("a:*:*:d:e:*:g:*:z", "a:b:c:d:e:f:g:h:z"));
+    }
+
+    #[test]
+    fn test_match_topic_empty_filter() {
+        assert!(!match_topic_str("", "abc"));
+    }
+}

src/options.rs

@@ -1,9 +1,11 @@
 use crate::errors::*;
+use serde::de::{self, Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
 use std::collections::HashMap;
+use std::result;
 use std::str::FromStr;
 
-
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Opt {
     key: String,
     value: String,
@@ -32,3 +34,26 @@ impl FromStr for Opt {
         }
     }
 }
+
+impl ToString for Opt {
+    fn to_string(&self) -> String {
+        format!("{}={}", self.key, self.value)
+    }
+}
+
+impl Serialize for Opt {
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+        where S: Serializer
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for Opt {
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+        where D: Deserializer<'de>
+    {
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}

src/runtime/db.rs

@@ -199,65 +199,63 @@ fn gen_changeset2<T: Model, U: UpdateToChangeset<C>, C: Updateable<T>>(object: L
     Ok((existing.id(), value, update))
 }
 
+fn run_update(state: Arc<dyn State>, family: &str, object: AnyLuaValue, update: AnyLuaValue) -> Result<Option<i32>> {
+    let family = Family::from_str(&family)?;
+    let object = LuaJsonValue::from(object);
+    let update = LuaJsonValue::from(update);
+
+    let (id, value, update) = match family {
+        Family::Domain => bail!("Domain doesn't have mutable fields"),
+        Family::Subdomain => gen_changeset::<Subdomain, SubdomainUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Subdomain(u))),
+        Family::Ipaddr => gen_changeset::<IpAddr, IpAddrUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::IpAddr(u))),
+        Family::SubdomainIpaddr => bail!("Subdomain-IpAddr doesn't have mutable fields"),
+        Family::Url => gen_changeset2::<Url, UrlUpdate, UrlChangeset>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Url(u))),
+        Family::Email => gen_changeset::<Email, EmailUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Email(u))),
+        Family::Phonenumber => gen_changeset::<PhoneNumber, PhoneNumberUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::PhoneNumber(u))),
+        Family::Device => gen_changeset::<Device, DeviceUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Device(u))),
+        Family::Network => gen_changeset::<Network, NetworkUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Network(u))),
+        Family::NetworkDevice => gen_changeset::<NetworkDevice, NetworkDeviceUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::NetworkDevice(u))),
+        Family::Account => gen_changeset::<Account, AccountUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Account(u))),
+        Family::Breach => bail!("Breach doesn't have mutable fields"),
+        Family::BreachEmail => gen_changeset::<BreachEmail, BreachEmailUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::BreachEmail(u))),
+        Family::Image => gen_changeset::<Image, ImageUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Image(u))),
+        Family::Port => gen_changeset::<Port, PortUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Port(u))),
+        Family::Netblock => gen_changeset::<Netblock, NetblockUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::Netblock(u))),
+        Family::Cryptoaddr => gen_changeset::<CryptoAddr, CryptoAddrUpdate>(object, update)
+            .map(|(id, v, u)| (id, v, Update::CryptoAddr(u))),
+    }?;
+
+    if update.is_dirty() {
+        let r = match state.db_update(family, value, update)? {
+            // Inserted and Found are technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
+    } else {
+        Ok(Some(id))
+    }
+}
+
 pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("db_update", hlua::function3(move |family: String, object: AnyLuaValue, update: AnyLuaValue| -> Result<Option<i32>> {
-        let family = Family::from_str(&family)
-            .map_err(|e| state.set_error(e.into()))?;
-        let object = LuaJsonValue::from(object);
-        let update = LuaJsonValue::from(update);
-
-        let update = match family {
-            Family::Domain => bail!("Domain doesn't have mutable fields"),
-            Family::Subdomain => gen_changeset::<Subdomain, SubdomainUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Subdomain(u))),
-            Family::Ipaddr => gen_changeset::<IpAddr, IpAddrUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::IpAddr(u))),
-            Family::SubdomainIpaddr => bail!("Subdomain-IpAddr doesn't have mutable fields"),
-            Family::Url => gen_changeset2::<Url, UrlUpdate, UrlChangeset>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Url(u))),
-            Family::Email => gen_changeset::<Email, EmailUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Email(u))),
-            Family::Phonenumber => gen_changeset::<PhoneNumber, PhoneNumberUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::PhoneNumber(u))),
-            Family::Device => gen_changeset::<Device, DeviceUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Device(u))),
-            Family::Network => gen_changeset::<Network, NetworkUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Network(u))),
-            Family::NetworkDevice => gen_changeset::<NetworkDevice, NetworkDeviceUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::NetworkDevice(u))),
-            Family::Account => gen_changeset::<Account, AccountUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Account(u))),
-            Family::Breach => bail!("Breach doesn't have mutable fields"),
-            Family::BreachEmail => gen_changeset::<BreachEmail, BreachEmailUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::BreachEmail(u))),
-            Family::Image => gen_changeset::<Image, ImageUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Image(u))),
-            Family::Port => gen_changeset::<Port, PortUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Port(u))),
-            Family::Netblock => gen_changeset::<Netblock, NetblockUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::Netblock(u))),
-            Family::Cryptoaddr => gen_changeset::<CryptoAddr, CryptoAddrUpdate>(object, update)
-                .map(|(id, v, u)| (id, v, Update::CryptoAddr(u))),
-        };
-
-        let (id, value, update) = update
-            .map_err(|e| state.set_error(e))?;
-
-        if update.is_dirty() {
-            let r = state.db_update(value, update)
-                .map_err(|e| state.set_error(e))?;
-
-            let r = match r {
-                // Inserted and Found are technically unreachable
-                DatabaseResponse::Inserted(id) => Some(id),
-                DatabaseResponse::Updated(id) => Some(id),
-                DatabaseResponse::NoChange(id) => Some(id),
-                DatabaseResponse::Found(id) => Some(id),
-                DatabaseResponse::None => None,
-            };
-            Ok(r)
-        } else {
-            Ok(Some(id))
-        }
+        run_update(state.clone(), &family, object, update)
+            .map_err(|e| state.set_error(e))
     }))
 }

src/sandbox/seccomp.rs

@@ -57,7 +57,12 @@ pub fn init() -> Result<()> {
     #[cfg(target_arch = "x86")]
     ctx.allow_syscall(Syscall::time)?;
     ctx.allow_syscall(Syscall::clock_gettime)?;
+    #[cfg(target_arch = "arm")]
+    ctx.allow_syscall(Syscall::clock_gettime64)?;
     ctx.allow_syscall(Syscall::nanosleep)?;
+    ctx.allow_syscall(Syscall::clock_nanosleep)?;
+    #[cfg(target_arch = "arm")]
+    ctx.allow_syscall(Syscall::clock_nanosleep_time64)?;
     ctx.allow_syscall(Syscall::exit)?;
     ctx.allow_syscall(Syscall::exit_group)?;
     ctx.allow_syscall(Syscall::brk)?;

src/schema.rs

@@ -228,6 +228,7 @@ table! {
         id -> Integer,
         family -> Text,
         key -> Integer,
+        value -> Text,
         expire -> Timestamp,
     }
 }

src/shell/mod.rs

@@ -48,6 +48,7 @@ pub enum Command {
     Use,
     Quickstart,
     Workspace,
+    Cal,
 
     Exit,
     Quit,
@@ -79,6 +80,7 @@ impl Command {
             Command::Quickstart => "quickstart",
             Command::Quit => "quit",
             Command::Workspace => "workspace",
+            Command::Cal => "cal",
             Command::Interrupt => unreachable!(),
         }
     }
@@ -92,19 +94,20 @@ impl Command {
                 Command::Autoscope.as_str(),
                 Command::Back.as_str(),
                 Command::Delete.as_str(),
+                Command::Exit.as_str(),
                 Command::Help.as_str(),
                 Command::Keyring.as_str(),
-                Command::Mod.as_str(),
                 Command::Noscope.as_str(),
                 Command::Pkg.as_str(),
                 Command::Run.as_str(),
                 Command::Scope.as_str(),
                 Command::Set.as_str(),
                 Command::Select.as_str(),
-                Command::Workspace.as_str(),
                 Command::Target.as_str(),
                 Command::Use.as_str(),
-                Command::Quickstart.as_str(),
+                Command::Quit.as_str(),
+                Command::Workspace.as_str(),
+                Command::Cal.as_str(),
             ];
         }
 
@@ -138,6 +141,7 @@ impl FromStr for Command {
             "quickstart"  => Ok(Command::Quickstart),
             "quit" => Ok(Command::Quit),
             "workspace" => Ok(Command::Workspace),
+            "cal" => Ok(Command::Cal),
             x => bail!("unknown command: {:?}, try \"help\"", x),
         }
     }
@@ -155,6 +159,7 @@ pub struct Shell<'a> {
     // autonoscope: RuleSet,
     options: Option<HashMap<String, String>>,
     signal_register: Arc<SignalRegister>,
+    cancel_twice: u8,
 }
 
 impl<'a> Shell<'a> {
@@ -175,6 +180,7 @@ impl<'a> Shell<'a> {
             keyring,
             options: None,
             signal_register: Arc::new(SignalRegister::new()),
+            cancel_twice: 0,
         };
 
         rl.reload_module_cache();
@@ -202,6 +208,11 @@ impl<'a> Shell<'a> {
         self.prompt.module.as_ref()
     }
 
+    #[inline(always)]
+    pub fn workspace(&self) -> &str {
+        self.prompt.workspace.as_str()
+    }
+
     #[inline(always)]
     pub fn options_mut(&mut self) -> Option<&mut HashMap<String, String>> {
         self.options.as_mut()
@@ -282,41 +293,56 @@ impl<'a> Shell<'a> {
 
     pub fn readline(&mut self) -> Option<(Command, Vec<String>)> {
         let readline = self.rl.readline(&self.prompt.to_string());
+
+        if readline.is_ok() {
+            self.cancel_twice = 0;
+        }
+
         match readline {
-            Ok(ref line) if line.is_empty() => None,
             Ok(line) => {
-                debug!("Readline returned {:?}", line);
+                self.cancel_twice = 0;
+                if line.is_empty() {
+                    None
+                } else {
+                    debug!("Readline returned {:?}", line);
 
-                self.rl.add_history_entry(line.as_str());
+                    self.rl.add_history_entry(line.as_str());
 
-                if line.starts_with('#') {
-                    return None;
-                }
-
-                let cmd = match shellwords::split(&line) {
-                    Ok(cmd) => cmd,
-                    Err(err) => {
-                        eprintln!("Error: {:?}", err);
+                    if line.starts_with('#') {
                         return None;
-                    },
-                };
-                debug!("shellwords returned {:?}", cmd);
+                    }
 
-                if cmd.is_empty() {
-                    return None;
-                }
+                    let cmd = match shellwords::split(&line) {
+                        Ok(cmd) => cmd,
+                        Err(err) => {
+                            eprintln!("Error: {:?}", err);
+                            return None;
+                        },
+                    };
+                    debug!("shellwords returned {:?}", cmd);
 
-                match Command::from_str(&cmd[0]) {
-                    Ok(x) => Some((x, cmd)),
-                    Err(err) => {
-                        eprintln!("Error: {}", err);
-                        None
-                    },
+                    if cmd.is_empty() {
+                        return None;
+                    }
+
+                    match Command::from_str(&cmd[0]) {
+                        Ok(x) => Some((x, cmd)),
+                        Err(err) => {
+                            eprintln!("Error: {}", err);
+                            None
+                        },
+                    }
                 }
             }
             Err(ReadlineError::Interrupted) => {
                 // ^C
-                Some((Command::Interrupt, vec![]))
+                self.cancel_twice += 1;
+
+                if self.cancel_twice > 1 {
+                    Some((Command::Interrupt, vec![]))
+                } else {
+                    None
+                }
             }
             Err(ReadlineError::Eof) => {
                 // ^D
@@ -461,7 +487,10 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
         Some((Command::Delete, args)) => delete_cmd::run(rl, &args)?,
         Some((Command::Help, args)) => help_cmd::run(rl, &args)?,
         Some((Command::Keyring, args)) => keyring_cmd::run(rl, &args)?,
-        Some((Command::Mod, args)) => cmd::<pkg_cmd::ArgsInteractive>(rl, &args)?,
+        Some((Command::Mod, args)) => {
+            term::warn("The \x1b[1mmod\x1b[0m command is deprecated, use \x1b[1mpkg\x1b[0m");
+            cmd::<pkg_cmd::ArgsInteractive>(rl, &args)?
+        },
         Some((Command::Noscope, args)) => noscope_cmd::run(rl, &args)?,
         Some((Command::Pkg, args)) => cmd::<pkg_cmd::ArgsInteractive>(rl, &args)?,
         Some((Command::Run, args)) => cmd::<run_cmd::Args>(rl, &args)?,
@@ -472,6 +501,7 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
         Some((Command::Use, args)) => use_cmd::run(rl, &args)?,
         Some((Command::Quickstart, args)) => quickstart_cmd::run(rl, &args)?,
         Some((Command::Workspace, args)) => cmd::<workspace_cmd::Args>(rl, &args)?,
+        Some((Command::Cal, args)) => cmd::<cal_cmd::Args>(rl, &args)?,
 
         Some((Command::Exit, _)) => return Ok(true),
         Some((Command::Quit, _)) => return Ok(true),
@@ -496,7 +526,6 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
     } else {
         Database::establish_quiet(workspace)?
     };
-    ttl::reap_expired(&db)?;
 
     let cache_dir = paths::cache_dir()?;
     let psl = PslReader::open_or_download(&cache_dir,
@@ -516,7 +545,9 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
     }
     autoupdate.check_background(&config, library.list());
 
-    let rl = Shell::new(&config, db, blobs, psl, library, keyring);
+    let mut rl = Shell::new(&config, db, blobs, psl, library, keyring);
+
+    ttl::reap_expired(&mut rl)?;
 
     Ok(rl)
 }

src/term.rs

@@ -221,6 +221,44 @@ pub fn error(line: &str) {
     eprintln!("\x1b[1m[\x1b[31m{}\x1b[0;1m]\x1b[0m {}", '-', line);
 }
 
+pub struct Term;
+
+impl SpinLogger for Term {
+    fn log(&mut self, line: &str) {
+        success(line)
+    }
+
+    fn debug(&mut self, line: &str) {
+        debug(line)
+    }
+
+    fn success(&mut self, line: &str) {
+        info(line)
+    }
+
+    fn error(&mut self, line: &str) {
+        error(line)
+    }
+
+    fn warn(&mut self, line: &str) {
+        warn(line)
+    }
+
+    fn warn_once(&mut self, _line: &str) {
+        unimplemented!()
+    }
+
+    #[inline]
+    fn status(&mut self, _status: String) {
+        unimplemented!()
+    }
+
+    #[inline]
+    fn stacked_status(&mut self, _name: &String, _status: String) {
+        unimplemented!()
+    }
+}
+
 pub struct Prompt {
     pub workspace: String,
     pub module: Option<Module>,

src/worker.rs

@@ -3,12 +3,13 @@ use crate::errors::*;
 use crate::blobs::Blob;
 use crate::channel;
 use crate::cmd::run_cmd::Params;
-use crate::db::{Database, DbChange, Family};
+use crate::db::{DbChange, Family};
 use crate::db::ttl::Ttl;
 use crate::engine::Module;
 use crate::ipc;
 use crate::ipc::parent::IpcParent;
 use crate::models::*;
+use crate::notify::{self, Notification};
 use serde_json;
 use crate::ratelimits::{Ratelimiter, RatelimitResponse};
 use crate::shell::Shell;
@@ -156,7 +157,7 @@ pub enum DatabaseEvent {
     InsertTtl((Insert, i32)),
     Activity(NewActivity),
     Select((Family, String)),
-    Update((String, Update)),
+    Update((Family, String, Update)),
 }
 
 impl EventWithCallback for DatabaseEvent {
@@ -168,7 +169,71 @@ impl EventWithCallback for DatabaseEvent {
 }
 
 impl DatabaseEvent {
-    pub fn insert<T: SpinLogger>(object: Insert, ttl: Option<i32>, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
+    fn notify<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, topic: &str, subject: String) {
+        if let Err(err) = notify::trigger_notify_event(rl, spinner, topic, &Notification {
+            subject,
+            body: None,
+        }) {
+            spinner.error(&format!("Failed to send notifications: {}", err));
+        }
+    }
+
+    fn on_insert<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, family: &str, value: &str) {
+        // TODO: also include fields, see update
+        let log = format!("Adding {} {:?}", family, value);
+        spinner.log(&log);
+
+        let subject = format!("Added {} {:?}", family, value);
+        let topic = format!("db:{}:{}:insert", family, value);
+        Self::notify(rl, spinner, &topic, subject);
+    }
+
+    fn on_update<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, family: &str, value: &str, update: &Update) {
+        spinner.log(&format!("Updating {} {:?} ({})", family, value, update.to_term_str()));
+
+        // TODO: in the future we could consider firing multiple events, one for each column
+        // TODO: this would be super noisy if a lot of fields change though
+        let subject = format!("Updated {} {:?} ({})", family, value, update.to_plain_str());
+        let topic = format!("db:{}:{}:update", family, value);
+        Self::notify(rl, spinner, &topic, subject);
+    }
+
+    fn on_activity<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, object: &NewActivity, verbose: u64) {
+        Self::spinner_log_new_activity(&object, spinner, verbose);
+
+        // TODO: we don't want to copy the match arms everywhere
+        let mut subject = format!("New activity: {:?}", object.topic);
+        if let Some(uniq) = &object.uniq {
+            subject += &format!(" ({:?})", uniq);
+        }
+        let topic = format!("activity:{}", object.topic);
+        Self::notify(rl, spinner, &topic, subject);
+    }
+
+    fn spinner_log_new_activity<T: SpinLogger>(object: &NewActivity, spinner: &mut T, verbose: u64) {
+        let mut log = format!("{:?} ", object.topic);
+        if let Some(uniq) = &object.uniq {
+            log.push_str(&format!("({:?}) ", uniq));
+        }
+        log.push_str(&format!("@ {}", object.time));
+
+        if let (Some(ref lat), Some(ref lon)) = (object.latitude, object.longitude) {
+            log.push_str(&format!(" ({}, {}", lat, lon));
+            if let Some(radius) = &object.radius {
+                log.push_str(&format!(" | {}m", radius));
+            }
+            log.push_str(")");
+        }
+
+        if verbose > 0 {
+            log.push_str(&format!(": {}", object.content));
+        }
+
+        spinner.log(&log);
+    }
+
+    fn insert<T: SpinLogger>(rl: &mut Shell, object: Insert, ttl: Option<i32>, tx: DbSender, spinner: &mut T, verbose: u64) {
+        let db = rl.db();
         if verbose >= 1 {
             spinner.debug(&format!("Inserting: {:?}", object));
         }
@@ -178,18 +243,21 @@ impl DatabaseEvent {
 
         let result = match result {
             Ok(Some((DbChange::Insert, id))) => {
-                if let Some(ttl) = ttl {
-                    if let Err(err) = Ttl::create(&object, id, ttl, db) {
-                        spinner.error(&format!("Failed to set ttl: {:?}", err));
+                match object.value(rl.db()) {
+                    Ok(value) => {
+                        if let Some(ttl) = ttl {
+                            if let Err(err) = Ttl::create(&object, id, value.to_string(), ttl, db) {
+                                spinner.error(&format!("Failed to set ttl: {:?}", err));
+                            }
+                        }
+
+                        Self::on_insert(rl, spinner, object.family(), &value);
+                    }
+                    Err(err) => {
+                        spinner.error(&format!("Failed to query necessary fields for {:?}: {:?}", object, err));
                     }
                 }
 
-                // TODO: replace id with actual object(?)
-                if let Ok(obj) = object.printable(db) {
-                    spinner.log(&obj.to_string()); // TODO: also include fields here
-                } else {
-                    spinner.error(&format!("Failed to query necessary fields for {:?}", object));
-                }
                 Ok(DatabaseResponse::Inserted(id))
             },
             Ok(Some((DbChange::Update(update), id))) => {
@@ -199,16 +267,14 @@ impl DatabaseEvent {
                     }
                 }
 
-                // TODO: replace id with actual object(?)
-                match object.label(&db) {
-                    Ok(label) => {
-                        spinner.log(&format!("Updating {} ({})", label, update));
-                    },
+                match object.value(rl.db()) {
+                    Ok(value) => Self::on_update(rl, spinner, object.family(), &value, &update),
                     Err(err) => {
                         // TODO: this should be unreachable
                         spinner.error(&format!("Failed to get label for {:?}: {:?}", object, err));
                     },
                 }
+
                 Ok(DatabaseResponse::Updated(id))
             },
             Ok(Some((DbChange::None, id))) => {
@@ -231,31 +297,15 @@ impl DatabaseEvent {
         tx.send(result).expect("Failed to send db result to channel");
     }
 
-    pub fn activity<T: SpinLogger>(object: NewActivity, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
+
+    pub fn activity<T: SpinLogger>(rl: &mut Shell, object: NewActivity, tx: DbSender, spinner: &mut T, verbose: u64) {
+        let db = rl.db();
         let result = db.insert_activity(object.clone());
         debug!("{:?} => {:?}", object, result);
 
         let result = match result {
             Ok(true) => {
-                let mut log = format!("{:?} ", object.topic);
-                if let Some(uniq) = &object.uniq {
-                    log.push_str(&format!("({:?}) ", uniq));
-                }
-                log.push_str(&format!("@ {}", object.time));
-
-                if let (Some(ref lat), Some(ref lon)) = (object.latitude, object.longitude) {
-                    log.push_str(&format!(" ({}, {}", lat, lon));
-                    if let Some(radius) = &object.radius {
-                        log.push_str(&format!(" | {}m", radius));
-                    }
-                    log.push_str(")");
-                }
-
-                if verbose > 0 {
-                    log.push_str(&format!(": {}", object.content));
-                }
-
-                spinner.log(&log);
+                Self::on_activity(rl, spinner, &object, verbose);
                 Ok(DatabaseResponse::Inserted(0))
             },
             Ok(false) => Ok(DatabaseResponse::NoChange(0)),
@@ -269,12 +319,37 @@ impl DatabaseEvent {
         tx.send(result).expect("Failed to send db result to channel");
     }
 
-    pub fn apply<T: SpinLogger>(self, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
+    pub fn update<T: SpinLogger>(rl: &mut Shell, family: &str, value: &str, update: &Update, tx: DbSender, spinner: &mut T, verbose: u64) {
+        let db = rl.db();
+        if verbose >= 1 {
+            spinner.debug(&format!("Updating: {:?}", update));
+        }
+
+        let result = db.update_generic(update);
+        debug!("{:?}: {:?} => {:?}", value, update, result);
+
+        let result = match result {
+            Ok(id) => {
+                Self::on_update(rl, spinner, family, &value, &update);
+                Ok(DatabaseResponse::Updated(id))
+            },
+            Err(err) => {
+                let err = err.to_string();
+                spinner.error(&err);
+                Err(err)
+            },
+        };
+
+        tx.send(result).expect("Failed to send db result to channel");
+    }
+
+    pub fn apply<T: SpinLogger>(self, rl: &mut Shell, tx: DbSender, spinner: &mut T, verbose: u64) {
         match self {
-            DatabaseEvent::Insert(object) => Self::insert(object, None, tx, spinner, db, verbose),
-            DatabaseEvent::InsertTtl((object, ttl)) => Self::insert(object, Some(ttl), tx, spinner, db, verbose),
-            DatabaseEvent::Activity(object) => Self::activity(object, tx, spinner, db, verbose),
+            DatabaseEvent::Insert(object) => Self::insert(rl, object, None, tx, spinner, verbose),
+            DatabaseEvent::InsertTtl((object, ttl)) => Self::insert(rl, object, Some(ttl), tx, spinner, verbose),
+            DatabaseEvent::Activity(object) => Self::activity(rl, object, tx, spinner, verbose),
             DatabaseEvent::Select((family, value)) => {
+                let db = rl.db();
                 let result = match db.get_opt(&family, &value) {
                     Ok(Some(id)) => Ok(DatabaseResponse::Found(id)),
                     Ok(None) => Ok(DatabaseResponse::None),
@@ -283,26 +358,7 @@ impl DatabaseEvent {
 
                 tx.send(result).expect("Failed to send db result to channel");
             },
-            DatabaseEvent::Update((object, update)) => {
-                if verbose >= 1 {
-                    spinner.debug(&format!("Updating: {:?}", update));
-                }
-
-                let result = db.update_generic(&update);
-                debug!("{:?}: {:?} => {:?}", object, update, result);
-                let result = result
-                    .map(DatabaseResponse::Updated)
-                    .map_err(|e| e.to_string());
-
-                if let Err(ref err) = result {
-                    spinner.error(&err);
-                } else {
-                    // TODO: bring this somewhat closer to upsert code
-                    spinner.log(&format!("Updating {:?} ({})", object, update));
-                }
-
-                tx.send(result).expect("Failed to send db result to channel");
-            },
+            DatabaseEvent::Update((family, value, update)) => Self::update(rl, family.as_str(), &value, &update, tx, spinner, verbose),
         }
     }
 }
@@ -443,7 +499,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, args: Vec<(serde_json::Value, Opti
                             stack.add(name, label);
                         },
                         Event2::Log(log) => log.apply(&mut stack.prefixed(name)),
-                        Event2::Database((db, tx)) => db.apply(tx, &mut stack.prefixed(name), rl.db(), verbose),
+                        Event2::Database((db, tx)) => db.apply(rl, tx, &mut stack.prefixed(name), verbose),
                         Event2::Ratelimit((req, tx)) => ratelimit.pass(tx, &req.key, req.passes, req.time),
                         Event2::Blob((blob, tx)) => rl.store_blob(tx, &blob),
                         Event2::Exit(event) => {