Release v0.26.0

Update dependencies

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [kpcyrd]
+patreon: kpcyrd

.github/workflows/docker-release.yml

@@ -0,0 +1,58 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Registry
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/kpcyrd/sn0int
+          tags: |
+            type=semver,pattern={{raw}}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push Docker images
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          file: Dockerfile
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      -
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker.yml

@@ -0,0 +1,30 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+        - name: sn0int
+          file: Dockerfile
+        - name: registry
+          file: sn0int-registry/Dockerfile
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build the Docker image
+      run: DOCKER_BUILDKIT=1 docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+    - name: Test the Docker image
+      run: docker run --rm ${{ matrix.build.name }} --help
+
+    - name: Show Docker images
+      run: docker images

.github/workflows/rust.yml

@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies (apt)
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-dev libseccomp-dev libsodium-dev
+    - name: Install dependencies (brew)
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config libsodium
+
+    - name: Build (sn0int)
+      run: cargo build --verbose
+    - name: Build (common)
+      run: cd sn0int-common && cargo build --verbose
+    - name: Build (std)
+      run: cd sn0int-std && cargo build --verbose
+    - name: Build (examples)
+      run: cargo build --verbose --examples
+
+    - name: Run tests (sn0int)
+      run: cargo test --verbose
+    - name: Run tests (sn0int, --ignored)
+      run: cargo test --verbose -- --ignored
+    - name: Run tests (common)
+      run: cd sn0int-common && cargo test --verbose
+    - name: Run tests (common, --ignored)
+      run: cd sn0int-common && cargo test --verbose -- --ignored
+    - name: Run tests (std)
+      run: cd sn0int-std && cargo test --verbose
+    - name: Run tests (std, --ignored)
+      run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'

.travis.yml

@@ -1,58 +0,0 @@
-language: rust
-
-# upload takes too long on windows and gets killed due to inactivity
-#cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
-
-before_install:
-- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
-- df -h
-- ci/run.sh $BUILD_MODE
-- df -h
-
-notifications:
-  irc:
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    #on_success: change # default: always
-    #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,90 +1,104 @@
 [package]
 name = "sn0int"
-version = "0.10.0"
+version = "0.26.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-edition = "2018"
-
-[profile.release]
-# skip lto to avoid compiler bug:
-# https://github.com/kpcyrd/sn0int/issues/77
-# https://github.com/rust-lang/rust/issues/58674
-opt-level = 1
-lto = false
-
-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
+edition = "2021"
 
 [workspace]
-# rocket is broken and blocks the whole workspace, removing
-#members = ["sn0int-registry/sn0int-common",
-#           "sn0int-registry"]
-members = ["sn0int-registry/sn0int-common"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
+
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
+
+[features]
+sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.5.0", path="sn0int-registry/sn0int-common" }
-rustyline = "3"
+sn0int-common = { version="0.14.0", path="sn0int-common" }
+sn0int-std = { version="=0.26.0", path="sn0int-std" }
+rustyline = "10.0"
 log = "0.4"
-env_logger = "0.6"
+env_logger = "0.10"
 hlua-badtouch = "0.4"
-structopt = "0.2"
+clap = { version = "4.3.11", features = ["derive", "env"] }
+clap_complete = "4.3.2"
 failure = "0.1"
-rand = "0.6"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+libsqlite3-sys = { version = "0.22.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "1.0"
-url = "1.7"
+dirs-next = "2.0"
+url = "2.0"
+percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.9"
-base64 = "0.10"
-kuchiki = "0.7.2"
-serde_urlencoded = "0.5"
-serde = "1.0"
-serde_derive = "1.0"
+chrootable-https = "0.16"
+data-encoding = "2.3.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_urlencoded = "0.7"
 serde_json = "1.0"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.3.0"
+opener = "0.6"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.2"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
-toml = "0.4"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
+toml = "0.7"
 threadpool = "1.7"
-x509-parser = "0.4.0"
-der-parser = "1.1.0"
-nom = "4.1.1"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
-semver = "0.9"
+semver = "1"
+bytes = "0.4"
+ipnetwork = "0.20"
+strum = "0.25"
+strum_macros = "0.25"
+embedded-triple = "0.1.0"
+humansize = "2"
+
+digest = "0.10"
+md-5 = "0.10"
+sha-1 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+hmac = "0.12"
+
+walkdir = "2.2"
+nude = "0.3"
+glob = "0.3.0"
+os-version = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
+caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.13"
+syscallz = "0.16"
+nix = { version = "0.27", features = ["fs"] }
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
-unveil = "0.2.0"
+pledge = "0.4"
+unveil = "0.3"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.9"
-
-#[patch.crates-io]
-#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
-#rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
+boxxy = "0.13"
+tempfile = "3.0"

Dockerfile

@@ -1,14 +1,18 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.18
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo build --release --verbose
-RUN strip target/release/sn0int
+RUN --mount=type=cache,target=/var/cache/buildkit \
+    CARGO_HOME=/var/cache/buildkit/cargo \
+    CARGO_TARGET_DIR=/var/cache/buildkit/target \
+    cargo build --release --locked --verbose && \
+    cp -v /var/cache/buildkit/target/release/sn0int /
+RUN strip /sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
-COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+FROM alpine:3.18
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
+COPY --from=0 /sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \
     XDG_CACHE_HOME=/cache

Makefile

@@ -1,21 +1,32 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
 	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
 	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,72 +1,188 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[twitter-img]:  https://img.shields.io/badge/twitter-@sn0int-blue.svg
+[twitter]:      https://twitter.com/sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
 
-sn0int is a semi-automatic OSINT framework and package manager. It was built
-for IT security professionals and bug hunters to gather intelligence about a
-given target or about yourself. sn0int is enumerating attack surface by
-semi-automatically processing public information and mapping the results in a
-unified format for followup investigations.
+sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
+
+[ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
-- Harvest subdomains from certificate transparency logs
-- Harvest subdomains from various passive dns logs
-- Sift through subdomain results for publicly accessible websites
-- Harvest emails from pgp keyservers
-- Enrich ip addresses with ASN and geoip info
-- Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Bruteforce interesting urls
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+and is fully opensource. None of the investigations listed above are hardcoded
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
-Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
 [![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
+## Installation
+
+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer:
+
+    sudo apt install curl sq
+    curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq dearmor | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    apt update
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
 ## Getting started
 
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
-  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Debian &gt;= bullseye, Ubuntu &gt;= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian &lt;= buster, Ubuntu &lt;= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
   - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
-  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
+- [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Publish your repo](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-repo)
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
   - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
 - [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
-  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
 - [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
   - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
@@ -74,35 +190,78 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
   - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [base64_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-decode)
+  - [base64_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-encode)
+  - [base64_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-decode)
+  - [base64_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-encode)
+  - [base32_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-decode)
+  - [base32_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-encode)
   - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
   - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
-  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
   - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
   - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
   - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_time](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time)
+  - [sn0int_time_from](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time-from)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -114,8 +273,16 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
   - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
@@ -123,7 +290,36 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
   - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
+
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
 
 ## License
 

ci/integration.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -47,10 +49,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +65,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,10 +94,17 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }
 
+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
     print('[*] running geoip')
     sn0int(tempdir, binary, [
         'use geoip',

ci/run.sh

@@ -1,41 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-registry/sn0int-common
-        cargo test --verbose
-        ;;
-    windows)
-        export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-        ci/run.sh "$2"
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-    docker)
-        docker build -t sn0int .
-        docker images
-        docker run --rm sn0int --help
-        ;;
-    docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
-        docker images
-        ;;
-esac

ci/setup.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
-        ;;
-    windows)
-        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-        7z e sqlite3.zip -y
-        "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-        ;;
-esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = int($1)-1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

contrib/html-toc2md.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-perl -n -e '/toctree-l(\d).*href="([^"]+)">(.+)<\/a/ && print $1==2?"  ":"", "- [$3](https://sn0int.readthedocs.io/en/latest/$2)\n"' < docs/_build/html/index.html

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -0,0 +1,83 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+.. code-block:: bash
+
+    $ brew install libsodium
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix pkg-config
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
+
+Docker
+~~~~~~
+
+.. code-block:: bash
+
+    $ DOCKER_BUILDKIT=1 docker build -t kpcyrd/sn0int .
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/config.rst

@@ -1,17 +1,54 @@
 Configuration
 =============
 
-This file documents the config file at ``~/.config/sn0int.toml``. By default
-this file does not exist and a default configuration is used instead.
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
 
-Configuring a proxy
--------------------
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
 
 To enable a proxy, add the following to your config file::
 
     [network]
     proxy = "127.0.0.1:9050"
 
-This forces everything through tor and restricts all other functions that
-depend on the network. For example the ``dns`` function is fully disabled if a
-proxy is configured.
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -36,10 +36,14 @@ Getting Started
    :glob:
 
    install
+   build
    usage
+   autonoscope
    scripting
    database
    structs
+   activity
+   notifications
    keyring
    config
    sandbox

docs/install.rst

@@ -1,7 +1,9 @@
 Installation
 ============
 
-If available, please prefer the package shipped by your linux distribution.
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
 
 Archlinux
 ---------
@@ -17,31 +19,42 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian testing/Debian sid/Kali
-------------------------------
+Debian >= bookwork, Ubuntu >= 22.10, Kali
+-----------------------------------------
 
-Note that debian `doesn't ship the geoip2-database
-<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
-download them automatically during the first run.
+There are prebuilt packages signed by a debian maintainer:
 
 .. code-block:: bash
 
-    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev publicsuffix
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ sudo apt install curl sq
+    $ curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq keyring filter -B --handle 64B13F7117D6E07D661BBCE0FE763A64F5E54FD6 | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ apt update
+    $ apt install sn0int
 
-Ubuntu/Debian stable
+Debian <= bullseye, Ubuntu <= 22.04
+-----------------------------------
+
+There are prebuilt packages signed by a debian maintainer:
+
+.. code-block:: bash
+
+    $ sudo apt install curl sq
+    $ curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq dearmor | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ apt update
+    $ apt install sn0int
+
+Fedora/CentOS/Redhat
 --------------------
 
-cargo in the repos is too old and the build is `going to fail
-<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
-most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
-instructions instead.
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -51,39 +64,47 @@ Docker
 
 .. code-block:: bash
 
-    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
 
 Alpine
 ------
 
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ apk add sn0int
 
 OpenBSD
 -------
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ pkg_add sn0int
+
+Gentoo
+------
+
+.. code-block:: bash
+
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
 
-This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
+
+Make sure rust is installed and setup.
 
 .. code-block:: bash
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-    $ 7z e sqlite3.zip -y
-    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
     $ cargo install -f --path .

docs/keyring.rst

@@ -52,12 +52,16 @@ If the user granted us access to those keys we can read them with ``keyring``:
 .. code-block:: lua
 
     creds = keyring('aws')
-    print(creds[1]['accesskey'])
-    print(creds[1]['secretkey'])
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
 
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/sandbox.rst

@@ -6,7 +6,7 @@ child process. It's important to note that there's a basic sandbox that's
 active on every operating system, and there's a second line of defense on
 supported operating systems.
 
-The first line of defense is the restrictive stdlib. It's assumed that and
+The first line of defense is the restrictive stdlib. It's assumed that an
 attacker gains full control over the lua code and is able to call any function
 with arbitrary arguments. The stdlib only provides functions that are
 considered safe, so for example it's not possible to start a process or open a
@@ -119,7 +119,7 @@ You might experience a sandbox failure, especially on architectures that are
 less popular. This usually looks like this::
 
     [sn0int][example][kpcyrd/ctlogs] > run
-    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [-] Failed "example.com": Sandbox child has crashed
     [+] Finished kpcyrd/ctlogs (1 errors)
 
 A module that never finishes could also mean an IO thread inside the worker got

docs/scripting.rst

@@ -1,11 +1,11 @@
-Scripting
-=========
+Writing your first module
+=========================
 
 Scripting is the core feature in sn0int. It's not strictly required, but if you
 want to write your own modules, this section is for you.
 
-Write your first module
------------------------
+Creating a repository
+---------------------
 
 It's highly recommended to use a VCS for development, so let's start by setting
 that up. We're going to assume you store your repos in ``~/repos`` but you're
@@ -13,7 +13,25 @@ free to change that to something else::
 
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
-    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+.. note::
+   If you're using github you can also create a repo from the `module repo
+   template`_.
+
+.. _module repo template: https://github.com/sn0int/sn0int-modules
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the following::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
 
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.
@@ -78,21 +96,26 @@ database.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
-        print(subdomain)
+        info(subdomain)
     end
 
-Combined with the header we wrote previously we can already execute this
-module. Make sure you've added a domain to scope with ``add domain
-example.com``, save your file and run it like this::
+This is already enough to execute it. Make sure you've added a domain to scope
+with ``add domain example.com``, save your file and run it like this::
 
     sn0int run -f ./first.lua
 
-We should see some output by our print function.
+We should see some output by our info function.
 
 .. note::
-   ``print`` is useful for development but must be removed before publishing.
+   ``info`` is useful for development but you usually want your module to run
+   quietly, so before publishing either remove it or replace it with ``debug``.
 
 Next, we want to actually resolve that name, we're going to use the ``dns``
 function for that. This function takes a name and a query type and returns a
@@ -102,6 +125,11 @@ truth-y.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -110,16 +138,22 @@ truth-y.
         })
         if last_err() then return end
 
-        print(records)
+        info(records)
     end
 
 If you run your module again you're going to see some output, either
 ``{"answers":[somedata],"error":null}`` or
-``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
-our scope and set it to resolvable if ``error`` is ``nil``.
+``{"answers":[],"error":"NXDomain"}``. If the dns reply doesn't indicate an
+error this means the subdomain exists and we can add it to our database with
+``resolvable`` being set to ``true``.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -140,41 +174,20 @@ our scope and set it to resolvable if ``error`` is ``nil``.
 .. hint::
    See the database section to understand how the database works in detail.
 
-If we execute our module one more time it's going to log that it discovered a
-subdomain, if it doesn't, try adding more domains to scope. Note that this only
-happens the first time. Modules that don't discover anything or don't discover
-anything new exit silently.
-
-After putting everything together, our final module looks like this:
-
-.. code-block:: lua
-
-    -- Description: ohai wurld
-    -- Version: 0.1.0
-    -- Source: domains
-    -- License: GPL-3.0
-
-    function run(arg)
-        subdomain = 'www.' .. arg['value']
-
-        records = dns(subdomain, {
-            record='A'
-        })
-        if last_err() then return end
-
-        if records['success'] ~= nil then
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=subdomain,
-                resolvable=true,
-            })
-        end
-    end
+If we execute our finished module one more time it's going to log that it
+discovered a subdomain, if it doesn't, try adding more domains to scope. Note
+that this only happens the first time. Modules that don't discover anything or
+don't discover anything new exit silently.
 
 There's still some room for improvement, for example, since we already resolved
 that record, we could also add the ip address to the scope and link it to the
 subdomain we added.
 
+.. hint::
+   For debugging purposes you can increase the verbosity with ``sn0int run -v``
+   so database operations are logged even if nothing was changed, or with
+   ``sn0int run -vv`` to enable ``debug()`` output.
+
 Publish your module
 -------------------
 
@@ -193,6 +206,24 @@ Afterwards publish your module with::
 
     sn0int publish ./first.lua
 
+Please also make sure you publish your repository to github so other people can
+submit pull requests. The recommended repository location is::
+
+    https://github.com/<your-username>/sn0int-modules
+
+Publish your repo
+-----------------
+
+It is highly recommended to publish your repository on github so people can
+file issues and pull requests for your module. If you've been following along
+with the github template you can simply commit your changes and push them.
+
+Your repository would look like one of these:
+
+- https://github.com/kpcyrd/sn0int-modules
+- https://github.com/ysf/sn0int-modules
+- https://github.com/cybiere/sn0int-modules
+
 Reading data from stdin
 -----------------------
 

docs/structs.rst

@@ -86,6 +86,8 @@ Emails
 
 ``value``
     The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
 ``valid``
     Whether that email address is valid or has been disabled.
 
@@ -143,6 +145,8 @@ connected to.
     Latitude of the networks location.
 ``longitude``
     Longitude of the networks location.
+``description``
+    A human readable description in case the value is a technical identifier.
 
 Accounts
 --------
@@ -164,6 +168,12 @@ A users account or profile on a webservice, like github or instagram.
     The url of the public profile if available.
 ``last_seen``
     The last time this account has been active/online.
+``birthday``
+    The users birthday set on the account.
+``phonenumber``
+    The phonenumber associated with the account.
+``profile_pic``
+    The blob identifier of the users current profile picture.
 
 Breaches
 --------
@@ -174,6 +184,110 @@ notification service.
 ``value``
     The name of the breach, breach compilation or notification service.
 
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Netblocks
+---------
+
+A netblock is a network address range that has been allocated to an individual,
+organization or company. Those are commonly found when running whois lookups on
+an ip address.
+
+Consider the following example: Running a whois lookup on ``140.82.118.4`` (one
+of the addresses currently in use by github) returns that this address belongs
+to the netrange ``140.82.112.0 - 140.82.127.255``, so the netblock in this case
+is ``140.82.112.0/20``.
+
+``family``
+    This is either ``4`` or ``6`` and populated automatically.
+``value``
+    This is the network range in CIDR notation.
+``asn``
+    The number of the autonomous system this network belongs to.
+``as_org``
+    The organization of the autonomous system this network belongs to.
+``description``
+    This field isn't strictly defined and meant to be used as a human
+    meaningful name if available.
+
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------

examples/boxxy.rs

@@ -1,6 +1,7 @@
-#[macro_use] extern crate boxxy;
-extern crate sn0int;
+#[macro_use]
+extern crate boxxy;
 extern crate env_logger;
+extern crate sn0int;
 
 fn stage1(sh: &mut boxxy::Shell, _args: Vec<String>) -> Result<(), boxxy::Error> {
     shprintln!(sh, "[*] starting stage1");
@@ -14,8 +15,6 @@ fn main() {
 
     println!("stage1        activate sandbox");
 
-    let toolbox = boxxy::Toolbox::new().with(vec![
-            ("stage1", stage1),
-        ]);
+    let toolbox = boxxy::Toolbox::new().with(vec![("stage1", stage1)]);
     boxxy::Shell::new(toolbox).run()
 }

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
+use clap::Parser;
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
-use structopt::StructOpt;
+use std::path::Path;
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
-    #[structopt(name="asn")]
+    #[command(name = "asn")]
     Asn(AsnArgs),
-    #[structopt(name="geoip")]
+    #[command(name = "geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -64,14 +30,15 @@ impl AsnArgs {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct GeoIPArgs {
     ip: IpAddr,
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -80,14 +47,13 @@ impl GeoIPArgs {
     }
 }
 
-
 fn run() -> Result<()> {
-    let args = Args::from_args();
+    let args = Args::parse();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

examples/spinners.rs

@@ -1,21 +1,20 @@
+use clap::Parser;
+use sn0int::term::{Spinner, StackedSpinners, SPINNERS};
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
-use structopt::StructOpt;
 
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="single")]
+    #[command(name = "single")]
     Single(Single),
-    #[structopt(name="stacked")]
+    #[command(name = "stacked")]
     Stacked(Stacked),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Single {
     idx: usize,
-    #[structopt(long="ticks", default_value="100")]
+    #[structopt(long = "ticks", default_value = "100")]
     ticks: usize,
 }
 
@@ -32,9 +31,8 @@ impl Single {
     }
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Stacked {
-}
+#[derive(Debug, Parser)]
+pub struct Stacked {}
 
 impl Stacked {
     fn run(&self) {
@@ -59,7 +57,7 @@ impl Stacked {
 }
 
 fn main() {
-    let args = Args::from_args();
+    let args = Args::parse();
     match args {
         Args::Single(args) => args.run(),
         Args::Stacked(args) => args.run(),

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

migrations/2019-07-04-233515_autonoscope/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE autonoscope;

migrations/2019-07-04-233515_autonoscope/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE autonoscope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    object VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    scoped BOOLEAN NOT NULL,
+    CONSTRAINT autonoscope_unique UNIQUE (object, value)
+);

migrations/2019-07-27-152215_netblocks/down.sql

@@ -0,0 +1 @@
+DROP TABLE netblocks;

migrations/2019-07-27-152215_netblocks/up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE netblocks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    asn INTEGER,
+    as_org VARCHAR,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2019-08-11-073709_misc-fields/down.sql

@@ -0,0 +1,41 @@
+PRAGMA foreign_keys=off;
+
+-- accounts
+CREATE TABLE _accounts_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);
+
+INSERT INTO _accounts_new (id, value, service, username, displayname, email, url, last_seen, unscoped)
+    SELECT id, value, service, username, displayname, email, url, last_seen, unscoped
+    FROM accounts;
+
+DROP TABLE accounts;
+ALTER TABLE _accounts_new RENAME TO accounts;
+
+-- networks
+CREATE TABLE _networks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+INSERT INTO _networks_new (id, value, unscoped, latitude, longitude)
+    SELECT id, value, unscoped, latitude, longitude
+    FROM networks;
+
+DROP TABLE networks;
+ALTER TABLE _networks_new RENAME TO networks;
+
+PRAGMA foreign_keys=on;

migrations/2019-08-11-073709_misc-fields/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE accounts ADD COLUMN phonenumber VARCHAR;
+ALTER TABLE accounts ADD COLUMN profile_pic VARCHAR;
+ALTER TABLE accounts ADD COLUMN birthday VARCHAR;
+ALTER TABLE networks ADD COLUMN description VARCHAR;

migrations/2019-11-19-154056_cryptoaddr/down.sql

@@ -0,0 +1 @@
+DROP TABLE cryptoaddrs;

migrations/2019-11-19-154056_cryptoaddr/up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE cryptoaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    currency VARCHAR,
+    denominator INTEGER,
+    balance BIGINT,
+    received BIGINT,
+    first_seen DATETIME,
+    last_withdrawal DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/dev/arp-scan.lua

@@ -1,45 +0,0 @@
--- Description: Parse arp-scan output
--- Version: 0.3.0
--- License: GPL-3.0
-
--- sudo arp-scan -qglI wlp3s0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        m = regex_find('(.+)\t(.+)', x)
-        if m ~= nil then
-            ipaddr = m[2]
-            mac = m[3]
-            now = datetime()
-
-            device_id = db_add('device', {
-                value=mac,
-                last_seen=now,
-            })
-            if last_err() then return end
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 300)
-            if last_err() then return end
-        end
-    end
-end

modules/dev/asn.lua

@@ -1,16 +0,0 @@
--- Description: Run a asn lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = asn_lookup(arg['value'])
-    if last_err() then return end
-
-    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
-        db_update('ipaddr', arg, {
-            asn=lookup['asn'],
-            as_org=lookup['as_org'],
-        })
-    end
-end

modules/dev/axfr.lua

@@ -1,159 +0,0 @@
--- Description: Try a zone transfer for subdomains
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function strip_root_dot(name)
-    m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function add_pointer(name)
-    local domain, domain_id, subdomain_id
-
-    -- select psl+1
-    domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function iter_axfr(zone, arg)
-    local name, r, m, domain
-
-    debug(arg)
-
-    name = arg[1]
-    r = arg[2]
-
-    -- select psl+1
-    domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-
-    -- this is a A record
-    if r['A'] ~= nil then
-        -- add the name and ip
-        ipaddr_id = db_add('ipaddr', {
-            family='4',
-            value=r['A'],
-        })
-        if last_err() then return end
-
-        db_add('subdomain-ipaddr', {
-            subdomain_id=subdomain_id,
-            ip_addr_id=ipaddr_id,
-        })
-        if last_err() then return end
-    end
-
-    if r['CNAME'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['CNAME'])
-        add_pointer(name)
-    end
-
-    if r['NS'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['NS'])
-        add_pointer(name)
-    end
-
-    if r['MX'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['MX'][2])
-        add_pointer(name:lower())
-    end
-end
-
-function iter_a(zone, arg)
-    local i, records, r
-
-    if arg == nil then return end
-
-    debug('nameserver: ' .. arg)
-    records = dns(zone, {
-        record='AXFR',
-        nameserver=arg .. ':53',
-        tcp=true,
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        iter_axfr(zone, records[i])
-        if last_err() then return end
-        i = i+1
-    end
-end
-
-function iter_ns(zone, arg)
-    local i, records, r
-
-    if arg == nil then return end
-
-    records = dns(arg, {
-        record='A',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        iter_a(zone, r['A'])
-        if last_err() then return end
-        i = i+1
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        iter_ns(arg['value'], r['NS'])
-        if last_err() then return end
-        i = i+1
-    end
-end

modules/dev/cname-harvest.lua

@@ -1,50 +0,0 @@
--- Description: Query for CNAMES to find subdomains
--- Version: 0.2.0
--- Source: subdomains
--- License: GPL-3.0
-
-function iter(r)
-    if r == nil then
-        return
-    end
-
-    m = regex_find("(.+)\\.$", r)
-    if last_err() then return end
-
-    if m == nil then
-        return
-    end
-    r = m[2]
-
-    domain = psl_domain_from_dns_name(r)
-    if last_err() then return end
-
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=r,
-        })
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        iter(r['CNAME'])
-        if last_err() then return end
-        i = i+1
-    end
-end

modules/dev/ctlogs.lua

@@ -1,106 +0,0 @@
--- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.4.0
--- Source: domains
--- License: GPL-3.0
-
-function each_name(name)
-    local domain_id, psl_domain
-
-    if seen[name] == 1 then
-        return
-    end
-    seen[name] = 1
-    debug(name)
-
-    if name:find('*.') == 1 then
-        -- ignore wildcard domains
-        return
-    end
-
-    -- the cert might be valid for subdomains that do not belong to the
-    -- domain we started with
-    psl_domain = psl_domain_from_dns_name(name)
-    domain_id = domains[psl_domain]
-    if domain_id == nil then
-        if any_domain then
-            -- unknown domains should be added to database
-            domain_id = db_add('domain', {
-                value=psl_domain,
-            })
-        else
-            -- only use domains that are already in scope
-            domain_id = db_select('domain', psl_domain)
-        end
-
-        -- if we didn't get a valid id, skip
-        if domain_id == nil then
-            return
-        end
-
-        domains[psl_domain] = domain_id
-    end
-
-    db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-end
-
-function run(arg)
-    full = getopt('full') ~= nil
-    any_domain = getopt('any-domain') ~= nil
-
-    domains = {}
-    domains[arg['value']] = arg['id']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://crt.sh/', {
-        query={
-            q='%.' .. arg['value'],
-            output='json'
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    certs = json_decode(resp['text'])
-    if last_err() then return end
-
-    seen = {}
-
-    i = 1
-    while i <= #certs do
-        c = certs[i]
-        debug(c)
-
-        if full then
-            -- fetch certificate
-            id = c['min_cert_id']
-            req = http_request(session, 'GET', 'https://crt.sh/', {
-                query={
-                    d=id .. '', -- TODO: find nicer way for tostring
-                }
-            })
-            resp = http_send(req)
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            -- iterate over all valid names
-            crt = x509_parse_pem(resp['text'])
-            if last_err() then return end
-            names = crt['valid_names']
-
-            j = 1
-            while j <= #names do
-                each_name(names[j])
-                j = j+1
-            end
-        else
-            each_name(c['name_value'])
-        end
-
-        i = i+1
-    end
-end

modules/dev/ddwrt-dhcp.lua

@@ -1,91 +0,0 @@
--- Description: Export dhcp leases from ddwrt webinterface
--- Version: 0.1.0
--- License: GPL-3.0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    skip_redacted = not getopt('use-redacted')
-
-    router = getopt('router') -- http://192.0.2.1/
-    if not router then
-        return 'router option is missing (http://192.0.2.1/)'
-    end
-    username = getopt('user')
-    password = getopt('password')
-
-    options = {}
-    if username and password then
-        options['basic_auth'] = {username, password}
-    end
-
-    -- request status page
-    session = http_mksession()
-    url = url_join(router, '/Info.live.htm')
-    req = http_request(session, 'GET', url, options)
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    txt = resp['text']
-    debug(txt)
-
-    -- extract leases from response
-    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
-    if last_err() then return end
-    if not dhcp_section then
-        return 'Failed to get dhcp lease section'
-    end
-
-    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
-    if last_err() then return end
-
-    now = datetime()
-
-    -- add devices to database
-    local i = 1
-    while i <= #leases do
-        local hostname = leases[i][2]
-        local ipaddr = leases[i][3]
-        local macaddr = leases[i][4]
-
-        debug({
-            hostname=hostname,
-            ipaddr=ipaddr,
-            macaddr=macaddr,
-        })
-
-        if skip_redacted and macaddr:match('^xx:xx:') then
-            info('Skipping redacted macaddr')
-        else
-            local device = {
-                value=macaddr,
-                last_seen=now,
-            }
-            if hostname ~= '*' then
-                device['hostname'] = hostname
-            end
-
-            local device_id = db_add('device', device)
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 120)
-        end
-
-        i = i+1
-    end
-end

modules/dev/dns-ptr.lua

@@ -1,32 +0,0 @@
--- Description: Run reverse dns lookups
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    if arg['family'] == '4' then
-        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
-
-        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
-        debug('Resolving: ' .. q)
-
-        records = dns(q, {
-            record='PTR',
-        })
-        if last_err() then return end
-        if records['error'] ~= nil then return end
-        records = records['answers']
-
-        i = 1
-        while records[i] ~= nil do
-            r = records[i][2]
-            if r['PTR'] then
-                db_update('ipaddr', arg, {
-                    reverse_dns=r['PTR'],
-                })
-                if last_err() then return end
-            end
-            i = i+1
-        end
-    end
-end

modules/dev/dns-resolve.lua

@@ -1,43 +0,0 @@
--- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.2.0
--- Source: subdomains
--- License: GPL-3.0
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    -- update subdomain
-    resolvable = records['error'] == nil
-    if arg['resolvable'] ~= resolvable then
-        -- TODO: pass arg to function as well
-        db_update('subdomain', arg, {
-            resolvable=resolvable
-        })
-    end
-
-    if not resolvable then
-        return
-    end
-
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        if r['A'] ~= nil then
-            ipaddr_id = db_add('ipaddr', {
-                family='4',
-                value=r['A'],
-            })
-            if last_err() then return end
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-            if last_err() then return end
-        end
-        i = i+1
-    end
-end

modules/dev/geoip.lua

@@ -1,10 +0,0 @@
--- Description: Run a geoip lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = geoip_lookup(arg['value'])
-    if last_err() then return end
-    db_update('ipaddr', arg, lookup)
-end

modules/dev/git-webroot.lua

@@ -1,28 +0,0 @@
--- Description: Search for git checkouts in webroot
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    url = url_join(arg['value'], '.git/HEAD')
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return
-    end
-
-    if not regex_find('^ref: ', reply['text']) then
-        return
-    end
-
-    db_add('url', {
-        subdomain_id=arg['subdomain_id'],
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    })
-end

modules/dev/github.lua

@@ -1,76 +0,0 @@
--- Description: Collect data from github profiles
--- Version: 0.1.0
--- Source: accounts:github.com
--- License: GPL-3.0
-
-function api_get(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-
-    local data = json_decode(resp['text'])
-    if last_err() then return end
-    return data
-end
-
-function scan4email(username)
-    local url = 'https://api.github.com/users/' .. username .. '/repos'
-    local repos = api_get(url)
-    if last_err() then return end
-
-    i = 1
-    while i <= #repos do
-        local repo = repos[i]
-        debug(repo)
-        local commits = api_get(repo['url'] .. '/commits')
-        if last_err() then return end
-
-        j = 1
-        while j <= #commits do
-            local commit = commits[j]
-            debug(commit)
-
-            if commit['author']['login'] == username then
-                -- name = commit['commit']['author']['name']
-                local email = commit['commit']['author']['email']
-                return email
-            end
-
-            if commit['committer']['login'] == username then
-                -- name = commit['commit']['committer']['name']
-                local email = commit['commit']['committer']['email']
-                return email
-            end
-
-            j = j+1
-        end
-
-        i = i+1
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://api.github.com/users/' .. arg['username']
-
-    local data = api_get(url)
-    if last_err() then return end
-    debug(data)
-
-    -- company = data['company']
-    -- location = data['location']
-    -- homepage = data['blog']
-
-    local email = data['email']
-    if not email and not arg['email'] then
-        email = scan4email(arg['username'])
-        if last_err() then return end
-    end
-
-    db_update('account', arg, {
-        url=data['html_url'],
-        displayname=data['name'],
-        email=email,
-    })
-end

modules/dev/hackertarget-subdomains.lua

@@ -1,30 +0,0 @@
--- Description: Query hackertarget for subdomains of a domain
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
-        query={
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
-
-    i = 1
-    while i <= #m do
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=m[i][2]
-        })
-
-        i = i+1
-    end
-end

modules/dev/instagram.lua

@@ -1,36 +0,0 @@
--- Description: Collect data from instagram profiles
--- Version: 0.1.0
--- Source: accounts:instagram.com
--- License: GPL-3.0
-
-function run(arg)
-    local session = http_mksession()
-    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-    local html = resp['text']
-
-    local ld = html_select(html, 'script[type="application/ld+json"]')
-    if last_err() then return end
-    debug(ld)
-
-    local ld = json_decode(ld['text'])
-    if last_err() then return end
-    debug(ld)
-
-    if ld['email'] then
-        db_add('email', {
-            value=ld['email'],
-        })
-    end
-
-    -- homepage=ld['url']
-
-    db_update('account', arg, {
-        displayname=ld['name'],
-        email=ld['email'],
-        url=url,
-    })
-end

modules/dev/isc-dhcpd-leases.lua

@@ -1,78 +0,0 @@
--- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.2.0
--- License: GPL-3.0
-
--- cat /var/lib/dhcpd/dhcpd.leases
-
-function add(lease)
-    if not lease['active'] then return end
-
-    now = datetime()
-
-    device_id = db_add('device', {
-        value=lease['mac'],
-        hostname=lease['hostname'],
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=lease['ipaddr'],
-        last_seen=now,
-    }, 180)
-    if last_err() then return end
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^lease (\\S+) \\{\n$', x)
-    if m then
-        lease = {}
-        debug('ipaddr=' .. m[2])
-        lease['ipaddr'] = m[2]
-    end
-    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
-    if m then
-        debug('mac=' .. m[2])
-        lease['mac'] = m[2]
-    end
-    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
-    if m then
-        debug('hostname=' .. m[2])
-        lease['hostname'] = m[2]
-    end
-    m = regex_find('^\\s*binding state active;\n$', x)
-    if m then
-        debug('active=true')
-        lease['active'] = true
-    end
-    m = regex_find('^\\}\n$', x)
-    if m then
-        add(lease)
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
-            each_line(x)
-        end
-    end
-end

modules/dev/iw-station-dump.lua

@@ -1,77 +0,0 @@
--- Description: Parse iw station dump
--- Version: 0.2.0
--- License: GPL-3.0
-
--- iw dev wlan0 station dump
-
-function add(client)
-    if
-        client['authenticated'] == 'yes' and
-        client['authorized'] == 'yes' and
-        client['mac']
-    then
-        debug(client)
-
-        now = datetime()
-
-        device_id = db_add('device', {
-            value=client['mac'],
-            last_seen=now,
-        })
-        if last_err() then return end
-
-        db_add_ttl('network-device', {
-            network_id=network_id,
-            device_id=device_id,
-            last_seen=now,
-        }, 180)
-        if last_err() then return end
-    end
-
-    client = nil
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^Station (\\S+)', x)
-    if m then
-        if client then
-            add(client)
-        end
-        client = {}
-        client['mac'] = m[2]
-        debug('mac=' .. m[2])
-    end
-
-    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
-    if m and client then
-        client[m[2]] = m[3]
-        debug(m[2] .. '=' .. m[3])
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    client = nil
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        each_line(x)
-    end
-
-    if client then
-        add(client)
-    end
-end

modules/dev/namechk.lua

@@ -1,90 +0,0 @@
--- Description: Find accounts by username with namechk.com
--- Version: 0.1.0
--- Source: accounts
--- License: GPL-3.0
-
-function get_services(html)
-    local divs = html_select_list(html, '.service')
-    if last_err() then return end
-
-    local services = {}
-
-    local i = 1
-    while i <= #divs do
-        services[i] = divs[i]['attrs']['data-name']
-        i = i+1
-    end
-
-    return services
-end
-
-function run(arg)
-    -- setup session
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://namechk.com/', {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
-    local auth_token = token['attrs']['value']
-
-        local headers = {}
-        headers['X-CSRF-Token'] = authenticity_token
-
-    local services = get_services(resp['text'])
-    debug({
-        auth_token=auth_token,
-        services=services,
-    })
-
-    -- trigger the scan
-    local req = http_request(session, 'POST', 'https://namechk.com/', {
-            headers=headers,
-        form={
-            authenticity_token=auth_token,
-            q=arg['username'],
-        }
-    })
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-    debug(resp)
-
-    local scan = json_decode(resp['text'])
-    if last_err() then return end
-    local scan_token = scan['valid']
-
-    -- get results
-    local i = 1
-    while i <= #services do
-        debug(services[i])
-
-        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
-            headers=headers,
-            form={
-                token=scan_token,
-                fat=auth_token,
-                service=services[i],
-            }
-        })
-        local resp = http_send(req)
-        if last_err() then return end
-
-        if resp['status'] == 200 then
-            local acc = json_decode(resp['text'])
-            if last_err() then return end
-            debug(acc)
-
-            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
-                db_add('account', {
-                    service=services[i],
-                    username=arg['username'],
-                    url=acc['callback_url'],
-                })
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/otx-subdomains.lua

@@ -1,32 +0,0 @@
--- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
-
-    req = http_request(session, 'GET', url, {})
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['passive_dns']
-
-    i = 1
-    while o[i] do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x['hostname'],
-        })
-
-        i = i+1
-    end
-end

modules/dev/passive-arp.lua

@@ -1,79 +0,0 @@
--- Description: Passive arp-scanner with sniffglue
--- Version: 0.1.0
--- License: GPL-3.0
-
--- sudo sniffglue -jv enp0s25
-
-function each_frame(frame)
-    if not frame['Ether'] then return end
-
-    local arp = frame['Ether'][2]['Arp']
-    if not arp then return end
-
-    if arp['Request'] then
-        arp = arp['Request']
-    elseif arp['Reply'] then
-        arp = arp['Reply']
-    else
-        -- unknown, abort
-        return
-    end
-
-    debug(arp)
-
-    -- TODO: this might change to a string in the future
-    local mac = mac(arp['src_mac'])
-    local ipaddr = arp['src_addr']
-    debug({src_mac=mac, src_addr=ipaddr})
-
-    local now = datetime()
-
-    local device_id = db_add('device', {
-        value=mac,
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=ipaddr,
-        last_seen=now,
-    }, 120)
-    if last_err() then return end
-end
-
-function mac(m)
-    return
-        hex({m[1]}) .. ':' ..
-        hex({m[2]}) .. ':' ..
-        hex({m[3]}) .. ':' ..
-        hex({m[4]}) .. ':' ..
-        hex({m[5]}) .. ':' ..
-        hex({m[6]})
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        local x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        local frame = json_decode(x)
-        if last_err() then return end
-
-        each_frame(frame)
-        if last_err() then return end
-    end
-end

modules/dev/passive-spider.lua

@@ -1,63 +0,0 @@
--- Description: Scrape known http responses for urls
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function entry(parent, href)
-    -- TODO: parse mailto:foo@example.com?subject=asdf
-    -- TODO: parse tel:+4912345
-    -- TODO: allow discovering 3rd-party domains
-    -- TODO: maybe record urls as well
-
-    local psl, parts, url, host
-
-    if href == nil then
-        return
-    end
-
-    url = url_join(parent, href)
-    if last_err() then return clear_err() end
-    if url:match('^https?://') == nil then
-        return
-    end
-
-    parts = url_parse(url)
-    if last_err() then return end
-    host = parts['host']
-    psl = psl_domain_from_dns_name(host)
-
-
-    domain_id = db_select('domain', psl)
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=host,
-        })
-    end
-end
-
-function run(arg)
-    if arg['body'] == nil or #arg['body'] == 0 then
-        return
-    end
-
-    body = utf8_decode(arg['body'])
-    if last_err() then return end
-
-    links = html_select_list(body, 'a')
-    if last_err() then return end
-
-    if #links == 0 then
-        return
-    end
-
-    -- process html links
-    i = 1
-    while i <= #links do
-        href = links[i]['attrs']['href']
-
-        entry(arg['value'], href)
-
-        i = i+1
-    end
-end

modules/dev/pgp-keyserver.lua

@@ -1,58 +0,0 @@
--- Description: Query pgp keyserver for email addresses
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    lookup_url = 'https://sks-keyservers.net/pks/lookup'
-
-    req = http_request(session, 'GET', lookup_url, {
-        query={
-            search=arg['value'],
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    links = html_select_list(resp['text'], 'a')
-    i = 1
-    while i <= #links do
-        href = links[i]['attrs']['href']
-
-        if href:find('/pks/lookup%?op=get&search=') == 1 then
-            url = url_join(lookup_url, href)
-
-            req = http_request(session, 'GET', url, {})
-
-            resp = http_send(req)
-            -- TODO: do not abort script if one attempt fails
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            pubkey = pgp_pubkey_armored(resp['text'])
-
-            -- print(pubkey)
-
-            -- TODO: ensure at least one email matches our target domain
-            if pubkey['uids'] then
-                j = 1
-                while j <= #pubkey['uids'] do
-                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                    if m then
-                        db_add('email', {
-                            value=m[2],
-                        })
-                    end
-                    j = j+1
-                end
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/phpmyadmin.lua

@@ -1,106 +0,0 @@
--- Description: Search for phpmyadmin
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    paths = {
-        "phpmyadmin/index.php",
-        "phpMyAdmin/index.php",
-        "pmd/index.php",
-        "pma/index.php",
-        "PMA/index.php",
-        "PMA2/index.php",
-        "pmamy/index.php",
-        "pmamy2/index.php",
-        "mysql/index.php",
-        "admin/index.php",
-        "db/index.php",
-        "dbadmin/index.php",
-        "web/phpMyAdmin/index.php",
-        "admin/pma/index.php",
-        "admin/PMA/index.php",
-        "admin/mysql/index.php",
-        "admin/mysql2/index.php",
-        "admin/phpmyadmin/index.php",
-        "admin/phpMyAdmin/index.php",
-        "admin/phpmyadmin2/index.php",
-        "mysqladmin/index.php",
-        "mysql-admin/index.php",
-        "mysql_admin/index.php",
-        "phpadmin/index.php",
-        "phpAdmin/index.php",
-        "phpmyadmin0/index.php",
-        "phpmyadmin1/index.php",
-        "phpmyadmin2/index.php",
-        "phpMyAdmin-4.4.0/index.php",
-        "myadmin/index.php",
-        "myadmin2/index.php",
-        "xampp/phpmyadmin/index.php",
-        "phpMyadmin_bak/index.php",
-        "www/phpMyAdmin/index.php",
-        "tools/phpMyAdmin/index.php",
-        "phpmyadmin-old/index.php",
-        "phpMyAdminold/index.php",
-        "phpMyAdmin.old/index.php",
-        "pma-old/index.php",
-        "claroline/phpMyAdmin/index.php",
-        "typo3/phpmyadmin/index.php",
-        "phpma/index.php",
-        "phpmyadmin/phpmyadmin/index.php",
-        "phpMyAdmin/phpMyAdmin/index.php",
-        "phpMyAbmin/index.php",
-        "phpMyAdmin__/index.php",
-        "phpMyAdmin+++---/index.php",
-        "v/index.php",
-        "phpmyadm1n/index.php",
-        "phpMyAdm1n/index.php",
-        "shaAdmin/index.php",
-        "phpMyadmi/index.php",
-        "phpMyAdmion/index.php",
-        "MyAdmin/index.php",
-        "phpMyAdmin1/index.php",
-        "phpMyAdmin123/index.php",
-        "pwd/index.php",
-        "phpMyAdmina/index.php",
-        "program/index.php",
-        "shopdb/index.php",
-        "phppma/index.php",
-        "phpmy/index.php",
-        "mysql/admin/index.php",
-        "mysql/dbadmin/index.php",
-        "mysql/sqlmanager/index.php",
-        "mysql/mysqlmanager/index.php",
-        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
-    }
-
-    session = http_mksession()
-
-    i = 1
-    while i <= #paths do
-        p = paths[i]
-        url = url_join(arg['value'], p)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            if reply['status'] == 200 then
-                db_add('url', {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                })
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/smtp-check.lua

@@ -1,90 +0,0 @@
--- Description: Verify email address by asking the smtp server
--- Version: 0.1.0
--- Source: emails
--- License: GPL-3.0
-
-function find_mx(domain)
-    local records, i, r
-
-    records = dns(domain, {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    i = 1
-    while i <= #records do
-        r = records[i][2]['MX']
-        if r then
-            debug('mx: ' .. r[2])
-            return r[2]
-        end
-        i = i+1
-    end
-end
-
-function run(arg)
-    -- extract domain
-    domain = arg['value']:match('@(.*)')
-    if doman ~= nil then
-        -- malformed domain
-        return
-    end
-
-    -- mx lookup
-    mx = find_mx(domain)
-    if last_err() then return end
-    if not mx then return end
-
-    -- create connection
-    c = sock_connect(mx, 25, {})
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send hello
-    sock_sendline(c, 'ehlo localhost')
-    if last_err() then return end
-
-    l = sock_recvline_regex(c, '^250 ')
-    if last_err() then return end
-    debug(l)
-
-    -- send email
-    sock_sendline(c, 'mail from:<root@localhost>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send rcpt
-    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- check status
-    verified = nil
-    if l:match('^2') then
-        debug('email is valid')
-        verified = true
-    elseif l:match('^5') then
-        debug('email is invalid')
-        verified = false
-    elseif l:match('^4') then
-        debug('unknown status, temporary delivery failure')
-    end
-
-    if verified ~= nil then
-        db_update('email', arg, {
-            valid=verified,
-        })
-    end
-end

modules/dev/threatminer-ipaddr.lua

@@ -1,52 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.2.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: add option to filter old entries based on last_seen
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
-        query={
-            rt='2',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    i = 1
-    while o[i] do
-        x = o[i]
-
-        domain = psl_domain_from_dns_name(x['domain'])
-        -- TODO: if this fails, skip this entry instead
-        if last_err() then return end
-
-        domain_id = db_add('domain', {
-            value=domain,
-        })
-
-        if domain_id ~= nil then
-            subdomain_id = db_add('subdomain', {
-                domain_id=domain_id,
-                value=x['domain'],
-            })
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=subdomain_id,
-                ip_addr_id=arg['id'],
-            })
-        end
-
-        i = i+1
-    end
-end

modules/dev/threatminer-subdomains.lua

@@ -1,35 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
-        query={
-            rt='5',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    i = 1
-    while o[i] do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x,
-        })
-
-        i = i+1
-    end
-end

modules/dev/thunderbird-autoconfig.lua

@@ -1,48 +0,0 @@
--- Description: Query thunderbird autoconfig db for subdomains
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- check if an autoconfig exists without disclosing our target yet
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    if resp['status'] ~= 200 then
-        return 'index request failed'
-    end
-
-    if resp['text']:find(arg['value'], 1, true) == nil then
-        debug('no autoconfig available')
-        return
-    end
-
-    -- request config
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
-
-    i = 1
-    while i <= #m do
-        subdomain = m[i][2]
-
-        domain = psl_domain_from_dns_name(subdomain)
-        if last_err() then return end
-
-        domain_id = db_select('domain', domain)
-        if last_err() then return end
-
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=subdomain,
-        })
-        if last_err() then return end
-
-        i = i+1
-    end
-end

modules/dev/twilio-lookup.lua

@@ -1,48 +0,0 @@
--- Description: Retrieve additional information about a phone number
--- Version: 0.1.0
--- Source: phonenumbers
--- Keyring-Access: twilio
--- License: GPL-3.0
-
-function run(arg)
-    number = url_escape(arg['value'])
-    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
-    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
-
-    --debug(url)
-
-    key = keyring('twilio')[1]
-    if not key then
-        return 'Missing required twilio access key'
-    end
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {
-        basic_auth={key['access_key'], key['secret_key']},
-    })
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return 'api returned error'
-    end
-
-    v = json_decode(reply['text'])
-    if last_err() then return end
-    debug(v)
-
-    update = {}
-    update['country'] = v['country_code']
-
-    if v['carrier'] then
-        update['carrier'] = v['carrier']['name']
-        update['line'] = v['carrier']['type']
-    end
-
-    if v['caller_name'] then
-        update['caller_name'] = v['caller_name']['caller_name']
-        update['caller_type'] = v['caller_name']['caller_type']
-    end
-
-    db_update('phonenumber', arg, update)
-end

modules/dev/url-scan.lua

@@ -1,45 +0,0 @@
--- Description: Scan subdomains for websites
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function request(subdomain_id, url)
-    req = http_request(session, 'GET', url, {
-        timeout=5000
-    })
-    reply = http_send(req)
-
-    if last_err() then
-        clear_err()
-        return
-    end
-
-    obj = {
-        subdomain_id=subdomain_id,
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    }
-
-    redirect = reply['headers']['location']
-    if redirect then
-        obj['redirect'] = url_join(url, redirect)
-    end
-
-    db_add('url', obj)
-
-    -- debug(reply['status'])
-    -- debug(reply['headers']['location'])
-    -- debug(reply['text'])
-end
-
-function run(arg)
-    domain = arg['value']
-
-    session = http_mksession()
-
-    request(arg['id'], 'http://' .. domain .. '/')
-    if last_err() then return end
-    request(arg['id'], 'https://' .. domain .. '/')
-    if last_err() then return end
-end

modules/dev/waybackurls.lua

@@ -1,57 +0,0 @@
--- Description: Discover subdomains from wayback machine
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-
-    domain = arg['value']
-    url = 'https://web.archive.org/cdx/search/cdx?url=*.' .. domain .. '/*&output=json&collapse=urlkey'
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-
-    -- no known urls
-    if o[1] == nil then
-        return
-    end
-
-    -- ensure the api response is still what we expect
-    if o[1][3] == nil then
-        return 'api returned unexpected json format'
-    end
-
-    seen = {}
-
-    i = 2
-    while o[i] do
-        url = o[i][3]
-        debug(url)
-        parts = url_parse(url)
-
-        if last_err() then
-            clear_err()
-            error("Failed to parse url: " .. json_encode(url))
-        else
-            subdomain = parts['host']
-            subdomain, _ = subdomain:gsub('%.$', '')
-
-            if seen[subdomain] == nil then
-                db_add('subdomain', {
-                    domain_id=arg['id'],
-                    value=parts['host'],
-                })
-                if last_err() then return end
-                seen[subdomain] = 1
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/well-known-uris.lua

@@ -1,64 +0,0 @@
--- Description: Scan for known /.well-known/ locations
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
-    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
-
-    -- TODO: check if every location causes a 200/redirect
-
-    locations = {
-        {path='security.txt'}, -- expect 200
-        {path='dnt-policy.txt'}, -- expect 200
-        {path='caldav', redirect=true}, -- expect redirect
-        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
-        {path='assetlinks.json'}, -- expect 200
-        {path='apple-app-site-association'},  -- expect 200
-        {path='keybase.txt'}, -- expect 200
-        {path='apple-developer-merchantid-domain-association'}, -- expect 200
-        {path='openpgpkey'}, -- expect 200
-        {path='change-password', redirect=true}, -- expect redirect
-    }
-
-    session = http_mksession()
-
-    i = 1
-    while i <= #locations do
-        path = locations[i]['path']
-        expect_redirect = locations[i]['redirect']
-
-        url = url_join(arg['value'], '/.well-known/' .. path)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000,
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            status = reply['status']
-            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
-                obj = {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                }
-
-                redirect = reply['headers']['location']
-                if redirect then
-                    obj['redirect'] = url_join(url, redirect)
-                end
-
-                db_add('url', obj)
-            end
-        end
-
-        i = i+1
-    end
-end

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/add-all.lua

@@ -0,0 +1,107 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('adding domain')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding subdomain')
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding ipaddr')
+    ipaddr_id = db_add('ipaddr', {
+        value='192.0.2.1',
+    })
+    if last_err() then return end
+
+    info('adding device')
+    device_id = db_add('device', {
+        value='ff:ff:ff:ff:ff:ff',
+    })
+    if last_err() then return end
+
+    info('adding network')
+    network_id = db_add('network', {
+        value='myssid',
+    })
+    if last_err() then return end
+
+    info('adding email')
+    email_id = db_add('email', {
+        value='foo@example.com',
+    })
+    if last_err() then return end
+
+    info('adding phonenumber')
+    phonenumber_id = db_add('phonenumber', {
+        value='+4912345678',
+    })
+    if last_err() then return end
+
+    info('adding breach')
+    breach_id = db_add('breach', {
+        value='hack the planet',
+    })
+    if last_err() then return end
+
+    info('adding account')
+    account_id = db_add('account', {
+        service='github.com',
+        username='kpcyrd',
+    })
+    if last_err() then return end
+
+    info('adding image')
+    blob = create_blob('abc')
+    image_id = db_add('image', {
+        value=blob,
+    })
+    if last_err() then return end
+
+    info('adding port')
+    port_id = db_add('port', {
+        ip_addr_id=ipaddr_id,
+        ip_addr='192.0.2.1',
+        port=443,
+        protocol='tcp',
+        status='open',
+    })
+    if last_err() then return end
+
+    info('adding url')
+    url_id = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://www.example.com/a/b',
+        body='<html></html>',
+    })
+    if last_err() then return end
+
+    info('adding breach_email')
+    db_add('breach-email', {
+        breach_id=breach_id,
+        email_id=email_id,
+    })
+    if last_err() then return end
+
+    info('adding network_device')
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+    })
+    if last_err() then return end
+
+    info('adding subdomain_ipaddr')
+    db_add('subdomain-ipaddr', {
+        subdomain_id=subdomain_id,
+        ip_addr_id=ipaddr_id,
+    })
+    if last_err() then return end
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/dummy-resolve.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end

modules/harness/dummy-subdomains.lua

@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

modules/harness/import-subdomains.lua

@@ -0,0 +1,29 @@
+-- Description: Import subdomains from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lua
+
+    while true do
+        local line = stdin_readline()
+        if line == nil then
+            break
+        end
+        -- strip newline
+        local m = regex_find('.+', line)
+        if m then
+            local subdomain = m[1]
+
+            local domain = psl_domain_from_dns_name(subdomain)
+            local domain_id = db_add('domain', {
+                value=domain,
+            })
+
+            db_add('subdomain', {
+                domain_id=domain_id,
+                value=subdomain,
+            })
+        end
+    end
+end

modules/harness/notify-ohai.lua

@@ -0,0 +1,8 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-ratelimit.lua

@@ -0,0 +1,9 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    ratelimit_throttle('foo', 3, 10000)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-slow.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    -- TODO your code here
+    sleep(1)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/onion-http.lua

@@ -0,0 +1,17 @@
+-- Description: Send a request to a hidden service
+-- Version: 0.1.0
+-- Stealth: passive
+-- License: GPL-3.0
+
+function run()
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'http://expyuzz4wqqyqhjn.onion/', {
+        proxy='127.0.0.1:9050',
+    })
+    local r = http_fetch(req)
+    if last_err() then return end
+
+    local title = html_select(r['text'], 'title')
+    if last_err() then return end
+    info(title['html'])
+end

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/pgp-stdin.lua

@@ -0,0 +1,9 @@
+-- Description: Decode armored pgp key from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local pubkey = stdin_read_to_end()
+    pubkey = pgp_pubkey_armored(pubkey)
+    info(pubkey)
+end

modules/harness/port.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    ip_addr = '192.168.1.2'
+    ip_addr_id = db_add('ipaddr', {
+        value=ip_addr,
+    })
+    db_add('port', {
+        ip_addr_id=ip_addr_id,
+        ip_addr=ip_addr,
+        protocol='tcp',
+        port=4444,
+        status='open',
+    })
+end