Merge pull request #269 from Setsuwaa/patch-1

Fixed a typo

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [kpcyrd]
+patreon: kpcyrd

.github/workflows/docker-release.yml

@@ -0,0 +1,58 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Registry
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/kpcyrd/sn0int
+          tags: |
+            type=semver,pattern={{raw}}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push Docker images
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          file: Dockerfile
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      -
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker.yml

@@ -0,0 +1,30 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+        - name: sn0int
+          file: Dockerfile
+        - name: registry
+          file: sn0int-registry/Dockerfile
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build the Docker image
+      run: DOCKER_BUILDKIT=1 docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+    - name: Test the Docker image
+      run: docker run --rm ${{ matrix.build.name }} --help
+
+    - name: Show Docker images
+      run: docker images

.github/workflows/rust.yml

@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies (apt)
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-dev libseccomp-dev libsodium-dev
+    - name: Install dependencies (brew)
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config libsodium
+
+    - name: Build (sn0int)
+      run: cargo build --verbose
+    - name: Build (common)
+      run: cd sn0int-common && cargo build --verbose
+    - name: Build (std)
+      run: cd sn0int-std && cargo build --verbose
+    - name: Build (examples)
+      run: cargo build --verbose --examples
+
+    - name: Run tests (sn0int)
+      run: cargo test --verbose
+    - name: Run tests (sn0int, --ignored)
+      run: cargo test --verbose -- --ignored
+    - name: Run tests (common)
+      run: cd sn0int-common && cargo test --verbose
+    - name: Run tests (common, --ignored)
+      run: cd sn0int-common && cargo test --verbose -- --ignored
+    - name: Run tests (std)
+      run: cd sn0int-std && cargo test --verbose
+    - name: Run tests (std, --ignored)
+      run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'

.travis.yml

@@ -1,58 +0,0 @@
-language: rust
-
-# upload takes too long on windows and gets killed due to inactivity
-#cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
-
-before_install:
-- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
-- df -h
-- ci/run.sh $BUILD_MODE
-- df -h
-
-notifications:
-  irc:
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    #on_success: change # default: always
-    #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,107 +1,104 @@
 [package]
 name = "sn0int"
-version = "0.13.0"
+version = "0.26.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-edition = "2018"
-
-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
+edition = "2021"
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
+
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
 
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.8.0", path="sn0int-registry/sn0int-common" }
-rustyline = "5.0"
+sn0int-common = { version="0.14.0", path="sn0int-common" }
+sn0int-std = { version="=0.26.0", path="sn0int-std" }
+rustyline = "10.0"
 log = "0.4"
-env_logger = "0.6"
+env_logger = "0.11"
 hlua-badtouch = "0.4"
-structopt = "0.2"
+clap = { version = "4.3.11", features = ["derive", "env"] }
+clap_complete = "4.3.2"
 failure = "0.1"
-rand = "0.7"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = "0.12.0"
+libsqlite3-sys = { version = "0.22.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "2.0"
+dirs-next = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.11"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-webpki = "0.21"
-webpki-roots = "0.17"
-pem = "0.6.0"
-base64 = "0.10"
-data-encoding = "2.1.2"
-kuchiki = "0.7.2"
-serde_urlencoded = "0.6"
-serde = "1.0"
-serde_derive = "1.0"
+chrootable-https = "0.16"
+data-encoding = "2.3.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_urlencoded = "0.7"
 serde_json = "1.0"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.4"
+opener = "0.6"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.5"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
-toml = "0.5"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
+toml = "0.7"
 threadpool = "1.7"
-x509-parser = "0.6.0"
-der-parser = "3.0"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
-semver = "0.9"
+semver = "1"
 bytes = "0.4"
-xml-rs = "0.8"
-bytesize = "1.0"
-ipnetwork = "0.14"
-strum = "0.15"
-strum_macros = "0.15"
+ipnetwork = "0.20"
+strum = "0.25"
+strum_macros = "0.25"
+embedded-triple = "0.1.0"
+humansize = "2"
 
-digest = "0.8.0"
-bs58 = "0.2.2"
-blake2 = "0.8.0"
-md-5 = "0.8.0"
-sha-1 = "0.8.1"
-sha2 = "0.8.0"
-sha3 = "0.8.0"
-hmac = "0.7"
+digest = "0.10"
+md-5 = "0.10"
+sha-1 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+hmac = "0.12"
 
-image = "0.22"
-kamadak-exif = "0.3.1"
 walkdir = "2.2"
-nude = "0.2"
+nude = "0.3"
+glob = "0.3.0"
+os-version = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
+caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.15"
+syscallz = "0.16"
+nix = { version = "0.27", features = ["fs"] }
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
-unveil = "0.2.0"
+pledge = "0.4"
+unveil = "0.3"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.11"
+boxxy = "0.13"
 tempfile = "3.0"

Dockerfile

@@ -1,15 +1,18 @@
-FROM rust:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.20
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo build --release --verbose
-RUN strip target/release/sn0int
+RUN --mount=type=cache,target=/var/cache/buildkit \
+    CARGO_HOME=/var/cache/buildkit/cargo \
+    CARGO_TARGET_DIR=/var/cache/buildkit/target \
+    cargo build --release --locked --verbose && \
+    cp -v /var/cache/buildkit/target/release/sn0int /
+RUN strip /sn0int
 
-FROM debian:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+FROM alpine:3.20
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
+COPY --from=0 /sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \
     XDG_CACHE_HOME=/cache

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -1,7 +1,5 @@
-# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
@@ -16,33 +14,34 @@
 [registry]:     https://sn0int.com/
 
 sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
-package manager. It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
 
 [ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Attempt to bypass cloudflare with shodan
-- Harvest data and images from instagram profiles
-- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
@@ -50,6 +49,8 @@ For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https:/
 
 ## Installation
 
+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
 Archlinux
 
     pacman -S sn0int
@@ -58,6 +59,36 @@ Mac OSX
 
     brew install sn0int
 
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer:
+
+    sudo apt install curl sq
+    curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq dearmor | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    sudo apt update
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" ghcr.io/kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
 For everything else please have a look at the [detailed list][1].
 
 [1]: https://sn0int.readthedocs.io/en/latest/install.html
@@ -67,11 +98,14 @@ For everything else please have a look at the [detailed list][1].
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Debian &gt;= bullseye, Ubuntu &gt;= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian &lt;= buster, Ubuntu &lt;= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
   - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
@@ -89,16 +123,19 @@ For everything else please have a look at the [detailed list][1].
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
-    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
-    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
-    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
-- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
-  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
+- [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Publish your repo](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-repo)
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -115,10 +152,29 @@ For everything else please have a look at the [detailed list][1].
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
   - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
   - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
     - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
     - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
     - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -146,6 +202,7 @@ For everything else please have a look at the [detailed list][1].
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -167,18 +224,28 @@ For everything else please have a look at the [detailed list][1].
   - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
   - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
@@ -189,9 +256,12 @@ For everything else please have a look at the [detailed list][1].
   - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
   - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_time](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time)
+  - [sn0int_time_from](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time-from)
   - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -203,10 +273,15 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
@@ -217,6 +292,14 @@ For everything else please have a look at the [detailed list][1].
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
   - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
   - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
   - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
   - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)

ci/integration.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -47,10 +49,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +65,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,10 +94,17 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }
 
+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
     print('[*] running geoip')
     sn0int(tempdir, binary, [
         'use geoip',

ci/run.sh

@@ -1,41 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-registry/sn0int-common
-        cargo test --verbose
-        ;;
-    windows)
-        cargo build --verbose --features=sqlite-bundled
-        cargo build --verbose --examples --features=sqlite-bundled
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-    docker)
-        docker build -t sn0int .
-        docker images
-        docker run --rm sn0int --help
-        ;;
-    docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
-        docker images
-        ;;
-esac

ci/setup.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
-        ;;
-esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -18,19 +18,21 @@ Archlinux
 
 .. code-block:: bash
 
-    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
 
 Mac OSX
 ~~~~~~~
 
-None.
+.. code-block:: bash
+
+    $ brew install libsodium
 
 Debian/Ubuntu/Kali
 ~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix pkg-config
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -40,21 +42,28 @@ Alpine
 
 .. code-block:: bash
 
-    $ apk add sqlite-dev libseccomp-dev
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
+
+Docker
+~~~~~~
+
+.. code-block:: bash
+
+    $ DOCKER_BUILDKIT=1 docker build -t kpcyrd/sn0int .
 
 OpenBSD
 ~~~~~~~
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3 geolite2-city geolite2-asn
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
 
 Gentoo
 ~~~~~~
 
 .. code-block:: bash
 
-    emerge --ask sys-libs/libseccomp dev-db/sqlite
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
 
 Windows
 ~~~~~~~
@@ -71,10 +80,4 @@ After all dependencies have been installed, simply build the binary:
 
     $ cargo build --release
 
-Note that you need a different command on windows:
-
-.. code-block:: bash
-
-    $ cargo build --release --features=sqlite-bundled
-
 After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -38,9 +38,12 @@ Getting Started
    install
    build
    usage
+   autonoscope
    scripting
    database
    structs
+   activity
+   notifications
    keyring
    config
    sandbox

docs/install.rst

@@ -19,12 +19,34 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian/Ubuntu/Kali
-------------------
+Debian >= bookwork, Ubuntu >= 22.10, Kali
+-----------------------------------------
 
-Note that debian `doesn't ship the geoip2-database
-<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
-download them automatically during the first run.
+There are prebuilt packages signed by a debian maintainer:
+
+.. code-block:: bash
+
+    $ sudo apt install curl sq
+    $ curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq keyring filter -B --handle 64B13F7117D6E07D661BBCE0FE763A64F5E54FD6 | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ apt update
+    $ apt install sn0int
+
+Debian <= bullseye, Ubuntu <= 22.04
+-----------------------------------
+
+There are prebuilt packages signed by a debian maintainer:
+
+.. code-block:: bash
+
+    $ sudo apt install curl sq
+    $ curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq dearmor | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ apt update
+    $ apt install sn0int
+
+Fedora/CentOS/Redhat
+--------------------
 
 Using rust+cargo from the repos might work for you, but we only officially
 support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
@@ -32,7 +54,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -47,8 +69,6 @@ Docker
 Alpine
 ------
 
-On alpine edge, with enabled testing repositories:
-
 .. code-block:: bash
 
     $ apk add sn0int
@@ -56,8 +76,6 @@ On alpine edge, with enabled testing repositories:
 OpenBSD
 -------
 
-On -current:
-
 .. code-block:: bash
 
     $ pkg_add sn0int
@@ -67,8 +85,15 @@ Gentoo
 
 .. code-block:: bash
 
-    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
-    emerge --ask net-analyzer/sn0int
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
@@ -82,4 +107,4 @@ Make sure rust is installed and setup.
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f --path . --features=sqlite-bundled
+    $ cargo install -f --path .

docs/keyring.rst

@@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/reference.rst

@@ -120,6 +120,10 @@ for ``DATETIME`` database fields.
     This format is sn0int specific, to get the current time for scripting use
     time_unix_ instead.
 
+.. warning::
+    This function is going to be deprecated at some point. Prefer sn0int_time_
+    for new scripts.
+
 db_add
 ------
 
@@ -149,6 +153,28 @@ ttl.
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -357,11 +383,20 @@ options are set. The following options are available:
   The request body that should be json encoded.
 ``form``
   The request body that should be form encoded.
+``follow_redirects``
+  Automatically follow redirects, up to the specified number. If set to 1, only
+  one redirect is going to be followed. Defaults to 0 so redirects aren't
+  followed.
 ``body``
   The raw request body as string.
 ``into_blob``
   If true, the response body is stored in blob storage and a blob reference is
   returned as ``blob`` instead of the full body.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -389,6 +424,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -467,6 +504,39 @@ Extract exif metadata from an image.
     if last_err() then return end
     debug(exif)
 
+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------
 
@@ -488,6 +558,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -522,6 +599,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -556,6 +644,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -636,6 +782,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 
@@ -759,6 +921,30 @@ only used for debugging.
 
     sleep(1)
 
+sn0int_time
+-----------
+
+Return current time in UTC. This function is suitable to determine datetimes
+for ``DATETIME`` database fields.
+
+.. code-block:: lua
+
+    now = sn0int_time()
+
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+sn0int_time_from
+----------------
+
+Identical to sn0int_time_ but uses a unix timestamp in seconds instead of the
+current time. This function is compatible with time_unix_ and strptime_.
+
+.. code-block:: lua
+
+    time = sn0int_time_from(1567931337)
+
 sn0int_version
 --------------
 
@@ -784,6 +970,16 @@ The following options are available:
   **Danger**: disable tls verification. This disables all security on the
   connection. Note that sn0int is still rather strict, you're going to run into
   issues if you need support for insecure ciphers.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
 
 .. code-block:: lua
 
@@ -815,6 +1011,23 @@ discarded when using sock_connect_ directly with ``tls=true``.
 
     info(tls)
 
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
+
 sock_send
 ---------
 
@@ -916,6 +1129,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -937,6 +1171,46 @@ Read a line from stdin. The final newline is not removed.
 .. note::
    This only works with `sn0int run --stdin`.
 
+.. TODO: add stdin_read_line and deprecate stdin_readline
+
+stdin_read_to_end
+-----------------
+
+Read stdin until EOF as a utf-8 string.
+
+.. code-block:: lua
+
+    stdin_read_to_end()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
 strftime
 --------
 
@@ -957,6 +1231,13 @@ Parse a date into a unix timestamp, see `strftime rules`_.
 
 .. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
 
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
 time_unix
 ---------
 
@@ -1077,6 +1358,111 @@ a ``run`` execution.
     warn_once('ohai')
     warn_once('ohai')
 
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 

docs/sandbox.rst

@@ -6,7 +6,7 @@ child process. It's important to note that there's a basic sandbox that's
 active on every operating system, and there's a second line of defense on
 supported operating systems.
 
-The first line of defense is the restrictive stdlib. It's assumed that and
+The first line of defense is the restrictive stdlib. It's assumed that an
 attacker gains full control over the lua code and is able to call any function
 with arbitrary arguments. The stdlib only provides functions that are
 considered safe, so for example it's not possible to start a process or open a
@@ -119,7 +119,7 @@ You might experience a sandbox failure, especially on architectures that are
 less popular. This usually looks like this::
 
     [sn0int][example][kpcyrd/ctlogs] > run
-    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [-] Failed "example.com": Sandbox child has crashed
     [+] Finished kpcyrd/ctlogs (1 errors)
 
 A module that never finishes could also mean an IO thread inside the worker got

docs/scripting.rst

@@ -1,11 +1,11 @@
-Scripting
-=========
+Writing your first module
+=========================
 
 Scripting is the core feature in sn0int. It's not strictly required, but if you
 want to write your own modules, this section is for you.
 
-Write your first module
------------------------
+Creating a repository
+---------------------
 
 It's highly recommended to use a VCS for development, so let's start by setting
 that up. We're going to assume you store your repos in ``~/repos`` but you're
@@ -14,6 +14,12 @@ free to change that to something else::
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
 
+.. note::
+   If you're using github you can also create a repo from the `module repo
+   template`_.
+
+.. _module repo template: https://github.com/sn0int/sn0int-modules
+
 We need to add this folder to the sn0int config file so it's correctly detected
 when starting sn0int. Open the `config file <config.html>`_ in your prefered
 editor. Note that the file does not exist by default and the path is different
@@ -90,21 +96,26 @@ database.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
-        print(subdomain)
+        info(subdomain)
     end
 
-Combined with the header we wrote previously we can already execute this
-module. Make sure you've added a domain to scope with ``add domain
-example.com``, save your file and run it like this::
+This is already enough to execute it. Make sure you've added a domain to scope
+with ``add domain example.com``, save your file and run it like this::
 
     sn0int run -f ./first.lua
 
-We should see some output by our print function.
+We should see some output by our info function.
 
 .. note::
-   ``print`` is useful for development but must be removed before publishing.
+   ``info`` is useful for development but you usually want your module to run
+   quietly, so before publishing either remove it or replace it with ``debug``.
 
 Next, we want to actually resolve that name, we're going to use the ``dns``
 function for that. This function takes a name and a query type and returns a
@@ -114,6 +125,11 @@ truth-y.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -122,16 +138,22 @@ truth-y.
         })
         if last_err() then return end
 
-        print(records)
+        info(records)
     end
 
 If you run your module again you're going to see some output, either
 ``{"answers":[somedata],"error":null}`` or
-``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
-our scope and set it to resolvable if ``error`` is ``nil``.
+``{"answers":[],"error":"NXDomain"}``. If the dns reply doesn't indicate an
+error this means the subdomain exists and we can add it to our database with
+``resolvable`` being set to ``true``.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -152,41 +174,20 @@ our scope and set it to resolvable if ``error`` is ``nil``.
 .. hint::
    See the database section to understand how the database works in detail.
 
-If we execute our module one more time it's going to log that it discovered a
-subdomain, if it doesn't, try adding more domains to scope. Note that this only
-happens the first time. Modules that don't discover anything or don't discover
-anything new exit silently.
-
-After putting everything together, our final module looks like this:
-
-.. code-block:: lua
-
-    -- Description: ohai wurld
-    -- Version: 0.1.0
-    -- Source: domains
-    -- License: GPL-3.0
-
-    function run(arg)
-        subdomain = 'www.' .. arg['value']
-
-        records = dns(subdomain, {
-            record='A'
-        })
-        if last_err() then return end
-
-        if records['success'] ~= nil then
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=subdomain,
-                resolvable=true,
-            })
-        end
-    end
+If we execute our finished module one more time it's going to log that it
+discovered a subdomain, if it doesn't, try adding more domains to scope. Note
+that this only happens the first time. Modules that don't discover anything or
+don't discover anything new exit silently.
 
 There's still some room for improvement, for example, since we already resolved
 that record, we could also add the ip address to the scope and link it to the
 subdomain we added.
 
+.. hint::
+   For debugging purposes you can increase the verbosity with ``sn0int run -v``
+   so database operations are logged even if nothing was changed, or with
+   ``sn0int run -vv`` to enable ``debug()`` output.
+
 Publish your module
 -------------------
 
@@ -205,6 +206,24 @@ Afterwards publish your module with::
 
     sn0int publish ./first.lua
 
+Please also make sure you publish your repository to github so other people can
+submit pull requests. The recommended repository location is::
+
+    https://github.com/<your-username>/sn0int-modules
+
+Publish your repo
+-----------------
+
+It is highly recommended to publish your repository on github so people can
+file issues and pull requests for your module. If you've been following along
+with the github template you can simply commit your changes and push them.
+
+Your repository would look like one of these:
+
+- https://github.com/kpcyrd/sn0int-modules
+- https://github.com/ysf/sn0int-modules
+- https://github.com/cybiere/sn0int-modules
+
 Reading data from stdin
 -----------------------
 

docs/structs.rst

@@ -259,6 +259,35 @@ is ``140.82.112.0/20``.
     This field isn't strictly defined and meant to be used as a human
     meaningful name if available.
 
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------
@@ -256,85 +256,3 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
-
-Autonoscope
------------
-
-Instead of manually unscoping everything you can also define so called
-autonoscope rules. Those are executed from most specific to least specific and
-the first match wins. If no rule matches, the default is in-scope::
-
-    [sn0int][demo] > # add the domain first
-    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
-    [sn0int][demo] > add domain example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > # automatically noscope all subdomains
-    [sn0int][demo] > autonoscope add domain example.com
-    [sn0int][demo] > # except subdomains of prod.example.com 
-    [sn0int][demo] > autoscope add domain prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > autonoscope list 
-      scope domain "prod.example.com"
-    noscope domain "example.com"
-    [sn0int][demo] > 
-    [sn0int][demo] > # this is going to be out-of-scope
-    [sn0int][demo] > add subdomain www.example.com
-    [sn0int][demo] > # this is going to be in-scope
-    [sn0int][demo] > add subdomain db.prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > select subdomains 
-    #1, "www.example.com"
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=0
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=1
-    #1, "www.example.com"
-    [sn0int][demo] > 
-
-Domains
-~~~~~~~
-
-Autonoscope rules for domains are applied to the following structs:
-
-- domains
-- subdomains
-- urls
-
-Example rules::
-
-    autonoscope add domain example.com
-    autonoscope add domain staging.example.com
-    autonoscope add domain com
-    autonoscope add domain .
-
-IPs
-~~~
-
-Autonoscope rules for IPs are applied to the following structs:
-
-- ipaddrs
-- netblocks
-- ports
-
-Example rules::
-
-    autonoscope add ip 0.0.0.0/0
-    autonoscope add ip ::/0
-    autonoscope add ip 192.168.0.0/16
-    autonoscope add ip 10.13.33.37/32
-
-URLs
-~~~~
-
-Autonoscope rules for urls are applied to the following structs:
-
-- urls
-
-Note that these rules are specific to a certain origin (like
-``https://example.com``) and are used to filter paths.
-
-Example rules::
-
-    autonoscope add url https://example.com/
-    autonoscope add url https://example.com/admin/
-    autonoscope add url https://example.com/a/b/c/d

examples/boxxy.rs

@@ -1,6 +1,7 @@
-#[macro_use] extern crate boxxy;
-extern crate sn0int;
+#[macro_use]
+extern crate boxxy;
 extern crate env_logger;
+extern crate sn0int;
 
 fn stage1(sh: &mut boxxy::Shell, _args: Vec<String>) -> Result<(), boxxy::Error> {
     shprintln!(sh, "[*] starting stage1");
@@ -14,8 +15,6 @@ fn main() {
 
     println!("stage1        activate sandbox");
 
-    let toolbox = boxxy::Toolbox::new().with(vec![
-            ("stage1", stage1),
-        ]);
+    let toolbox = boxxy::Toolbox::new().with(vec![("stage1", stage1)]);
     boxxy::Shell::new(toolbox).run()
 }

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
+use clap::Parser;
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
-use structopt::StructOpt;
+use std::path::Path;
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
-    #[structopt(name="asn")]
+    #[command(name = "asn")]
     Asn(AsnArgs),
-    #[structopt(name="geoip")]
+    #[command(name = "geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -64,14 +30,15 @@ impl AsnArgs {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct GeoIPArgs {
     ip: IpAddr,
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -80,14 +47,13 @@ impl GeoIPArgs {
     }
 }
 
-
 fn run() -> Result<()> {
-    let args = Args::from_args();
+    let args = Args::parse();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

examples/spinners.rs

@@ -1,21 +1,20 @@
+use clap::Parser;
+use sn0int::term::{Spinner, StackedSpinners, SPINNERS};
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
-use structopt::StructOpt;
 
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="single")]
+    #[command(name = "single")]
     Single(Single),
-    #[structopt(name="stacked")]
+    #[command(name = "stacked")]
     Stacked(Stacked),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Single {
     idx: usize,
-    #[structopt(long="ticks", default_value="100")]
+    #[structopt(long = "ticks", default_value = "100")]
     ticks: usize,
 }
 
@@ -32,9 +31,8 @@ impl Single {
     }
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Stacked {
-}
+#[derive(Debug, Parser)]
+pub struct Stacked {}
 
 impl Stacked {
     fn run(&self) {
@@ -59,7 +57,7 @@ impl Stacked {
 }
 
 fn main() {
-    let args = Args::from_args();
+    let args = Args::parse();
     match args {
         Args::Single(args) => args.run(),
         Args::Stacked(args) => args.run(),

migrations/2019-11-19-154056_cryptoaddr/down.sql

@@ -0,0 +1 @@
+DROP TABLE cryptoaddrs;

migrations/2019-11-19-154056_cryptoaddr/up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE cryptoaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    currency VARCHAR,
+    denominator INTEGER,
+    balance BIGINT,
+    received BIGINT,
+    first_seen DATETIME,
+    last_withdrawal DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/dev/apex-domain.lua

@@ -1,11 +0,0 @@
--- Description: Create subdomain entries for apex domains
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function run(arg)
-    db_add('subdomain', {
-        domain_id=arg['id'],
-        value=arg['value'],
-    })
-end

modules/dev/arp-scan.lua

@@ -1,45 +0,0 @@
--- Description: Parse arp-scan output
--- Version: 0.3.0
--- License: GPL-3.0
-
--- sudo arp-scan -qglI wlp3s0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        m = regex_find('(.+)\t(.+)', x)
-        if m ~= nil then
-            ipaddr = m[2]
-            mac = m[3]
-            now = datetime()
-
-            device_id = db_add('device', {
-                value=mac,
-                last_seen=now,
-            })
-            if last_err() then return end
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 300)
-            if last_err() then return end
-        end
-    end
-end

modules/dev/asn.lua

@@ -1,16 +0,0 @@
--- Description: Run a asn lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = asn_lookup(arg['value'])
-    if last_err() then return end
-
-    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
-        db_update('ipaddr', arg, {
-            asn=lookup['asn'],
-            as_org=lookup['as_org'],
-        })
-    end
-end

modules/dev/axfr.lua

@@ -1,145 +0,0 @@
--- Description: Try a zone transfer for subdomains
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function add_pointer(name)
-    -- select psl+1
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function iter_axfr(zone, arg)
-    debug(arg)
-
-    local name = arg[1]
-    local r = arg[2]
-
-    -- select psl+1
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-
-    -- this is a A record
-    if r['A'] ~= nil then
-        -- add the name and ip
-        local ipaddr_id = db_add('ipaddr', {
-            family='4',
-            value=r['A'],
-        })
-        if last_err() then return end
-
-        db_add('subdomain-ipaddr', {
-            subdomain_id=subdomain_id,
-            ip_addr_id=ipaddr_id,
-        })
-        if last_err() then return end
-    end
-
-    if r['CNAME'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['CNAME'])
-        add_pointer(name)
-    end
-
-    if r['NS'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['NS'])
-        add_pointer(name)
-    end
-
-    if r['MX'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['MX'][2])
-        add_pointer(name:lower())
-    end
-end
-
-function iter_a(zone, arg)
-    if arg == nil then return end
-
-    debug('nameserver: ' .. arg)
-    local records = dns(zone, {
-        record='AXFR',
-        nameserver=arg .. ':53',
-        tcp=true,
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        iter_axfr(zone, records[i])
-        if last_err() then return end
-    end
-end
-
-function iter_ns(zone, arg)
-    if arg == nil then return end
-
-    local records = dns(arg, {
-        record='A',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        iter_a(zone, r['A'])
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    local records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        local r = records[i][2]
-        iter_ns(arg['value'], r['NS'])
-        if last_err() then return end
-    end
-end

modules/dev/chaturbate.lua

@@ -1,66 +0,0 @@
--- Description: Collect information from chaturbate streams
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts:chaturbate.com
-
-function resolve_external_link(href)
-    local link = url_join('https://chaturbate.com/', href)
-    local parts = url_parse(link)
-    return parts['params']['url']
-end
-
-service_map = {}
-service_map['www.instagram.com']    = {'^/([^/]+)', 'instagram.com'}
-service_map['twitter.com']          = {'^/([^/]+)', 'twitter.com'}
-service_map['www.patreon.com']      = {'^/([^/]+)', 'patreon.com'}
-service_map['www.twitch.tv']        = {'^/([^/]+)', 'twitch.tv'}
-
-function detect_account(link)
-    local parts = url_parse(link)
-    if last_err() then return clear_err() end
-    local host = parts['host']
-
-    local service = service_map[host]
-    if service then
-        local m = regex_find(service[1], parts['path'])
-        if m then
-            db_add('account', {
-                service=service[2],
-                username=m[2],
-            })
-        end
-    end
-end
-
-function run(arg)
-    local session = http_mksession()
-
-    local url = 'https://chaturbate.com/' .. arg['username'] .. '/'
-    local req = http_request(session, 'GET', url, {})
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-
-    local last_seen = nil
-    if #html_select_list(r['text'], '.offline_tipping') > 0 then
-        debug('is offline')
-    else
-        debug('is online')
-        last_seen = datetime()
-    end
-
-    db_update('account', arg, {
-        last_seen=last_seen,
-        url='https://chaturbate.com/' .. arg['username'] .. '/',
-    })
-
-    local links = html_select_list(r['text'], 'a[href^="/external_link/"]')
-    for i=1, #links do
-        local link = resolve_external_link(links[i]['attrs']['href'])
-        debug(link)
-        detect_account(link)
-        if last_err() then return end
-    end
-end

modules/dev/cname-harvest.lua

@@ -1,48 +0,0 @@
--- Description: Query for CNAMES to find subdomains
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function iter(r)
-    if r == nil then
-        return
-    end
-
-    m = regex_find("(.+)\\.$", r)
-    if last_err() then return end
-
-    if m == nil then
-        return
-    end
-    r = m[2]
-
-    domain = psl_domain_from_dns_name(r)
-    if last_err() then return end
-
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=r,
-        })
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        iter(r['CNAME'])
-        if last_err() then return end
-    end
-end

modules/dev/ctlogs.lua

@@ -1,101 +0,0 @@
--- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.5.0
--- Source: domains
--- License: GPL-3.0
-
-function each_name(name)
-    local domain_id, psl_domain
-
-    if seen[name] == 1 then
-        return
-    end
-    seen[name] = 1
-    debug(name)
-
-    if name:find('*.') == 1 then
-        -- ignore wildcard domains
-        return
-    end
-
-    -- the cert might be valid for subdomains that do not belong to the
-    -- domain we started with
-    psl_domain = psl_domain_from_dns_name(name)
-    domain_id = domains[psl_domain]
-    if domain_id == nil then
-        if any_domain then
-            -- unknown domains should be added to database
-            domain_id = db_add('domain', {
-                value=psl_domain,
-            })
-        else
-            -- only use domains that are already in scope
-            domain_id = db_select('domain', psl_domain)
-        end
-
-        -- if we didn't get a valid id, skip
-        if domain_id == nil then
-            return
-        end
-
-        domains[psl_domain] = domain_id
-    end
-
-    db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-end
-
-function run(arg)
-    full = getopt('full') ~= nil
-    any_domain = getopt('any-domain') ~= nil
-
-    domains = {}
-    domains[arg['value']] = arg['id']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://crt.sh/', {
-        query={
-            q='%.' .. arg['value'],
-            output='json'
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    certs = json_decode(resp['text'])
-    if last_err() then return end
-
-    seen = {}
-
-    for i=1, #certs do
-        c = certs[i]
-        debug(c)
-
-        if full then
-            -- fetch certificate
-            id = c['min_cert_id']
-            req = http_request(session, 'GET', 'https://crt.sh/', {
-                query={
-                    d=id .. '', -- TODO: find nicer way for tostring
-                }
-            })
-            resp = http_send(req)
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            -- iterate over all valid names
-            crt = x509_parse_pem(resp['text'])
-            if last_err() then return end
-            names = crt['valid_names']
-
-            for j=1, #names do
-                each_name(names[j])
-            end
-        else
-            each_name(c['name_value'])
-        end
-    end
-end

modules/dev/cve-2014-8244.lua

@@ -1,89 +0,0 @@
--- Description: Get connected devices from linksys router via JNAP
--- Version: 0.1.0
--- License: GPL-3.0
-
-function run()
-    local network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-    local network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    local router = getopt('router')
-    if not router then
-        return 'router option is missing (http://192.0.2.1:9000/)'
-    end
-
-    local target = url_join(router, 'JNAP/')
-    debug(target)
-
-    local session = http_mksession()
-    local headers = {}
-    headers['Content-Type'] = 'application/json; charset=UTF-8'
-    headers['X-JNAP-Action'] = 'http://linksys.com/jnap/core/Transaction'
-    headers['X-JNAP-Authorization'] = 'null'
-    local req = http_request(session, 'POST', target, {
-        headers=headers,
-        body='[{"action":"http://linksys.com/jnap/devicelist/GetDevices","request":{"sinceRevision":0}},{"action":"http://linksys.com/jnap/networkconnections/GetNetworkConnections","request":{}}]',
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-    -- debug(r)
-
-    body = json_decode(r['text'])
-    if last_err() then return end
-
-    if body['result'] ~= 'OK' then
-        return 'JNAP request failed'
-    end
-
-    local now = datetime()
-    for i=1,#body['responses'] do
-        rr = body['responses'][i]['output']
-        if rr['devices'] then
-            for j=1,#rr['devices'] do
-                device = rr['devices'][j]
-                debug(device)
-
-                name=device['friendlyName']
-                macs=device['knownMACAddresses']
-                connections=device['connections']
-
-                for k=1,#macs do
-                    -- this is an historic entry, do not bump last_seen
-                    device_id = db_add('device', {
-                        value=macs[k],
-                        hostname=name,
-                    })
-                    if last_err() then return end
-                end
-
-                for k=1,#connections do
-                    ipaddr = connections[k]['ipAddress']
-                    mac = connections[k]['macAddress']
-
-                    -- this is an active connection
-                    device_id = db_add('device', {
-                        value=mac,
-                        last_seen=now,
-                    })
-                    if last_err() then return end
-
-                    db_add_ttl('network-device', {
-                        network_id=network_id,
-                        device_id=device_id,
-                        ipaddr=ipaddr,
-                        last_seen=now,
-                    }, 180)
-                    if last_err() then return end
-                end
-            end
-        end
-    end
-end

modules/dev/dasoertliche-reverse.lua

@@ -1,32 +0,0 @@
--- Description: Run phonenumber reverse lookups on dasoertliche.de
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: phonenumbers
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://www.dasoertliche.de/Controller', {
-        query={
-            form_name='search_inv',
-            ph=arg['value'],
-        }
-    })
-    r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-    x = html_select(r['text'], '#entry_1 a')
-    if not x then
-        -- no results
-        return clear_err()
-    end
-
-    -- strip  
-    m = regex_find('[^\\xa0]+', x['text'])
-
-    if m then
-        db_update('phonenumber', arg, {
-            caller_name=m[1],
-        })
-    end
-end

modules/dev/ddwrt-dhcp.lua

@@ -1,88 +0,0 @@
--- Description: Export dhcp leases from ddwrt webinterface
--- Version: 0.2.0
--- License: GPL-3.0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    skip_redacted = not getopt('use-redacted')
-
-    router = getopt('router') -- http://192.0.2.1/
-    if not router then
-        return 'router option is missing (http://192.0.2.1/)'
-    end
-    username = getopt('user')
-    password = getopt('password')
-
-    options = {}
-    if username and password then
-        options['basic_auth'] = {username, password}
-    end
-
-    -- request status page
-    session = http_mksession()
-    url = url_join(router, '/Info.live.htm')
-    req = http_request(session, 'GET', url, options)
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    txt = resp['text']
-    debug(txt)
-
-    -- extract leases from response
-    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
-    if last_err() then return end
-    if not dhcp_section then
-        return 'Failed to get dhcp lease section'
-    end
-
-    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
-    if last_err() then return end
-
-    now = datetime()
-
-    -- add devices to database
-    for i=1, #leases do
-        local hostname = leases[i][2]
-        local ipaddr = leases[i][3]
-        local macaddr = leases[i][4]
-
-        debug({
-            hostname=hostname,
-            ipaddr=ipaddr,
-            macaddr=macaddr,
-        })
-
-        if skip_redacted and macaddr:match('^xx:xx:') then
-            info('Skipping redacted macaddr')
-        else
-            local device = {
-                value=macaddr,
-                last_seen=now,
-            }
-            if hostname ~= '*' then
-                device['hostname'] = hostname
-            end
-
-            local device_id = db_add('device', device)
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 120)
-        end
-    end
-end

modules/dev/dns-mx-domains.lua

@@ -1,44 +0,0 @@
--- Description: Discover mail server from MX records for domain
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function run(arg)
-    local records = dns(arg['value'], {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    for i=1, #records do
-        local r = records[i][2]['MX']
-        if r then
-            local mx = strip_root_dot(r[2])
-            local domain = psl_domain_from_dns_name(mx)
-
-            domain_id = db_add('domain', {
-                value=domain,
-            })
-            if last_err() then return end
-
-            db_add('subdomain', {
-                domain_id=domain_id,
-                value=mx,
-            })
-            if last_err() then return end
-        end
-    end
-end

modules/dev/dns-mx-emails.lua

@@ -1,54 +0,0 @@
--- Description: Discover mail server from MX records for emails
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: emails
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function run(arg)
-    -- extract domain
-    local domain = arg['value']:match('@(.*)')
-    if doman ~= nil then
-        -- malformed domain
-        return
-    end
-
-    -- mx lookup
-    local records = dns(domain, {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    for i=1, #records do
-        local r = records[i][2]['MX']
-        if r then
-            local mx = strip_root_dot(r[2])
-            domain = psl_domain_from_dns_name(mx)
-
-            domain_id = db_add('domain', {
-                value=domain,
-            })
-            if last_err() then return end
-
-            if domain_id then
-                db_add('subdomain', {
-                    domain_id=domain_id,
-                    value=mx,
-                })
-                if last_err() then return end
-            end
-        end
-    end
-end

modules/dev/dns-ns.lua

@@ -1,51 +0,0 @@
--- Description: Add a domains NS records to scope
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function each(r)
-    local name = strip_root_dot(r)
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function run(arg)
-    local records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        local r = records[i][2]
-        debug(r)
-        each(r['NS'])
-        if last_err() then return end
-    end
-end

modules/dev/dns-ptr.lua

@@ -1,30 +0,0 @@
--- Description: Run reverse dns lookups
--- Version: 0.2.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    if arg['family'] == '4' then
-        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
-
-        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
-        debug('Resolving: ' .. q)
-
-        records = dns(q, {
-            record='PTR',
-        })
-        if last_err() then return end
-        if records['error'] ~= nil then return end
-        records = records['answers']
-
-        for i=1, #records do
-            r = records[i][2]
-            if r['PTR'] then
-                db_update('ipaddr', arg, {
-                    reverse_dns=r['PTR'],
-                })
-                if last_err() then return end
-            end
-        end
-    end
-end

modules/dev/dns-resolve.lua

@@ -1,41 +0,0 @@
--- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    -- update subdomain
-    resolvable = records['error'] == nil
-    if arg['resolvable'] ~= resolvable then
-        -- TODO: pass arg to function as well
-        db_update('subdomain', arg, {
-            resolvable=resolvable
-        })
-    end
-
-    if not resolvable then
-        return
-    end
-
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        if r['A'] ~= nil then
-            ipaddr_id = db_add('ipaddr', {
-                family='4',
-                value=r['A'],
-            })
-            if last_err() then return end
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-            if last_err() then return end
-        end
-    end
-end

modules/dev/exif.lua

@@ -1,12 +0,0 @@
--- Description: Extract exif data from images
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    exif = img_exif(arg['value'])
-    if last_err() then return end
-    debug(exif)
-
-    db_update('image', arg, exif)
-end

modules/dev/geoip.lua

@@ -1,10 +0,0 @@
--- Description: Run a geoip lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = geoip_lookup(arg['value'])
-    if last_err() then return end
-    db_update('ipaddr', arg, lookup)
-end

modules/dev/git-webroot.lua

@@ -1,28 +0,0 @@
--- Description: Search for git checkouts in webroot
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    url = url_join(arg['value'], '.git/HEAD')
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return
-    end
-
-    if not regex_find('^ref: ', reply['text']) then
-        return
-    end
-
-    db_add('url', {
-        subdomain_id=arg['subdomain_id'],
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    })
-end

modules/dev/github.lua

@@ -1,107 +0,0 @@
--- Description: Collect data from github profiles
--- Version: 0.2.0
--- Source: accounts:github.com
--- License: GPL-3.0
-
-function api_get(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    -- TODO: set_error(?)
-    if resp['status'] == 403 then return 'ratelimit exceeded' end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-
-    local data = json_decode(resp['text'])
-    if last_err() then return end
-    return data
-end
-
-function import_gpg(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-
-    local key = pgp_pubkey_armored(resp['text'])
-    if not key['uids'] then return end
-
-    for i=1, #key['uids'] do
-        local k = key['uids'][i]
-        debug(k)
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
-        if m then
-            db_add('email', {
-                value=m[3],
-                displayname=m[2],
-            })
-        end
-    end
-end
-
-function scan4email(username)
-    local url = 'https://api.github.com/users/' .. username .. '/repos'
-    local repos = api_get(url)
-    if last_err() then return end
-
-    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
-
-    for i=1, #repos do
-        local repo = repos[i]
-        debug(repo)
-        local commits = api_get(repo['url'] .. '/commits')
-        if last_err() then return end
-
-        for j=1, #commits do
-            local commit = commits[j]
-            debug(commit)
-
-            if commit['author'] and commit['author']['login'] == username then
-                local name = commit['commit']['author']['name']
-                local email = commit['commit']['author']['email']
-                db_add('email', {
-                    value=email,
-                    displayname=name,
-                })
-                return email
-            end
-
-            if commit['committer'] and commit['committer']['login'] == username then
-                local name = commit['commit']['committer']['name']
-                local email = commit['commit']['committer']['email']
-                db_add('email', {
-                    value=email,
-                    displayname=name,
-                })
-                return email
-            end
-        end
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://api.github.com/users/' .. arg['username']
-
-    local data = api_get(url)
-    if last_err() then return end
-    debug(data)
-
-    -- company = data['company']
-    -- location = data['location']
-    -- homepage = data['blog']
-
-    url = 'https://github.com/' .. arg['username'] .. '.gpg'
-    import_gpg(url)
-    if last_err() then return end
-
-    local email = data['email']
-    if not email and not arg['email'] then
-        email = scan4email(arg['username'])
-        if last_err() then return end
-    end
-
-    db_update('account', arg, {
-        url=data['html_url'],
-        displayname=data['name'],
-        email=email,
-    })
-end

modules/dev/hackertarget-subdomains.lua

@@ -1,27 +0,0 @@
--- Description: Query hackertarget for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
-        query={
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
-
-    for i=1, #m do
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=m[i][2]
-        })
-    end
-end

modules/dev/home-assistant-devices.lua

@@ -1,56 +0,0 @@
--- Description: Query device location from home assistant
--- Version: 0.1.0
--- License: GPL-3.0
--- Keyring-Access: home-assistant
-
-function run()
-    -- parsing options
-    instance = getopt('instance')
-    if not instance then
-        return 'instance option is missing'
-    end
-
-    host = url_parse(instance)
-    if last_err() then return end
-    host = host['host']
-
-    entity = getopt('entity')
-    if not entity then
-        return 'entity option is missing'
-    end
-
-    -- fetching credentials
-    creds = keyring('home-assistant:' .. host)
-    if creds[1] == nil then
-        profile = url_join(instance, 'profile')
-        return 'missing home-assistant:' .. host .. ' Long-Lived Access Token, open ' .. profile
-    end
-    token = creds[1]['secret_key']
-
-    headers = {}
-    headers['Authorization'] = 'Bearer ' .. token
-    headers['Content-Type'] = 'application/json'
-
-    -- requesting status
-    session = http_mksession()
-    url = url_join(instance, 'api/states/' .. entity)
-    req = http_request(session, 'GET', url, {
-        headers=headers
-    })
-    r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-
-    m = json_decode(r['text'])
-    if last_err() then return end
-    debug(m)
-
-    info({
-        gps_accuracy=m['attributes']['gps_accuracy'],
-        longitude=m['attributes']['longitude'],
-        latitude=m['attributes']['latitude'],
-        last_updated=m['last_updated'],
-    })
-end

modules/dev/images.lua

@@ -1,9 +0,0 @@
--- Description: Parse image metadata
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    local img = img_load(arg['value'])
-    db_update('image', arg, img)
-end

modules/dev/instagram.lua

@@ -1,150 +0,0 @@
--- Description: Collect data from instagram profiles
--- Version: 0.2.0
--- Source: accounts:instagram.com
--- License: GPL-3.0
-
-PAGE_SIZE = 50
-
-function get_shared_data(html)
-    local s = html_select_list(html, 'script')
-
-    for i=1, #s do
-        local m = regex_find('^window\\._sharedData = (.+);$', s[i]['text'])
-        if m then
-            return json_decode(m[2])
-        end
-    end
-end
-
-function sign_request(rhx, json_params)
-    local magic = rhx .. ':' .. json_params
-    local x_instagram_gis = hex(md5(magic))
-    return x_instagram_gis
-end
-
-function download_image(node)
-    local url = node['display_url']
-    debug(url)
-
-    local req = http_request(session, 'GET', url, {
-        into_blob=true,
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-    db_add('image', {
-        value=r['blob'],
-    })
-end
-
-function pull_graphql(page)
-    local end_cursor = page['page_info']['end_cursor']
-
-    for i=1, #page['edges'] do
-        -- shortcode = page['edges'][i]['shortcode']
-        local node = page['edges'][i]['node']
-        node['thumbnail_resources'] = nil
-        node['media_preview'] = nil
-        -- debug(node)
-
-        -- if node['__typename'] == 'GraphImage'
-
-        -- node['dimensions']['height']
-        -- node['dimensions']['width']
-        -- ^ not sure how to get that picture
-
-        -- node['taken_at_timestamp']
-        -- location = node['location']
-
-        local err = download_image(node)
-        if last_err() then return end
-        if err ~= nil then return err end
-
-        todo_posts = todo_posts -1
-        debug('posts left: ' .. todo_posts .. '/' .. total_posts)
-    end
-
-    if page['page_info']['has_next_page'] then
-        debug('requesting next page=' .. end_cursor)
-
-        variables = json_encode({
-            id=user['id'],
-            first=PAGE_SIZE,
-            after=end_cursor
-        })
-
-        local headers = {}
-        headers['X-Instagram-GIS'] = sign_request(rhx_gis, variables)
-
-        local req = http_request(session, 'GET', 'https://www.instagram.com/graphql/query/', {
-            query={
-                query_hash='42323d64886122307be10013ad2dcc44',
-                variables=variables,
-            },
-            headers=headers,
-        })
-        r = http_send(req)
-        if last_err() then return end
-        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-        x = json_decode(r['text'])
-        if last_err() then return end
-        return pull_graphql(x['data']['user']['edge_owner_to_timeline_media'])
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-    local html = resp['text']
-
-    local ld = html_select(html, 'script[type="application/ld+json"]')
-    if last_err() then return end
-
-    local ld = json_decode(ld['text'])
-    if last_err() then return end
-    --debug(ld)
-
-    if ld['email'] then
-        db_add('email', {
-            value=ld['email'],
-        })
-    end
-
-    -- homepage=ld['url']
-
-    db_update('account', arg, {
-        displayname=ld['name'],
-        email=ld['email'],
-        url=url,
-    })
-
-    -- download images
-    local sd = get_shared_data(html)
-    if last_err() then return end
-    -- debug(sd)
-
-    rhx_gis = sd['rhx_gis']
-    user = sd['entry_data']['ProfilePage'][1]['graphql']['user']
-
-    -- user['full_name']
-    -- user['id']
-    -- user['is_business_account']
-    -- user['is_private']
-    -- user['is_verified']
-    -- user['has_blocked_viewer']
-    -- user['connected_fb_page']
-    -- user['country_block']
-
-    local page = user['edge_owner_to_timeline_media']
-    total_posts = page['count']
-    todo_posts = total_posts
-
-    -- TODO: fast-update abort if image has been downloaded already
-    return pull_graphql(page)
-end

modules/dev/isc-dhcpd-leases.lua

@@ -1,78 +0,0 @@
--- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.2.0
--- License: GPL-3.0
-
--- cat /var/lib/dhcpd/dhcpd.leases
-
-function add(lease)
-    if not lease['active'] then return end
-
-    now = datetime()
-
-    device_id = db_add('device', {
-        value=lease['mac'],
-        hostname=lease['hostname'],
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=lease['ipaddr'],
-        last_seen=now,
-    }, 180)
-    if last_err() then return end
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^lease (\\S+) \\{\n$', x)
-    if m then
-        lease = {}
-        debug('ipaddr=' .. m[2])
-        lease['ipaddr'] = m[2]
-    end
-    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
-    if m then
-        debug('mac=' .. m[2])
-        lease['mac'] = m[2]
-    end
-    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
-    if m then
-        debug('hostname=' .. m[2])
-        lease['hostname'] = m[2]
-    end
-    m = regex_find('^\\s*binding state active;\n$', x)
-    if m then
-        debug('active=true')
-        lease['active'] = true
-    end
-    m = regex_find('^\\}\n$', x)
-    if m then
-        add(lease)
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
-            each_line(x)
-        end
-    end
-end

modules/dev/iw-station-dump.lua

@@ -1,77 +0,0 @@
--- Description: Parse iw station dump
--- Version: 0.2.0
--- License: GPL-3.0
-
--- iw dev wlan0 station dump
-
-function add(client)
-    if
-        client['authenticated'] == 'yes' and
-        client['authorized'] == 'yes' and
-        client['mac']
-    then
-        debug(client)
-
-        now = datetime()
-
-        device_id = db_add('device', {
-            value=client['mac'],
-            last_seen=now,
-        })
-        if last_err() then return end
-
-        db_add_ttl('network-device', {
-            network_id=network_id,
-            device_id=device_id,
-            last_seen=now,
-        }, 180)
-        if last_err() then return end
-    end
-
-    client = nil
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^Station (\\S+)', x)
-    if m then
-        if client then
-            add(client)
-        end
-        client = {}
-        client['mac'] = m[2]
-        debug('mac=' .. m[2])
-    end
-
-    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
-    if m and client then
-        client[m[2]] = m[3]
-        debug(m[2] .. '=' .. m[3])
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    client = nil
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        each_line(x)
-    end
-
-    if client then
-        add(client)
-    end
-end

modules/dev/keybase-domains.lua

@@ -1,28 +0,0 @@
--- Description: Find keybase proofs for domains
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query={
-            domain=arg['value'],
-        }
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    db_add('account', {
-        service='keybase.io',
-        username=them['basics']['username'],
-    })
-end

modules/dev/keybase-profiles.lua

@@ -1,44 +0,0 @@
--- Description: Find keybase proofs for online accounts
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts
-
-function run(arg)
-    service = arg['service']
-    if service == 'twitter.com' then
-        service = 'twitter'
-    elseif service == 'github.com' then
-        service = 'github'
-    elseif service == 'reddit.com' then
-        service = 'reddit'
-    elseif service == 'news.ycombinator.com' then
-        service = 'hackernews'
-    elseif service == 'facebook.com' then
-        service = 'facebook'
-    else
-        return
-    end
-
-    query = {}
-    query[service] = arg['username']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query=query,
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    db_add('account', {
-        service='keybase.io',
-        username=them['basics']['username'],
-    })
-end

modules/dev/keybase.lua

@@ -1,85 +0,0 @@
--- Description: Collect accounts and emails from keybase accounts
--- Version: 0.2.0
--- License: GPL-3.0
--- Source: accounts:keybase.io
-
-function extract_mails(pubkey)
-    for j=1, #pubkey['uids'] do
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-        if m then
-            db_add('email', {
-                value=m[3],
-                displayname=m[2],
-            })
-        end
-    end
-end
-
-function add_domain(dns)
-    local domain = psl_domain_from_dns_name(dns)
-    if last_err() then return end
-
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    if domain ~= dns then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=dns,
-        })
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query={
-            usernames=arg['username'],
-        }
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    -- update keybase profile
-    db_update('account', arg, {
-        displayname=them['profile']['full_name'],
-        url='https://keybase.io/'..arg['username'],
-    })
-
-    -- collect emails
-    pubkey = pgp_pubkey_armored(them['public_keys']['primary']['bundle'])
-    debug(pubkey)
-    extract_mails(pubkey)
-
-    -- collect profiles
-    profiles = them['proofs_summary']['all']
-
-    for i=1, #profiles do
-        profile = profiles[i]
-        debug(profile)
-
-        if
-            profile['proof_type'] == 'generic_web_site' or
-            profile['proof_type'] == 'dns'
-        then
-            add_domain(profile['nametag'])
-        else
-            db_add('account', {
-                service=profile['proof_type'],
-                username=profile['nametag'],
-                url=profile['service_url'],
-            })
-        end
-    end
-end

modules/dev/namechk.lua

@@ -1,85 +0,0 @@
--- Description: Find accounts by username with namechk.com
--- Version: 0.3.0
--- Source: accounts
--- License: GPL-3.0
-
-function get_services(html)
-    local divs = html_select_list(html, '.service')
-    if last_err() then return end
-
-    local services = {}
-
-    for i=1, #divs do
-        services[i] = divs[i]['attrs']['data-name']
-    end
-
-    return services
-end
-
-function run(arg)
-    -- setup session
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://namechk.com/', {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
-    local auth_token = token['attrs']['value']
-
-        local headers = {}
-        headers['X-CSRF-Token'] = authenticity_token
-
-    local services = get_services(resp['text'])
-    debug({
-        auth_token=auth_token,
-        services=services,
-    })
-
-    -- trigger the scan
-    local req = http_request(session, 'POST', 'https://namechk.com/', {
-        headers=headers,
-        form={
-            authenticity_token=auth_token,
-            q=arg['username'],
-        }
-    })
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-    debug(resp)
-
-    local scan = json_decode(resp['text'])
-    if last_err() then return end
-    local scan_token = scan['valid']
-
-    -- get results
-    for i=1, #services do
-        debug(services[i])
-
-        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
-            headers=headers,
-            form={
-                token=scan_token,
-                fat=auth_token,
-                service=services[i],
-            }
-        })
-        local resp = http_send(req)
-        if last_err() then return end
-
-        if resp['status'] == 200 then
-            local acc = json_decode(resp['text'])
-            if last_err() then return end
-            debug(acc)
-
-            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
-                db_add('account', {
-                    service=services[i],
-                    username=arg['username'],
-                    url=acc['callback_url'],
-                })
-            end
-        end
-    end
-end

modules/dev/nudity.lua

@@ -1,12 +0,0 @@
--- Description: Scan collected images for nudity
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    local nudity = img_nudity(arg['value'])
-    debug(nudity)
-    db_update('image', arg, {
-        nudity=nudity['score'],
-    })
-end

modules/dev/otx-subdomains.lua

@@ -1,29 +0,0 @@
--- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
-
-    req = http_request(session, 'GET', url, {})
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['passive_dns']
-
-    for i=1, #o do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x['hostname'],
-        })
-    end
-end

modules/dev/passive-arp.lua

@@ -1,79 +0,0 @@
--- Description: Passive arp-scanner with sniffglue
--- Version: 0.1.0
--- License: GPL-3.0
-
--- sudo sniffglue -jv enp0s25
-
-function each_frame(frame)
-    if not frame['Ether'] then return end
-
-    local arp = frame['Ether'][2]['Arp']
-    if not arp then return end
-
-    if arp['Request'] then
-        arp = arp['Request']
-    elseif arp['Reply'] then
-        arp = arp['Reply']
-    else
-        -- unknown, abort
-        return
-    end
-
-    debug(arp)
-
-    -- TODO: this might change to a string in the future
-    local mac = mac(arp['src_mac'])
-    local ipaddr = arp['src_addr']
-    debug({src_mac=mac, src_addr=ipaddr})
-
-    local now = datetime()
-
-    local device_id = db_add('device', {
-        value=mac,
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=ipaddr,
-        last_seen=now,
-    }, 120)
-    if last_err() then return end
-end
-
-function mac(m)
-    return
-        hex({m[1]}) .. ':' ..
-        hex({m[2]}) .. ':' ..
-        hex({m[3]}) .. ':' ..
-        hex({m[4]}) .. ':' ..
-        hex({m[5]}) .. ':' ..
-        hex({m[6]})
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        local x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        local frame = json_decode(x)
-        if last_err() then return end
-
-        each_frame(frame)
-        if last_err() then return end
-    end
-end

modules/dev/passive-spider.lua

@@ -1,91 +0,0 @@
--- Description: Scrape known http responses for urls
--- Version: 0.2.0
--- Source: urls
--- License: GPL-3.0
-
-function entry(parent, href)
-    -- TODO: parse mailto:foo@example.com?subject=asdf
-    -- TODO: parse tel:+4912345
-    -- TODO: allow discovering 3rd-party domains
-    -- TODO: maybe record urls as well
-
-    local psl, parts, url, host
-
-    if href == nil then
-        return
-    end
-
-    url = url_join(parent, href)
-    if last_err() then return clear_err() end
-    if url:match('^https?://') == nil then
-        return
-    end
-
-    parts = url_parse(url)
-    if last_err() then return end
-    host = parts['host']
-    psl = psl_domain_from_dns_name(host)
-
-    domain_id = db_select('domain', psl)
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=host,
-        })
-    end
-
-    -- whatsapp links
-    if host == 'wa.me' then
-        m = regex_find('^/([0-9]+)$', parts['path'])
-        if m then
-            db_add('phonenumber', {
-                value='+' .. m[2],
-            })
-        end
-    end
-
-    -- instagram
-    if host == 'www.instagram.com' then
-        m = regex_find('^/([^/]+)', parts['path'])
-        if m then
-            db_add('account', {
-                service='instagram.com',
-                username=m[2],
-            })
-        end
-    end
-
-    -- facebook
-    if host == 'www.facebook.com' then
-        m = regex_find('^/([^/]+)', parts['path'])
-        if m then
-            db_add('account', {
-                service='facebook.com',
-                username=m[2],
-            })
-        end
-    end
-end
-
-function run(arg)
-    if arg['body'] == nil or #arg['body'] == 0 then
-        return
-    end
-
-    body = utf8_decode(arg['body'])
-    if last_err() then return end
-
-    links = html_select_list(body, 'a')
-    if last_err() then return end
-
-    if #links == 0 then
-        return
-    end
-
-    -- process html links
-    for i=1, #links do
-        href = links[i]['attrs']['href']
-
-        entry(arg['value'], href)
-    end
-end

modules/dev/pgp-keyserver.lua

@@ -1,74 +0,0 @@
--- Description: Query pgp keyserver for email addresses
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
--- TODO: rename to pgp-pks-domains
-
-function run(arg)
-    local domain = arg['value']
-
-    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    local lookup_url = 'https://sks-keyservers.net/pks/lookup'
-
-    session = http_mksession()
-    local req = http_request(session, 'GET', lookup_url, {
-        query={
-            search=domain,
-        }
-    })
-
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    links = html_select_list(resp['text'], 'a')
-    debug('keys found: ' .. #links)
-    for i=1, #links do
-        href = links[i]['attrs']['href']
-
-        if href:find('/pks/lookup%?op=get&search=') == 1 then
-            url = url_join(lookup_url, href)
-
-            debug(url)
-            req = http_request(session, 'GET', url, {})
-            resp = http_send(req)
-            -- TODO: do not abort script if one attempt fails
-            if last_err() then return end
-
-            if resp['status'] ~= 200 then
-                error('http error: ' .. url .. ' => ' .. resp['status'])
-            else
-                pubkey = pgp_pubkey_armored(resp['text'])
-                -- print(pubkey)
-
-                emails = {}
-                domain_matched = false
-
-                -- TODO: ensure at least one email matches our target domain
-                if pubkey['uids'] then
-                    for j=1, #pubkey['uids'] do
-                        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                        if m then
-                            local email = m[3]:lower()
-                            emails[#emails+1] = {
-                                value=email,
-                                displayname=m[2],
-                            }
-
-                            if email:match('@' .. domain .. '$') then
-                                domain_matched = true
-                            end
-                        end
-                    end
-                end
-
-                if domain_matched then
-                    for j=1, #emails do
-                        db_add('email', emails[j])
-                    end
-                end
-            end
-        end
-    end
-end

modules/dev/pgp-vks.lua

@@ -1,31 +0,0 @@
--- Description: Find alternative emails with vks lookups
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: emails
-
-function run(arg)
-    local keyserver = 'https://keys.openpgp.org'
-
-    local url = keyserver .. '/vks/v1/by-email/' .. arg['value']
-    local session = http_mksession()
-    local req = http_request(session, 'GET', url, {})
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then return end
-
-    local pubkey = pgp_pubkey_armored(r['text'])
-
-    for i=1, #pubkey['uids'] do
-        local uid = pubkey['uids'][i]
-        debug(uid)
-
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", uid)
-        if m then
-            local email = m[3]:lower()
-            db_add('email', {
-                value=email,
-                displayname=m[2],
-            })
-        end
-    end
-end

modules/dev/phpmyadmin.lua

@@ -1,103 +0,0 @@
--- Description: Search for phpmyadmin
--- Version: 0.2.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    paths = {
-        "phpmyadmin/index.php",
-        "phpMyAdmin/index.php",
-        "pmd/index.php",
-        "pma/index.php",
-        "PMA/index.php",
-        "PMA2/index.php",
-        "pmamy/index.php",
-        "pmamy2/index.php",
-        "mysql/index.php",
-        "admin/index.php",
-        "db/index.php",
-        "dbadmin/index.php",
-        "web/phpMyAdmin/index.php",
-        "admin/pma/index.php",
-        "admin/PMA/index.php",
-        "admin/mysql/index.php",
-        "admin/mysql2/index.php",
-        "admin/phpmyadmin/index.php",
-        "admin/phpMyAdmin/index.php",
-        "admin/phpmyadmin2/index.php",
-        "mysqladmin/index.php",
-        "mysql-admin/index.php",
-        "mysql_admin/index.php",
-        "phpadmin/index.php",
-        "phpAdmin/index.php",
-        "phpmyadmin0/index.php",
-        "phpmyadmin1/index.php",
-        "phpmyadmin2/index.php",
-        "phpMyAdmin-4.4.0/index.php",
-        "myadmin/index.php",
-        "myadmin2/index.php",
-        "xampp/phpmyadmin/index.php",
-        "phpMyadmin_bak/index.php",
-        "www/phpMyAdmin/index.php",
-        "tools/phpMyAdmin/index.php",
-        "phpmyadmin-old/index.php",
-        "phpMyAdminold/index.php",
-        "phpMyAdmin.old/index.php",
-        "pma-old/index.php",
-        "claroline/phpMyAdmin/index.php",
-        "typo3/phpmyadmin/index.php",
-        "phpma/index.php",
-        "phpmyadmin/phpmyadmin/index.php",
-        "phpMyAdmin/phpMyAdmin/index.php",
-        "phpMyAbmin/index.php",
-        "phpMyAdmin__/index.php",
-        "phpMyAdmin+++---/index.php",
-        "v/index.php",
-        "phpmyadm1n/index.php",
-        "phpMyAdm1n/index.php",
-        "shaAdmin/index.php",
-        "phpMyadmi/index.php",
-        "phpMyAdmion/index.php",
-        "MyAdmin/index.php",
-        "phpMyAdmin1/index.php",
-        "phpMyAdmin123/index.php",
-        "pwd/index.php",
-        "phpMyAdmina/index.php",
-        "program/index.php",
-        "shopdb/index.php",
-        "phppma/index.php",
-        "phpmy/index.php",
-        "mysql/admin/index.php",
-        "mysql/dbadmin/index.php",
-        "mysql/sqlmanager/index.php",
-        "mysql/mysqlmanager/index.php",
-        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
-    }
-
-    session = http_mksession()
-
-    for i=1, #paths do
-        p = paths[i]
-        url = url_join(arg['value'], p)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            if reply['status'] == 200 then
-                db_add('url', {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                })
-            end
-        end
-    end
-end

modules/dev/shodan-certs.lua

@@ -1,53 +0,0 @@
--- Description: Find IPs using certificates for target subdomains
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: subdomains
--- Keyring-Access: shodan
-
-function contains(list, search)
-    for i=1, #list do
-        if list[i] == search then
-            return true
-        end
-    end
-    return false
-end
-
-function run(arg)
-    local key = keyring('shodan')[1]
-    if not key then
-        return 'Missing required shodan access key'
-    end
-    local access_key = key['access_key']
-
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://api.shodan.io/shodan/host/search', {
-        query={
-            key=access_key,
-            query='ssl:' .. arg['value'],
-        }
-    })
-    local data = http_fetch_json(req)
-    if last_err() then return end
-
-    for i=1, #data['matches'] do
-        local host = data['matches'][i]
-        -- debug(host)
-
-        local crt = x509_parse_pem(host['ssl']['chain'][1])
-        if last_err() then return end
-
-        ipaddr_id = db_add('ipaddr', {
-            value=host['ip_str'],
-        })
-        if last_err() then return end
-
-        if contains(crt['valid_names'], arg['value']) then
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-            if last_err() then return end
-        end
-    end
-end

modules/dev/smtp-check.lua

@@ -1,86 +0,0 @@
--- Description: Verify email address by asking the smtp server
--- Version: 0.2.0
--- Source: emails
--- License: GPL-3.0
-
-function find_mx(domain)
-    local records = dns(domain, {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    for i=1, #records do
-        local r = records[i][2]['MX']
-        if r then
-            debug('mx: ' .. r[2])
-            return r[2]
-        end
-    end
-end
-
-function run(arg)
-    -- extract domain
-    domain = arg['value']:match('@(.*)')
-    if doman ~= nil then
-        -- malformed domain
-        return
-    end
-
-    -- mx lookup
-    mx = find_mx(domain)
-    if last_err() then return end
-    if not mx then return end
-
-    -- create connection
-    c = sock_connect(mx, 25, {})
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send hello
-    sock_sendline(c, 'ehlo localhost')
-    if last_err() then return end
-
-    l = sock_recvline_regex(c, '^250 ')
-    if last_err() then return end
-    debug(l)
-
-    -- send email
-    sock_sendline(c, 'mail from:<root@localhost>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send rcpt
-    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- check status
-    verified = nil
-    if l:match('^2') then
-        debug('email is valid')
-        verified = true
-    elseif l:match('^5') then
-        debug('email is invalid')
-        verified = false
-    elseif l:match('^4') then
-        debug('unknown status, temporary delivery failure')
-    end
-
-    if verified ~= nil then
-        db_update('email', arg, {
-            valid=verified,
-        })
-    end
-end

modules/dev/spyse-subdomains.lua

@@ -1,49 +0,0 @@
--- Description: Request subdomains from spyse.com
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
--- Keyring-Access: spyse
-
-function run(arg)
-    local key = keyring('spyse')[1]
-    if not key then
-        return 'Missing required spyse access key'
-    end
-    local access_key = key['access_key']
-
-    local session = http_mksession()
-
-    local page=1
-    while true do
-        local req = http_request(session, 'GET', 'https://api.spyse.com/v1/subdomains', {
-            query={
-                api_token=key['access_key'],
-                domain=arg['value'],
-                page=''..page,
-            }
-        })
-        debug('sending request for page #' .. page)
-        local r = http_send(req)
-        if last_err() then return end
-        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-        local data = json_decode(r['text'])
-        if last_err() then return end
-
-        if #data['records'] == 0 then
-            break
-        end
-
-        for i=1, #data['records'] do
-            local record = data['records'][i]
-            local subdomain = record['domain']
-            -- info(record['ip']['ip'])
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=subdomain,
-            })
-        end
-
-        page = page+1
-    end
-end

modules/dev/telefonbuch-reverse.lua

@@ -1,29 +0,0 @@
--- Description: Run phonenumber reverse lookups on dastelefonbuch.de
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: phonenumbers
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: fix url encoding for +
-    query = arg['value']
-    query = query:gsub('+', '%%2B')
-    url = 'https://www.dastelefonbuch.de/R%C3%BCckw%C3%A4rts-Suche/' .. query
-
-    req = http_request(session, 'GET', url, {})
-    r = http_send(req)
-    if last_err() then return end
-
-    -- no data available
-    if r['status'] == 410 then return end
-    -- an error occured
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-    x = html_select(r['text'], '#entry_1 div')
-    name = x['attrs']['title']
-
-    db_update('phonenumber', arg, {
-        caller_name=name,
-    })
-end

modules/dev/threatminer-ipaddr.lua

@@ -1,49 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.3.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: add option to filter old entries based on last_seen
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
-        query={
-            rt='2',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    for i=1, #o do
-        x = o[i]
-
-        domain = psl_domain_from_dns_name(x['domain'])
-        -- TODO: if this fails, skip this entry instead
-        if last_err() then return end
-
-        domain_id = db_add('domain', {
-            value=domain,
-        })
-
-        if domain_id ~= nil then
-            subdomain_id = db_add('subdomain', {
-                domain_id=domain_id,
-                value=x['domain'],
-            })
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=subdomain_id,
-                ip_addr_id=arg['id'],
-            })
-        end
-    end
-end

modules/dev/threatminer-subdomains.lua

@@ -1,32 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
-        query={
-            rt='5',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    for i=1, #o do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x,
-        })
-    end
-end

modules/dev/thunderbird-autoconfig.lua

@@ -1,45 +0,0 @@
--- Description: Query thunderbird autoconfig db for subdomains
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- check if an autoconfig exists without disclosing our target yet
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    if resp['status'] ~= 200 then
-        return 'index request failed'
-    end
-
-    if resp['text']:find(arg['value'], 1, true) == nil then
-        debug('no autoconfig available')
-        return
-    end
-
-    -- request config
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
-
-    for i=1, #m do
-        subdomain = m[i][2]
-
-        domain = psl_domain_from_dns_name(subdomain)
-        if last_err() then return end
-
-        domain_id = db_select('domain', domain)
-        if last_err() then return end
-
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=subdomain,
-        })
-        if last_err() then return end
-    end
-end

modules/dev/tld-scan.lua

@@ -1,46 +0,0 @@
--- Description: Search for the same domain base on all TLDs
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function run(arg)
-    local m = regex_find('^([^\\.]+)\\.', arg['value'])
-    local base = m[2]
-
-    -- TODO: we need a way to cache this
-    -- TODO: .co.uk is missing
-    local url = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'
-
-    local session = http_mksession()
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    local tlds = regex_find_all('([^\n]+)', resp['text'])
-
-    for i=1, #tlds do
-        local tld = tlds[i][1]:lower()
-
-        if not tld:match('^#') then
-            local domain = base .. '.' .. tld
-
-            debug(domain)
-            records = dns(domain, {
-                record='NS',
-            })
-            if last_err() then
-                clear_err()
-            else
-                if records['error'] == nil and records['answers'][1] then
-                    debug(records)
-                    db_add('domain', {
-                        value=domain,
-                    })
-                end
-            end
-        end
-    end
-end

modules/dev/twilio-lookup.lua

@@ -1,48 +0,0 @@
--- Description: Retrieve additional information about a phone number
--- Version: 0.1.0
--- Source: phonenumbers
--- Keyring-Access: twilio
--- License: GPL-3.0
-
-function run(arg)
-    number = url_escape(arg['value'])
-    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
-    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
-
-    --debug(url)
-
-    key = keyring('twilio')[1]
-    if not key then
-        return 'Missing required twilio access key'
-    end
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {
-        basic_auth={key['access_key'], key['secret_key']},
-    })
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return 'api returned error'
-    end
-
-    v = json_decode(reply['text'])
-    if last_err() then return end
-    debug(v)
-
-    update = {}
-    update['country'] = v['country_code']
-
-    if v['carrier'] then
-        update['carrier'] = v['carrier']['name']
-        update['line'] = v['carrier']['type']
-    end
-
-    if v['caller_name'] then
-        update['caller_name'] = v['caller_name']['caller_name']
-        update['caller_type'] = v['caller_name']['caller_type']
-    end
-
-    db_update('phonenumber', arg, update)
-end

modules/dev/twitch.lua

@@ -1,116 +0,0 @@
--- Description: Collect information from twitch streams
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts:twitch.tv
-
-function twitch_get(url)
-    local headers = {}
-    -- twitch web client id
-    headers['Client-ID'] = 'jzkbprff40iqj646a697cyrvl0zt2m6'
-    local req = http_request(session, 'GET', url, {
-        headers=headers
-    })
-    return http_send(req)
-end
-
-function scrape_panels(uid)
-    local headers = {}
-    headers['Client-ID'] = 'jzkbprff40iqj646a697cyrvl0zt2m6'
-    local req = http_request(session, 'POST', 'https://gql.twitch.tv/gql', {
-        headers=headers,
-        body='[{"operationName":"ChannelPanels","variables":{"id":"' .. uid .. '"},"extensions":{"persistedQuery":{"version":1,"sha256Hash":"236b0ec07489e5172ee1327d114172f27aceca206a1a8053106d60926a7f622e"}}}]'
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end -- TODO: set_err
-    local data = json_decode(r['text'])
-    if last_err() then return end
-
-    if not data[1] then return end
-    local panels = data[1]['data']['user']['panels']
-
-    for i=1, #panels do
-        detect_account(panels[i]['linkURL'])
-        harvest_description(panels[i]['description'])
-    end
-end
-
-function harvest_description(text)
-    if not text then return end
-
-    local m = regex_find_all('(http|https)://([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?', text)
-    for i=1, #m do
-        detect_account(m[i][1])
-    end
-
-    local m = regex_find_all('[a-zA-Z0-9_\\+\\-\\.]+@[a-zA-Z0-9_\\-\\.]+', text)
-    for i=1, #m do
-        db_add('email', {
-            value=m[i][1],
-        })
-    end
-end
-
-service_map = {}
-service_map['www.instagram.com']    = {'^/([^/]+)', 'instagram.com'}
-service_map['twitter.com']          = {'^/([^/]+)', 'twitter.com'}
-service_map['www.patreon.com']      = {'^/([^/]+)', 'patreon.com'}
-service_map['www.tipeeestream.com'] = {'^/([^/]+)', 'tipeeestream.com'}
-
-function detect_account(link)
-    if not link then return end
-    debug(link)
-
-    local parts = url_parse(link)
-    if last_err() then return clear_err() end
-    local host = parts['host']
-
-    local service = service_map[host]
-    if service then
-        local m = regex_find(service[1], parts['path'])
-        if m then
-            db_add('account', {
-                service=service[2],
-                username=m[2],
-            })
-        end
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-
-    update = {}
-
-    local r = twitch_get('https://api.twitch.tv/kraken/channels/' .. arg['username'])
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-    local data = json_decode(r['text'])
-    if last_err() then return end
-
-    -- info(data)
-    uid = data['_id']
-    update['displayname'] = data['display_name']
-    update['url'] = data['url']
-    debug(data['logo']) -- TODO: support linking images as profile pic
-    -- debug(data['created_at'])
-
-    local r = twitch_get('https://api.twitch.tv/kraken/streams/' .. arg['username'])
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-    local data = json_decode(r['text'])
-    if last_err() then return end
-
-    -- info(data)
-
-    if data['stream'] and data['stream']['stream_type'] == 'live' then
-        -- data['stream']['created_at']
-        debug('online!')
-        update['last_seen'] = datetime()
-    end
-
-    db_update('account', arg, update)
-    if last_err() then return end
-
-    scrape_panels(uid)
-end

modules/dev/url-scan.lua

@@ -1,45 +0,0 @@
--- Description: Scan subdomains for websites
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function request(subdomain_id, url)
-    req = http_request(session, 'GET', url, {
-        timeout=5000
-    })
-    reply = http_send(req)
-
-    if last_err() then
-        clear_err()
-        return
-    end
-
-    obj = {
-        subdomain_id=subdomain_id,
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    }
-
-    redirect = reply['headers']['location']
-    if redirect then
-        obj['redirect'] = url_join(url, redirect)
-    end
-
-    db_add('url', obj)
-
-    -- debug(reply['status'])
-    -- debug(reply['headers']['location'])
-    -- debug(reply['text'])
-end
-
-function run(arg)
-    domain = arg['value']
-
-    session = http_mksession()
-
-    request(arg['id'], 'http://' .. domain .. '/')
-    if last_err() then return end
-    request(arg['id'], 'https://' .. domain .. '/')
-    if last_err() then return end
-end

modules/dev/venmo.lua

@@ -1,68 +0,0 @@
--- Description: Fetch data from venmo profiles
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts:venmo.com
-
--- based on https://github.com/mportatoes/venemy
-
-function download_blob(session, url)
-    while true do
-        local req = http_request(session, 'GET', url, {
-            into_blob=true,
-        })
-        local r = http_send(req)
-        if last_err() then return end
-
-        if r['status'] == 200 then
-            return r
-        elseif r['headers']['location'] then
-            url = r['headers']['location']
-        else
-            return 'http error: ' .. r['status']
-        end
-    end
-end
-
-function run(arg)
-    local session = http_mksession()
-
-    local url = 'https://api.venmo.com/v1/users/' .. arg['username']
-    local req = http_request(session, 'GET', url, {})
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-
-    local data = json_decode(r['text'])
-    if last_err() then return end
-    data = data['data']
-    debug(data)
-
-    if data['profile_picture_url'] then
-        local url = data['profile_picture_url']
-
-        -- enhance facebook images
-        local m = regex_find('https://graph\\.facebook\\.com/v2\\.10/(\\d+)/picture\\?type=square', url)
-        if m then
-            url = 'https://graph.facebook.com/v2.10/' .. m[2] .. '/picture?type=large'
-            db_add('account', {
-                service='facebook.com',
-                username=''..m[2],
-            })
-        end
-
-        r = download_blob(session, url)
-        if last_err() then return end
-
-        -- TODO: link image to profile
-        db_add('image', {
-            value=r['blob'],
-        })
-    end
-
-    db_update('account', arg, {
-        displayname=data['display_name'],
-        url='https://venmo.com/' .. arg['value'],
-    })
-end

modules/dev/viewdns-reverseip.lua

@@ -1,29 +0,0 @@
--- Description: Search for domains pointing to ip
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: ipaddrs
-
--- XXX: the api ignores subdomains
-function run(arg)
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://viewdns.info/reverseip/', {
-        query={
-            host=arg['value'],
-            t='1',
-        }
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-    local html = r['text']
-
-    local domains = html_select_list(html, '#null table td:first-child')
-    for i=2, #domains do
-        local domain = domains[i]['text']
-        db_add('domain', {
-            value=domain,
-        })
-    end
-end

modules/dev/waybackurls.lua

@@ -1,54 +0,0 @@
--- Description: Discover subdomains from wayback machine
--- Version: 0.4.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-
-    domain = arg['value']
-    url = 'https://web.archive.org/cdx/search/cdx?url=*.' .. domain .. '/*&output=json&collapse=urlkey'
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-
-    -- no known urls
-    if o[1] == nil then
-        return
-    end
-
-    -- ensure the api response is still what we expect
-    if o[1][3] == nil then
-        return 'api returned unexpected json format'
-    end
-
-    seen = {}
-
-    for i=2, #o do
-        url = o[i][3]
-        debug(url)
-        parts = url_parse(url)
-
-        if last_err() then
-            clear_err()
-            error("Failed to parse url: " .. json_encode(url))
-        else
-            subdomain = parts['host']
-            subdomain, _ = subdomain:gsub('%.$', '')
-
-            if seen[subdomain] == nil then
-                db_add('subdomain', {
-                    domain_id=arg['id'],
-                    value=parts['host'],
-                })
-                if last_err() then return end
-                seen[subdomain] = 1
-            end
-        end
-    end
-end

modules/dev/well-known-uris.lua

@@ -1,61 +0,0 @@
--- Description: Scan for known /.well-known/ locations
--- Version: 0.2.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
-    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
-
-    -- TODO: check if every location causes a 200/redirect
-
-    locations = {
-        {path='security.txt'}, -- expect 200
-        {path='dnt-policy.txt'}, -- expect 200
-        {path='caldav', redirect=true}, -- expect redirect
-        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
-        {path='assetlinks.json'}, -- expect 200
-        {path='apple-app-site-association'},  -- expect 200
-        {path='keybase.txt'}, -- expect 200
-        {path='apple-developer-merchantid-domain-association'}, -- expect 200
-        {path='openpgpkey'}, -- expect 200
-        {path='change-password', redirect=true}, -- expect redirect
-    }
-
-    session = http_mksession()
-
-    for i=1, #locations do
-        path = locations[i]['path']
-        expect_redirect = locations[i]['redirect']
-
-        url = url_join(arg['value'], '/.well-known/' .. path)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000,
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            status = reply['status']
-            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
-                obj = {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                }
-
-                redirect = reply['headers']['location']
-                if redirect then
-                    obj['redirect'] = url_join(url, redirect)
-                end
-
-                db_add('url', obj)
-            end
-        end
-    end
-end

modules/dev/wigle-ssid-location.lua

@@ -1,50 +0,0 @@
--- Description: Use triangulated network location from wardriving data
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: networks
--- Keyring-Access: wigle
-
-function run(arg)
-    local key = keyring('wigle')[1]
-    if not key then
-        return 'wigle api key is required'
-    end
-
-    local session = http_mksession()
-    local url = 'https://api.wigle.net/api/v2/network/search'
-    local req = http_request(session, 'GET', url, {
-        basic_auth={key['access_key'], key['secret_key']},
-        query={
-            ssid=arg['value'],
-        }
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'unexpected status code: ' .. r['status']
-    end
-
-    local data = json_decode(r['text'])
-    if last_err() then return end
-
-    if not data['success'] then
-        return "api didn't signal success"
-    end
-
-    -- skip if ssid isn't unique
-    if data['totalResults'] ~= 1 then
-        return
-    end
-
-    -- ssid is globally unique
-    local network = data['results'][1]
-    debug(network)
-
-    -- the location is triangulated, only use this if we don't overwrite anything
-    if not arg['latitude'] and not arg['longitude'] then
-        db_update('network', arg, {
-            latitude=network['trilat'],
-            longitude=network['trilong'],
-        })
-    end
-end

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/dummy-resolve.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end

modules/harness/dummy-subdomains.lua

@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

modules/harness/import-subdomains.lua

@@ -3,7 +3,7 @@
 -- License: GPL-3.0
 
 function run()
-    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lua
 
     while true do
         local line = stdin_readline()