Release v0.17.1

seccomp: whitelist ppoll instead of poll on aarch64

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.travis.yml

@@ -5,6 +5,14 @@ language: rust
 
 matrix:
   include:
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker-registry
   - os: linux
     rust: stable
     env:
@@ -13,19 +21,14 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=std
   - os: linux
     rust: stable
     env:
     - BUILD_MODE=boxxy
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  # https://github.com/rust-lang/rust/issues/68264
-  #- os: linux
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE=docker-registry
   - os: osx
     rust: stable
     env:
@@ -34,6 +37,10 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=std
   #- os: windows
   #  rust: stable
   #  env:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.16.0"
+version = "0.17.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -13,15 +13,17 @@ edition = "2018"
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
 
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-registry/sn0int-common" }
-rustyline = "5.0"
+sn0int-common = { version="0.10.0", path="sn0int-common" }
+sn0int-std = { version="0.17.1", path="sn0int-std" }
+rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
 hlua-badtouch = "0.4"
@@ -31,7 +33,6 @@ rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 libsqlite3-sys = "0.16.0"
@@ -41,14 +42,8 @@ url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
 chrootable-https = "0.14"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-webpki = "0.21"
-webpki-roots = "0.18"
-ct-logs = "0.6"
-pem = "0.7"
 base64 = "0.11"
 data-encoding = "2.1.2"
-kuchiki = "0.7.2"
 serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
@@ -61,41 +56,30 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
-maxminddb = "0.13"
 threadpool = "1.7"
-x509-parser = "0.6.0"
-der-parser = "3.0"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
 semver = "0.9"
 bytes = "0.4"
-xml-rs = "0.8"
 bytesize = "1.0"
-ipnetwork = "0.15"
+ipnetwork = "0.16"
 strum = "0.17"
 strum_macros = "0.17"
-tungstenite = { version = "0.9", default-features = false }
 
 digest = "0.8.0"
-bs58 = "0.3"
-blake2 = "0.8.0"
 md-5 = "0.8.0"
 sha-1 = "0.8.1"
 sha2 = "0.8.0"
 sha3 = "0.8.0"
 hmac = "0.7"
 
-image = "0.22"
-kamadak-exif = "0.4"
 walkdir = "2.2"
-nude = "0.2"
+nude = "0.3"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
 syscallz = "0.12"
-nix = "0.16"
+nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -218,6 +218,8 @@ For everything else please have a look at the [detailed list][1].
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)

ci/integration.py

@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/run.sh

@@ -6,14 +6,18 @@ case "$1" in
         cargo build --verbose --examples
         ;;
     test)
-        ci/run.sh build
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
     common)
-        cd sn0int-registry/sn0int-common
+        cd sn0int-common
         cargo test --verbose
         ;;
+    std)
+        cd sn0int-std
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
     windows)
         cargo build --verbose --features=sqlite-bundled
         cargo build --verbose --examples --features=sqlite-bundled
@@ -31,7 +35,7 @@ case "$1" in
         docker run --rm sn0int --help
         ;;
     docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
+        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
         docker images
         ;;
 esac

docs/activity.rst

@@ -68,7 +68,7 @@ Logging an event that has a location attached could look like this:
         topic='harness/activity-ping:dummy',
         time=sn0int_time(),
         latitude=40.726662,
-        longitude=-74.036677
+        longitude=-74.036677,
         radius=50,
         content={
             a='b',

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/reference.rst

@@ -153,6 +153,28 @@ ttl.
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -1025,6 +1047,32 @@ Read stdin until EOF as a utf-8 string.
 .. note::
    This only works with `sn0int run --stdin`.
 
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
 strftime
 --------
 

examples/maxmind.rs

@@ -1,16 +1,8 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
@@ -27,8 +19,8 @@ pub struct AsnArgs {
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let path = AsnDB::cache_path()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
         let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
@@ -44,8 +36,8 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let path = GeoIP::cache_path()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
         let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
@@ -59,9 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

sn0int-common/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "sn0int-common"
-version = "0.9.0"
-description = "Common code for sn0int"
+version = "0.10.0"
+description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-common" }
+sn0int-common = { version="0.10.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-registry/Dockerfile

@@ -2,16 +2,16 @@ FROM rust:buster
 RUN apt-get update -q && apt-get install -yq llvm libclang-dev \
     && rm -rf /var/lib/apt/lists/*
 RUN rustup install nightly
-WORKDIR /usr/src/sn0int-registry
+WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo +nightly build --release --verbose
+RUN cd sn0int-registry && cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
 FROM debian:buster
 RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
-COPY templates /templates
+COPY --from=0 /usr/src/sn0int/target/release/sn0int-registry /usr/local/bin/sn0int-registry
+COPY sn0int-registry/templates /templates
 ENV ROCKET_ENV=prod \
     ROCKET_ADDRESS=0.0.0.0 \
     ROCKET_PORT=8000

sn0int-std/Cargo.toml

@@ -0,0 +1,54 @@
+[package]
+name = "sn0int-std"
+version = "0.17.1"
+description = "sn0int - stdlib"
+authors = ["kpcyrd <git@rxv.cc>"]
+repository = "https://github.com/kpcyrd/sn0int"
+license = "GPL-3.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4"
+failure = "0.1"
+hlua-badtouch = "0.4"
+tokio = "0.1.14"
+rand = "0.7"
+regex = "1.0"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+serde_urlencoded = "0.6"
+webpki = "0.21"
+webpki-roots = "0.19.0"
+ct-logs = "0.6"
+chrootable-https = "0.14"
+http = "0.2"
+bufstream = "0.1.4"
+pem = "0.7"
+url = "2.0"
+tungstenite = { version = "0.10.1", default-features = false }
+kuchiki = "0.8.0"
+maxminddb = "0.13"
+# x509-parser 0.6.1 is broken
+x509-parser = "0.5.1"
+der-parser = "2.0"
+publicsuffix = { version="1.5", default-features=false }
+xml-rs = "0.8"
+geo = "0.12"
+bytes = "0.4"
+base64 = "0.11"
+chrono = { version = "0.4", features = ["serde"] }
+
+image = "0.23.0"
+kamadak-exif = "0.5.1"
+
+bs58 = "0.3"
+digest = "0.8.0"
+blake2 = "0.8.0"
+
+[dev-dependencies]
+env_logger = "0.7"
+maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -0,0 +1,118 @@
+use bytes::Bytes;
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::result;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl Serialize for Blob {
+    #[inline]
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub trait BlobState {
+    fn register_blob(&self, blob: Blob) -> String;
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

sn0int-std/src/engine/mod.rs

@@ -0,0 +1 @@
+pub mod structs;

sn0int-std/src/errors.rs

@@ -0,0 +1,3 @@
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
+pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -0,0 +1,111 @@
+use crate::errors::*;
+
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use geo::{LineString, Polygon, Coordinate};
+use geo::prelude::*;
+
+
+#[derive(Debug, Deserialize)]
+pub struct Point {
+    lon: f64,
+    lat: f64,
+}
+
+impl Point {
+    pub fn try_from(x: AnyLuaValue) -> Result<Point> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
+    let ring = ring.iter()
+        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .collect::<Vec<_>>();
+
+    let polygon = Polygon::new(LineString::from(ring), vec![]);
+    let point = geo::Point::new(p.lon, p.lat);
+
+    polygon.contains(&point)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn hamburg_polygon() -> &'static [Point] {
+        &[
+            Point { lat: 53.63975308945899,  lon: 9.764785766601562  },
+            Point { lat: 53.59494998253459,  lon: 9.827270507812     },
+            Point { lat: 53.663153974456456, lon: 9.9151611328125    },
+            Point { lat: 53.65582987649682,  lon: 9.976272583007812  },
+            Point { lat: 53.68613523817129,  lon: 9.992752075195312  },
+            Point { lat: 53.68674518938816,  lon: 10.051460266113281 },
+            Point { lat: 53.72495117617815,  lon: 10.075492858886719 },
+            Point { lat: 53.71946627930625,  lon: 10.118408203125    },
+            Point { lat: 53.743635083157756, lon: 10.164413452148438 },
+            Point { lat: 53.73104466704585,  lon: 10.202865600585938 },
+            Point { lat: 53.676781546441546, lon: 10.16304016113281  },
+            Point { lat: 53.632832079199474, lon: 10.235824584960938 },
+            Point { lat: 53.608803292930894, lon: 10.2008056640625   },
+            Point { lat: 53.578646152866504, lon: 10.208358764648438 },
+            Point { lat: 53.57212285981298,  lon: 10.163726806640625 },
+            Point { lat: 53.52071674896369,  lon: 10.18707275390625  },
+            Point { lat: 53.52643162253097,  lon: 10.224151611328125 },
+            Point { lat: 53.44062753992289,  lon: 10.347747802734375 },
+            Point { lat: 53.38824275010831,  lon: 10.248870849609375 },
+            Point { lat: 53.38824275010831,  lon: 10.15960693359375  },
+            Point { lat: 53.44635321212876,  lon: 10.064849853515625 },
+            Point { lat: 53.40595029739904,  lon: 9.985198974609375  },
+            Point { lat: 53.42385506057106,  lon: 9.951210021972656  },
+            Point { lat: 53.41843327091211,  lon: 9.944171905517578  },
+            Point { lat: 53.41812635648326,  lon: 9.927349090576172  },
+            Point { lat: 53.412294561442884, lon: 9.917736053466797  },
+            Point { lat: 53.41464783813818,  lon: 9.901256561279297  },
+            Point { lat: 53.443490472483326, lon: 9.912586212158201  },
+            Point { lat: 53.45177144115704,  lon: 9.897651672363281  },
+            Point { lat: 53.43633277935392,  lon: 9.866924285888672  },
+            Point { lat: 53.427639673754776, lon: 9.866409301757812  },
+            Point { lat: 53.427639673754776, lon: 9.858856201171875  },
+            Point { lat: 53.46710230573499,  lon: 9.795513153076172  },
+            Point { lat: 53.49039461941655,  lon: 9.795341491699219  },
+            Point { lat: 53.49029248806277,  lon: 9.77903366088867   },
+            Point { lat: 53.49856433088649,  lon: 9.780235290527344  },
+            Point { lat: 53.5078554643033,   lon: 9.758434295654297  },
+            Point { lat: 53.545407634092975, lon: 9.759807586669922  },
+            Point { lat: 53.568147234570084, lon: 9.633293151855469  },
+            Point { lat: 53.58802162343514,  lon: 9.655780792236328  },
+            Point { lat: 53.568351121879815, lon: 9.727706909179688  },
+            Point { lat: 53.60921067445695,  lon: 9.737663269042969  },
+        ]
+    }
+
+    #[test]
+    fn test_polygon_hamburg_contains_hamburg() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 53.551085,
+            lon: 9.993682,
+        });
+        assert!(contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_berlin() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 52.52437,
+            lon: 13.41053,
+        });
+        assert!(!contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_ny() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 40.726662,
+            lon: -74.036677,
+        });
+        assert!(!contains);
+    }
+}

sn0int-std/src/geoip/mod.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 use crate::lazy::LazyInit;
-use crate::paths;
+// use crate::paths;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
@@ -19,7 +19,7 @@ pub trait Maxmind: Sized {
 
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
-    fn cache_path() -> Result<PathBuf> {
+    fn cache_path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         for path in &[
             // Archlinux
@@ -36,7 +36,7 @@ pub trait Maxmind: Sized {
         }
 
         // use cache path
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join(Self::filename());
         Ok(path)
     }
@@ -52,8 +52,8 @@ pub trait Maxmind: Sized {
         Self::from_buf(buf)
     }
 
-    fn try_open_reader() -> Result<Option<MaxmindReader>> {
-        let path = Self::cache_path()?;
+    fn try_open_reader(cache_dir: &Path) -> Result<Option<MaxmindReader>> {
+        let path = Self::cache_path(cache_dir)?;
 
         if path.exists() {
             let db = MaxmindReader::open_path(path)?;

sn0int-std/src/gfx/exif.rs

@@ -61,8 +61,9 @@ impl LocationBuilder {
 }
 
 pub fn gps(img: &[u8]) -> Result<Option<Location>> {
-    let mut buf = io::BufReader::new(img);
-    let reader = exif::Reader::new(&mut buf)?;
+    let mut buf = io::Cursor::new(img);
+    let reader = exif::Reader::new()
+        .read_from_container(&mut buf)?;
     let fields = reader.fields();
 
     let location = Location::try_from_iter(fields).ok();

sn0int-std/src/gfx/mod.rs

@@ -6,13 +6,13 @@ pub mod exif;
 
 #[derive(Debug)]
 pub enum ImageFormat {
-    PNG,
-    JPEG,
-    GIF,
-    WEBP,
-    TIFF,
-    BMP,
-    ICO,
+    Png,
+    Jpeg,
+    Gif,
+    WebP,
+    Tiff,
+    Bmp,
+    Ico,
 }
 
 impl ImageFormat {
@@ -20,28 +20,27 @@ impl ImageFormat {
         // https://www.iana.org/assignments/media-types/media-types.xhtml#image
         // /etc/nginx/mime.types
         match self {
-            ImageFormat::PNG => "image/png",
-            ImageFormat::JPEG => "image/jpeg",
-            ImageFormat::GIF => "image/gif",
-            ImageFormat::WEBP => "image/webp",
-            ImageFormat::TIFF => "image/tiff",
-            ImageFormat::BMP => "image/bmp",
-            ImageFormat::ICO => "image/vnd.microsoft.icon",
+            ImageFormat::Png => "image/png",
+            ImageFormat::Jpeg => "image/jpeg",
+            ImageFormat::Gif => "image/gif",
+            ImageFormat::WebP => "image/webp",
+            ImageFormat::Tiff => "image/tiff",
+            ImageFormat::Bmp => "image/bmp",
+            ImageFormat::Ico => "image/vnd.microsoft.icon",
         }
     }
 
     pub fn try_from(format: &image::ImageFormat) -> Result<ImageFormat> {
         use image::ImageFormat::*;
         match format {
-            PNG => Ok(ImageFormat::PNG),
-            JPEG => Ok(ImageFormat::JPEG),
-            GIF => Ok(ImageFormat::GIF),
-            WEBP => Ok(ImageFormat::WEBP),
-            TIFF => Ok(ImageFormat::TIFF),
-            BMP => Ok(ImageFormat::BMP),
-            ICO => Ok(ImageFormat::ICO),
-
-            HDR | PNM | TGA => bail!("Unsupported format: {:?}", format),
+            Png => Ok(ImageFormat::Png),
+            Jpeg => Ok(ImageFormat::Jpeg),
+            Gif => Ok(ImageFormat::Gif),
+            WebP => Ok(ImageFormat::WebP),
+            Tiff => Ok(ImageFormat::Tiff),
+            Bmp => Ok(ImageFormat::Bmp),
+            Ico => Ok(ImageFormat::Ico),
+            _ => bail!("Unsupported format: {:?}", format),
         }
     }
 }
@@ -49,13 +48,13 @@ impl ImageFormat {
 impl Into<image::ImageFormat> for ImageFormat {
     fn into(self) -> image::ImageFormat {
         match self {
-            ImageFormat::PNG => image::ImageFormat::PNG,
-            ImageFormat::JPEG => image::ImageFormat::JPEG,
-            ImageFormat::GIF => image::ImageFormat::GIF,
-            ImageFormat::WEBP => image::ImageFormat::WEBP,
-            ImageFormat::TIFF => image::ImageFormat::TIFF,
-            ImageFormat::BMP => image::ImageFormat::BMP,
-            ImageFormat::ICO => image::ImageFormat::ICO,
+            ImageFormat::Png => image::ImageFormat::Png,
+            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
+            ImageFormat::Gif => image::ImageFormat::Gif,
+            ImageFormat::WebP => image::ImageFormat::WebP,
+            ImageFormat::Tiff => image::ImageFormat::Tiff,
+            ImageFormat::Bmp => image::ImageFormat::Bmp,
+            ImageFormat::Ico => image::ImageFormat::Ico,
         }
     }
 }
@@ -114,7 +113,7 @@ mod tests {
 
     #[test]
     fn verify_gfx_load_ico() {
-        let ico = fs::read("./sn0int-registry/assets/favicon.ico").expect("fs::read");
+        let ico = fs::read("../sn0int-registry/assets/favicon.ico").expect("fs::read");
         let img = load(&ico).expect("gfx::load");
 
         assert_eq!("image/vnd.microsoft.icon", img.mime());

sn0int-std/src/html.rs

@@ -102,6 +102,7 @@ pub fn html_form(html: &str) -> Result<HashMap<String, String>> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn test_html_select() {

sn0int-std/src/lib.rs

@@ -0,0 +1,24 @@
+#[macro_use] extern crate serde_derive;
+use hlua_badtouch as hlua;
+
+pub mod blobs;
+pub mod crt;
+mod errors;
+pub mod engine;
+pub mod geo;
+pub mod geoip;
+pub mod gfx;
+pub mod html;
+pub mod json;
+pub mod lazy;
+pub mod psl;
+pub mod ratelimits;
+pub mod sockets;
+pub mod web;
+pub mod websockets;
+pub mod xml;
+
+#[cfg(test)]
+fn test_init() {
+    let _ = env_logger::builder().is_test(true).try_init();
+}

sn0int-std/src/psl.rs

@@ -4,9 +4,7 @@ use chrootable_https::Client;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::paths;
 use crate::lazy::LazyInit;
-use crate::worker;
 
 
 #[derive(Debug, PartialEq)]
@@ -23,22 +21,24 @@ pub enum PslReader {
 }
 
 impl PslReader {
-    pub fn open_or_download() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open_or_download<F>(cache_dir: &Path, indicator: F) -> Result<PslReader>
+        where
+            F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>
+    {
+        let path = Self::path(cache_dir)?;
         let reader = match Self::open_from(&path) {
             Ok(r) => r,
-            Err(_) => worker::spawn_fn("Downloading public suffix list", || {
+            Err(_) => indicator(Box::new(move || {
                 PslReader::download(&path, publicsuffix::LIST_URL)?;
                 Self::open_from(&path)
-            }, false)?,
+            }))?,
         };
         Ok(reader)
     }
 
-    pub fn open() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open(cache_dir: &Path) -> Result<PslReader> {
+        let path = Self::path(cache_dir)?;
         Self::open_from(&path)
-
     }
 
     pub fn open_from(path: &Path) -> Result<PslReader> {
@@ -46,7 +46,7 @@ impl PslReader {
         Ok(PslReader::Reader(file))
     }
 
-    pub fn path() -> Result<PathBuf> {
+    pub fn path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         let path = Path::new("/usr/share/publicsuffix/public_suffix_list.dat");
         if path.exists() {
@@ -54,7 +54,7 @@ impl PslReader {
         }
 
         // else, use local cache
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join("public_suffix_list.dat");
         Ok(path)
     }

sn0int-std/src/ratelimits.rs

@@ -1,8 +1,11 @@
 use chrono::prelude::*;
-use crate::worker::RatelimitSender;
 use std::collections::HashMap;
+use std::result;
+use std::sync::mpsc;
 use std::time::Duration;
 
+pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
+
 pub struct Ratelimiter {
     buckets: HashMap<String, Bucket>,
 }

sn0int-std/src/web.rs

@@ -1,17 +1,17 @@
 pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 
-use crate::blobs::Blob;
+use crate::blobs::{Blob, BlobState};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 use std::ops::Deref;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use crate::engine::ctx::State;
 use serde_json;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use std::fmt;
 use std::net::SocketAddr;
+use std::sync::Arc;
 use serde::Serialize;
 use crate::engine::structs::LuaMap;
 use crate::json::LuaJsonValue;
@@ -37,6 +37,12 @@ pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri>
         .map_err(Error::from)
 }
 
+pub trait WebState {
+    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+
+    fn register_in_jar(&self, session: &str, key: String, value: String);
+}
+
 #[derive(Debug)]
 pub struct HttpSession {
     id: String,
@@ -134,7 +140,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn State) -> Result<Response> {
+    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -228,7 +234,9 @@ impl HttpRequest {
         Ok(res)
     }
 
-    pub fn response_to_lua(&self, state: &dyn State, res: Response) -> Result<LuaMap> {
+    pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
+        where S: WebState + BlobState
+    {
         // map result to LuaMap
         let mut resp = LuaMap::new();
         resp.insert_num("status", f64::from(res.status));
@@ -254,7 +262,7 @@ impl HttpRequest {
         Ok(resp)
     }
 
-    fn register_cookies_on_state(session: &str, state: &dyn State, cookie: &str) {
+    fn register_cookies_on_state(session: &str, state: &dyn WebState, cookie: &str) {
         let mut key = String::new();
         let mut value = String::new();
         let mut in_key = true;

sn0int-std/src/websockets.rs

@@ -3,11 +3,10 @@ use crate::errors::*;
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
 use crate::sockets::{Stream, SocketOptions};
-use std::borrow::Cow;
+use http::Request;
 use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::io;
-use tungstenite::handshake::client::Request;
 use tungstenite::protocol::{self, Message};
 use url::Url;
 
@@ -45,19 +44,17 @@ pub struct WebSocket {
 
 impl WebSocket {
     pub fn negotiate(stream: Stream, url: Url, headers: Option<&HashMap<String, String>>) -> Result<WebSocket> {
-        let extra_headers = headers.map(|headers| {
-            headers.iter()
-                .map(|(k, v)| (Cow::Borrowed(k.as_str()), Cow::Borrowed(v.as_str())))
-                .collect()
-        });
+        let mut req = Request::get(url.to_string()); // TODO: don't re-parse here
 
-        let (sock, _resp) = tungstenite::client::client(
-            Request {
-                url,
-                extra_headers,
-            },
-            stream,
-        )?;
+        if let Some(headers) = headers {
+            for (k, v) in headers {
+                req = req.header(k, v);
+            }
+        }
+
+        let req = req.body(()).unwrap();
+
+        let (sock, _resp) = tungstenite::client::client(req, stream)?;
         Ok(WebSocket {
             sock,
         })

sn0int-std/src/xml.rs

@@ -124,6 +124,7 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn verify_xml_decode_empty() {

src/args.rs

@@ -48,6 +48,9 @@ pub enum SubCommand {
     /// Search in the registry
     #[structopt(name="search")]
     Search(Search),
+    /// The sn0int package manager
+    #[structopt(name="pkg")]
+    Pkg(cmd::pkg_cmd::Args),
     /// Insert into the database
     #[structopt(name="add")]
     Add(cmd::add_cmd::Args),
@@ -78,6 +81,9 @@ pub enum SubCommand {
     /// Run a lua repl
     #[structopt(name="repl")]
     Repl,
+    /// Show paths of various file system locations
+    #[structopt(name="paths")]
+    Paths,
     /// Generate shell completions
     #[structopt(name="completions")]
     Completions(Completions),
@@ -118,6 +124,9 @@ pub struct Run {
     /// Narrow down targeted entities
     #[structopt(short="t", long="target")]
     pub target: Option<String>,
+    /// Dump the sandbox init message to stdout instead of running a child process
+    #[structopt(long="dump-sandbox-init-msg")]
+    pub dump_sandbox_init_msg: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -11,7 +11,7 @@ use crate::term;
 
 
 fn path() -> Result<PathBuf> {
-    let path = paths::data_dir()?;
+    let path = paths::sn0int_dir()?;
     Ok(path.join("auth"))
 }
 

src/blobs.rs

@@ -3,44 +3,13 @@ use crate::paths;
 use crate::worker::{EventWithCallback, Event2};
 use crate::workspaces::Workspace;
 
-use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
 use bytes::Bytes;
-use serde::ser::{Serialize, Serializer};
-use serde::de::{self, Deserialize, Deserializer};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::mpsc;
 
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Blob {
-    pub id: String,
-    pub bytes: Bytes,
-}
-
-impl Blob {
-    pub fn create(bytes: Bytes) -> Blob {
-        let id = Self::hash(&bytes);
-        Blob {
-            id,
-            bytes,
-        }
-    }
-
-    pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
-    }
-
-    #[inline]
-    fn encode_hash(bytes: &[u8]) -> String {
-        let x = bs58::encode(bytes).into_string();
-        format!("{:0<44}", x)
-    }
-}
+pub use sn0int_std::blobs::Blob;
 
 impl EventWithCallback for Blob {
     type Payload = ();
@@ -51,30 +20,6 @@ impl EventWithCallback for Blob {
     }
 }
 
-impl Serialize for Blob {
-    #[inline]
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let s = base64::encode(&self.bytes);
-        serializer.serialize_str(&s)
-    }
-}
-
-impl<'de> Deserialize<'de> for Blob {
-    #[inline]
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
-            .map_err(de::Error::custom)?;
-        Ok(Blob::create(Bytes::from(bytes)))
-    }
-}
-
 pub struct BlobStorage {
     path: PathBuf,
 }
@@ -155,7 +100,6 @@ impl BlobStorage {
 mod tests {
     use super::*;
     use tempfile;
-    use serde_json;
 
     #[inline]
     fn blob() -> (Bytes, Blob) {
@@ -163,15 +107,6 @@ mod tests {
         (bytes.clone(), Blob::create(bytes))
     }
 
-    #[test]
-    fn verify_create_blob() {
-        let (bytes, blob) = blob();
-        assert_eq!(blob, Blob {
-            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
-            bytes,
-        });
-    }
-
     #[test]
     fn test_blobstorage_save() {
         let dir = tempfile::tempdir().expect("tempdir");
@@ -204,38 +139,4 @@ mod tests {
 
         assert!(result.is_err());
     }
-
-    #[test]
-    fn test_serialize() {
-        let (_, blob) = blob();
-        let json = serde_json::to_string(&blob).unwrap();
-        assert_eq!(&json, "\"YXNkZg==\"");
-    }
-
-    #[test]
-    fn test_deserialize() {
-        let (_, blob1) = blob();
-        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
-        assert_eq!(blob1, blob2);
-    }
-
-    #[test]
-    fn test_hash_encoding() {
-        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
-    }
-
-    #[test]
-    fn test_hash_encoding_padding() {
-        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
-    }
 }

src/cmd/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use crate::shell::Shell;
+use crate::config::Config;
 
 pub trait Cmd: structopt::StructOpt + Sized {
     fn run(self, rl: &mut Shell) -> Result<()>;
@@ -11,6 +12,10 @@ pub trait Cmd: structopt::StructOpt + Sized {
     }
 }
 
+pub trait LiteCmd: structopt::StructOpt + Sized {
+    fn run(self, config: &Config) -> Result<()>;
+}
+
 pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
@@ -23,8 +28,8 @@ pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
-pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;

src/cmd/pkg_cmd.rs

@@ -1,6 +1,9 @@
 use crate::errors::*;
 
 use crate::args;
+use crate::config::Config;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::engine::Library;
 use crate::registry::{self, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
@@ -20,6 +23,13 @@ pub struct Args {
     pub subcommand: SubCommand,
 }
 
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct ArgsInteractive {
+    #[structopt(subcommand)]
+    pub subcommand: SubCommandInteractive,
+}
+
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
     /// List installed modules
@@ -31,9 +41,6 @@ pub enum SubCommand {
     /// Search modules in registry
     #[structopt(name="search")]
     Search(args::Search),
-    /// Reload modules
-    #[structopt(name="reload")]
-    Reload(Reload),
     /// Update modules
     #[structopt(name="update")]
     Update(Update),
@@ -42,6 +49,15 @@ pub enum SubCommand {
     Uninstall(Uninstall),
 }
 
+#[derive(Debug, StructOpt)]
+pub enum SubCommandInteractive {
+    #[structopt(flatten)]
+    Base(SubCommand),
+    /// Reload modules
+    #[structopt(name="reload")]
+    Reload(Reload),
+}
+
 #[derive(Debug, StructOpt)]
 pub struct List {
     /// Only show modules with a specific input source
@@ -65,15 +81,18 @@ pub struct Uninstall {
     module: ModuleID,
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
+#[derive(PartialEq)]
+enum ModuleReload {
+    Yes,
+    No,
+}
 
-    match args.subcommand {
+fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
+    match subcommand {
         SubCommand::List(list) => {
             let autoupdate = AutoUpdater::load()?;
 
-            for module in rl.engine().list() {
+            for module in library.list() {
                 if let Some(source) = &list.source {
                     if !module.source_equals(&source) {
                         continue;
@@ -93,31 +112,22 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
                 println!("{}", out);
                 println!("\t{}", module.description());
             }
+            Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
             registry::run_install(install, &config)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
-        SubCommand::Search(search) => registry::run_search(rl.engine(), &search, &config)?,
-        SubCommand::Reload(_) => {
-            let current = rl.take_module()
-                            .map(|m| m.canonical());
-
-            rl.engine_mut().reload_modules()?;
-            rl.reload_module_cache();
-
-            if let Some(module) = current {
-                if let Ok(module) = rl.engine().get(&module).map(|x| x.to_owned()) {
-                    rl.set_module(module);
-                }
-            }
+        SubCommand::Search(search) => {
+            registry::run_search(library, &search, &config)?;
+            Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
             let updater = Arc::new(Updater::new(&config)?);
 
-            let modules = rl.engine().list()
+            let modules = library.list()
                 .into_iter()
                 .filter_map(|module| {
                     let canonical = module.canonical();
@@ -138,15 +148,34 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
             autoupdate.save()?;
 
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
             let updater = Updater::new(&config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
     }
-
-    Ok(())
+}
+
+impl LiteCmd for Args {
+    fn run(self, config: &Config) -> Result<()> {
+        let library = Library::new(false, &config)?;
+        run_subcommand(self.subcommand, &library, config)?;
+        Ok(())
+    }
+}
+
+impl Cmd for ArgsInteractive {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let action = match self.subcommand {
+            SubCommandInteractive::Base(subcommand) => run_subcommand(subcommand, rl.library(), rl.config())?,
+            SubCommandInteractive::Reload(_) => ModuleReload::Yes,
+        };
+        if action == ModuleReload::Yes {
+            rl.reload_modules()?;
+        }
+        Ok(())
+    }
 }

src/cmd/quickstart_cmd.rs

@@ -2,7 +2,6 @@ use crate::errors::*;
 
 use crate::args::Install;
 use crate::api::Client;
-use crate::cmd::mod_cmd;
 use crate::registry::{InstallTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
@@ -47,7 +46,7 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     autoupdate.save()?;
 
     // trigger reload
-    mod_cmd::run(rl, &[String::from("mod"), String::from("reload")])?;
+    rl.reload_modules()?;
 
     Ok(())
 }

src/cmd/run_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use chrootable_https::dns::Resolver;
 use crate::args;
 use crate::blobs::{Blob, BlobStorage};
 use crate::cmd::Cmd;
 use crate::db::{ttl, Filter};
 use crate::engine::Module;
+use crate::ipc::common::StartCommand;
 use crate::models::*;
 use crate::shell::Shell;
 use crate::keyring::KeyRing;
@@ -113,16 +115,10 @@ fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> R
     Ok(())
 }
 
-pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
-
-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-
+fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
     let filter = rl.scoped_targets();
 
-    let args = match module.source() {
+    match module.source() {
         Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
         Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
@@ -152,7 +148,44 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
             }
         },
         None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
-    }?;
+    }
+}
+
+pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let keyring = rl.keyring().request_keys(&module);
+
+    let dns_config = Resolver::from_system()?;
+    let proxy = rl.config().network.proxy.clone();
+
+    let args = get_args(rl, &module)?;
+    for (arg, _pretty_arg, blobs) in args {
+        let start_cmd = StartCommand::new(params.verbose,
+                                          keyring.clone(),
+                                          dns_config.clone(),
+                                          proxy.clone(),
+                                          options.clone(),
+                                          module.clone(),
+                                          arg,
+                                          blobs);
+        let out = serde_json::to_string(&start_cmd)?;
+        println!("{}", out);
+    }
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
     let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);

src/cmd/use_cmd.rs

@@ -14,7 +14,7 @@ pub struct Args {
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
 
-    let module = rl.engine().get(&args.module)?.clone();
+    let module = rl.library().get(&args.module)?.clone();
     rl.set_module(module);
 
     Ok(())

src/cmd/workspace_cmd.rs

@@ -1,7 +1,8 @@
 use crate::errors::*;
 
 use crate::blobs::BlobStorage;
-use crate::cmd::Cmd;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::config::Config;
 use crate::db::Database;
 use crate::shell::Shell;
 use crate::term;
@@ -52,6 +53,8 @@ fn usage(workspace: Option<Workspace>) -> Result<()> {
 }
 
 fn change(rl: &mut Shell, workspace: Workspace) -> Result<()> {
+    workspace.migrate()?;
+
     let blobs = BlobStorage::workspace(&workspace)?;
     let db = Database::establish(workspace)?;
     rl.set_blobstorage(blobs);
@@ -66,29 +69,42 @@ fn list() -> Result<()> {
     Ok(())
 }
 
-impl Cmd for Args {
-    fn run(self, rl: &mut Shell) -> Result<()> {
-        if self.delete {
-            if let Some(workspace) = self.workspace {
+fn run(args: Args, rl: Option<&mut Shell>) -> Result<()> {
+    if args.delete {
+        if let Some(workspace) = args.workspace {
+            if let Some(rl) = rl {
                 if *rl.db().workspace() == workspace {
                     bail!("Can't delete current workspace")
                 }
-
-                delete(workspace, self.force)
-            } else {
-                bail!("--delete requires workspace")
             }
-        } else if self.usage {
-            usage(self.workspace)
-        } else if let Some(workspace) = self.workspace {
+
+            delete(workspace, args.force)
+        } else {
+            bail!("--delete requires workspace")
+        }
+    } else if args.usage {
+        usage(args.workspace)
+    } else if let Some(workspace) = args.workspace {
+        if let Some(rl) = rl {
             change(rl, workspace)
         } else {
-            list()
+            Ok(())
         }
+    } else {
+        list()
     }
 }
 
-#[inline]
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    Args::run_str(rl, args)
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run(self, Some(rl))
+    }
+}
+
+impl LiteCmd for Args {
+    #[inline]
+    fn run(self, _config: &Config) -> Result<()> {
+        run(self, None)
+    }
 }

src/engine/ctx.rs

@@ -1,8 +1,7 @@
 use crate::errors::*;
 
-use crate::blobs::Blob;
 use crate::db::Family;
-use crate::engine::{Environment, Reporter};
+use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
@@ -13,10 +12,12 @@ use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
 use crate::websockets::{WebSocket, WebSocketOptions};
-use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent, RatelimitEvent};
+use crate::worker::{Event, LogEvent, DatabaseEvent, DatabaseResponse, StdioEvent, RatelimitEvent};
 use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
+use sn0int_std::blobs::{Blob, BlobState};
+use sn0int_std::web::WebState;
 use std::collections::HashMap;
 use std::result;
 use std::net::SocketAddr;
@@ -71,48 +72,52 @@ pub trait State {
         self.send(&Event::Log(LogEvent::Status(msg)))
     }
 
-    fn db_insert(&self, object: Insert) -> Result<Option<i32>> {
+    fn db_recv(&self) -> Result<DatabaseResponse> {
+        let reply = self.recv()?;
+        let reply: result::Result<DatabaseResponse, String> = serde_json::from_value(reply)?;
+        reply.map_err(|err| format_err!("Database error: {:?}", err))
+    }
+
+    fn db_insert(&self, object: Insert) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Insert(object)));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<Option<i32>> {
+    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_activity(&self, activity: InsertActivity) -> Result<()> {
+    fn db_activity(&self, activity: InsertActivity) -> Result<bool> {
         let activity = activity.try_into_new()?;
 
         self.send(&Event::Database(DatabaseEvent::Activity(activity)));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
+        let r = self.db_recv()
+            .context("Failed to log activity")?;
 
-        reply
-            .map(|_| ())
-            .map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        match r {
+            DatabaseResponse::Inserted(_) => Ok(false),
+            DatabaseResponse::NoChange(_) => Ok(true),
+            _ => bail!("Unexpected database response for db_activity: {:?}", r),
+        }
     }
 
-    fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
+    fn db_select(&self, family: Family, value: String) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Select((family, value))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to query database: {:?}", err))
+        self.db_recv()
+            .context("Failed to query database")
+            .map_err(Error::from)
     }
 
-    fn db_update(&self, object: String, update: Update) -> Result<Option<i32>> {
+    fn db_update(&self, object: String, update: Update) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Update((object, update))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to update database: {:?}", err))
+        self.db_recv()
+            .context("Failed to update database")
+            .map_err(Error::from)
     }
 
     fn stdin_read_line(&self) -> Result<Option<String>> {
@@ -173,16 +178,10 @@ pub trait State {
 
     fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>>;
 
-    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
-
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
 
-    fn register_in_jar(&self, session: &str, key: String, value: String);
-
-    fn register_blob(&self, blob: Blob) -> String;
-
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>>;
 
     fn persist_blob(&self, id: &str) -> Result<()> {
@@ -197,7 +196,7 @@ pub trait State {
 // #[derive(Debug)]
 pub struct LuaState {
     error: Mutex<Option<Error>>,
-    logger: Arc<Mutex<Box<dyn Reporter>>>,
+    logger: Arc<Mutex<Box<dyn IpcChild>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
     ws_sessions: Mutex<HashMap<String, Arc<Mutex<WebSocket>>>>,
     blobs: Mutex<HashMap<String, Arc<Blob>>>,
@@ -341,6 +340,29 @@ impl State for LuaState {
         sock.clone()
     }
 
+    fn http_mksession(&self) -> String {
+        let mut mtx = self.http_sessions.lock().unwrap();
+        let (id, session) = HttpSession::new();
+        mtx.insert(id.clone(), session);
+        id
+    }
+
+    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
+        let mtx = self.http_sessions.lock().unwrap();
+        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
+
+        HttpRequest::new(&session, method, url, options)
+    }
+
+    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
+        let mtx = self.blobs.lock().unwrap();
+        let blob = mtx.get(id)
+            .ok_or_else(|| format_err!("Invalid blob reference"))?;
+        Ok(blob.clone())
+    }
+}
+
+impl WebState for LuaState {
     fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>> {
         let proxy = self.resolve_proxy_options(proxy)?;
 
@@ -367,27 +389,15 @@ impl State for LuaState {
         }
     }
 
-    fn http_mksession(&self) -> String {
-        let mut mtx = self.http_sessions.lock().unwrap();
-        let (id, session) = HttpSession::new();
-        mtx.insert(id.clone(), session);
-        id
-    }
-
-    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
-        let mtx = self.http_sessions.lock().unwrap();
-        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
-
-        HttpRequest::new(&session, method, url, options)
-    }
-
     fn register_in_jar(&self, session: &str, key: String, value: String) {
         let mut mtx = self.http_sessions.lock().unwrap();
         if let Some(session) = mtx.get_mut(session) {
             session.cookies.register_in_jar(key, value);
         }
     }
+}
 
+impl BlobState for LuaState {
     fn register_blob(&self, blob: Blob) -> String {
         let id = blob.id.clone();
 
@@ -397,13 +407,6 @@ impl State for LuaState {
 
         id
     }
-
-    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
-        let mtx = self.blobs.lock().unwrap();
-        let blob = mtx.get(id)
-            .ok_or_else(|| format_err!("Invalid blob reference"))?;
-        Ok(blob.clone())
-    }
 }
 
 impl LuaState {
@@ -423,7 +426,7 @@ pub struct Script {
     code: String,
 }
 
-pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
+pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
@@ -473,6 +476,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::debug(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
+    runtime::geo_polygon_contains(&mut lua, state.clone());
     runtime::geoip_lookup(&mut lua, state.clone());
     runtime::getopt(&mut lua, state.clone());
     runtime::hex(&mut lua, state.clone());
@@ -535,6 +539,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::status(&mut lua, state.clone());
     runtime::stdin_read_line(&mut lua, state.clone());
     runtime::stdin_read_to_end(&mut lua, state.clone());
+    runtime::str_find(&mut lua, state.clone());
+    runtime::str_replace(&mut lua, state.clone());
+    runtime::strval(&mut lua, state.clone());
     runtime::strftime(&mut lua, state.clone());
     runtime::strptime(&mut lua, state.clone());
     runtime::time_unix(&mut lua, state.clone());
@@ -585,7 +592,7 @@ impl Script {
     }
 
     pub fn run(&self, env: Environment,
-                      tx: Arc<Mutex<Box<dyn Reporter>>>,
+                      tx: Arc<Mutex<Box<dyn IpcChild>>>,
                       arg: AnyLuaValue,
     ) -> Result<()> {
         let (mut lua, state) = ctx(env, tx);
@@ -616,8 +623,9 @@ impl Script {
 
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
-        use crate::engine::DummyReporter;
+        use crate::ipc::child::DummyIpcChild;
         use crate::geoip::Maxmind;
+        use crate::paths;
         let keyring = Vec::new();
         let dns_config = Resolver::from_system()?;
         let proxy = None;
@@ -629,8 +637,9 @@ com
 a.prod.fastly.net
 // ===END PRIVATE DOMAINS===
 "#.into());
-        let geoip = GeoIP::try_open_reader()?;
-        let asn = AsnDB::try_open_reader()?;
+        let cache_dir = paths::cache_dir()?;
+        let geoip = GeoIP::try_open_reader(&cache_dir)?;
+        let asn = AsnDB::try_open_reader(&cache_dir)?;
 
         let env = Environment {
             verbose: 0,
@@ -643,6 +652,6 @@ a.prod.fastly.net
             geoip,
             asn,
         };
-        self.run(env, DummyReporter::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
     }
 }

src/engine/isolation.rs

@@ -1,252 +0,0 @@
-use crate::errors::*;
-use chrootable_https::dns::Resolver;
-use crate::blobs::Blob;
-use crate::engine::{Environment, Module, Reporter};
-use crate::geoip::MaxmindReader;
-use crate::keyring::KeyRingEntry;
-use crate::psl::PslReader;
-use serde_json;
-use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
-use std::collections::HashMap;
-use std::env;
-use std::ffi::OsString;
-use std::io::prelude::*;
-use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
-use std::net::SocketAddr;
-use std::sync::{mpsc, Arc, Mutex};
-use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct StartCommand {
-    verbose: u64,
-    keyring: Vec<KeyRingEntry>,
-    dns_config: Resolver,
-    proxy: Option<SocketAddr>,
-    options: HashMap<String, String>,
-    module: Module,
-    arg: serde_json::Value,
-    blobs: Vec<Blob>,
-}
-
-impl StartCommand {
-    pub fn new(verbose: u64,
-               keyring: Vec<KeyRingEntry>,
-               dns_config: Resolver,
-               proxy: Option<SocketAddr>,
-               options: HashMap<String, String>,
-               module: Module,
-               arg: serde_json::Value,
-               blobs: Vec<Blob>,
-    ) -> StartCommand {
-        StartCommand {
-            verbose,
-            keyring,
-            dns_config,
-            proxy,
-            options,
-            module,
-            arg,
-            blobs,
-        }
-    }
-}
-
-pub struct Supervisor {
-    child: Child,
-    stdin: ChildStdin,
-    stdout: BufReader<ChildStdout>,
-}
-
-impl Supervisor {
-    pub fn setup(module: &Module) -> Result<Supervisor> {
-        let exe = match env::current_exe() {
-            Ok(exe) => exe.into_os_string(),
-            _ => OsString::from("sn0int"),
-        };
-
-        let mut child = Command::new(exe)
-            .arg("sandbox")
-            .arg(&module.canonical())
-            .stdin(Stdio::piped())
-            .stdout(Stdio::piped())
-            .spawn()
-            .context("Failed to spawn child process")?;
-
-        let stdin = child.stdin.take().expect("Failed to take child stdin");
-        let stdout = child.stdout.take().expect("Failed to take child stdout");
-        let stdout = BufReader::new(stdout);
-
-        Ok(Supervisor {
-            child,
-            stdin,
-            stdout,
-        })
-    }
-
-    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
-        self.send(&start)?;
-        Ok(())
-    }
-
-    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
-        let mut value = serde_json::to_string(value)?;
-        value.push('\n');
-        self.stdin.write_all(value.as_bytes())?;
-        debug!("Supervisor sent: {:?}", value);
-        Ok(())
-    }
-
-    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
-        let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
-            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
-        }
-    }
-
-    pub fn recv(&mut self) -> Result<Event> {
-        let mut line = String::new();
-        let len = self.stdout.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Supervisor received: {:?}", event);
-        Ok(event)
-    }
-
-    pub fn wait(&mut self) -> Result<()> {
-        let exit = self.child.wait()
-            .context("Failed to wait for child")?;
-
-        if exit.success() {
-            Ok(())
-        } else {
-            bail!("Child signaled error")
-        }
-    }
-
-    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
-        where <T as EventWithCallback>::Payload: serde::Serialize
-    {
-        let (tx2, rx2) = mpsc::channel();
-        tx.send(event.with_callback(tx2));
-        let reply = rx2.recv().unwrap();
-
-        self.send_struct(reply, tx);
-    }
-}
-
-#[derive(Debug)]
-pub struct StdioReporter {
-    stdin: Stdin,
-    stdout: Stdout,
-}
-
-impl StdioReporter {
-    pub fn setup() -> StdioReporter {
-        let stdin = io::stdin();
-        let stdout = io::stdout();
-
-        StdioReporter {
-            stdin,
-            stdout,
-        }
-    }
-
-    pub fn recv_start(&mut self) -> Result<StartCommand> {
-        let value = self.recv()?;
-        let event = serde_json::from_value(value)?;
-        Ok(event)
-    }
-}
-
-impl Reporter for StdioReporter {
-    fn send(&mut self, event: &Event) -> Result<()> {
-        let mut event = serde_json::to_string(&event)?;
-        event.push('\n');
-        self.stdout.write_all(event.as_bytes())?;
-        debug!("Reporter sent: {:?}", event);
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        let mut line = String::new();
-        let len = self.stdin.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Reporter received: {:?}", event);
-        Ok(event)
-    }
-}
-
-pub fn spawn_module(module: Module,
-                    tx: &EventSender,
-                    arg: serde_json::Value,
-                    keyring: Vec<KeyRingEntry>,
-                    verbose: u64,
-                    has_stdin: bool,
-                    proxy: Option<SocketAddr>,
-                    options: HashMap<String, String>,
-                    blobs: Vec<Blob>,
-) -> Result<ExitEvent> {
-    let dns_config = Resolver::from_system()?;
-
-    let mut reader = if has_stdin {
-        Some(BufReader::new(stdin()))
-    } else {
-        None
-    };
-
-    let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
-
-    let exit = loop {
-        match supervisor.recv()? {
-            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => supervisor.send_event_callback(object, &tx),
-            Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
-            Event::Ratelimit(req) => supervisor.send_event_callback(req, &tx),
-            Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
-            Event::Exit(event) => {
-                if let ExitEvent::Err(err) = &event {
-                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
-                }
-                break event;
-            },
-        }
-    };
-
-    supervisor.wait()?;
-
-    Ok(exit)
-}
-
-pub fn run_worker(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
-    let mut reporter = StdioReporter::setup();
-    let start = reporter.recv_start()?;
-
-    let environment = Environment {
-        verbose: start.verbose,
-        keyring: start.keyring,
-        dns_config: start.dns_config,
-        proxy: start.proxy,
-        options: start.options,
-        blobs: start.blobs,
-        psl,
-        geoip,
-        asn,
-    };
-
-    let mtx: Arc<Mutex<Box<dyn Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
-    let result = start.module.run(environment,
-                                  mtx.clone(),
-                                  start.arg.into());
-    let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
-                        .into_inner().expect("Failed to consume Mutex");
-
-    let event = result.into();
-    reporter.send(&Event::Exit(event))?;
-
-    Ok(())
-}

src/engine/mod.rs

@@ -5,7 +5,6 @@ use crate::config::Config;
 use crate::geoip::MaxmindReader;
 use crate::json::LuaJsonValue;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use std::fs;
 use std::fmt::Debug;
 use std::path::PathBuf;
@@ -13,6 +12,7 @@ use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
+use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::Resolver;
@@ -21,11 +21,10 @@ use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker::{self, Event};
+use crate::worker;
 
 pub mod ctx;
-pub mod isolation;
-pub mod structs;
+pub use sn0int_std::engine::structs;
 
 
 /// Data that is passed to every script
@@ -43,29 +42,29 @@ pub struct Environment {
 }
 
 #[derive(Debug)]
-pub struct Engine<'a> {
+pub struct Library<'a> {
     path: PathBuf,
     modules: HashMap<String, Vec<Module>>,
     config: &'a Config
 }
 
-impl<'a> Engine<'a> {
-    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Engine> {
+impl<'a> Library<'a> {
+    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Library> {
         let path = paths::module_dir()?;
 
-        let mut engine = Engine {
+        let mut library = Library {
             path,
             modules: HashMap::new(),
             config,
         };
 
         if verbose_init {
-            engine.reload_modules()?;
+            library.reload_modules()?;
         } else {
-            engine.reload_modules_quiet()?;
+            library.reload_modules_quiet()?;
         }
 
-        Ok(engine)
+        Ok(library)
     }
 
     pub fn reload_modules(&mut self) -> Result<usize> {
@@ -299,9 +298,9 @@ impl Module {
         self.private_module
     }
 
-    pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<dyn Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(env, reporter, arg.into())
+        self.script.run(env, ipc_child, arg.into())
     }
 
     #[inline]
@@ -320,28 +319,3 @@ impl Module {
         }
     }
 }
-
-pub trait Reporter: Debug {
-    fn send(&mut self, event: &Event) -> Result<()>;
-
-    fn recv(&mut self) -> Result<serde_json::Value>;
-}
-
-#[derive(Debug)]
-pub struct DummyReporter;
-
-impl DummyReporter {
-    pub fn new() -> Arc<Mutex<Box<dyn Reporter>>> {
-        Arc::new(Mutex::new(Box::new(DummyReporter)))
-    }
-}
-
-impl Reporter for DummyReporter {
-    fn send(&mut self, _event: &Event) -> Result<()> {
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        unimplemented!("DummyReporter::recv doesn't exist")
-    }
-}

src/fmt.rs

@@ -141,8 +141,7 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
-        self.push_into_group()?;
+    pub fn direct_display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::display(self, v)
         } else {
@@ -151,8 +150,19 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         self.push_into_group()?;
+        self.direct_display::<C, _>(v)
+    }
+
+    pub fn display_label<C: Color, D: fmt::Display>(&mut self, label: &str, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        write!(self, "{}=", label)?;
+        self.direct_display::<C, _>(v)
+    }
+
+    #[inline]
+    pub fn direct_debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::debug(self, v)
         } else {
@@ -160,6 +170,12 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         }
     }
 
+    #[inline]
+    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        self.direct_debug::<C, _>(v)
+    }
+
     pub fn opt_debug<C: Color, D: fmt::Debug>(&mut self, v: &Option<D>) -> fmt::Result {
         if let Some(v) = &v {
             self.debug::<C, _>(v)?;
@@ -167,6 +183,15 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         Ok(())
     }
 
+    pub fn opt_debug_label<C: Color, D: fmt::Debug>(&mut self, label: &str, v: &Option<D>) -> fmt::Result {
+        if let Some(v) = &v {
+            self.push_into_group()?;
+            write!(self, "{}=", label)?;
+            self.direct_debug::<C, _>(v)?;
+        }
+        Ok(())
+    }
+
     #[inline]
     pub fn clear(&mut self) -> fmt::Result {
         if self.scoped {

src/ipc/child.rs

@@ -0,0 +1,110 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use crate::engine::Environment;
+use crate::geoip::MaxmindReader;
+use crate::psl::PslReader;
+use serde_json;
+use crate::worker::Event;
+
+use std::fmt::Debug;
+use std::io::prelude::*;
+use std::io::{self, Stdin, Stdout};
+use std::sync::{Arc, Mutex};
+
+
+pub trait IpcChild: Debug {
+    fn send(&mut self, event: &Event) -> Result<()>;
+
+    fn recv(&mut self) -> Result<serde_json::Value>;
+}
+
+#[derive(Debug)]
+pub struct StdioIpcChild {
+    stdin: Stdin,
+    stdout: Stdout,
+}
+
+impl StdioIpcChild {
+    pub fn setup() -> StdioIpcChild {
+        let stdin = io::stdin();
+        let stdout = io::stdout();
+
+        StdioIpcChild {
+            stdin,
+            stdout,
+        }
+    }
+
+    pub fn recv_start(&mut self) -> Result<StartCommand> {
+        let value = self.recv()?;
+        let event = serde_json::from_value(value)?;
+        Ok(event)
+    }
+}
+
+impl IpcChild for StdioIpcChild {
+    fn send(&mut self, event: &Event) -> Result<()> {
+        let mut event = serde_json::to_string(&event)?;
+        event.push('\n');
+        debug!("IpcChild sends: {:?}", event);
+        self.stdout.write_all(event.as_bytes())?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        let mut line = String::new();
+        let len = self.stdin.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcChild received: {:?}", event);
+        Ok(event)
+    }
+}
+
+#[derive(Debug)]
+pub struct DummyIpcChild;
+
+impl DummyIpcChild {
+    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+        Arc::new(Mutex::new(Box::new(DummyIpcChild)))
+    }
+}
+
+impl IpcChild for DummyIpcChild {
+    fn send(&mut self, _event: &Event) -> Result<()> {
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        unimplemented!("DummyIpcChild::recv doesn't exist")
+    }
+}
+
+pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
+    let mut ipc_child = StdioIpcChild::setup();
+    let start = ipc_child.recv_start()?;
+
+    let environment = Environment {
+        verbose: start.verbose,
+        keyring: start.keyring,
+        dns_config: start.dns_config,
+        proxy: start.proxy,
+        options: start.options,
+        blobs: start.blobs,
+        psl,
+        geoip,
+        asn,
+    };
+
+    let mtx: Arc<Mutex<Box<dyn IpcChild>>> = Arc::new(Mutex::new(Box::new(ipc_child)));
+    let result = start.module.run(environment,
+                                  mtx.clone(),
+                                  start.arg.into());
+    let mut ipc_child = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
+                        .into_inner().expect("Failed to consume Mutex");
+
+    let event = result.into();
+    ipc_child.send(&Event::Exit(event))?;
+
+    Ok(())
+}

src/ipc/common.rs

@@ -0,0 +1,44 @@
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+
+use std::collections::HashMap;
+use std::net::SocketAddr;
+
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StartCommand {
+    pub verbose: u64,
+    pub keyring: Vec<KeyRingEntry>,
+    pub dns_config: Resolver,
+    pub proxy: Option<SocketAddr>,
+    pub options: HashMap<String, String>,
+    pub module: Module,
+    pub arg: serde_json::Value,
+    pub blobs: Vec<Blob>,
+}
+
+impl StartCommand {
+    pub fn new(verbose: u64,
+               keyring: Vec<KeyRingEntry>,
+               dns_config: Resolver,
+               proxy: Option<SocketAddr>,
+               options: HashMap<String, String>,
+               module: Module,
+               arg: serde_json::Value,
+               blobs: Vec<Blob>,
+    ) -> StartCommand {
+        StartCommand {
+            verbose,
+            keyring,
+            dns_config,
+            proxy,
+            options,
+            module,
+            arg,
+            blobs,
+        }
+    }
+}

src/ipc/mod.rs

@@ -0,0 +1,3 @@
+pub mod child;
+pub mod common;
+pub mod parent;

src/ipc/parent.rs

@@ -0,0 +1,144 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
+
+use std::collections::HashMap;
+use std::env;
+use std::ffi::OsString;
+use std::io::prelude::*;
+use std::io::{BufReader, BufRead, stdin};
+use std::net::SocketAddr;
+use std::sync::mpsc;
+use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
+
+
+pub struct IpcParent {
+    child: Child,
+    stdin: ChildStdin,
+    stdout: BufReader<ChildStdout>,
+}
+
+impl IpcParent {
+    pub fn setup(module: &Module) -> Result<IpcParent> {
+        let exe = match env::current_exe() {
+            Ok(exe) => exe.into_os_string(),
+            _ => OsString::from("sn0int"),
+        };
+
+        let mut child = Command::new(exe)
+            .arg("sandbox")
+            .arg(&module.canonical())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .context("Failed to spawn child process")?;
+
+        let stdin = child.stdin.take().expect("Failed to take child stdin");
+        let stdout = child.stdout.take().expect("Failed to take child stdout");
+        let stdout = BufReader::new(stdout);
+
+        Ok(IpcParent {
+            child,
+            stdin,
+            stdout,
+        })
+    }
+
+    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
+        let start = serde_json::to_value(&start)?;
+        self.send(&start)?;
+        Ok(())
+    }
+
+    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
+        let mut value = serde_json::to_string(value)?;
+        value.push('\n');
+        self.stdin.write_all(value.as_bytes())?;
+        debug!("IpcParent sent: {:?}", value);
+        Ok(())
+    }
+
+    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
+        let value = serde_json::to_value(value).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
+
+    pub fn recv(&mut self) -> Result<Event> {
+        let mut line = String::new();
+        let len = self.stdout.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcParent received: {:?}", event);
+        Ok(event)
+    }
+
+    pub fn wait(&mut self) -> Result<()> {
+        let exit = self.child.wait()
+            .context("Failed to wait for child")?;
+
+        if exit.success() {
+            Ok(())
+        } else {
+            bail!("Child signaled error")
+        }
+    }
+
+    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
+        where <T as EventWithCallback>::Payload: serde::Serialize
+    {
+        let (tx2, rx2) = mpsc::channel();
+        tx.send(event.with_callback(tx2));
+        let reply = rx2.recv().unwrap();
+
+        self.send_struct(reply, tx);
+    }
+}
+
+pub fn run(module: Module,
+           tx: &EventSender,
+           arg: serde_json::Value,
+           keyring: Vec<KeyRingEntry>,
+           verbose: u64,
+           has_stdin: bool,
+           proxy: Option<SocketAddr>,
+           options: HashMap<String, String>,
+           blobs: Vec<Blob>,
+) -> Result<ExitEvent> {
+    let dns_config = Resolver::from_system()?;
+
+    let mut reader = if has_stdin {
+        Some(BufReader::new(stdin()))
+    } else {
+        None
+    };
+
+    let mut ipc_parent = IpcParent::setup(&module)?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+
+    let exit = loop {
+        match ipc_parent.recv()? {
+            Event::Log(event) => tx.send(Event2::Log(event)),
+            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Exit(event) => {
+                if let ExitEvent::Err(err) = &event {
+                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
+                }
+                break event;
+            },
+        }
+    };
+
+    ipc_parent.wait()?;
+
+    Ok(exit)
+}

src/keyring.rs

@@ -74,7 +74,7 @@ pub struct KeyRing {
 
 impl KeyRing {
     pub fn path() -> Result<PathBuf> {
-        let path = paths::data_dir()?;
+        let path = paths::sn0int_dir()?;
         let path = path.join("keyring.json");
         Ok(path)
     }

src/lib.rs

@@ -16,24 +16,26 @@ pub mod autonoscope;
 pub mod blobs;
 pub mod cmd;
 pub mod config;
-pub mod crt;
+use sn0int_std::crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
 pub mod filters;
 pub mod fmt;
-pub mod geoip;
-pub mod gfx;
-pub mod html;
-pub mod json;
+use sn0int_std::geo;
+pub use sn0int_std::geoip;
+use sn0int_std::gfx;
+use sn0int_std::html;
+use sn0int_std::json;
+pub mod ipc;
 pub mod keyring;
-pub mod lazy;
+use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
 pub mod paths;
-pub mod psl;
+pub use sn0int_std::psl;
 pub mod options;
-pub mod ratelimits;
+use sn0int_std::ratelimits;
 pub mod registry;
 pub mod repl;
 pub mod runtime;
@@ -41,17 +43,12 @@ pub mod sandbox;
 pub mod schema;
 pub mod ser;
 pub mod shell;
-pub mod sockets;
+use sn0int_std::sockets;
 pub mod term;
 pub mod update;
 pub mod utils;
-pub mod web;
-pub mod websockets;
+use sn0int_std::web;
+use sn0int_std::websockets;
 pub mod worker;
 pub mod workspaces;
-pub mod xml;
-
-#[cfg(test)]
-fn test_init() {
-    let _ = env_logger::builder().is_test(true).try_init();
-}
+use sn0int_std::xml;

src/main.rs

@@ -1,13 +1,15 @@
 use env_logger::{self, Env};
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
-use sn0int::cmd;
+use sn0int::cmd::{self, LiteCmd};
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
-use sn0int::engine::{self, Engine, Module};
+use sn0int::engine::Module;
 use sn0int::geoip::{GeoIP, AsnDB, Maxmind};
+use sn0int::ipc;
 use sn0int::options::Opt;
+use sn0int::paths;
 use sn0int::psl::PslReader;
 use sn0int::registry;
 use sn0int::repl;
@@ -32,7 +34,7 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         Module::load(&path.to_path_buf(), "anonymous", &filename, true)
             .context(format!("Failed to parse {:?}", path))?
     } else {
-        rl.engine().get(&args.module)?
+        rl.library().get(&args.module)?
             .clone()
     };
 
@@ -45,17 +47,22 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         rl.set_target(Some(target));
     }
 
-    cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    if args.dump_sandbox_init_msg {
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+    } else {
+        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    }
 }
 
 fn run_sandbox() -> Result<()> {
-    let geoip = GeoIP::try_open_reader()?;
-    let asn = AsnDB::try_open_reader()?;
-    let psl = PslReader::open()?;
+    let cache_dir = paths::cache_dir()?;
+    let geoip = GeoIP::try_open_reader(&cache_dir)?;
+    let asn = AsnDB::try_open_reader(&cache_dir)?;
+    let psl = PslReader::open(&cache_dir)?;
 
     sandbox::init()
         .context("Failed to init sandbox")?;
-    engine::isolation::run_worker(geoip, asn, psl)
+    ipc::child::run(geoip, asn, psl)
 }
 
 fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {
@@ -100,21 +107,26 @@ fn run() -> Result<()> {
         Some(SubCommand::Login(_)) => auth::run_login(&config),
         Some(SubCommand::New(new)) => run_new(&args, &new),
         Some(SubCommand::Publish(publish)) => registry::run_publish(&args, &publish, &config),
-        Some(SubCommand::Install(install)) => registry::run_install(install, &config),
-        Some(SubCommand::Search(search)) => {
-            let engine = Engine::new(false, &config)?;
-            registry::run_search(&engine, &search, &config)
-        },
+        // TODO: deprecate
+        Some(SubCommand::Install(install)) => cmd::pkg_cmd::Args {
+            subcommand: cmd::pkg_cmd::SubCommand::Install(install),
+        }.run(&config),
+        // TODO: deprecate
+        Some(SubCommand::Search(search)) => cmd::pkg_cmd::Args {
+            subcommand: cmd::pkg_cmd::SubCommand::Search(search),
+        }.run(&config),
+        Some(SubCommand::Pkg(pkg)) => pkg.run(&config),
         Some(SubCommand::Add(add)) => run_cmd(&args, add, &config),
         Some(SubCommand::Select(select)) => run_cmd(&args, select, &config),
         Some(SubCommand::Delete(delete)) => run_cmd(&args, delete, &config),
         Some(SubCommand::Activity(activity)) => run_cmd(&args, activity, &config),
         Some(SubCommand::Scope(scope)) => run_cmd(&args, scope, &config),
         Some(SubCommand::Noscope(noscope)) => run_cmd(&args, noscope, &config),
-        Some(SubCommand::Workspace(workspace)) => run_cmd(&args, workspace, &config),
+        Some(SubCommand::Workspace(workspace)) => workspace.run(&config),
         Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
         Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),
         Some(SubCommand::Repl) => repl::run(&config),
+        Some(SubCommand::Paths) => paths::run(&config),
         Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),
         None => shell::run(&args, &config),
     }

src/models/account.rs

@@ -179,12 +179,14 @@ impl DisplayableDetailed for DetailedAccount {
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.displayname)?;
-        w.opt_debug::<Yellow, _>(&self.email)?;
         w.opt_debug::<Yellow, _>(&self.url)?;
-        w.opt_debug::<Yellow, _>(&self.last_seen)?;
-        w.opt_debug::<Yellow, _>(&self.birthday)?;
-        w.opt_debug::<Yellow, _>(&self.phonenumber)?;
-        w.opt_debug::<Yellow, _>(&self.profile_pic)?;
+        w.end_group()?;
+        w.start_group();
+        w.opt_debug_label::<Yellow, _>("last_seen", &self.last_seen)?;
+        w.opt_debug_label::<Yellow, _>("email", &self.email)?;
+        w.opt_debug_label::<Yellow, _>("phonenumber", &self.phonenumber)?;
+        w.opt_debug_label::<Yellow, _>("birthday", &self.birthday)?;
+        w.opt_debug_label::<Yellow, _>("profile_pic", &self.profile_pic)?;
         w.end_group()?;
 
         Ok(())

src/models/cryptoaddr.rs

@@ -165,14 +165,14 @@ pub struct DetailedCryptoAddr {
 }
 
 #[inline]
-fn add_currency(w: &mut fmt::DetailFormatter, num: &Option<i64>, denominator: &Option<i32>) -> fmt::Result {
+fn add_currency(w: &mut fmt::DetailFormatter, label: &str, num: &Option<i64>, denominator: &Option<i32>) -> fmt::Result {
     if let Some(&num) = num.as_ref() {
         let denominator = denominator.unwrap_or(0);
         let display = display_currency(num as u64, denominator as usize);
         if num > 0 {
-            w.display::<Green, _>(display)?;
+            w.display_label::<Green, _>(label, display)?;
         } else {
-            w.display::<Red, _>(display)?;
+            w.display_label::<Red, _>(label, display)?;
         }
     }
     Ok(())
@@ -191,14 +191,14 @@ impl DisplayableDetailed for DetailedCryptoAddr {
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.currency)?;
-        add_currency(w, &self.balance, &self.denominator)?;
-        add_currency(w, &self.received, &self.denominator)?;
+        add_currency(w, "balance", &self.balance, &self.denominator)?;
+        add_currency(w, "received", &self.received, &self.denominator)?;
         w.end_group()?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.first_seen)?;
-        w.opt_debug::<Yellow, _>(&self.last_withdrawal)?;
-        w.opt_debug::<Yellow, _>(&self.description)?;
+        w.opt_debug_label::<Yellow, _>("first_seen", &self.first_seen)?;
+        w.opt_debug_label::<Yellow, _>("last_withdrawal", &self.last_withdrawal)?;
+        w.opt_debug_label::<Yellow, _>("description", &self.description)?;
         w.end_group()?;
 
         Ok(())

src/models/device.rs

@@ -171,15 +171,15 @@ impl DisplayableDetailed for DetailedDevice {
         w.debug::<Green, _>(&self.value)?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.name)?;
-        w.opt_debug::<Yellow, _>(&self.hostname)?;
-        w.opt_debug::<Yellow, _>(&self.vendor)?;
-        w.opt_debug::<Yellow, _>(&self.last_seen)?;
+        w.opt_debug_label::<Yellow, _>("name", &self.name)?;
+        w.opt_debug_label::<Yellow, _>("hostname", &self.hostname)?;
+        w.opt_debug_label::<Yellow, _>("vendor", &self.vendor)?;
+        w.opt_debug_label::<Yellow, _>("last_seen", &self.last_seen)?;
         w.end_group()?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.network)?;
-        w.opt_debug::<Yellow, _>(&self.ipaddr)?;
+        w.opt_debug_label::<Yellow, _>("network", &self.network)?;
+        w.opt_debug_label::<Yellow, _>("ipaddr", &self.ipaddr)?;
         w.end_group()?;
 
         Ok(())

src/paths.rs

@@ -1,3 +1,4 @@
+use crate::config::Config;
 use crate::errors::*;
 use crate::workspaces::Workspace;
 
@@ -6,7 +7,7 @@ use std::fs;
 use std::path::PathBuf;
 
 
-pub fn data_dir() -> Result<PathBuf> {
+pub fn sn0int_dir() -> Result<PathBuf> {
     let path = dirs::data_dir()
         .ok_or_else(|| format_err!("Failed to find data directory"))?;
     let path = path.join("sn0int");
@@ -16,24 +17,39 @@ pub fn data_dir() -> Result<PathBuf> {
 }
 
 pub fn history_path() -> Result<PathBuf> {
-    let path = data_dir()?;
+    let path = sn0int_dir()?;
     let path = path.join("history");
     Ok(path)
 }
 
 pub fn module_dir() -> Result<PathBuf> {
-    let path = data_dir()?;
+    let path = sn0int_dir()?;
     let path = path.join("modules");
     fs::create_dir_all(&path)
         .context("Failed to create module directory")?;
     Ok(path)
 }
 
+pub fn data_dir() -> Result<PathBuf> {
+    let path = sn0int_dir()?
+        .join("data");
+    fs::create_dir_all(&path)
+        .context("Failed to create module directory")?;
+    Ok(path)
+}
+
+pub fn workspace_dir(workspace: &Workspace) -> Result<PathBuf> {
+    let path = sn0int_dir()?
+        .join("data")
+        .join(workspace.as_str());
+    fs::create_dir_all(&path)
+        .context("Failed to create module directory")?;
+    Ok(path)
+}
+
 pub fn blobs_dir(workspace: &Workspace) -> Result<PathBuf> {
-    let path = data_dir()?;
-    let path = path
-        .join("blobs")
-        .join(workspace.to_string());
+    let path = workspace_dir(workspace)?
+        .join("blobs");
     fs::create_dir_all(&path)
         .context("Failed to create module directory")?;
     Ok(path)
@@ -47,3 +63,20 @@ pub fn cache_dir() -> Result<PathBuf> {
         .context("Failed to create cache directory")?;
     Ok(path)
 }
+
+fn print_path<D: std::fmt::Debug>(k: &str, v: D) {
+    println!("{:30}: {:?}", k, v);
+}
+
+pub fn run(config: &Config) -> Result<()> {
+    print_path("config_file", Config::path()?);
+    print_path("data_dir", data_dir()?);
+    print_path("module_dir", module_dir()?);
+
+    for (k, v) in &config.namespaces {
+        print_path(&format!("modules({})", k), v);
+    }
+
+    print_path("cache_dir", cache_dir()?);
+    Ok(())
+}

src/registry.rs

@@ -3,7 +3,7 @@ use crate::args::{Args, Publish, Install, Search};
 use crate::api::Client;
 use crate::auth;
 use crate::config::Config;
-use crate::engine::{Engine, Module};
+use crate::engine::{Library, Module};
 use colored::Colorize;
 use separator::Separatable;
 use sn0int_common::ModuleID;
@@ -242,7 +242,7 @@ impl Task for UpdateTask {
     }
 }
 
-pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<()> {
+pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result<()> {
     let client = Client::new(&config)?;
 
     let label = format!("Searching {:?}", search.query);
@@ -252,7 +252,7 @@ pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<(
 
     for module in &modules {
         let canonical = module.canonical();
-        let installed = engine.get_opt(&canonical)?;
+        let installed = library.get_opt(&canonical)?;
 
         if search.new && installed.is_some() {
             continue;

src/repl/complete.rs

@@ -18,6 +18,7 @@ impl ReplCompleter {
 }
 
 impl rustyline::Helper for ReplCompleter {}
+impl rustyline::validate::Validator for ReplCompleter {}
 
 impl Completer for ReplCompleter {
     type Candidate = String;

src/repl/mod.rs

@@ -1,9 +1,11 @@
 use crate::config::Config;
 use crate::errors::*;
-use crate::engine::{ctx, Environment, DummyReporter};
+use crate::engine::{ctx, Environment};
+use crate::ipc::child::DummyIpcChild;
 use crate::engine::ctx::{State, LuaState};
 use crate::geoip::{Maxmind, AsnDB, GeoIP};
 use crate::hlua::{Lua, AnyLuaValue};
+use crate::paths;
 use crate::psl::PslReader;
 use crate::shell::readline::Readline;
 use crate::runtime::format_lua;
@@ -82,12 +84,14 @@ pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
     let dns_config = Resolver::from_system()?;
     let proxy = config.network.proxy;
-    let psl = PslReader::open()?;
-    let geoip = GeoIP::try_open_reader()?;
-    let asn = AsnDB::try_open_reader()?;
+
+    let cache_dir = paths::cache_dir()?;
+    let psl = PslReader::open(&cache_dir)?;
+    let geoip = GeoIP::try_open_reader(&cache_dir)?;
+    let asn = AsnDB::try_open_reader(&cache_dir)?;
 
     let env = Environment {
-        verbose: 0, // this doesn't do anything since we use a dummy reporter
+        verbose: 0, // this doesn't do anything since we use a dummy ipc child
         keyring,
         dns_config,
         proxy,
@@ -98,7 +102,7 @@ pub fn run(config: &Config) -> Result<()> {
         asn,
     };
 
-    let tx = DummyReporter::new();
+    let tx = DummyIpcChild::new();
     let (lua, state) = ctx::ctx(env, tx);
     let mut repl = Repl::new(lua, state);
 

src/runtime/blobs.rs

@@ -1,12 +1,14 @@
 use crate::errors::*;
-use crate::blobs::Blob;
 use crate::engine::ctx::State;
 use crate::engine::structs::byte_array;
 use crate::hlua::{self, AnyLuaValue};
+use sn0int_std::blobs::{Blob, BlobState};
 use std::sync::Arc;
 
 
-pub fn create_blob(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn create_blob<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + BlobState + 'static
+{
     lua.set("create_blob", hlua::function1(move |bytes: AnyLuaValue| -> Result<String> {
         let bytes = byte_array(bytes)
             .map_err(|err| state.set_error(err))?;

src/runtime/db.rs

@@ -8,6 +8,7 @@ use crate::hlua::{self, AnyLuaValue};
 use std::str::FromStr;
 use std::sync::Arc;
 use crate::models::*;
+use crate::worker::DatabaseResponse;
 use crate::json::LuaJsonValue;
 
 
@@ -85,8 +86,18 @@ pub fn db_add(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let object = into_insert(family, object, &state)
             .map_err(|e| state.set_error(e))?;
 
-        state.db_insert(object)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_insert(object)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Found is technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
@@ -99,13 +110,23 @@ pub fn db_add_ttl(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let object = into_insert(family, object, &state)
             .map_err(|e| state.set_error(e))?;
 
-        state.db_insert_ttl(object, ttl)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_insert_ttl(object, ttl)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Found is technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
 pub fn db_activity(lua: &mut hlua::Lua, state: Arc<dyn State>) {
-    lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<()> {
+    lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<bool> {
         let v: LuaJsonValue = v.into();
         let v: serde_json::Value = v.into();
         let activity: InsertActivity = serde_json::from_value(v)
@@ -121,8 +142,18 @@ pub fn db_select(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let family = Family::from_str(&family)
             .map_err(|e| state.set_error(e.into()))?;
 
-        state.db_select(family, value)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_select(family, value)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Everything except Found and None are technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
@@ -191,8 +222,18 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             .map_err(|e| state.set_error(e))?;
 
         if update.is_dirty() {
-            state.db_update(value, update)
-                .map_err(|e| state.set_error(e))
+            let r = state.db_update(value, update)
+                .map_err(|e| state.set_error(e))?;
+
+            let r = match r {
+                // Inserted and Found are technically unreachable
+                DatabaseResponse::Inserted(id) => Some(id),
+                DatabaseResponse::Updated(id) => Some(id),
+                DatabaseResponse::NoChange(id) => Some(id),
+                DatabaseResponse::Found(id) => Some(id),
+                DatabaseResponse::None => None,
+            };
+            Ok(r)
         } else {
             Ok(Some(id))
         }

src/runtime/geo.rs

@@ -0,0 +1,126 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::geo::{self, Point};
+use crate::hlua::{self, AnyLuaValue};
+use std::sync::Arc;
+
+
+pub fn geo_polygon_contains(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("geo_polygon_contains", hlua::function2(move |polygon: Vec<AnyLuaValue>, point: AnyLuaValue| -> Result<bool> {
+        let polygon = polygon.into_iter()
+            .map(|p| Point::try_from(p))
+            .collect::<Result<Vec<_>>>()
+            .context("Invalid polygon")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        let point = Point::try_from(point)
+            .context("Invalid point")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        Ok(geo::polygon_contains(&polygon, &point))
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::engine::ctx::Script;
+
+    fn hamburg_polygon() -> &'static str {
+        "
+            { lat=53.63975308945899,  lon=9.764785766601562  },
+            { lat=53.59494998253459,  lon=9.827270507812     },
+            { lat=53.663153974456456, lon=9.9151611328125    },
+            { lat=53.65582987649682,  lon=9.976272583007812  },
+            { lat=53.68613523817129,  lon=9.992752075195312  },
+            { lat=53.68674518938816,  lon=10.051460266113281 },
+            { lat=53.72495117617815,  lon=10.075492858886719 },
+            { lat=53.71946627930625,  lon=10.118408203125    },
+            { lat=53.743635083157756, lon=10.164413452148438 },
+            { lat=53.73104466704585,  lon=10.202865600585938 },
+            { lat=53.676781546441546, lon=10.16304016113281  },
+            { lat=53.632832079199474, lon=10.235824584960938 },
+            { lat=53.608803292930894, lon=10.2008056640625   },
+            { lat=53.578646152866504, lon=10.208358764648438 },
+            { lat=53.57212285981298,  lon=10.163726806640625 },
+            { lat=53.52071674896369,  lon=10.18707275390625  },
+            { lat=53.52643162253097,  lon=10.224151611328125 },
+            { lat=53.44062753992289,  lon=10.347747802734375 },
+            { lat=53.38824275010831,  lon=10.248870849609375 },
+            { lat=53.38824275010831,  lon=10.15960693359375  },
+            { lat=53.44635321212876,  lon=10.064849853515625 },
+            { lat=53.40595029739904,  lon=9.985198974609375  },
+            { lat=53.42385506057106,  lon=9.951210021972656  },
+            { lat=53.41843327091211,  lon=9.944171905517578  },
+            { lat=53.41812635648326,  lon=9.927349090576172  },
+            { lat=53.412294561442884, lon=9.917736053466797  },
+            { lat=53.41464783813818,  lon=9.901256561279297  },
+            { lat=53.443490472483326, lon=9.912586212158201  },
+            { lat=53.45177144115704,  lon=9.897651672363281  },
+            { lat=53.43633277935392,  lon=9.866924285888672  },
+            { lat=53.427639673754776, lon=9.866409301757812  },
+            { lat=53.427639673754776, lon=9.858856201171875  },
+            { lat=53.46710230573499,  lon=9.795513153076172  },
+            { lat=53.49039461941655,  lon=9.795341491699219  },
+            { lat=53.49029248806277,  lon=9.77903366088867   },
+            { lat=53.49856433088649,  lon=9.780235290527344  },
+            { lat=53.5078554643033,   lon=9.758434295654297  },
+            { lat=53.545407634092975, lon=9.759807586669922  },
+            { lat=53.568147234570084, lon=9.633293151855469  },
+            { lat=53.58802162343514,  lon=9.655780792236328  },
+            { lat=53.568351121879815, lon=9.727706909179688  },
+            { lat=53.60921067445695,  lon=9.737663269042969  },
+        "
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_contains_hamburg() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=53.551085,
+                lon=9.993682,
+            }})
+            if last_err() then return end
+            if not x then
+                return 'hamburg should be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_not_contains_berlin() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=52.52437,
+                lon=13.41053,
+            }})
+            if last_err() then return end
+            if x then
+                return 'berlin should not be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_not_contains_ny() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=40.726662,
+                lon=-74.036677,
+            }})
+            if last_err() then return end
+            if x then
+                return 'ny should not be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+}

src/runtime/http.rs

@@ -3,6 +3,8 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::hlua::{self, AnyLuaValue, AnyHashableLuaValue};
 use crate::json;
+use sn0int_std::blobs::BlobState;
+use sn0int_std::web::WebState;
 use std::sync::Arc;
 use std::collections::HashMap;
 use crate::web::{RequestOptions, HttpRequest};
@@ -25,7 +27,9 @@ pub fn http_request(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
-pub fn http_send(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn http_send<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + WebState + BlobState + 'static
+{
     lua.set("http_send", hlua::function1(move |request: AnyLuaValue| -> Result<HashMap<AnyHashableLuaValue, AnyLuaValue>> {
         let req = HttpRequest::try_from(request)
             .context("invalid http request object")
@@ -40,7 +44,9 @@ pub fn http_send(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
-pub fn http_fetch(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn http_fetch<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + WebState + BlobState + 'static
+{
     lua.set("http_fetch", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
         let req = HttpRequest::try_from(request)
             .context("invalid http request object")
@@ -59,7 +65,9 @@ pub fn http_fetch(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
-pub fn http_fetch_json(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn http_fetch_json<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + WebState + 'static
+{
     lua.set("http_fetch_json", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
         let req = HttpRequest::try_from(request)
             .context("invalid http request object")

src/runtime/mod.rs

@@ -11,6 +11,7 @@ import_fns!(db);
 import_fns!(dns);
 import_fns!(encoding);
 import_fns!(error);
+import_fns!(geo);
 import_fns!(geoip);
 import_fns!(gfx);
 import_fns!(hashes);
@@ -30,6 +31,7 @@ import_fns!(semver);
 import_fns!(sleep);
 import_fns!(sock);
 import_fns!(stdio);
+import_fns!(str);
 import_fns!(url);
 import_fns!(utf8);
 import_fns!(websockets);

src/runtime/str.rs

@@ -0,0 +1,121 @@
+use crate::errors::*;
+use crate::engine::ctx::State;
+use crate::hlua::{self, AnyLuaValue};
+use std::sync::Arc;
+use crate::json;
+
+
+pub fn str_find(lua: &mut hlua::Lua, _state: Arc<dyn State>) {
+    lua.set("str_find", hlua::function2(move |s: String, needle: String| -> Option<u32> {
+        s.find(&needle)
+            .map(|i| i as u32 + 1)
+    }))
+}
+
+pub fn str_replace(lua: &mut hlua::Lua, _state: Arc<dyn State>) {
+    lua.set("str_replace", hlua::function3(move |s: String, search: String, replace: String| -> String {
+        s.replace(&search, &replace)
+    }))
+}
+
+pub fn strval(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("strval", hlua::function1(move |x: AnyLuaValue| -> Result<AnyLuaValue> {
+        let x = match x {
+            AnyLuaValue::LuaNil => x,
+            AnyLuaValue::LuaBoolean(v) => if v {
+                AnyLuaValue::LuaString("true".into())
+            } else {
+                AnyLuaValue::LuaString("false".into())
+            },
+            AnyLuaValue::LuaString(_) => x,
+            AnyLuaValue::LuaNumber(v) => {
+                // this is needed or every number is detected as float
+                let v = if v % 1f64 == 0f64 {
+                    (v as u64).to_string()
+                } else {
+                    v.to_string()
+                };
+                AnyLuaValue::LuaString(v)
+            },
+            AnyLuaValue::LuaAnyString(_) => x,
+            AnyLuaValue::LuaArray(_) => {
+                let v = json::encode(x)
+                    .map_err(|err| state.set_error(err))?;
+                AnyLuaValue::LuaString(v)
+            },
+            AnyLuaValue::LuaOther => x,
+        };
+        Ok(x)
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::engine::ctx::Script;
+
+    #[test]
+    fn verify_str_find() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = str_find('asdf', 'asdf')
+            if x ~= 1 then return 'error 1' end
+            x = str_find('asdf', 'a')
+            if x ~= 1 then return 'error 2' end
+            x = str_find('asdf', 's')
+            if x ~= 2 then return 'error 3' end
+            x = str_find('asdf', 'x')
+            if x ~= nil then return 'error 4' end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_str_replace() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = str_replace('asdf', 'asdf', '')
+            if x ~= '' then return 'error 1: ' .. x end
+
+            x = str_replace('asdf', '', 'x')
+            if x ~= 'xaxsxdxfx' then return 'error 2: ' .. x end
+
+            x = str_replace('asdf', 's', 'x')
+            if x ~= 'axdf' then return 'error 3: ' .. x end
+
+            x = str_replace('asdf', 'sd', 'xxx')
+            if x ~= 'axxxf' then return 'error 4: ' .. x end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_strval() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = strval("asdf")
+            if x ~= "asdf" then return 'error 1: ' .. x end
+
+            x = strval(1234)
+            if x ~= "1234" then return 'error 2: ' .. x end
+
+            x = strval(313.37)
+            if x ~= "313.37" then return 'error 3: ' .. x end
+
+            x = strval(true)
+            if x ~= "true" then return 'error 4: ' .. x end
+
+            x = strval(false)
+            if x ~= "false" then return 'error 5: ' .. x end
+
+            x = strval(nil)
+            if x ~= nil then return 'error 6: ' .. x end
+
+            x = strval({a=1})
+            if x ~= '{"a":1}' then return 'error 7: ' .. x end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+}

src/sandbox/seccomp.rs

@@ -17,6 +17,10 @@ pub fn init() -> Result<()> {
     ctx.allow_syscall(Syscall::fcntl64)?;
     ctx.allow_syscall(Syscall::uname)?;
     ctx.allow_syscall(Syscall::close)?;
+    #[cfg(not(target_arch = "aarch64"))]
+    ctx.allow_syscall(Syscall::poll)?;
+    #[cfg(target_arch = "aarch64")]
+    ctx.allow_syscall(Syscall::ppoll)?;
     ctx.allow_syscall(Syscall::epoll_create1)?;
     ctx.allow_syscall(Syscall::pipe2)?;
     ctx.allow_syscall(Syscall::epoll_ctl)?;

src/shell/complete.rs

@@ -190,6 +190,21 @@ impl Completer for CmdCompleter {
                         ], &cmd[1]))
                     }
                 },
+                Command::Pkg => {
+                    // we can only complete the 2nd argument
+                    if args != 2 {
+                        Ok((0, vec![]))
+                    } else {
+                        Ok(filter_options("pkg", &[
+                            "list",
+                            "install",
+                            "search",
+                            "reload",
+                            "update",
+                            "uninstall",
+                        ], &cmd[1]))
+                    }
+                },
                 Command::Noscope => self.filter("noscope", &cmd),
                 Command::Use => {
                     // we can only complete the 2nd argument
@@ -257,6 +272,7 @@ impl Highlighter for CmdCompleter {
 }
 
 impl rustyline::Helper for CmdCompleter {}
+impl rustyline::validate::Validator for CmdCompleter {}
 
 pub fn run_generate(args: &Completions) -> Result<()> {
     Args::clap().gen_completions_to("sn0int", args.shell, &mut stdout());

src/shell/mod.rs

@@ -6,10 +6,10 @@ use crate::cmd::*;
 use crate::config::Config;
 use crate::db::ttl;
 use crate::keyring::KeyRing;
-use crate::worker::VoidSender;
+use crate::worker::{self, VoidSender};
 use colored::Colorize;
 use crate::db::{self, Database};
-use crate::engine::{Engine, Module};
+use crate::engine::{Library, Module};
 use crate::update::AutoUpdater;
 use std::collections::HashMap;
 use std::str::FromStr;
@@ -39,6 +39,7 @@ pub enum Command {
     Keyring,
     Mod,
     Noscope,
+    Pkg,
     Run,
     Scope,
     Set,
@@ -48,6 +49,9 @@ pub enum Command {
     Quickstart,
     Workspace,
 
+    Exit,
+    Quit,
+
     Interrupt,
 }
 
@@ -60,10 +64,12 @@ impl Command {
             Command::Autoscope => "autoscope",
             Command::Back => "back",
             Command::Delete => "delete",
+            Command::Exit => "exit",
             Command::Help => "help",
             Command::Keyring => "keyring",
             Command::Mod => "mod",
             Command::Noscope => "noscope",
+            Command::Pkg => "pkg",
             Command::Run => "run",
             Command::Scope => "scope",
             Command::Set => "set",
@@ -71,6 +77,7 @@ impl Command {
             Command::Target => "target",
             Command::Use => "use",
             Command::Quickstart => "quickstart",
+            Command::Quit => "quit",
             Command::Workspace => "workspace",
             Command::Interrupt => unreachable!(),
         }
@@ -89,6 +96,7 @@ impl Command {
                 Command::Keyring.as_str(),
                 Command::Mod.as_str(),
                 Command::Noscope.as_str(),
+                Command::Pkg.as_str(),
                 Command::Run.as_str(),
                 Command::Scope.as_str(),
                 Command::Set.as_str(),
@@ -115,10 +123,12 @@ impl FromStr for Command {
             "autoscope" => Ok(Command::Autoscope),
             "back" => Ok(Command::Back),
             "delete" => Ok(Command::Delete),
+            "exit" => Ok(Command::Exit),
             "help" => Ok(Command::Help),
             "keyring" => Ok(Command::Keyring),
             "mod" => Ok(Command::Mod),
             "noscope" => Ok(Command::Noscope),
+            "pkg"  => Ok(Command::Pkg),
             "run"  => Ok(Command::Run),
             "scope"  => Ok(Command::Scope),
             "set"  => Ok(Command::Set),
@@ -126,6 +136,7 @@ impl FromStr for Command {
             "target"  => Ok(Command::Target),
             "use"  => Ok(Command::Use),
             "quickstart"  => Ok(Command::Quickstart),
+            "quit" => Ok(Command::Quit),
             "workspace" => Ok(Command::Workspace),
             x => bail!("unknown command: {:?}, try \"help\"", x),
         }
@@ -139,7 +150,7 @@ pub struct Shell<'a> {
     blobs: BlobStorage,
     psl: Lazy<PslReader, Arc<Psl>>,
     config: &'a Config,
-    engine: Engine<'a>,
+    library: Library<'a>,
     keyring: KeyRing,
     // autonoscope: RuleSet,
     options: Option<HashMap<String, String>>,
@@ -147,7 +158,7 @@ pub struct Shell<'a> {
 }
 
 impl<'a> Shell<'a> {
-    pub fn new(config: &'a Config, db: Database, blobs: BlobStorage, psl: PslReader, engine: Engine<'a>, keyring: KeyRing) -> Shell<'a> {
+    pub fn new(config: &'a Config, db: Database, blobs: BlobStorage, psl: PslReader, library: Library<'a>, keyring: KeyRing) -> Shell<'a> {
         let h = CmdCompleter::default();
         let rl = Readline::with(h);
 
@@ -160,7 +171,7 @@ impl<'a> Shell<'a> {
             blobs,
             psl: Lazy::from(psl),
             config,
-            engine,
+            library,
             keyring,
             options: None,
             signal_register: Arc::new(SignalRegister::new()),
@@ -250,13 +261,13 @@ impl<'a> Shell<'a> {
     }
 
     #[inline(always)]
-    pub fn engine(&self) -> &Engine {
-        &self.engine
+    pub fn library(&self) -> &Library {
+        &self.library
     }
 
     #[inline(always)]
-    pub fn engine_mut(&mut self) -> &mut Engine<'a> {
-        &mut self.engine
+    pub fn library_mut(&mut self) -> &mut Library<'a> {
+        &mut self.library
     }
 
     #[inline(always)]
@@ -318,10 +329,27 @@ impl<'a> Shell<'a> {
         }
     }
 
+    pub fn reload_modules(&mut self) -> Result<()> {
+        let current = self.take_module()
+                        .map(|m| m.canonical());
+
+        self.library_mut().reload_modules()?;
+        self.reload_module_cache();
+
+        if let Some(module) = current {
+            if let Ok(module) = self.library().get(&module) {
+                let module = module.clone();
+                self.set_module(module);
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn reload_module_cache(&mut self) {
         if let Some(helper) = self.rl.helper_mut() {
             helper.modules.clear();
-            for module in self.engine.variants() {
+            for module in self.library.variants() {
                 helper.modules.push(module);
             }
         }
@@ -433,8 +461,9 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
         Some((Command::Delete, args)) => delete_cmd::run(rl, &args)?,
         Some((Command::Help, args)) => help_cmd::run(rl, &args)?,
         Some((Command::Keyring, args)) => keyring_cmd::run(rl, &args)?,
-        Some((Command::Mod, args)) => mod_cmd::run(rl, &args)?,
+        Some((Command::Mod, args)) => cmd::<pkg_cmd::ArgsInteractive>(rl, &args)?,
         Some((Command::Noscope, args)) => noscope_cmd::run(rl, &args)?,
+        Some((Command::Pkg, args)) => cmd::<pkg_cmd::ArgsInteractive>(rl, &args)?,
         Some((Command::Run, args)) => cmd::<run_cmd::Args>(rl, &args)?,
         Some((Command::Scope, args)) => scope_cmd::run(rl, &args)?,
         Some((Command::Set, args)) => set_cmd::run(rl, &args)?,
@@ -442,7 +471,10 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
         Some((Command::Target, args)) => target_cmd::run(rl, &args)?,
         Some((Command::Use, args)) => use_cmd::run(rl, &args)?,
         Some((Command::Quickstart, args)) => quickstart_cmd::run(rl, &args)?,
-        Some((Command::Workspace, args)) => workspace_cmd::run(rl, &args)?,
+        Some((Command::Workspace, args)) => cmd::<workspace_cmd::Args>(rl, &args)?,
+
+        Some((Command::Exit, _)) => return Ok(true),
+        Some((Command::Quit, _)) => return Ok(true),
         Some((Command::Interrupt, _)) => return Ok(true),
         None => (),
     }
@@ -456,6 +488,8 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
         None => Workspace::from_str("default").unwrap(),
     };
 
+    workspace.migrate()?;
+
     let blobs = BlobStorage::workspace(&workspace)?;
     let db = if verbose_init {
         Database::establish(workspace)?
@@ -464,12 +498,14 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
     };
     ttl::reap_expired(&db)?;
 
-    let psl = PslReader::open_or_download()
+    let cache_dir = paths::cache_dir()?;
+    let psl = PslReader::open_or_download(&cache_dir,
+            |cb| worker::spawn_fn("Downloading public suffix list", cb, false))
         .context("Failed to download public suffix list")?;
-    let engine = Engine::new(verbose_init, &config)?;
+    let library = Library::new(verbose_init, &config)?;
     let keyring = KeyRing::init()?;
 
-    if verbose_init && engine.list().is_empty() {
+    if verbose_init && library.list().is_empty() {
         term::success("No modules found, run quickstart to install default modules");
     }
 
@@ -477,9 +513,9 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
     if autoupdate.outdated() > 0 {
         term::warn(&format!("{} modules are outdated, run: mod update", autoupdate.outdated()));
     }
-    autoupdate.check_background(&config, engine.list());
+    autoupdate.check_background(&config, library.list());
 
-    let rl = Shell::new(&config, db, blobs, psl, engine, keyring);
+    let rl = Shell::new(&config, db, blobs, psl, library, keyring);
 
     Ok(rl)
 }

src/update.rs

@@ -26,7 +26,7 @@ pub struct AutoUpdater {
 impl AutoUpdater {
     #[inline]
     fn path() -> Result<PathBuf> {
-        let path = paths::data_dir()?;
+        let path = paths::sn0int_dir()?;
         Ok(path.join("autoupdate.json"))
     }
 

src/worker.rs

@@ -5,12 +5,14 @@ use crate::channel;
 use crate::cmd::run_cmd::Params;
 use crate::db::{Database, DbChange, Family};
 use crate::db::ttl::Ttl;
-use crate::engine::{self, Module};
-use crate::engine::isolation::Supervisor;
+use crate::engine::Module;
+use crate::ipc;
+use crate::ipc::parent::IpcParent;
 use crate::models::*;
 use serde_json;
 use crate::ratelimits::{Ratelimiter, RatelimitResponse};
 use crate::shell::Shell;
+use sn0int_std::ratelimits::RatelimitSender;
 use std::collections::HashMap;
 use std::result;
 use std::sync::{mpsc, Arc, Mutex};
@@ -22,9 +24,17 @@ use crate::term::{Spinner, StackedSpinners, SpinLogger};
 use threadpool::ThreadPool;
 
 
-type DbSender = mpsc::Sender<result::Result<Option<i32>, String>>;
+type DbSender = mpsc::Sender<result::Result<DatabaseResponse, String>>;
 pub type VoidSender = mpsc::Sender<result::Result<(), String>>;
-pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum DatabaseResponse {
+    Inserted(i32),
+    Updated(i32),
+    Found(i32),
+    NoChange(i32),
+    None,
+}
 
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Event {
@@ -150,7 +160,7 @@ pub enum DatabaseEvent {
 }
 
 impl EventWithCallback for DatabaseEvent {
-    type Payload = Option<i32>;
+    type Payload = DatabaseResponse;
 
     fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
         Event2::Database((self, tx))
@@ -176,11 +186,11 @@ impl DatabaseEvent {
 
                 // TODO: replace id with actual object(?)
                 if let Ok(obj) = object.printable(db) {
-                    spinner.log(&obj.to_string());
+                    spinner.log(&obj.to_string()); // TODO: also include fields here
                 } else {
                     spinner.error(&format!("Failed to query necessary fields for {:?}", object));
                 }
-                Ok(Some(id))
+                Ok(DatabaseResponse::Inserted(id))
             },
             Ok(Some((DbChange::Update(update), id))) => {
                 if let Some(ttl) = ttl {
@@ -199,7 +209,7 @@ impl DatabaseEvent {
                         spinner.error(&format!("Failed to get label for {:?}: {:?}", object, err));
                     },
                 }
-                Ok(Some(id))
+                Ok(DatabaseResponse::Updated(id))
             },
             Ok(Some((DbChange::None, id))) => {
                 if let Some(ttl) = ttl {
@@ -208,9 +218,9 @@ impl DatabaseEvent {
                     }
                 }
 
-                Ok(Some(id))
+                Ok(DatabaseResponse::NoChange(id))
             },
-            Ok(None) => Ok(None),
+            Ok(None) => Ok(DatabaseResponse::None),
             Err(err) => {
                 let err = err.to_string();
                 spinner.error(&err);
@@ -242,9 +252,9 @@ impl DatabaseEvent {
                 }
 
                 spinner.log(&log);
-                Ok(None)
+                Ok(DatabaseResponse::Inserted(0))
             },
-            Ok(false) => Ok(None),
+            Ok(false) => Ok(DatabaseResponse::NoChange(0)),
             Err(err) => {
                 let err = err.to_string();
                 spinner.error(&err);
@@ -261,8 +271,11 @@ impl DatabaseEvent {
             DatabaseEvent::InsertTtl((object, ttl)) => Self::insert(object, Some(ttl), tx, spinner, db, verbose),
             DatabaseEvent::Activity(object) => Self::activity(object, tx, spinner, db, verbose),
             DatabaseEvent::Select((family, value)) => {
-                let result = db.get_opt(&family, &value)
-                    .map_err(|e| e.to_string());
+                let result = match db.get_opt(&family, &value) {
+                    Ok(Some(id)) => Ok(DatabaseResponse::Found(id)),
+                    Ok(None) => Ok(DatabaseResponse::None),
+                    Err(e) => Err(e.to_string()),
+                };
 
                 tx.send(result).expect("Failed to send db result to channel");
             },
@@ -274,7 +287,7 @@ impl DatabaseEvent {
                 let result = db.update_generic(&update);
                 debug!("{:?}: {:?} => {:?}", object, update, result);
                 let result = result
-                    .map(Some)
+                    .map(DatabaseResponse::Updated)
                     .map_err(|e| e.to_string());
 
                 if let Err(ref err) = result {
@@ -328,13 +341,13 @@ impl StdioEvent {
         }
     }
 
-    pub fn apply(self, supervisor: &mut Supervisor, tx: &EventSender, reader: &mut Option<BufReader<Stdin>>) {
+    pub fn apply(self, ipc_parent: &mut IpcParent, tx: &EventSender, reader: &mut Option<BufReader<Stdin>>) {
         let reply = match self {
             StdioEvent::Readline => Self::read_line(reader),
             StdioEvent::ToEnd => Self::read_to_end(reader),
         };
         let reply = reply.map_err(|e| e.to_string());
-        supervisor.send_struct(reply, tx);
+        ipc_parent.send_struct(reply, tx);
     }
 }
 
@@ -400,7 +413,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, args: Vec<(serde_json::Value, Opti
             }
 
             tx.send(Event2::Start);
-            let event = match engine::isolation::spawn_module(module, &tx, arg, keyring, verbose, has_stdin, proxy, options, blobs) {
+            let event = match ipc::parent::run(module, &tx, arg, keyring, verbose, has_stdin, proxy, options, blobs) {
                 Ok(exit) => exit,
                 Err(err) => ExitEvent::SetupFailed(err.to_string()),
             };

src/workspaces.rs

@@ -3,13 +3,14 @@ use crate::errors::*;
 use crate::blobs::BlobStorage;
 use crate::paths;
 use regex::Regex;
+use std::collections::HashSet;
 use std::ffi::OsStr;
 use std::fs;
 use std::path::PathBuf;
 use std::str::FromStr;
 
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub struct Workspace {
     s: String,
 }
@@ -17,7 +18,8 @@ pub struct Workspace {
 impl Workspace {
     #[inline]
     pub fn db_path(&self) -> Result<PathBuf> {
-        Ok(paths::data_dir()?.join(self.s.to_string() + ".db"))
+        Ok(paths::workspace_dir(&self)?
+            .join("db.sqlite"))
     }
 
     #[inline]
@@ -38,9 +40,40 @@ impl Workspace {
     }
 
     pub fn delete(&self) -> Result<()> {
-        let blobs = BlobStorage::workspace(self)?;
-        fs::remove_dir_all(blobs.path())?;
-        fs::remove_file(self.db_path()?)?;
+        self.migrate()?;
+
+        let path = paths::workspace_dir(&self)?;
+        fs::remove_dir_all(path)?;
+
+        Ok(())
+    }
+
+    pub fn migrate(&self) -> Result<()> {
+        // relocate old paths
+        let old_db = paths::sn0int_dir()?
+            .join(self.to_string() + ".db");
+        if old_db.exists() {
+            let new_db = paths::workspace_dir(self)?
+                .join("db.sqlite");
+
+            fs::rename(old_db, new_db)
+                .context("Failed to migrate old db to new location")?;
+        }
+
+        let old_blobs_parent = paths::sn0int_dir()?
+            .join("blobs");
+        let old_blobs = old_blobs_parent
+            .join(self.as_str());
+        if old_blobs.exists() {
+            let new_blobs = paths::workspace_dir(self)?
+                .join("blobs");
+            fs::rename(old_blobs, new_blobs)
+                .context("Failed to migrate old blob folder to new location")?;
+        }
+
+        // cleanup
+        fs::remove_dir(old_blobs_parent).ok();
+
         Ok(())
     }
 }
@@ -76,9 +109,26 @@ impl Deref for Workspace {
 }
 
 pub fn list() -> Result<Vec<Workspace>> {
-    let mut workspaces = Vec::new();
+    let mut workspaces = HashSet::new();
 
     for entry in fs::read_dir(paths::data_dir()?)? {
+        let entry = entry?;
+        let path = entry.path();
+        if !path.is_dir() {
+            continue;
+        }
+
+        let name = path.file_name()
+            .ok_or_else(|| format_err!("read_dir returned file with no name"))?;
+        let name = name.to_str()
+            .ok_or_else(|| format_err!("Workspace has invalid name: {:?}", name))?;
+
+        if let Ok(workspace) = Workspace::from_str(name) {
+            workspaces.insert(workspace);
+        }
+    }
+
+    for entry in fs::read_dir(paths::sn0int_dir()?)? {
         let entry = entry?;
         let path = entry.path();
         if path.is_dir() {
@@ -97,10 +147,12 @@ pub fn list() -> Result<Vec<Workspace>> {
             .ok_or_else(|| format_err!("Workspace has invalid name: {:?}", name))?;
 
         if let Ok(workspace) = Workspace::from_str(name) {
-            workspaces.push(workspace);
+            workspaces.insert(workspace);
         }
     }
 
+    let mut workspaces = workspaces.into_iter().collect::<Vec<_>>();
+    workspaces.sort();
     Ok(workspaces)
 }