From ecd843c74f8b1cd37cbbe4de1fe5993cd9e4917d Mon Sep 17 00:00:00 2001 From: kpcyrd Date: Sat, 7 Mar 2020 14:59:05 +0100 Subject: [PATCH] docs: extract autonoscope section --- README.md | 13 ++++--- docs/autonoscope.rst | 81 +++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/reference.rst | 4 +++ docs/usage.rst | 82 -------------------------------------------- 5 files changed, 95 insertions(+), 86 deletions(-) create mode 100644 docs/autonoscope.rst diff --git a/README.md b/README.md index 8b49470..4008770 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,10 @@ For everything else please have a look at the [detailed list][1]. - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module) - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results) - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities) - - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope) - - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains) - - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips) - - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls) +- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html) + - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains) + - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips) + - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls) - [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html) - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository) - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module) @@ -100,6 +100,8 @@ For everything else please have a look at the [detailed list][1]. - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin) - [Database](https://sn0int.readthedocs.io/en/latest/database.html) - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add) + - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl) + - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity) - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update) - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select) - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html) @@ -154,6 +156,7 @@ For everything else please have a look at the [detailed list][1]. - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime) - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add) - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl) + - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity) - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select) - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update) - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns) @@ -177,6 +180,7 @@ For everything else please have a look at the [detailed list][1]. - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif) - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity) - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info) + - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval) - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode) - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream) - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode) @@ -222,6 +226,7 @@ For everything else please have a look at the [detailed list][1]. - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace) - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime) - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime) + - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval) - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix) - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode) - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode) diff --git a/docs/autonoscope.rst b/docs/autonoscope.rst new file mode 100644 index 0000000..969a80f --- /dev/null +++ b/docs/autonoscope.rst @@ -0,0 +1,81 @@ +Autonoscope +=========== + +Instead of manually unscoping everything you can also define so called +autonoscope rules. Those are executed from most specific to least specific and +the first match wins. If no rule matches, the default is in-scope:: + + [sn0int][demo] > # add the domain first + [sn0int][demo] > # this is necessary because we only want to partially unscope example.com + [sn0int][demo] > add domain example.com + [sn0int][demo] > + [sn0int][demo] > # automatically noscope all subdomains + [sn0int][demo] > autonoscope add domain example.com + [sn0int][demo] > # except subdomains of prod.example.com + [sn0int][demo] > autoscope add domain prod.example.com + [sn0int][demo] > + [sn0int][demo] > autonoscope list + scope domain "prod.example.com" + noscope domain "example.com" + [sn0int][demo] > + [sn0int][demo] > # this is going to be out-of-scope + [sn0int][demo] > add subdomain www.example.com + [sn0int][demo] > # this is going to be in-scope + [sn0int][demo] > add subdomain db.prod.example.com + [sn0int][demo] > + [sn0int][demo] > select subdomains + #1, "www.example.com" + #2, "db.prod.example.com" + [sn0int][demo] > select subdomains where unscoped=0 + #2, "db.prod.example.com" + [sn0int][demo] > select subdomains where unscoped=1 + #1, "www.example.com" + [sn0int][demo] > + +Domains +------- + +Autonoscope rules for domains are applied to the following structs: + +- domains +- subdomains +- urls + +Example rules:: + + autonoscope add domain example.com + autonoscope add domain staging.example.com + autonoscope add domain com + autonoscope add domain . + +IPs +--- + +Autonoscope rules for IPs are applied to the following structs: + +- ipaddrs +- netblocks +- ports + +Example rules:: + + autonoscope add ip 0.0.0.0/0 + autonoscope add ip ::/0 + autonoscope add ip 192.168.0.0/16 + autonoscope add ip 10.13.33.37/32 + +URLs +---- + +Autonoscope rules for urls are applied to the following structs: + +- urls + +Note that these rules are specific to a certain origin (like +``https://example.com``) and are used to filter paths. + +Example rules:: + + autonoscope add url https://example.com/ + autonoscope add url https://example.com/admin/ + autonoscope add url https://example.com/a/b/c/d diff --git a/docs/index.rst b/docs/index.rst index 5712413..33bdd01 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,6 +38,7 @@ Getting Started install build usage + autonoscope scripting database structs diff --git a/docs/reference.rst b/docs/reference.rst index 15f0088..9121cdc 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -391,6 +391,8 @@ options are set. The following options are available: ``proxy`` Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works if it doesn't conflict with the global proxy settings. +``binary`` + Set to ``true`` to get the http response as raw bytes. This function may fail. @@ -418,6 +420,8 @@ the following keys: A table of headers ``text`` The response body as string +``binary`` + The response body as bytes (if ``binary=true``) ``blob`` If ``into_blob`` was enabled for the request the body is downloaded into blob storage with a reference to the body in this field. diff --git a/docs/usage.rst b/docs/usage.rst index 1a59c93..651e582 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -256,85 +256,3 @@ You can reverse this using the scope command:: .. hint:: All entities have this field, you can refer to it in queries using ``unscoped=1``. - -Autonoscope ------------ - -Instead of manually unscoping everything you can also define so called -autonoscope rules. Those are executed from most specific to least specific and -the first match wins. If no rule matches, the default is in-scope:: - - [sn0int][demo] > # add the domain first - [sn0int][demo] > # this is necessary because we only want to partially unscope example.com - [sn0int][demo] > add domain example.com - [sn0int][demo] > - [sn0int][demo] > # automatically noscope all subdomains - [sn0int][demo] > autonoscope add domain example.com - [sn0int][demo] > # except subdomains of prod.example.com - [sn0int][demo] > autoscope add domain prod.example.com - [sn0int][demo] > - [sn0int][demo] > autonoscope list - scope domain "prod.example.com" - noscope domain "example.com" - [sn0int][demo] > - [sn0int][demo] > # this is going to be out-of-scope - [sn0int][demo] > add subdomain www.example.com - [sn0int][demo] > # this is going to be in-scope - [sn0int][demo] > add subdomain db.prod.example.com - [sn0int][demo] > - [sn0int][demo] > select subdomains - #1, "www.example.com" - #2, "db.prod.example.com" - [sn0int][demo] > select subdomains where unscoped=0 - #2, "db.prod.example.com" - [sn0int][demo] > select subdomains where unscoped=1 - #1, "www.example.com" - [sn0int][demo] > - -Domains -~~~~~~~ - -Autonoscope rules for domains are applied to the following structs: - -- domains -- subdomains -- urls - -Example rules:: - - autonoscope add domain example.com - autonoscope add domain staging.example.com - autonoscope add domain com - autonoscope add domain . - -IPs -~~~ - -Autonoscope rules for IPs are applied to the following structs: - -- ipaddrs -- netblocks -- ports - -Example rules:: - - autonoscope add ip 0.0.0.0/0 - autonoscope add ip ::/0 - autonoscope add ip 192.168.0.0/16 - autonoscope add ip 10.13.33.37/32 - -URLs -~~~~ - -Autonoscope rules for urls are applied to the following structs: - -- urls - -Note that these rules are specific to a certain origin (like -``https://example.com``) and are used to filter paths. - -Example rules:: - - autonoscope add url https://example.com/ - autonoscope add url https://example.com/admin/ - autonoscope add url https://example.com/a/b/c/d