maq/LinkScope_Client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Modules from Core application.

Browse Source
This commit is contained in:
AccentuSoft
2023-04-28 13:38:59 +03:00
parent 7c3cb92a34
commit 379e81548d
172 changed files with 0 additions and 20869 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
Modules/Aircraft_Inquiry/AircraftInquiryByDealer.py
View File

@@ -1,44 +0,0 @@
#!/usr/bin/env python3
class AircraftInquiryByDealer:
name = "Aircraft Inquiry By Dealer"
category = "Aircraft"
description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
originTypes = {"Company"}
resultTypes = {'Phrase', 'Company'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import pandas as pd
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
futures = []
uidList = []
return_result = []
submit_url = "https://registry.faa.gov/aircraftinquiry/Search/DealerResult"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
futures.append(session.post(submit_url, data={"Dealertxt": entity['Company Name']}))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
df_list = pd.read_html(future.result().text)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
except ValueError:
return "Error occurred when checking the data returned from the endpoint."
df = df_list[0]
for certificate_index in range(len(df["Certificate Number"])):
index_of_child = len(return_result)
return_result.append([{'Company Name': df["Name"][certificate_index],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Aircraft Dealer', 'Notes': ''}}])
return_result.append([{'Phrase': df["Certificate Number"][certificate_index],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Aircraft Certificate Number', 'Notes': ''}}])
return return_result

50
Modules/Aircraft_Inquiry/AircraftInquiryByEngine.py
View File

@@ -1,50 +0,0 @@
#!/usr/bin/env python3
class AircraftInquiryByEngine:
name = "Aircraft Inquiry By Engine"
category = "Aircraft"
description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
originTypes = {"Phrase"}
resultTypes = {"Phrase"}
parameters = {'Manufacturer': {'description': "Enter the Manufacturer of the Engine Model",
'type': 'String',
'value': 'None'}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
import pandas as pd
Manufacturer = parameters['Manufacturer']
futures = []
uidList = []
return_result = []
submit_url = "https://registry.faa.gov/aircraftinquiry/Search/EngineReferenceResult"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
futures.append(session.post(submit_url, data={"Modeltxt": entity['Phrase'],
"MfrNametxt": Manufacturer}))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
df_list = pd.read_html(future.result().text)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
except ValueError:
return "No results retrieved"
df = df_list[0]
return_result.append([{'Phrase': f"Model Code:str({df['Mfr/Mdl Code']})",
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aircraft Model Code', 'Notes': ''}}])
return_result.append([{'Phrase': f"Engine Type:{df['Type Engine']}",
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aircraft Engine Type', 'Notes': ''}}])
return_result.append([{'Phrase': f"Horse Power:str({df['Horsepower']})",
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aircraft Engine Horsepower', 'Notes': ''}}])
return return_result

59
Modules/Aircraft_Inquiry/AircraftInquiryByNNumber.py
View File

@@ -1,59 +0,0 @@
#!/usr/bin/env python3
class AircraftInquiryByNNumber:
name = "Aircraft Inquiry By N-Number"
category = "Aircraft"
description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
originTypes = {"Phrase"}
resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company', 'Country', 'City'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
import pandas as pd
futures = []
uidList = []
return_result = []
submit_url = "https://registry.faa.gov/aircraftinquiry/Search/NNumberResult"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
futures.append(session.post(submit_url, data={"NNumbertxt": entity['Phrase']}))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
df_list = pd.read_html(future.result().text)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
except ValueError:
return "No results retrieved"
df1 = df_list[0]
df2 = df_list[1]
df3 = df_list[2]
return_result.append([{'ID Number': df1[1][0],
'Entity Type': 'Identification Number'},
{uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
return_result.append([{'Company Name': df1[1][1],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Aircraft Company', 'Notes': ''}}])
return_result.append([{'Full Name': df2[1][0],
'Entity Type': 'Person'},
{uid: {'Resolution': 'Aircraft Owner', 'Notes': ''}}])
return_result.append([{'City Name': df2[1][2],
'Entity Type': 'City'},
{uid: {'Resolution': "Aircraft Owner's City", 'Notes': ''}}])
return_result.append([{'Country Name': df2[1][4],
'Entity Type': 'Country'},
{uid: {'Resolution': "Aircraft Owner's Country", 'Notes': ''}}])
return_result.append([{'Phrase': df3[1][1],
'Entity Type': 'Phrase'},
{uid: {'Resolution': "Aircraft Engine Series", 'Notes': ''}}])
return_result.append([{'Phrase': df3[1][2],
'Entity Type': 'Phrase'},
{uid: {'Resolution': "Aircraft Engine Motor", 'Notes': ''}}])
return return_result

46
Modules/Aircraft_Inquiry/AircraftInquiryByPersonName.py
View File

@@ -1,46 +0,0 @@
#!/usr/bin/env python3
class AircraftInquiryByPersonName:
name = "Aircraft Inquiry By Person Name"
category = "Aircraft"
description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
originTypes = {"Person"}
resultTypes = {'Phrase', 'Identification Number', 'Company'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
import pandas as pd
futures = []
uidList = []
return_result = []
submit_url = "https://registry.faa.gov/aircraftinquiry/Search/NameResult"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
futures.append(session.post(submit_url, data={"nametxt": entity['Full Name'], "sort_option": "1"}))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
df_list = pd.read_html(future.result().text)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
except ValueError:
return "No results retrieved"
df = df_list[0]
for index in range(len(df["N-Number"])):
return_result.append([{'Phrase': df["N-Number"][index],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
return_result.append([{'ID Number': str(df['Serial Number'][index]),
'Entity Type': 'Identification Number'},
{uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
return_result.append([{'Company Name': df['Manufacturer Name Model'][index],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Aircraft Manufacturer Name', 'Notes': ''}}])
return return_result

46
Modules/Aircraft_Inquiry/AircraftInquiryBySerialNumber.py
View File

@@ -1,46 +0,0 @@
#!/usr/bin/env python3
class AircraftInquiryBySerialNumber:
name = "Aircraft Inquiry By Serial Number"
category = "Aircraft"
description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
originTypes = {"Identification Number"}
resultTypes = {'Phrase', 'Company'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
import pandas as pd
futures = []
uidList = []
return_result = []
submit_url = "https://registry.faa.gov/aircraftinquiry/Search/SerialResult"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
futures.append(session.post(submit_url, data={"Serialtxt": entity['ID Number'], "sort_option": "1"}))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
df_list = pd.read_html(future.result().text)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
except ValueError:
return "No results retrieved"
df = df_list[0]
for index in range(len(df["N-Number"])):
return_result.append([{'Company Name': df["Manufacturer Name"][index],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Aircraft Manufacturer', 'Notes': ''}}])
return_result.append([{'Phrase': df["N-Number"][index],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
return_result.append([{'Phrase': df["Model"][index],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aircraft Model', 'Notes': ''}}])
return return_result

3
Modules/Aircraft_Inquiry/requirements.txt
View File

@@ -1,3 +0,0 @@
requests
pandas
requests-futures

18
Modules/Aleph/Aleph.xml
View File

@@ -1,18 +0,0 @@
<Aleph>
<Aleph_ID>
<Attributes>
<Attribute default="Aleph Default ID" check="String" primary="True">ID</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Aleph_ID>
<Aleph_Collection_ID>
<Attributes>
<Attribute default="Aleph Default Collection ID" check="String" primary="True">ID</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Aleph_Collection_ID>
</Aleph>

272
Modules/Aleph/Aleph_Entity_Search.py
View File

@@ -1,272 +0,0 @@
#!/usr/bin/env python3
class Aleph_Entity_Search:
name = "Aleph Entity Search"
category = "Aleph OCCRP"
description = "Find information about a given search parameter"
originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
resultTypes = {'Phrase', 'Person', 'Company', 'Address', 'Country', 'Aleph ID', 'Aleph Collection ID',
'Phone Number', 'Organization'}
parameters = {'Max Results': {'description': 'The maximum number of results to return.',
'type': 'String',
'value': 'Enter the number of results you want returned',
'default': '1'},
'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
'It is not intended to amount to advice on which you should '
'place sole and entire reliance.\n'
'We recommend that you conduct your own independent fact '
'checking against the data and materials that you '
'access on Aleph.\n'
'Aleph API is not a replacement for traditional due diligence '
'checks and know-your-customer background checks.',
'type': 'String',
'value': 'Type "Accept" (without quotes) to confirm your understanding.',
'global': True}
}
def resolution(self, entityJsonList, parameters):
import requests
import pycountry
import time
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
return_result = []
uidList = []
futures = []
url = "https://aleph.occrp.org/api/2/entities"
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
gender = "None"
if parameters['Aleph Disclaimer'] != 'Accept':
return "Please Accept the Terms for Aleph."
try:
max_results = int(parameters['Max Results'])
except ValueError:
return "The value for parameter 'Max Results' is not a valid integer."
if max_results <= 0:
return []
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]].strip()
crafted_url = f"{url}?q={primary_field}&filter:schemata=Thing&limit={max_results}"
time.sleep(1)
futures.append(session.get(crafted_url, headers=headers))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
for schema in response['results']:
index_of_child = len(return_result)
try:
if schema['schema'] == "Person":
if schema['properties'].get('gender') is not None:
if schema['properties'].get('gender')[0] == "F":
gender = "Female"
elif schema['properties'].get('gender')[0] == "M":
gender = "Male"
if schema['properties'].get('legalForm') is not None:
return_result.append(
[{'Full Name': schema['properties']['name'][0],
'Gender': gender,
'Date Of Birth': str(schema['properties'].get('birthDate')),
'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}",
'Entity Type': 'Person'},
{uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
else:
return_result.append(
[{'Full Name': str(schema['properties']['name'][0]),
'Gender': gender,
'Date Of Birth': str(schema['properties']['birthDate'][0]),
'Notes': schema['links']['self'],
'Entity Type': 'Person'},
{uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
if schema['properties'].get('registrationNumber') is not None:
return_result.append(
[{'Registration Number': str(schema['properties']['registrationNumber'][0]),
'Notes': '',
'Entity Type': 'Company'},
{index_of_child: {'Resolution': 'Aleph Registration Number', 'Notes': ''}}])
if schema['properties'].get('country') is not None:
return_result.append(
[{'Country Name': str(
pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
'Notes': '',
'Entity Type': 'Country'},
{index_of_child: {'Resolution': 'Country of Origin', 'Notes': ''}}])
if schema['properties'].get('addressEntity'):
return_result.append(
[{'Street Address': str(
schema['properties']['addressEntity'][0]['properties']['full'][0]),
'Notes': '',
'Entity Type': 'Address'},
{index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
else:
return_result.append(
[{'Street Address': str(schema['properties']['address'][0]),
'Notes': '',
'Entity Type': 'Address'},
{index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
return_result.append(
[{'ID': str(schema['id']),
'Notes': '',
'Entity Type': 'Aleph ID'},
{index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['collection']['label']),
'Notes': str(schema['collection']['summary']),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
return_result.append(
[{'ID': str(schema['collection']['collection_id']),
'Notes': '',
'Entity Type': 'Aleph Collection ID'},
{index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
elif schema['schema'] == "Organization":
return_result.append(
[{'Organization Name': str(schema['properties']['name'][0]),
'Registration Number': str(schema['properties']['registrationNumber'][0]),
'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}\n"
f"Source URL: {schema['properties']['sourceUrl'][0]}",
'Entity Type': 'Organization'},
{uid: {'Resolution': 'Aleph Organisation Entity', 'Notes': ''}}])
return_result.append(
[{'Country Name': str(
pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
'Notes': '',
'Entity Type': 'Country'},
{index_of_child: {'Resolution': "Aleph Organisation Country", 'Notes': ''}}])
return_result.append(
[{'Street Address': str(schema['properties']['address'][0]),
'Notes': '',
'Entity Type': 'Address'},
{index_of_child: {'Resolution': "Aleph Organisation Address", 'Notes': ''}}])
return_result.append(
[{'ID': str(schema['id']),
'Notes': '',
'Entity Type': 'Aleph ID'},
{index_of_child: {'Resolution': "Aleph Organisation ID", 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['collection']['label']),
'Notes': str(schema['collection']['summary']),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
return_result.append(
[{'Phone Number': str(schema['properties']['phone'][0]),
'Notes': '',
'Entity Type': 'Phone Number'},
{index_of_child: {'Resolution': 'Phone Number', 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['properties']['classification'][0]),
'Notes': '',
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Organisation Classification', 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['collection']['collection_id']),
'Notes': '',
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
elif schema['schema'] == "Pages":
if 'updated_at' in schema:
date_created = schema['updated_at']
else:
date_created = schema['created_at']
doc_name = 'Document: ' + schema['properties']['title'][0]
entity_link = schema['links']['self']
file_link = schema['links']['file']
source_url = schema['properties']['sourceUrl'][0]
return_result.append(
[{'Phrase': doc_name,
'Source': source_url,
'Notes': 'Link to Aleph Entity: ' + entity_link + '\n\n' +
'Link to document: ' + file_link,
'Entity Type': 'Phrase',
'Date Created': date_created},
{uid: {'Resolution': 'Aleph Document', 'Notes': ''}}])
elif schema['properties']['parent'][0]['schema'] == "Person":
gender = str(schema['properties']['parent'][0]['properties'].get('gender')[0])
if schema['properties']['parent'][0]['properties'].get('legalForm') is not None:
return_result.append(
[{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
'Gender': gender,
'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
'Notes': f"{schema['properties']['parent'][0]['links']['self']}\nLegal Form: "
f"{schema['properties']['parent'][0]['properties']['legalForm'][0]}",
'Entity Type': 'Person'},
{uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
else:
return_result.append(
[{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
'Gender': gender,
'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
'Notes': schema['properties']['parent'][0]['links']['self'],
'Entity Type': 'Person'},
{uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
if schema['properties']['parent'][0]['properties'].get('registrationNumber') is not None:
return_result.append(
[{'Registration Number': str(
schema['properties']['parent'][0]['properties']['registrationNumber'][0]),
'Notes': '',
'Entity Type': 'Company'},
{index_of_child: {'Resolution': 'Company Registration Number', 'Notes': ''}}])
if schema['properties']['parent'][0]['properties'].get('country') is not None:
return_result.append(
[{'Country Name': str(
pycountry.countries.get(
alpha_2=schema['properties']['parent'][0]['properties']['country'][0]).name),
'Notes': '',
'Entity Type': 'Country'},
{index_of_child: {'Resolution': 'Country', 'Notes': ''}}])
return_result.append(
[{'ID': str(schema['properties']['parent'][0]['id']),
'Notes': '',
'Entity Type': 'Aleph ID'},
{index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['properties']['parent'][0]['collection']['label']),
'Notes': str(schema['properties']['parent'][0]['collection']['summary']),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['properties']['parent'][0]['collection']['collection_id']),
'Notes': '',
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
index_of_child_of_child = len(return_result)
return_result.append(
[{'Company Name': str(schema['properties']['name'][0]),
'Notes': schema['links']['self'],
'Entity Type': 'Company'},
{index_of_child: {'Resolution': 'Aleph Company Entity', 'Notes': ''}}])
return_result.append(
[{'Street Address': str(schema['properties']['addressEntity'][0]['properties']['full'][0]),
'Notes': '',
'Entity Type': 'Address'},
{index_of_child_of_child: {'Resolution': 'Aleph Person Address', 'Notes': ''}}])
for country_code in schema['collection']['countries']:
return_result.append(
[{'Country Name': str(pycountry.countries.get(alpha_2=country_code).name),
'Notes': '',
'Entity Type': 'Country'},
{index_of_child_of_child: {'Resolution': 'Country', 'Notes': ''}}])
return_result.append(
[{'Phrase': str(schema['collection']['label']),
'Notes': str(schema['collection']['summary']),
'Entity Type': 'Phrase'},
{index_of_child_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
return_result.append(
[{'ID': str(schema['collection']['collection_id']),
'Notes': '',
'Entity Type': 'Aleph Collection ID'},
{index_of_child_of_child: {'Resolution': 'Aleph Entity Search', 'Notes': ''}}])
except (TypeError, KeyError):
continue
return return_result

99
Modules/Aleph/GetCollectionByID.py
View File

@@ -1,99 +0,0 @@
#!/usr/bin/env python3
class GetCollectionByID:
name = "Get Collection for Phrase"
category = "Aleph OCCRP"
description = "Get the Collections that contain info related to the given Phrases."
originTypes = {'Phrase'}
resultTypes = {'Phrase', 'Person', 'Address', 'Phone Number', 'Email Address', 'Country', 'Bank Account'}
parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
'It is not intended to amount to advice on which you should '
'place sole and entire reliance.\n'
'We recommend that you conduct your own independent fact '
'checking against the data and materials that you '
'access on Aleph.\n'
'Aleph API is not a replacement for traditional due diligence '
'checks and know-your-customer background checks.',
'type': 'String',
'value': 'Type "Accept" (without quotes) to confirm your understanding.',
'global': True}}
def resolution(self, entityJsonList, parameters):
import time
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
returnResults = []
futures = []
uidList = []
if parameters['Aleph Disclaimer'] != 'Accept':
return "Please Accept the Terms for Aleph."
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]].strip()
url = f"https://aleph.occrp.org/api/2/collections/{primary_field}"
time.sleep(1)
futures.append(session.get(url, headers=headers))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
if response['statistics']['names'].get('values') is not None:
nameKeys = list(response['statistics']['names'].get('values').keys())
for nameKey in nameKeys:
returnResults.append([{'Full Name': str(nameKey),
'Entity Type': 'Person'},
{uid: {'Resolution': 'Person Entity',
'Notes': ''}}])
if response['statistics']['addresses'].get('values') is not None:
addressKeys = list(response['statistics']['addresses'].get('values').keys())
for addressKey in addressKeys:
returnResults.append([{'Street Address': str(addressKey),
'Entity Type': 'Address'},
{uid: {'Resolution': 'Address Entity',
'Notes': ''}}])
if response['statistics']['phones'].get('values') is not None:
phoneKeys = list(response['statistics']['phones'].get('values').keys())
for phoneKey in phoneKeys:
returnResults.append([{'Phone Number': str(phoneKey),
'Entity Type': 'Phone Number'},
{uid: {'Resolution': 'Phone Number Entity',
'Notes': ''}}])
if response['statistics']['emails'].get('values') is not None:
emailKeys = list(response['statistics']['emails'].get('values').keys())
for emailKey in emailKeys:
returnResults.append([{'Email Address': str(emailKey),
'Entity Type': 'Email Address'},
{uid: {'Resolution': 'Email Address Entity',
'Notes': ''}}])
if response['statistics']['countries'].get('values') is not None:
countriesKeys = list(response['statistics']['countries'].get('values').keys())
for countriesKey in countriesKeys:
returnResults.append([{'Country Name': str(countriesKey),
'Entity Type': 'Country'},
{uid: {'Resolution': 'Country Entity',
'Notes': ''}}])
if response['statistics']['languages'].get('values') is not None:
languagesKeys = list(response['statistics']['languages'].get('values').keys())
for languagesKey in languagesKeys:
returnResults.append([{'Phrase': str(languagesKey),
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Language Entity',
'Notes': ''}}])
if response['statistics']['ibans'].get('values') is not None:
ibansKeys = list(response['statistics']['ibans'].get('values').keys())
for ibansKey in ibansKeys:
returnResults.append([{'Account Number': str(ibansKey),
'Entity Type': 'Bank Account'},
{uid: {'Resolution': 'IBAN Entity',
'Notes': ''}}])
return returnResults

73
Modules/Aleph/GetCollectionsInfo.py
View File

@@ -1,73 +0,0 @@
#!/usr/bin/env python3
class GetCollectionsInfo:
name = "Get Collections Info"
category = "Aleph OCCRP"
description = "Find information about Collections and their IDs"
originTypes = {'Phrase'}
resultTypes = {'Phrase, Aleph ID'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.',
'type': 'String',
'default': ''},
'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
'It is not intended to amount to advice on which you should place'
'sole and entire reliance.\n'
'We recommend that you conduct your own independent fact checking'
'against the data and materials that you access on Aleph.\n'
'Aleph API is not a replacement for traditional due diligence '
'checks and know-your-customer background checks.',
'type': 'String',
'value': 'Type "Accept" (without quotes) to confirm your understanding.',
'global': True}
}
def resolution(self, entityJsonList, parameters):
import time
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
returnResults = []
futures = []
uidList = []
if parameters['Aleph Disclaimer'] != 'Accept':
return "Please Accept the Terms for Aleph."
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
try:
maxResults = int(parameters['Max Results'])
except ValueError:
return "The value for parameter 'Max Results' is not a valid integer."
if maxResults <= 0:
return []
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
url = f"https://aleph.occrp.org/api/2/collections?offset=0&limit=300&q={entity['Phrase']}"
time.sleep(1)
futures.append(session.get(url, headers=headers))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
collections = response['results'][:maxResults]
for collection in collections:
index_of_child = len(returnResults)
returnResults.append([{'Phrase': collection['label'],
'Notes': str(collection.get('summary')),
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Aleph Collection Name',
'Notes': ''}}])
returnResults.append([{'ID': collection['id'],
'Entity Type': 'Aleph ID'},
{index_of_child: {'Resolution': 'Aleph Collection ID',
'Notes': ''}}])
return returnResults

103
Modules/Aleph/GetSimilarEntities.py
View File

@@ -1,103 +0,0 @@
#!/usr/bin/env python3
class GetSimilarEntities:
name = "Get Similar Entities"
category = "Aleph OCCRP"
description = "Find information about similar entities"
originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
resultTypes = {'Phrase', 'Person', 'Address', 'Aleph ID', 'Company'}
parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
'It is not intended to amount to advice on which you should '
'place sole and entire reliance.\n'
'We recommend that you conduct your own independent fact '
'checking against the data and materials that you '
'access on Aleph.\n'
'Aleph API is not a replacement for traditional due diligence '
'checks and know-your-customer background checks.',
'type': 'String',
'value': 'Type "Accept" (without quotes) to confirm your understanding.',
'global': True}}
def resolution(self, entityJsonList, parameters):
import time
import requests
import pycountry
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
returnResults = []
futures = []
uidList = []
if parameters['Aleph Disclaimer'] != 'Accept':
return "Please Accept the Terms for Aleph."
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]].strip()
url = f"https://aleph.occrp.org/api/2/entities/{primary_field}/similar"
time.sleep(1)
futures.append(session.get(url, headers=headers))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
entities = response['results']
for schema in entities:
if schema['entity']['schema'] == 'Person':
index_of_child = len(returnResults)
returnResults.append([{'Full Name': ' '.join(map(str, schema['entity']['properties']['name'])),
'Gender': ' '.join(map(str, schema['entity']['properties']['gender'])),
'Notes': ' '.join(map(str, schema['entity']['properties']['legalForm'])),
'Entity Type': 'Person'},
{uid: {'Resolution': 'Person Entity',
'Notes': ''}}])
country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
returnResults.append([{'Street Address': schema['entity']['properties']['addressEntity'][0]
['properties']['full'][0],
'Postal Code': schema['entity']['properties']['addressEntity'][0]
['properties']['postalCode'][0],
'Country': country,
'Entity Type': 'Address'},
{index_of_child: {'Resolution': 'Address',
'Notes': ''}}])
returnResults.append([{'Phrase': schema['entity']['collection']['label'],
'Notes': schema['entity']['collection']['summary'],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Location in Database',
'Notes': ''}}])
returnResults.append([{'ID': schema['entity']['id'],
'Entity Type': 'Aleph ID'},
{index_of_child: {'Resolution': 'ID in Database',
'Notes': ''}}])
elif schema['entity']['schema'] == 'Company':
index_of_child = len(returnResults)
returnResults.append([{'Company Name': ' '.join(map(str, schema['entity']['properties']['name'])),
'Notes': str(schema['entity']['properties']['status']),
'Entity Type': 'Company'},
{uid: {'Resolution': 'Company Entity',
'Notes': ''}}])
country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
returnResults.append([{'Street Address': str(schema['entity']['properties'].get('address')),
'Country': country,
'Entity Type': 'Address'},
{index_of_child: {'Resolution': 'Address',
'Notes': ''}}])
returnResults.append([{'Phrase': schema['entity']['collection']['label'],
'Notes': schema['entity']['collection']['summary'],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Location in Database',
'Notes': ''}}])
returnResults.append([{'ID': schema['entity']['id'],
'Entity Type': 'Aleph ID'},
{index_of_child: {'Resolution': 'ID in Database',
'Notes': ''}}])
return returnResults

3
Modules/Aleph/requirements.txt
View File

@@ -1,3 +0,0 @@
requests
pycountry
requests-futures

280
Modules/Amass/Amass_Domain.py
View File

@@ -1,280 +0,0 @@
#!/usr/bin/env python3
# User needs to be in docker group or to have root privileges
class Amass_Domain:
name = "Amass Domain Scan"
category = "Network Infrastructure"
description = "Find information about a particular domain. Requires Docker to be installed."
originTypes = {'Domain'}
resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain', 'IPv6 Address'}
parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://virustotal.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'AlienVault API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://otx.alienvault.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'BinaryEdge API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://app.binaryedge.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'C99 API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://c99.nl.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Censys API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://censys.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Chaos API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://chaos.projectdiscovery.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Cloudflare API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://cloudflare.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'DNSDB API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://dnsdb.info.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'GitHub API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://github.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Hunter API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://hunter.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'IPInfo Access Token': {'description': 'Enter your api key under your profile after'
' signing up on https://ipinfo.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'NetworksDB API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://networksdb.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'PassiveTotal API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://passivetotal.com .',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ReconDev API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://recon.dev.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'SecurityTrails API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://securitytrails.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Shodan API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://shodan.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Spyse API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://spyse.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ThreatBook API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://threatbook.cn.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Umbrella API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://umbrella.cisco.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'URLScan API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://urlscan.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'WhoisXMLAPI API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://whoisxmlapi.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ZETAlytics API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://zetalytics.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ZoomEye API Key': {'description': 'Please Enter the Username and password with a space '
'in between',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'FacebookCT API Key': {'description': 'Please Enter the api key and secret with a space '
'in between. Obtain them at https://developer.facebook.com',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Twitter API Key': {'description': 'Please Enter the api key and secret with a space '
'in between. Obtain them at https://developer.twitter.com',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ReconDev.free API Key': {
'description':
'Please Enter the api key under your profile after signing up on https://recon.dev',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ReconDev.paid API Key': {
'description':
'Please Enter the api key under your profile after signing up on https://recon.dev',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
from pathlib import Path
import json
from ipaddress import ip_address, IPv4Address, IPv6Address
import docker
import tempfile
from docker.errors import APIError
return_result = []
# Generate Config as a temporary file:
with tempfile.TemporaryDirectory() as tempDir:
tempPath = Path(tempDir).absolute()
config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
suffix='Config',
dir=tempPath)
config.write("share = true\n")
config.write("[scope]\n")
config.write("port = 80\n")
config.write("port = 443\n")
config.write("[data_sources]\n")
config.write("minimum_ttl = 1440\n")
for parameter in self.parameters:
if parameters[f'{parameter}'] != 'None':
field1 = f"[data_sources.{parameter}]"
field2 = f"[data_sources.{parameter}.Credentials]"
if parameter == "ZoomEye":
username, password = parameters[parameter].split(' ', 1)
config.write(f"{field1}\n")
config.write(f"{field2}\n")
config.write(f"username = {username}\n")
config.write(f"password = {password}\n")
elif parameter == "FacebookCT":
field3, secret = parameters[parameter].split(' ', 1)
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.app1\n")
config.write(f"apikey = \"{field3}\"\n")
config.write(f"secret = {secret}\n")
elif parameter == "Twitter":
field3, secret = parameters[parameter].split(' ', 1)
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.account1\n")
config.write(f"apikey = \"{field3}\"\n")
config.write(f"secret = {secret}\n")
elif parameter == "ReconDev.paid":
field3 = parameters[f'{parameter}']
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.paid\n")
config.write(f"apikey = \"{field3}\"\n")
elif parameter == "ReconDev.free":
field3 = parameters[f'{parameter}']
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.free\n")
config.write(f"apikey = \"{field3}\"\n")
else:
field3 = parameters[f'{parameter}']
config.write(f"{field1}\n")
config.write(f"{field2}\n")
config.write(f"apikey = \"{field3}\"\n")
path_to_config = Path(config.name).name
config.seek(0)
for entity in entityJsonList:
primary_field = entity["Domain Name"].strip()
try:
client = docker.from_env()
container = client.containers.run("caffix/amass:latest",
f"enum -src -d {primary_field} "
f"-config /.config/amass/{path_to_config}",
volumes={
str(tempPath): {'bind': '/.config/amass',
'mode': 'rw'}},
remove=True)
jsonFile = tempPath / 'amass.json'
jsonContents = ""
if jsonFile.exists():
with open(jsonFile, 'r') as jsonFileHandler:
jsonContents = jsonFileHandler.read()
client.close()
except (APIError, docker.errors.ContainerError) as error:
return "Something happened to the docker container - Cannot continue: " + str(error)
uid = entity['uid']
for dictionary in jsonContents.splitlines():
index_of_child = len(return_result)
line_dictionary = json.loads(dictionary)
size = len(line_dictionary['addresses'])
return_result.append([{'Domain Name': str(line_dictionary['name']),
'Entity Type': 'Domain'},
{uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
for ip in range(size):
if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
'Entity Type': 'IP Address'},
{index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
'Entity Type': 'IPv6 Address'},
{index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
'Entity Type': 'Autonomous System'},
{index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Amass Domain Scan Description',
'Notes': ''}}])
config.close()
return return_result

285
Modules/Amass/Amass_Intel.py
View File

@@ -1,285 +0,0 @@
#!/usr/bin/env python3
# User needs to be in docker group or have root privileges
class Amass_Intel:
name = "Amass Intel Scan"
category = "Network Infrastructure"
description = "Find information about a particular domain. Requires Docker to be installed."
originTypes = {'Domain', 'IP Address', 'Autonomous System'}
resultTypes = {'Domain'}
parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://virustotal.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'AlienVault API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://otx.alienvault.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'BinaryEdge API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://app.binaryedge.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'C99 API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://c99.nl.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Censys API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://censys.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Chaos API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://chaos.projectdiscovery.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Cloudflare API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://cloudflare.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'DNSDB API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://dnsdb.info.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'GitHub API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://github.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Hunter API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://hunter.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'IPInfo Access Token': {'description': 'Enter your api key under your profile after'
' signing up on https://ipinfo.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'NetworksDB API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://networksdb.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'PassiveTotal API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://passivetotal.com .',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ReconDev API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://recon.dev.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'SecurityTrails API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://securitytrails.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Shodan API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://shodan.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Spyse API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://spyse.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ThreatBook API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://threatbook.cn.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Umbrella API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://umbrella.cisco.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'URLScan API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://urlscan.io.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'WhoisXMLAPI API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://whoisxmlapi.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ZETAlytics API Key': {'description': 'Enter your api key under your profile after'
' signing up on https://zetalytics.com.',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ZoomEye API Key': {'description': 'Please Enter the Username and password with a space '
'in between',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'FacebookCT API Key': {'description': 'Please Enter the api key and secret with a space '
'in between. Obtain them at https://developer.facebook.com',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'Twitter API Key': {'description': 'Please Enter the api key and secret with a space '
'in between. Obtain them at https://developer.twitter.com',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ReconDev.free API Key': {
'description':
'Please Enter the api key under your profile after signing up on https://recon.dev',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'},
'ReconDev.paid API Key': {
'description':
'Please Enter the api key under your profile after signing up on https://recon.dev',
'type': 'String',
'value': 'None',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
from pathlib import Path
import json
from ipaddress import ip_address, IPv4Address, IPv6Address
import docker
import tempfile
from docker.errors import APIError
return_result = []
# Generate Config as a temporary file:
with tempfile.TemporaryDirectory() as tempDir:
tempPath = Path(tempDir).absolute()
config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
suffix='Config',
dir=tempPath)
config.write("share = true\n")
config.write("[scope]\n")
config.write("port = 80\n")
config.write("port = 443\n")
config.write("[data_sources]\n")
config.write("minimum_ttl = 1440\n")
for parameter in parameters:
if parameters[f'{parameter}'] != 'None':
field1 = f"[data_sources.{parameter}]"
field2 = f"[data_sources.{parameter}.Credentials]"
if parameter == "ZoomEye":
username, password = parameters[parameter].split(' ', 1)
config.write(f"{field1}\n")
config.write(f"{field2}\n")
config.write(f"username = {username}\n")
config.write(f"password = {password}\n")
elif parameter == "FacebookCT":
field3, secret = parameters[parameter].split(' ', 1)
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.app1\n")
config.write(f"apikey = \"{field3}\"\n")
config.write(f"secret = {secret}\n")
elif parameter == "Twitter":
field3, secret = parameters[parameter].split(' ', 1)
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.account1\n")
config.write(f"apikey = \"{field3}\"\n")
config.write(f"secret = {secret}\n")
elif parameter == "ReconDev.paid":
field3 = parameters[f'{parameter}']
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.paid\n")
config.write(f"apikey = \"{field3}\"\n")
elif parameter == "ReconDev.free":
field3 = parameters[f'{parameter}']
config.write(f"{field1}\n")
config.write(f"[data_sources.{parameter}.free\n")
config.write(f"apikey = \"{field3}\"\n")
else:
field3 = parameters[f'{parameter}']
config.write(f"{field1}\n")
config.write(f"{field2}\n")
config.write(f"apikey = \"{field3}\"\n")
path_to_config = "/" + Path(config.name).name
config.seek(0)
for entity in entityJsonList:
primary_field = entity[list(entity)[1]].strip()
try:
client = docker.from_env()
if entity['Entity Type'] == "Domain":
container = client.containers.run("caffix/amass:latest",
f"intel -whois -d {primary_field} -config /.config/amass"
f"{path_to_config}",
volumes={
str(tempPath): {'bind': '/.config/amass',
'mode': 'rw'}},
remove=True)
elif entity['Entity Type'] == "IP Address":
try:
ip_address(primary_field)
except ValueError:
return "The Entity Provided isn't a valid IP Address"
container = client.containers.run("caffix/amass:latest",
f"intel -addr {primary_field} -config "
f"/.config/amass{path_to_config}",
volumes={
str(tempPath): {'bind': '/.config/amass',
'mode': 'rw'}},
remove=True)
elif entity['Entity Type'] == "Autonomous System":
if primary_field.startswith('AS'):
primary_field = primary_field[2:]
container = client.containers.run("caffix/amass:latest",
f"intel -asn {primary_field}"
f" -config /.config/amass{path_to_config}",
volumes={
str(tempPath): {'bind': '/.config/amass',
'mode': 'rw'}},
remove=True)
textFile = tempPath / 'amass.txt'
textContents = ""
if textFile.exists():
with open(textFile, 'r') as textFileHandler:
textContents = textFileHandler.read()
client.close()
except (APIError, docker.errors.ContainerError) as error:
return "Something happened to the docker container - Cannot continue: " + str(error)
uid = entity['uid']
for newDomain in textContents.splitlines():
return_result.append([{'Domain Name': newDomain.strip(),
'Entity Type': 'Domain'},
{uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
config.close()
return return_result

1
Modules/Amass/requirements.txt
View File

@@ -1 +0,0 @@
docker

66
Modules/BigMatch/BigMatch.py
View File

@@ -1,66 +0,0 @@
#!/usr/bin/env python3
class BigMatch:
name = "BigMatch Search"
category = "Secrets & Leaks"
description = "Find information about a file using https://bigmatch.rev.ng/static/index.html"
originTypes = {"Image", "Document", "Archive"}
resultTypes = {'Website'}
parameters = {}
def resolution(self, entityJsonList, parameters):
from pathlib import Path
from playwright.sync_api import sync_playwright, TimeoutError, Error
from bs4 import BeautifulSoup
return_result = []
url = "https://bigmatch.rev.ng/static/index.html"
failString = 'Too many strings in binary?'
successString = 'Results:'
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
for entity in entityJsonList:
uid = entity['uid']
file_path = Path(parameters['Project Files Directory']) / entity["File Path"]
file_path = file_path.absolute()
if not (file_path.exists() and file_path.is_file()):
continue
page.wait_for_timeout(3000)
for _ in range(3):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
inputLocator = page.locator("input")
inputLocator.set_input_files([str(file_path)])
page.wait_for_timeout(3000)
soup = BeautifulSoup(page.content(), 'lxml')
soupText = soup.get_text()
while (failString not in soupText) and (successString not in soupText):
page.wait_for_timeout(1000)
soup = BeautifulSoup(page.content(), 'lxml')
soupText = soup.get_text()
if failString in soupText:
return []
for link in soup.find_all('a'):
potentialLink = link.get('href', None)
if potentialLink is not None and 'github' in potentialLink:
return_result.append([{'URL': potentialLink, 'Entity Type': 'Website'},
{uid: {'Resolution': 'BigMatch Github Link', 'Notes': ''}}])
break
except TimeoutError:
pass
except Error:
break
page.close()
browser.close()
return return_result

2
Modules/BigMatch/requirements.txt
View File

@@ -1,2 +0,0 @@
bs4
playwright

64
Modules/BinaryEdge/BinaryEdgeHost.py
View File

@@ -1,64 +0,0 @@
#!/usr/bin/env python3
class BinaryEdgeHost:
name = "BinaryEdge Host Query"
category = "Network Infrastructure"
description = "Get information about a host from BinaryEdge."
originTypes = {"IP Address", "IPv6 Address"}
resultTypes = {'Port'}
parameters = {'BinaryEdge API Key': {'description': "Enter your BinaryEdge API key. Sign up for one at "
"https://www.binaryedge.io/",
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
import json
baseURL = 'https://api.binaryedge.io/v2/query/ip/'
requestHeaders = {'X-Key': parameters['BinaryEdge API Key'].strip()}
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
if entity['Entity Type'] == 'IP Address':
primaryField = entity['IP Address']
elif entity['Entity Type'] == 'IPv6 Address':
primaryField = entity['IPv6 Address']
else:
continue
infoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
statusCode = infoRequest.status_code
if statusCode == 401:
return "The BinaryEdge API key provided is not valid."
elif statusCode == 403:
return "The BinaryEdge API key provided does not have permission to access this resource."
elif statusCode != 200:
continue
requestContent = json.loads(infoRequest.content)
for event in requestContent['events']:
for result in event['results']:
resultDetails = result['result']
if 'state' not in resultDetails['data']:
# Discard return result if it doesn't actually give us useful info about the state of the port.
# This happens in cases where the API returns stuff like the ciphers used in an SSH service.
# There seems to always be a result with the simple port info, so we will use that one.
continue
originDetails = result['origin']
targetDetails = result['target']
returnResults.append([{'Port': f"{targetDetails['ip']}:{targetDetails['port']}:"
f"{targetDetails['protocol']}",
'State': resultDetails['data']['state']['state'],
'Banner': resultDetails['data']['service'].get('banner', 'N/A'),
'Product': resultDetails['data']['service'].get('product', 'Unknown'),
'Entity Type': 'Port'},
{uid: {'Resolution': 'BinaryEdge Scan Timestamp: ' + str(originDetails['ts']),
'Notes': ''}}])
return returnResults

1
Modules/BinaryEdge/requirements.txt
View File

@@ -1 +0,0 @@
requests

34
Modules/BreachData/HIBP.xml
View File

@@ -1,34 +0,0 @@
<HIBP>
<Data_Breach>
<Attributes>
<Attribute default="Breach Name" check="String" primary="True">Breach Name</Attribute>
<Attribute default="Breach Title" check="String" primary="False">Breach Title</Attribute>
<Attribute default="Breach Domain" check="String" primary="False">Breach Domain</Attribute>
<Attribute default="0" check="Numbers" primary="False">Breach Pwn Count</Attribute>
<Attribute default="Breach Description" check="String" primary="False">Breach Description</Attribute>
<Attribute default="False" check="String" primary="False">Breach Is Sensitive</Attribute>
<Attribute default="False" check="String" primary="False">Breach Is Verified</Attribute>
<Attribute default="False" check="String" primary="False">Breach Is Fabricated</Attribute>
<Attribute default="False" check="String" primary="False">Breach Is Retired</Attribute>
<Attribute default="False" check="String" primary="False">Breach Is Spam List</Attribute>
<Attribute default="False" check="String" primary="False">Breach Is Malware</Attribute>
<Attribute default="Unknown" check="String" primary="False">Breach Added Date</Attribute>
<Attribute default="Unknown" check="String" primary="False">Breach Modified Date</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Data_Breach>
<Paste_Data_Leak>
<Attributes>
<Attribute default="Paste Identifier" check="String" primary="True">Paste Identifier</Attribute>
<Attribute default="Paste Title" check="String" primary="False">Paste Title</Attribute>
<Attribute default="Paste Source" check="String" primary="False">Paste Source</Attribute>
<Attribute default="Paste ID" check="String" primary="False">Paste ID</Attribute>
<Attribute default="0" check="Numbers" primary="False">Paste Email Count</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Paste_Data_Leak>
</HIBP>

25
Modules/BreachData/HIBPBreachToDomain.py
View File

@@ -1,25 +0,0 @@
#!/usr/bin/env python3
class HIBPBreachToDomain:
name = "HIBP Breach To Domain"
category = "Leaked Data"
description = "Get the domain of the primary website that a data breach occurred on."
originTypes = {'Data Breach'}
resultTypes = {'Domain'}
parameters = {}
def resolution(self, entityJsonList, parameters):
returnResults = []
for entity in entityJsonList:
domainMaybe = entity.get('Breach Domain')
if isinstance(domainMaybe, str) and domainMaybe.strip() != '':
returnResults.append([{'Domain Name': domainMaybe,
'Entity Type': 'Domain'},
{entity['uid']: {'Resolution': 'Data Breach to Domain',
'Notes': ''}}])
return returnResults

85
Modules/BreachData/HaveIBeenPwnedBreachDomains.py
View File

@@ -1,85 +0,0 @@
#!/usr/bin/env python3
class HaveIBeenPwnedBreachDomains:
name = "HIBP Breach Domain Lookup"
category = "Leaked Data"
description = "Find breaches associated with a specified domain."
originTypes = {'Domain'}
resultTypes = {'Data Breach'}
parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
'You can get a key here: https://haveibeenpwned.com/API/Key',
'type': 'String',
'value': '',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
import requests
import json
from time import sleep
from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
from PySide6.QtGui import QImage
baseURL = "https://haveibeenpwned.com/api/v3/breaches?domain="
requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
returnResults = []
count = 0
while count < len(entityJsonList):
entity = entityJsonList[count]
primaryField = entity['Domain Name']
breachInfoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
statusCode = breachInfoRequest.status_code
if statusCode == 200:
breachContent = json.loads(breachInfoRequest.content)
for breach in breachContent:
try:
breachLogoIconRequest = requests.get(breach['LogoPath'])
breachIconByteArray = QByteArray(breachLogoIconRequest.content)
breachIconImageOriginal = QImage().fromData(breachIconByteArray)
breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
# Rotate the breach domain logo upside down
breachIconImageRotated = breachIconImageScaled.mirrored()
breachIconByteArrayFin = QByteArray()
breachImageBuffer = QBuffer(breachIconByteArrayFin)
breachImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
breachIconImageRotated.save(breachImageBuffer, "PNG")
breachImageBuffer.close()
except Exception:
breachIconByteArrayFin = None
# If Breach Date is None, then default to entity creation date.
returnResults.append([{'Breach Name': breach['Name'],
'Breach Title': breach['Title'],
'Breach Domain': breach['Domain'],
'Breach Pwn Count': str(breach['PwnCount']),
'Breach Description': breach['Description'],
'Breach Is Sensitive': str(breach['IsSensitive']),
'Breach Is Verified': str(breach['IsVerified']),
'Breach Is Fabricated': str(breach['IsFabricated']),
'Breach Is Retired': str(breach['IsRetired']),
'Breach Is Spam List': str(breach['IsSpamList']),
'Breach Is Malware': str(breach['IsMalware']),
'Breach Added Date': breach['AddedDate'],
'Breach Modified Date': breach['ModifiedDate'],
'Entity Type': 'Data Breach',
'Icon': breachIconByteArrayFin, # If None -> Default breach icon.
'Date Created': breach['BreachDate']},
{entity['uid']: {'Resolution': 'Contained in Breach',
'Notes': ''}}])
elif statusCode == 401:
return "The HIBP API Key provided is invalid."
elif statusCode == 429:
sleep(2)
continue
elif statusCode == 503:
return "The HIBP Service is unavailable."
sleep(1.7)
count += 1
return returnResults

88
Modules/BreachData/HaveIBeenPwnedBreaches.py
View File

@@ -1,88 +0,0 @@
#!/usr/bin/env python3
class HaveIBeenPwnedBreaches:
name = "HIBP Breach Lookup"
category = "Leaked Data"
description = "Find all breaches that an account has been involved in. Note that Date Created for breaches is an " \
"estimate."
originTypes = {'Email Address', 'Phone Number'}
resultTypes = {'Data Breach'}
parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
'You can get a key here: https://haveibeenpwned.com/API/Key',
'type': 'String',
'value': '',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
import requests
import json
from time import sleep
from urllib.parse import quote_plus
from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
from PySide6.QtGui import QImage
baseURL = "https://haveibeenpwned.com/api/v3/breachedaccount/"
requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
returnResults = []
count = 0
while count < len(entityJsonList):
entity = entityJsonList[count]
primaryField = entity[list(entity)[1]]
breachInfoRequest = requests.get(baseURL + quote_plus(primaryField) + '?truncateResponse=false',
headers=requestHeaders)
statusCode = breachInfoRequest.status_code
if statusCode == 200:
breachContent = json.loads(breachInfoRequest.content)
for breach in breachContent:
try:
breachLogoIconRequest = requests.get(breach['LogoPath'])
breachIconByteArray = QByteArray(breachLogoIconRequest.content)
breachIconImageOriginal = QImage().fromData(breachIconByteArray)
breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
# Rotate the breach domain logo upside down
breachIconImageRotated = breachIconImageScaled.mirrored()
breachIconByteArrayFin = QByteArray()
breachImageBuffer = QBuffer(breachIconByteArrayFin)
breachImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
breachIconImageRotated.save(breachImageBuffer, "PNG")
breachImageBuffer.close()
except Exception:
breachIconByteArrayFin = None
# If Breach Date is None, then default to entity creation date.
returnResults.append([{'Breach Name': breach['Name'],
'Breach Title': breach['Title'],
'Breach Domain': breach['Domain'],
'Breach Pwn Count': str(breach['PwnCount']),
'Breach Description': breach['Description'],
'Breach Is Sensitive': str(breach['IsSensitive']),
'Breach Is Verified': str(breach['IsVerified']),
'Breach Is Fabricated': str(breach['IsFabricated']),
'Breach Is Retired': str(breach['IsRetired']),
'Breach Is Spam List': str(breach['IsSpamList']),
'Breach Is Malware': str(breach['IsMalware']),
'Breach Added Date': breach['AddedDate'],
'Breach Modified Date': breach['ModifiedDate'],
'Entity Type': 'Data Breach',
'Icon': breachIconByteArrayFin, # If None -> Default breach icon.
'Date Created': breach['BreachDate']},
{entity['uid']: {'Resolution': 'Contained in Breach',
'Notes': ''}}])
elif statusCode == 401:
return "The HIBP API Key provided is invalid."
elif statusCode == 429:
sleep(2)
continue
elif statusCode == 503:
return "The HIBP Service is unavailable."
sleep(1.7)
count += 1
return returnResults

57
Modules/BreachData/HaveIBeenPwnedPassword.py
View File

@@ -1,57 +0,0 @@
#!/usr/bin/env python3
class HaveIBeenPwnedPassword:
name = "HIBP Password Lookup"
category = "Leaked Data"
description = "Check whether the given password was found in breaches."
originTypes = {'Phrase'}
resultTypes = {'Phrase'}
parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
'You can get a key here: https://haveibeenpwned.com/API/Key',
'type': 'String',
'value': '',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
import requests
from time import sleep
from hashlib import sha1
baseURL = "https://api.pwnedpasswords.com/range/"
requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
returnResults = []
count = 0
while count < len(entityJsonList):
entity = entityJsonList[count]
primaryField = sha1(entity[list(entity)[1]].encode('utf-8')).hexdigest().upper()
hashPrefix = primaryField[:5]
hashSuffix = primaryField[5:]
breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
statusCode = breachInfoRequest.status_code
if statusCode == 200:
pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
for password in pwnedPasswordContent:
if hashSuffix in password:
returnResults.append([{'Phrase': f"Password Hash Found {password.split(':')[1]} times "
f"in breach data.",
'Entity Type': 'Phrase'},
{entity['uid']: {'Resolution': 'Pwned Password',
'Notes': ''}}])
break
elif statusCode == 401:
return "The HIBP API Key provided is invalid."
elif statusCode == 429:
sleep(2)
continue
elif statusCode == 503:
return "The HIBP Service is unavailable."
sleep(1.7)
count += 1
return returnResults

58
Modules/BreachData/HaveIBeenPwnedPasswordHash.py
View File

@@ -1,58 +0,0 @@
#!/usr/bin/env python3
class HaveIBeenPwnedPasswordHash:
name = "HIBP Password Hash Lookup"
category = "Leaked Data"
description = "Check whether the given password hash was found in breaches."
originTypes = {'Hash', 'Phrase'}
resultTypes = {'Phrase'}
parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
'You can get a key here: https://haveibeenpwned.com/API/Key',
'type': 'String',
'value': '',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
import requests
from time import sleep
baseURL = "https://api.pwnedpasswords.com/range/"
requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
returnResults = []
count = 0
while count < len(entityJsonList):
entity = entityJsonList[count]
primaryField = entity[list(entity)[1]].upper()
if len(primaryField) != 40:
continue
hashPrefix = primaryField[:5]
hashSuffix = primaryField[5:]
breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
statusCode = breachInfoRequest.status_code
if statusCode == 200:
pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
for password in pwnedPasswordContent:
if hashSuffix in password:
returnResults.append([{'Phrase': f"Password Hash Found {password.split(':')[1]} times "
f"in breach data.",
'Entity Type': 'Phrase'},
{entity['uid']: {'Resolution': 'Pwned Password',
'Notes': ''}}])
break
elif statusCode == 401:
return "The HIBP API Key provided is invalid."
elif statusCode == 429:
sleep(2)
continue
elif statusCode == 503:
return "The HIBP Service is unavailable."
sleep(1.7)
count += 1
return returnResults

61
Modules/BreachData/HaveIBeenPwnedPastes.py
View File

@@ -1,61 +0,0 @@
#!/usr/bin/env python3
class HaveIBeenPwnedPastes:
name = "HIBP Paste Lookup"
category = "Leaked Data"
description = "Find all pastes that an account has been involved in."
originTypes = {'Email Address'}
resultTypes = {'Paste Data Leak'}
parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
'You can get a key here: https://haveibeenpwned.com/API/Key',
'type': 'String',
'value': '',
'global': True,
'default': 'None'}}
def resolution(self, entityJsonList, parameters):
import requests
import json
from time import sleep
from urllib.parse import quote_plus
baseURL = "https://haveibeenpwned.com/api/v3/pasteaccount/"
requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
returnResults = []
count = 0
while count < len(entityJsonList):
entity = entityJsonList[count]
emailAddress = entity['Email Address']
pasteInfoRequest = requests.get(baseURL + quote_plus(emailAddress), headers=requestHeaders)
statusCode = pasteInfoRequest.status_code
if statusCode == 200:
pasteContent = json.loads(pasteInfoRequest.content)
for paste in pasteContent:
pasteID = paste['Id']
pasteSource = paste['Source']
# If Paste Date is None, then default to entity creation date.
returnResults.append([{'Paste Identifier': f'{pasteSource} | {pasteID}',
'Paste Title': paste['Title'],
'Paste Source': pasteSource,
'Paste ID': pasteID,
'Paste Email Count': str(paste['EmailCount']),
'Entity Type': 'Paste Data Leak',
'Date Created': paste['Date']},
{entity['uid']: {'Resolution': 'Contained in Paste',
'Notes': ''}}])
elif statusCode == 401:
return "The HIBP API Key provided is invalid."
elif statusCode == 429:
sleep(2)
continue
elif statusCode == 503:
return "The HIBP Service is unavailable."
sleep(1.7)
count += 1
return returnResults

1
Modules/BreachData/requirements.txt
View File

@@ -1 +0,0 @@
requests

69
Modules/CryptoCurrency/BlockChainAddressDestinations.py
View File

@@ -1,69 +0,0 @@
#!/usr/bin/env python3
class BlockChainAddressDestinations:
name = "Get Outbound Transactions for Bitcoin Address"
category = "CryptoCurrency"
description = "Returns the Bitcoin transactions where cryptocurrency was sent from this address."
originTypes = {'BTC Address'}
resultTypes = {'BTC Transaction'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpointAddress = 'https://blockchain.info/rawaddr/'
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity['BTC Address']
try:
addressDetails = requests.get(apiEndpointAddress + primaryField).json()
if addressDetails.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
blockTransactions = addressDetails.get('txs', [])
for transaction in blockTransactions:
inputValue = 0
isInAddr = False
for transactionInput in transaction.get('inputs', []):
inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
if transactionInput['prev_out']['addr'] == primaryField:
isInAddr = True
if not isInAddr:
continue
outputValue = 0
for transactionOutput in transaction.get('out', []):
outputValue += (int(transactionOutput['value']) / 100000000)
timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
returnResults.append(
[{'Transaction Hash': transaction['hash'],
'Input Value (BTC)': str(inputValue),
'Output Value (BTC)': str(outputValue),
'Fee': str(transaction['fee']),
'Number of Inputs': str(transaction['vin_sz']),
'Number of Outputs': str(transaction['vout_sz']),
'Transaction Index': str(transaction['tx_index']),
'Size': str(transaction['size']),
'Height': str(transaction['block_height']),
'Entity Type': 'BTC Transaction',
'Date Created': timestamp},
{uid: {'Resolution': 'BTC Transaction',
'Notes': ''}}])
time.sleep(5)
return returnResults

73
Modules/CryptoCurrency/BlockChainAddressSources.py
View File

@@ -1,73 +0,0 @@
#!/usr/bin/env python3
class BlockChainAddressSources:
name = "Get Inbound Transactions for Bitcoin Address"
category = "CryptoCurrency"
description = "Returns the Bitcoin transactions where cryptocurrency was sent to this address."
originTypes = {'BTC Address'}
resultTypes = {'BTC Transaction'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
returnResultResolutions = {}
apiEndpointAddress = 'https://blockchain.info/rawaddr/'
for entity in entityJsonList:
primaryField = entity['BTC Address']
try:
addressDetails = requests.get(apiEndpointAddress + primaryField).json()
if addressDetails.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
blockTransactions = addressDetails.get('txs', [])
for transaction in blockTransactions:
outputValue = 0
isOutAddr = False
for transactionOutput in transaction.get('out', []):
outputValue += (int(transactionOutput['value']) / 100000000)
if transactionOutput['addr'] == primaryField:
isOutAddr = True
if not isOutAddr:
continue
inputValue = 0
for transactionInput in transaction.get('inputs', []):
inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
returnResultResolutions[len(returnResults)] = {'Resolution': 'BTC Transaction'}
returnResults.append(
[{'Transaction Hash': transaction['hash'],
'Input Value (BTC)': str(inputValue),
'Output Value (BTC)': str(outputValue),
'Fee': str(transaction['fee']),
'Number of Inputs': str(transaction['vin_sz']),
'Number of Outputs': str(transaction['vout_sz']),
'Transaction Index': str(transaction['tx_index']),
'Size': str(transaction['size']),
'Height': str(transaction['block_height']),
'Entity Type': 'BTC Transaction',
'Date Created': timestamp},
{'^^^': {'Resolution': 'NULL',
'Notes': ''}}])
time.sleep(5)
returnResults.append([{'BTC Address': primaryField,
'Entity Type': 'BTC Address'},
returnResultResolutions])
returnResultResolutions = {}
return returnResults

51
Modules/CryptoCurrency/BlockChainBlock.py
View File

@@ -1,51 +0,0 @@
#!/usr/bin/env python3
class BlockChainBlock:
name = "Get Bitcoin Block Details"
category = "CryptoCurrency"
description = "Returns the details of a particular bitcoin block."
originTypes = {'Hash', 'Phrase', 'BTC Block'}
resultTypes = {'BTC Block'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpoint = 'https://blockchain.info/rawblock/'
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity[list(entity)[1]]
try:
details = requests.get(apiEndpoint + primaryField).json()
if details.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
returnResults.append(
[{'Block Address': details['hash'],
'Previous Block': details['prev_block'],
'Merkle Root': details['mrkl_root'],
'Relayed By': details['relayed_by'],
'Nonce': str(details['nonce']),
'Bits': str(details['bits']),
'Size': str(details['size']),
'Block Index': str(details['block_index']),
'Height': str(details['height']),
'Main Chain': str(details['main_chain']),
'Entity Type': 'BTC Block',
'Date Created': timestamp},
{uid: {'Resolution': 'Bitcoin Block Details',
'Notes': ''}}])
time.sleep(5)
return returnResults

55
Modules/CryptoCurrency/BlockChainBlockHeight.py
View File

@@ -1,55 +0,0 @@
#!/usr/bin/env python3
class BlockChainBlockHeight:
name = "Get Bitcoin Blocks At Height"
category = "CryptoCurrency"
description = "Returns the details of all bitcoin blocks at the specified height."
originTypes = {'Phrase'}
resultTypes = {'BTC Block'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpoint = 'https://blockchain.info/block-height/'
for entity in entityJsonList:
uid = entity['uid']
try:
primaryField = int(entity['Phrase'])
except ValueError:
continue
try:
heightDetails = requests.get(apiEndpoint + str(primaryField)).json()
if heightDetails.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
for details in heightDetails['blocks']:
timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
returnResults.append(
[{'Block Address': details['hash'],
'Previous Block': details['prev_block'],
'Merkle Root': details['mrkl_root'],
'Relayed By': details['relayed_by'],
'Nonce': str(details['nonce']),
'Bits': str(details['bits']),
'Size': str(details['size']),
'Block Index': str(details['block_index']),
'Height': str(details['height']),
'Main Chain': str(details['main_chain']),
'Entity Type': 'BTC Block',
'Date Created': timestamp},
{uid: {'Resolution': 'Bitcoin Block Address',
'Notes': ''}}])
time.sleep(5)
return returnResults

57
Modules/CryptoCurrency/BlockChainBlockNext.py
View File

@@ -1,57 +0,0 @@
#!/usr/bin/env python3
class BlockChainBlockNext:
name = "Get Next Bitcoin Block"
category = "CryptoCurrency"
description = "Returns the details of the next bitcoin block."
originTypes = {'BTC Block'}
resultTypes = {'BTC Block'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpoint = 'https://blockchain.info/rawblock/'
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity['Block Address']
try:
currDetails = requests.get(apiEndpoint + primaryField).json()
if currDetails.get('error') is not None:
continue
nextBlockHashList = currDetails.get('next_block')
# Ignore nonexistent or indeterminate 'next' blocks.
if nextBlockHashList is None or len(nextBlockHashList) > 1:
continue
time.sleep(5)
details = requests.get(apiEndpoint + nextBlockHashList[0]).json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
returnResults.append(
[{'Block Address': details['hash'],
'Previous Block': details['prev_block'],
'Merkle Root': details['mrkl_root'],
'Relayed By': details['relayed_by'],
'Nonce': str(details['nonce']),
'Bits': str(details['bits']),
'Size': str(details['size']),
'Block Index': str(details['block_index']),
'Height': str(details['height']),
'Main Chain': str(details['main_chain']),
'Entity Type': 'BTC Block',
'Date Created': timestamp},
{uid: {'Resolution': 'Next BTC Block',
'Notes': ''}}])
time.sleep(5)
return returnResults

59
Modules/CryptoCurrency/BlockChainBlockPrev.py
View File

@@ -1,59 +0,0 @@
#!/usr/bin/env python3
class BlockChainBlockPrev:
name = "Get Previous Bitcoin Block"
category = "CryptoCurrency"
description = "Returns the details of the previous bitcoin block."
originTypes = {'BTC Block'}
resultTypes = {'BTC Block'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpoint = 'https://blockchain.info/rawblock/'
for entity in entityJsonList:
primaryField = entity['Block Address']
try:
currDetails = requests.get(apiEndpoint + primaryField).json()
if currDetails.get('error') is not None:
continue
prevBlockHash = currDetails.get('prev_block')
# Ignore first block.
if prevBlockHash == "0000000000000000000000000000000000000000000000000000000000000000":
continue
time.sleep(5)
details = requests.get(apiEndpoint + prevBlockHash).json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
returnResults.append(
[{'Block Address': details['hash'],
'Previous Block': details['prev_block'],
'Merkle Root': details['mrkl_root'],
'Relayed By': details['relayed_by'],
'Nonce': str(details['nonce']),
'Bits': str(details['bits']),
'Size': str(details['size']),
'Block Index': str(details['block_index']),
'Height': str(details['height']),
'Main Chain': str(details['main_chain']),
'Entity Type': 'BTC Block',
'Date Created': timestamp},
{'^^^': {'Resolution': 'NULL'}}])
returnResults.append([{'Block Address': primaryField,
'Entity Type': 'BTC Block'},
{len(returnResults) - 1: {'Resolution': 'Next BTC Block'}}])
time.sleep(5)
return returnResults

61
Modules/CryptoCurrency/BlockChainBlockTransactions.py
View File

@@ -1,61 +0,0 @@
#!/usr/bin/env python3
class BlockChainBlockTransactions:
name = "Get Bitcoin Block Transactions"
category = "CryptoCurrency"
description = "Returns the transactions that happened in a particular bitcoin block."
originTypes = {'BTC Block'}
resultTypes = {'BTC Transaction'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpoint = 'https://blockchain.info/rawblock/'
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity['Block Address']
try:
details = requests.get(apiEndpoint + primaryField).json()
if details.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
blockTransactions = details.get('tx', [])
for transaction in blockTransactions:
inputValue = 0
for transactionInput in transaction.get('inputs', []):
inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
outputValue = 0
for transactionOutput in transaction.get('out', []):
outputValue += (int(transactionOutput['value']) / 100000000)
timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
returnResults.append(
[{'Transaction Hash': transaction['hash'],
'Input Value (BTC)': str(inputValue),
'Output Value (BTC)': str(outputValue),
'Fee': str(transaction['fee']),
'Number of Inputs': str(transaction['vin_sz']),
'Number of Outputs': str(transaction['vout_sz']),
'Transaction Index': str(transaction['tx_index']),
'Size': str(transaction['size']),
'Height': str(transaction['block_height']),
'Entity Type': 'BTC Transaction',
'Date Created': timestamp},
{uid: {'Resolution': 'Bitcoin Block Address',
'Notes': ''}}])
time.sleep(5)
return returnResults

58
Modules/CryptoCurrency/BlockChainTransaction.py
View File

@@ -1,58 +0,0 @@
#!/usr/bin/env python3
class BlockChainTransaction:
name = "Get Bitcoin Transaction"
category = "CryptoCurrency"
description = "Returns the details of the specified Bitcoin transaction."
originTypes = {'BTC Transaction', 'Hash', 'Phrase'}
resultTypes = {'BTC Transaction'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
from datetime import datetime
returnResults = []
apiEndpoint = 'https://blockchain.info/rawtx/'
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity[list(entity)[1]]
try:
transaction = requests.get(apiEndpoint + primaryField).json()
if transaction.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
inputValue = 0
for transactionInput in transaction.get('inputs', []):
inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
outputValue = 0
for transactionOutput in transaction.get('out', []):
outputValue += (int(transactionOutput['value']) / 100000000)
returnResults.append(
[{'Transaction Hash': transaction['hash'],
'Input Value (BTC)': str(inputValue),
'Output Value (BTC)': str(outputValue),
'Fee': str(transaction['fee']),
'Number of Inputs': str(transaction['vin_sz']),
'Number of Outputs': str(transaction['vout_sz']),
'Transaction Index': str(transaction['tx_index']),
'Size': str(transaction['size']),
'Height': str(transaction['block_height']),
'Entity Type': 'BTC Transaction',
'Date Created': timestamp},
{uid: {'Resolution': 'Bitcoin Transaction Information',
'Notes': ''}}])
time.sleep(5)
return returnResults

55
Modules/CryptoCurrency/BlockChainTransactionDestinations.py
View File

@@ -1,55 +0,0 @@
#!/usr/bin/env python3
class BlockChainTransactionDestinations:
name = "Get Bitcoin Transaction Destinations"
category = "CryptoCurrency"
description = "Returns the Bitcoin addresses that received cryptocurrency in the specified transaction."
originTypes = {'BTC Transaction'}
resultTypes = {'BTC Address'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
returnResults = []
apiEndpointTransaction = 'https://blockchain.info/rawtx/'
apiEndpointAddress = 'https://blockchain.info/rawaddr/'
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity['Transaction Hash']
try:
transaction = requests.get(apiEndpointTransaction + primaryField).json()
if transaction.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
for transactionInput in transaction.get('out', []):
time.sleep(5)
inputAddress = transactionInput['addr']
try:
details = requests.get(apiEndpointAddress + inputAddress).json()
if details.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
returnResults.append(
[{'BTC Address': details['address'],
'Total Transactions': str(details['n_tx']),
'Unredeemed Transactions': str(details['n_unredeemed']),
'Total BTC Received': str(details['total_received'] / 100000000),
'Total BTC Sent': str(details['total_sent'] / 100000000),
'Current Balance': str(details['final_balance'] / 100000000),
'Entity Type': 'BTC Address'},
{uid: {'Resolution': 'Bitcoin Transaction',
'Notes': ''}}])
time.sleep(5)
return returnResults

62
Modules/CryptoCurrency/BlockChainTransactionSources.py
View File

@@ -1,62 +0,0 @@
#!/usr/bin/env python3
class BlockChainTransactionSources:
name = "Get Bitcoin Transaction Sources"
category = "CryptoCurrency"
description = "Returns the Bitcoin addresses that sent cryptocurrency in the specified transaction."
originTypes = {'BTC Transaction'}
resultTypes = {'BTC Address'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
returnResults = []
returnResultResolutions = {}
apiEndpointTransaction = 'https://blockchain.info/rawtx/'
apiEndpointAddress = 'https://blockchain.info/rawaddr/'
for entity in entityJsonList:
primaryField = entity['Transaction Hash']
try:
transaction = requests.get(apiEndpointTransaction + primaryField).json()
if transaction.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
for transactionInput in transaction.get('inputs', []):
time.sleep(5)
inputAddress = transactionInput['prev_out']['addr']
try:
details = requests.get(apiEndpointAddress + inputAddress).json()
if details.get('error') is not None:
continue
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
returnResultResolutions[len(returnResults)] = {'Resolution': 'BTC Transaction'}
returnResults.append(
[{'BTC Address': details['address'],
'Total Transactions': str(details['n_tx']),
'Unredeemed Transactions': str(details['n_unredeemed']),
'Total BTC Received': str(details['total_received'] / 100000000),
'Total BTC Sent': str(details['total_sent'] / 100000000),
'Current Balance': str(details['final_balance'] / 100000000),
'Entity Type': 'BTC Address'},
{'^^^': {'Resolution': 'NULL',
'Notes': ''}}])
time.sleep(5)
# Re-add the source entity so that we can point to it.
# Only include the primary field, in case the rest of the fields were updated in the meantime.
returnResults.append([{'Transaction Hash': primaryField,
'Entity Type': 'BTC Transaction'},
returnResultResolutions])
returnResultResolutions = {}
return returnResults

26
Modules/CryptoCurrency/CryptoCurrency.xml
View File

@@ -1,26 +0,0 @@
<CryptoCurrency>
<BTC_Block>
<Attributes>
<Attribute default="0000000000000000000000000000000000000000000000000000000000000000" check="String" primary="True">Block Address</Attribute>
</Attributes>
<Icon>
Bitcoin.svg
</Icon>
</BTC_Block>
<BTC_Transaction>
<Attributes>
<Attribute default="0000000000000000000000000000000000000000000000000000000000000000" check="String" primary="True">Transaction Hash</Attribute>
</Attributes>
<Icon>
Transaction.svg
</Icon>
</BTC_Transaction>
<BTC_Address>
<Attributes>
<Attribute default="1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" check="String" primary="True">BTC Address</Attribute>
</Attributes>
<Icon>
CryptoWallet.svg
</Icon>
</BTC_Address>
</CryptoCurrency>

40
Modules/CryptoCurrency/EtherScanGetBalance.py
View File

@@ -1,40 +0,0 @@
#!/usr/bin/env python3
class EtherScanGetBalance:
name = "EtherScan.io Get Balance"
category = "CryptoCurrency"
description = "EtherScan get the balance of the selected account"
originTypes = {"Crypto Wallet"}
resultTypes = {'Crypto Wallet'}
parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
"https://etherscan.io.",
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
import time
return_result = []
api_key = parameters['EtherScan API Key']
for entity in entityJsonList:
uid = entity['uid']
primary_field = entity['Wallet Address']
crafted_url = f"https://api.etherscan.io/api?module=account&action=balance" \
f"&address={primary_field}&tag=latest&apikey={api_key}"
try:
response = requests.get(crafted_url)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
response = response.json()
return_result.append([{'Amount': response['result'],
'Currency': 'Ethereum',
'Entity Type': 'Currency'},
{uid: {'Resolution': 'EtherScan.io Account Balance', 'Notes': ''}}])
time.sleep(0.2)
return return_result

39
Modules/CryptoCurrency/EtherScanGetBlocksMined.py
View File

@@ -1,39 +0,0 @@
#!/usr/bin/env python3
class EtherScanGetBlocksMined:
name = "EtherScan.io Get Blocks Mined"
category = "CryptoCurrency"
description = "EtherScan Blocks Mined from the selected account"
originTypes = {"Crypto Wallet"}
resultTypes = {'Crypto Wallet'}
parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
"https://etherscan.io.",
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
import time
return_result = []
api_key = parameters['EtherScan API Key']
for entity in entityJsonList:
uid = entity['uid']
primary_field = entity['Wallet Address']
crafted_url = f"https://api.etherscan.io/api?module=account&action=getminedblocks" \
f"&address={primary_field}&tag=latest&apikey={api_key}"
try:
response = requests.get(crafted_url)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
response = response.json()
return_result.append([{'Phrase': response['result'],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'EtherScan.io Blocks Mined', 'Notes': ''}}])
time.sleep(0.2)
return return_result

25
Modules/CryptoCurrency/ToCryptoWallet.py
View File

@@ -1,25 +0,0 @@
#!/usr/bin/env python3
class ToCryptoWallet:
name = "BTC Address To Crypto Wallet"
category = "CryptoCurrency"
description = "Convert BTC Address entities to Crypto Wallet entities."
originTypes = {'BTC Address'}
resultTypes = {'Crypto Wallet'}
parameters = {}
def resolution(self, entityJsonList, parameters):
returnResults = []
for entity in entityJsonList:
primaryField = entity['BTC Address']
returnResults.append([{'Wallet Address': primaryField,
'Currency Name': 'Bitcoin',
'Entity Type': 'Crypto Wallet'},
{entity['uid']: {'Resolution': 'To Crypto Wallet',
'Notes': ''}}])
return returnResults

2
Modules/CryptoCurrency/requirements.txt
View File

@@ -1,2 +0,0 @@
requests
datetime

107
Modules/EDGAR_&_EFD/CompanyInfo.py
View File

@@ -1,107 +0,0 @@
#!/usr/bin/env python3
class CompanyInfo:
# A string that is treated as the name of this resolution.
name = "Get Company Info"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes containing Company Information"
originTypes = {'Edgar ID'}
resultTypes = {'Phrase, SIC, EIN, Address'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(1)
search_url = f'https://data.sec.gov/submissions/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
time.sleep(1)
if r.status_code != 200:
return []
data = r.json()
exchanges = data['exchanges']
for exchange in exchanges:
returnResults.append([{'Exchange Name': exchange,
'Entity Type': 'Exchange'},
{uid: {'Resolution': 'Exchange',
'Notes': ''}}])
tickers = data['tickers']
for ticker in tickers:
returnResults.append([{'Ticker ID': ticker,
'Entity Type': 'Ticker'},
{uid: {'Resolution': 'Ticker',
'Notes': ''}}])
if data['insiderTransactionForOwnerExists'] == 1:
returnResults.append([{'Phrase': 'Insider Transaction For Owner Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Notes': ''}}])
else:
returnResults.append([{'Phrase': 'Insider Transaction For Owner Does Not Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Notes': ''}}])
if data['insiderTransactionForIssuerExists'] == 1:
returnResults.append([{'Phrase': 'Insider Transaction For Issuer Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Notes': ''}}])
else:
returnResults.append([{'Phrase': 'Insider Transaction For Issuer Does Not Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Notes': ''}}])
if data['sic'] is not None:
returnResults.append([{'SIC': str(data['sic']),
'Description': data['sicDescription'],
'Entity Type': 'SIC'},
{uid: {'Resolution': '',
'Notes': ''}}])
if data['ein'] is not None:
returnResults.append([{'EIN': str(data['ein']),
'Entity Type': 'EIN'},
{uid: {'Resolution': '',
'Notes': ''}}])
if data['addresses'] is not None:
returnResults.append([{'Street Address': data['addresses']['mailing']['street1'],
'Postal Code': data['addresses']['mailing']['zipCode'],
'Country': data['addresses']['mailing']['stateOrCountry'],
'Locality': data['addresses']['mailing']['city'],
'Entity Type': 'Address'},
{uid: {'Resolution': '',
'Notes': ''}}])
if data['addresses']['mailing']['street1'] != data['addresses']['business']['street1']:
returnResults.append([{'Street Address': data['addresses']['business']['street1'],
'Postal Code': data['addresses']['business']['zipCode'],
'Country': data['addresses']['business']['stateOrCountry'],
'Locality': data['addresses']['business']['city'],
'Entity Type': 'Address'},
{uid: {'Resolution': '',
'Notes': ''}}])
return returnResults

77
Modules/EDGAR_&_EFD/CompanyToCIK.py
View File

@@ -1,77 +0,0 @@
#!/usr/bin/env python3
class CompanyToCIK:
# A string that is treated as the name of this resolution.
name = "Get CIK ID From Company"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes of contact info for websites"
originTypes = {'Phrase', 'Company'}
resultTypes = {'Phrase'}
parameters = {}
def resolution(self, entityJsonList, parameters):
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright, TimeoutError, Error
returnResults = []
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080}
)
page = context.new_page()
for entity in entityJsonList:
page.wait_for_timeout(1000)
uid = entity['uid']
search_term = entity[list(entity)[1]]
pageResolved = False
for _ in range(3):
try:
page.goto(f'https://www.sec.gov/cgi-bin/browse-edgar?company={search_term}',
wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
continue
soup = BeautifulSoup(page.content(), 'lxml')
count = 0
temp = None
for td_element in soup.find_all('td'):
if td_element.text:
text = td_element.text
splitText = text.split('SIC')[0]
if count == 0:
temp = [{'CIK': splitText,
'Entity Type': 'Edgar ID'},
{len(returnResults): {'Resolution': 'CIK Edgar ID',
'Notes': ''}}]
elif count == 1:
returnResults.append([{'Company Name': splitText,
'Entity Type': 'Company'},
{uid: {'Resolution': 'Edgar Company',
'Notes': ''}}])
returnResults.append(temp)
elif count == 2:
returnResults.append([{'Phrase': "State: " + splitText,
'Entity Type': 'Phrase'},
{len(returnResults) - 1: {'Resolution': 'Edgar Company State',
'Notes': ''}}])
count = (count + 1) % 3
page.close()
browser.close()
return returnResults

198
Modules/EDGAR_&_EFD/EFDByFromDate.py
View File

@@ -1,198 +0,0 @@
#!/usr/bin/env python3
class EFDByFromDate:
name = 'Get EFD Reports From Date'
category = "US Senate Financial Info"
description = 'Get EFD reports starting from the date specified by the input entities.'
originTypes = {'Date'}
resultTypes = {'Politically Exposed Person', 'Website'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. '
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'},
'To Date': {'description': 'Records will be collected from the Start Date provided by the input '
'entities. NOTE: The Start Date is assumed to be in ISO format.\n'
'An End Date is required to complete the Date constraints. '
'Please input the End Date for the search in the format mm/dd/yyyy',
'type': 'String',
'value': ''},
'Filer Type': {'description': 'Please select the Office you wish to search records for.',
'type': 'MultiChoice',
'value': {'Senator',
'Candidate',
'Former Senator',
}},
'Report Type': {'description': 'Please select the Report Type you want to search for.',
'type': 'MultiChoice',
'value': {'Annual',
'Periodic Transactions',
'Due Date Extension',
'Blind Trusts',
'Other Documents',
}}}
def resolution(self, entityJsonList, parameters):
from datetime import datetime
from playwright.sync_api import sync_playwright, TimeoutError, Error
from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
returnResults = []
try:
maxResults = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter."
if maxResults <= 0:
return []
try:
toDate = datetime.strptime(parameters['To Date'], '%m/%d/%Y')
except ValueError:
return "Invalid End Date specified."
url = 'https://efdsearch.senate.gov/search/'
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
pageResolved = False
for _ in range(5):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
return "Could not access EFD Search website."
try:
page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
except TimeoutError:
return "The EFD search website is unresponsive."
except Error:
return "Connection Error."
page.wait_for_timeout(1000)
for entity in entityJsonList:
try:
# Assume ISO format - guessing
date = datetime.fromisoformat(entity['Date'])
except ValueError:
continue
if toDate < date:
continue
date = date.strftime('%m/%d/%Y')
toDate = toDate.strftime('%m/%d/%Y')
uid = entity['uid']
page.wait_for_timeout(1000)
pageResolved = False
for _ in range(3):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
continue
try:
page.fill("input[name=\"submitted_end_date\"]", toDate)
page.fill("input[name=\"submitted_start_date\"]", date)
if 'Senator' in parameters['Filer Type']:
page.click("label:has-text(\"Senator\")")
if 'Candidate' in parameters['Filer Type']:
page.click("label:has-text(\"Candidate\")")
if 'Former Senator' in parameters['Filer Type']:
page.click("label:has-text(\"Former Senator\")")
if 'Annual' in parameters['Report Type']:
page.click("text=Annual")
if 'Periodic Transactions' in parameters['Report Type']:
page.click("text=Periodic Transactions")
if 'Due Date Extension' in parameters['Report Type']:
page.click("text=Due Date Extension")
if 'Blind Trusts' in parameters['Report Type']:
page.click("text=Blind Trusts")
if 'Other Documents' in parameters['Report Type']:
page.click("text=Other Documents")
page.click("text=Search Reports")
entriesInfo = page.locator('#filedReports_info')
entriesInfo.wait_for(state='visible')
currentFirstIndex = 1
currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
lastIndex = int(entriesInfo.inner_text().split(" ")[5])
resultCount = 0
if lastIndex == 0:
continue
# Need to click twice to sort by most recent.
page.click("text=Date Received/Filed")
page.wait_for_timeout(500)
page.click("text=Date Received/Filed")
page.wait_for_timeout(500)
while True:
soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
for record in soup:
if isinstance(record, Tag) and record.get('class'):
recordFields = record.childGenerator()
senateName = next(recordFields).text
senateName += " " + next(recordFields).text
office = next(recordFields).text
report = next(recordFields)
reportType = report.text
reportLink = next(report.children).get('href')
dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
resultCount += 1
childIndex = len(returnResults)
returnResults.append([{'Full Name': senateName,
'Office': office,
'Entity Type': 'Politically Exposed Person'},
{uid: {'Resolution': 'EFD Reports', 'Notes': ''}}])
returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
'Report Type': reportType,
'Entity Type': 'Website'},
{childIndex: {'Resolution': 'Filed Disclosure Report',
'Notes': '',
'Date Created': dateCreated}}])
if resultCount == maxResults:
break
# Break if we've read enough records, or we ran out of records on this page.
if resultCount == maxResults or currentLastIndex == lastIndex:
break
# We've read all the available records, so we click next.
page.click("text=Next")
entriesInfo.wait_for(state='visible')
while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
page.wait_for_timeout(1000)
currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
lastIndex = int(entriesInfo.inner_text().split(" ")[5])
except TimeoutError:
continue
except Exception as e:
return "Resolution '" + self.name + "' encountered an error: " + str(e)
page.close()
browser.close()
return returnResults

166
Modules/EDGAR_&_EFD/EFDByName.py
View File

@@ -1,166 +0,0 @@
#!/usr/bin/env python3
class EFDByName:
name = 'Get EFD Reports by Name'
category = "US Senate Financial Info"
description = 'Get EFD reports concerning the people specified by the input entities.'
originTypes = {'Person', 'Politically Exposed Person'}
resultTypes = {'Website'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'},
'Report Type': {'description': 'Please select the Report Type you want to search for.',
'type': 'MultiChoice',
'value': {'Annual',
'Periodic Transactions',
'Due Date Extension',
'Blind Trusts',
'Other Documents',
}}}
def resolution(self, entityJsonList, parameters):
from datetime import datetime
from playwright.sync_api import sync_playwright, TimeoutError
from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
returnResults = []
try:
maxResults = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter."
if maxResults <= 0:
return []
url = 'https://efdsearch.senate.gov/search/'
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
pageResolved = False
for _ in range(5):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
if not pageResolved:
return "Could not access efdsearch website."
try:
page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
except TimeoutError:
return "The efdsearch website is unresponsive"
page.wait_for_timeout(1000)
for entity in entityJsonList:
lastName = entity['Full Name'].split(' ')[-1]
firstName = " ".join(entity['Full Name'].split(' ')[:-1])
uid = entity['uid']
page.wait_for_timeout(1000)
pageResolved = False
for _ in range(3):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
if not pageResolved:
continue
try:
personOccupation = entity['Occupation'].lower()
if personOccupation == 'senator':
page.click("label:has-text(\"Senator\")")
elif personOccupation == 'candidate':
page.click("label:has-text(\"Candidate\")")
elif personOccupation == 'former senator':
page.click("label:has-text(\"Former Senator\")")
if 'Annual' in parameters['Report Type']:
page.click("text=Annual")
if 'Periodic Transactions' in parameters['Report Type']:
page.click("text=Periodic Transactions")
if 'Due Date Extension' in parameters['Report Type']:
page.click("text=Due Date Extension")
if 'Blind Trusts' in parameters['Report Type']:
page.click("text=Blind Trusts")
if 'Other Documents' in parameters['Report Type']:
page.click("text=Other Documents")
page.fill("[placeholder=\"First name (starts with)\"]", firstName)
page.fill("[placeholder=\"Last name (starts with)\"]", lastName)
page.click("text=Search Reports")
entriesInfo = page.locator('#filedReports_info')
entriesInfo.wait_for(state='visible')
currentFirstIndex = 1
currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
lastIndex = int(entriesInfo.inner_text().split(" ")[5])
resultCount = 0
if lastIndex == 0:
continue
# Need to click twice to sort by most recent.
page.click("text=Date Received/Filed")
page.wait_for_timeout(500)
page.click("text=Date Received/Filed")
page.wait_for_timeout(500)
while True:
soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
for record in soup:
if isinstance(record, Tag) and record.get('class'):
recordFields = record.childGenerator()
senateName = next(recordFields).text
senateName += " " + next(recordFields).text
office = next(recordFields).text
report = next(recordFields)
reportType = report.text
reportLink = next(report.children).get('href')
dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
resultCount += 1
returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
'Report Type': reportType,
'Entity Type': 'Website'},
{uid: {'Resolution': 'Filed Disclosure Report',
'Notes': '',
'Date Created': dateCreated}}])
if resultCount == maxResults:
break
# Break if we've read enough records, or we ran out of records on this page.
if resultCount == maxResults or currentLastIndex == lastIndex:
break
# We've read all the available records, so we click next.
page.click("text=Next")
entriesInfo.wait_for(state='visible')
while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
page.wait_for_timeout(1000)
currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
lastIndex = int(entriesInfo.inner_text().split(" ")[5])
except TimeoutError:
continue
except Exception as e:
return "Resolution '" + self.name + "' encountered an error: " + str(e)
page.close()
browser.close()
return returnResults

198
Modules/EDGAR_&_EFD/EFDByToDate.py
View File

@@ -1,198 +0,0 @@
#!/usr/bin/env python3
class EFDByToDate:
name = 'Get EFD Reports To Date'
category = "US Senate Financial Info"
description = 'Get EFD reports ending at the date specified by the input entities.'
originTypes = {'Date'}
resultTypes = {'Politically Exposed Person', 'Website'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. '
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'},
'To Date': {'description': 'Records will be collected from the End Date provided by the input '
'entities. NOTE: The End Date is assumed to be in ISO format.\n'
'A Start Date is required to complete the Date constraints. '
'Please input the Start Date for the search in the format mm/dd/yyyy',
'type': 'String',
'value': ''},
'Filer Type': {'description': 'Please select the Office you wish to search records for.',
'type': 'MultiChoice',
'value': {'Senator',
'Candidate',
'Former Senator',
}},
'Report Type': {'description': 'Please select the Report Type you want to search for.',
'type': 'MultiChoice',
'value': {'Annual',
'Periodic Transactions',
'Due Date Extension',
'Blind Trusts',
'Other Documents',
}}}
def resolution(self, entityJsonList, parameters):
from datetime import datetime
from playwright.sync_api import sync_playwright, TimeoutError, Error
from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
returnResults = []
try:
maxResults = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter."
if maxResults <= 0:
return []
try:
date = datetime.strptime(parameters['To Date'], '%m/%d/%Y')
except ValueError:
return "Invalid End Date specified."
url = 'https://efdsearch.senate.gov/search/'
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
pageResolved = False
for _ in range(5):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
return "Could not access EFD Search website."
try:
page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
except TimeoutError:
return "The EFD search website is unresponsive."
except Error:
return "Connection Error."
page.wait_for_timeout(1000)
for entity in entityJsonList:
try:
# Assume ISO format - guessing
toDate = datetime.fromisoformat(entity['Date'])
except ValueError:
continue
if toDate < date:
continue
date = date.strftime('%m/%d/%Y')
toDate = toDate.strftime('%m/%d/%Y')
uid = entity['uid']
page.wait_for_timeout(1000)
pageResolved = False
for _ in range(3):
try:
page.goto(url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
continue
try:
page.fill("input[name=\"submitted_end_date\"]", toDate)
page.fill("input[name=\"submitted_start_date\"]", date)
if 'Senator' in parameters['Filer Type']:
page.click("label:has-text(\"Senator\")")
if 'Candidate' in parameters['Filer Type']:
page.click("label:has-text(\"Candidate\")")
if 'Former Senator' in parameters['Filer Type']:
page.click("label:has-text(\"Former Senator\")")
if 'Annual' in parameters['Report Type']:
page.click("text=Annual")
if 'Periodic Transactions' in parameters['Report Type']:
page.click("text=Periodic Transactions")
if 'Due Date Extension' in parameters['Report Type']:
page.click("text=Due Date Extension")
if 'Blind Trusts' in parameters['Report Type']:
page.click("text=Blind Trusts")
if 'Other Documents' in parameters['Report Type']:
page.click("text=Other Documents")
page.click("text=Search Reports")
entriesInfo = page.locator('#filedReports_info')
entriesInfo.wait_for(state='visible')
currentFirstIndex = 1
currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
lastIndex = int(entriesInfo.inner_text().split(" ")[5])
resultCount = 0
if lastIndex == 0:
continue
# Need to click twice to sort by most recent.
page.click("text=Date Received/Filed")
page.wait_for_timeout(500)
page.click("text=Date Received/Filed")
page.wait_for_timeout(500)
while True:
soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
for record in soup:
if isinstance(record, Tag) and record.get('class'):
recordFields = record.childGenerator()
senateName = next(recordFields).text
senateName += " " + next(recordFields).text
office = next(recordFields).text
report = next(recordFields)
reportType = report.text
reportLink = next(report.children).get('href')
dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
resultCount += 1
childIndex = len(returnResults)
returnResults.append([{'Full Name': senateName,
'Office': office,
'Entity Type': 'Politically Exposed Person'},
{uid: {'Resolution': 'EFD Reports', 'Notes': ''}}])
returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
'Report Type': reportType,
'Entity Type': 'Website'},
{childIndex: {'Resolution': 'Filed Disclosure Report',
'Notes': '',
'Date Created': dateCreated}}])
if resultCount == maxResults:
break
# Break if we've read enough records, or we ran out of records on this page.
if resultCount == maxResults or currentLastIndex == lastIndex:
break
# We've read all the available records, so we click next.
page.click("text=Next")
entriesInfo.wait_for(state='visible')
while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
page.wait_for_timeout(1000)
currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
lastIndex = int(entriesInfo.inner_text().split(" ")[5])
except TimeoutError:
continue
except Exception as e:
return "Resolution '" + self.name + "' encountered an error: " + str(e)
page.close()
browser.close()
return returnResults

119
Modules/EDGAR_&_EFD/Edgar.xml
View File

@@ -1,119 +0,0 @@
<Edgar>
<Edgar_ID>
<Attributes>
<Attribute default="000000000000" check="Numbers" primary="True">CIK</Attribute>
</Attributes>
</Edgar_ID>
<Form_Field>
<Attributes>
<Attribute default="Form Field" check="String" primary="True">Field Name</Attribute>
<Attribute default="Unknown" check="String" primary="False">Account Number</Attribute>
<Attribute default="Unknown" check="String" primary="False">Fiscal Year</Attribute>
<Attribute default="Unknown" check="String" primary="False">Fiscal Period</Attribute>
<Attribute default="Unknown" check="String" primary="False">Value</Attribute>
<Attribute default="Unknown" check="String" primary="False">Unit</Attribute>
<Attribute default="Unknown" check="String" primary="False">Taxonomy</Attribute>
</Attributes>
</Form_Field>
<Form13F>
<Attributes>
<Attribute default="Form Field" check="String" primary="True">Name Of Issuer</Attribute>
<Attribute default="Unknown" check="String" primary="False">Title Of Class</Attribute>
<Attribute default="000000000" check="CUSIP" primary="False">CUSIP</Attribute>
<Attribute default="Unknown" check="String" primary="False">Value</Attribute>
<Attribute default="Unknown" check="String" primary="False">Number Of Shares</Attribute>
<Attribute default="Unknown" check="String" primary="False">Ssh Prnamt Type</Attribute>
<Attribute default="Unknown" check="String" primary="False">Investment Discretion</Attribute>
</Attributes>
</Form13F>
<Form4>
<Attributes>
<Attribute default="Security Title" check="String" primary="True">Security Title</Attribute>
<Attribute default="Unknown" check="String" primary="False">Deemed Execution Date</Attribute>
<Attribute default="Unknown" check="String" primary="False">Equity Swap Involved</Attribute>
<Attribute default="Unknown" check="String" primary="False">Transaction Timeliness</Attribute>
<Attribute default="Unknown" check="String" primary="False">Transaction Shares</Attribute>
<Attribute default="Unknown" check="String" primary="False">Transaction Price Per Share</Attribute>
<Attribute default="Unknown" check="String" primary="False">Shares Owned Following Transaction</Attribute>
<Attribute default="Unknown" check="String" primary="False">Direct Or Indirect Ownership</Attribute>
</Attributes>
</Form4>
<Form3>
<Attributes>
<Attribute default="Security Title" check="String" primary="True">Security Title</Attribute>
<Attribute default="Unknown" check="String" primary="False">Shares Owned Following Transaction</Attribute>
<Attribute default="Unknown" check="String" primary="False">Direct Or Indirect Ownership</Attribute>
<Attribute default="Unknown" check="String" primary="False">Nature Of Ownership</Attribute>
</Attributes>
</Form3>
<FormD>
<Attributes>
<Attribute default="Company Name" check="String" primary="True">Company Name</Attribute>
<Attribute default="Unknown" check="String" primary="False">Industry Group Type</Attribute>
<Attribute default="Unknown" check="String" primary="False">Investment Fund Type</Attribute>
<Attribute default="Unknown" check="String" primary="False">Aggregate Net Asset Value Range</Attribute>
<Attribute default="Unknown" check="String" primary="False">Duration Of Offering</Attribute>
<Attribute default="Unknown" check="String" primary="False">Types Of Securities Offered</Attribute>
<Attribute default="Unknown" check="String" primary="False">Business Combination Transaction</Attribute>
<Attribute default="Unknown" check="String" primary="False">Minimum Investment Accepted</Attribute>
<Attribute default="Unknown" check="String" primary="False">Total Offering Amount</Attribute>
<Attribute default="Unknown" check="String" primary="False">Total Amount Sold</Attribute>
<Attribute default="Unknown" check="String" primary="False">Total Amount Remaining</Attribute>
<Attribute default="Unknown" check="String" primary="False">Has Non Accredited Investors</Attribute>
<Attribute default="Unknown" check="String" primary="False">Total Number Already Invested</Attribute>
<Attribute default="Unknown" check="String" primary="False">Sales Commissions</Attribute>
<Attribute default="Unknown" check="String" primary="False">Finders Fees</Attribute>
<Attribute default="Unknown" check="String" primary="False">Gross Proceeds Used</Attribute>
</Attributes>
</FormD>
<Exchange>
<Attributes>
<Attribute default="Exchange Name" check="String" primary="True">Exchange Name</Attribute>
</Attributes>
</Exchange>
<SIC>
<Attributes>
<Attribute default="0000" check="SIC/NAICS" primary="True">SIC</Attribute>
<Attribute default="Unknown" check="String" primary="False">Description</Attribute>
</Attributes>
</SIC>
<EIN>
<Attributes>
<Attribute default="00-0000000" check="EIN" primary="True">EIN</Attribute>
</Attributes>
</EIN>
<CUSIP>
<Attributes>
<Attribute default="000000000" check="CUSIP" primary="True">CUSIP</Attribute>
</Attributes>
</CUSIP>
<LEIID>
<Attributes>
<Attribute default="00000000000000000000" check="LEIID" primary="True">LEIID</Attribute>
</Attributes>
</LEIID>
<ISINID>
<Attributes>
<Attribute default="000000000000" check="ISINID" primary="True">ISINID</Attribute>
</Attributes>
</ISINID>
<FormNMFP2>
<Attributes>
<Attribute default="NMFP2 Form Field" check="String" primary="True">Field Name</Attribute>
<Attribute default="Unknown" check="String" primary="False">Friday 1</Attribute>
<Attribute default="Unknown" check="String" primary="False">Friday 2</Attribute>
<Attribute default="Unknown" check="String" primary="False">Friday 3</Attribute>
<Attribute default="Unknown" check="String" primary="False">Friday 4</Attribute>
<Attribute default="Unknown" check="String" primary="False">Friday 5</Attribute>
</Attributes>
</FormNMFP2>
<Collateral_Issuer>
<Attributes>
<Attribute default="Issuer Name" check="String" primary="True">Name</Attribute>
<Attribute default="Unknown" check="String" primary="False">Coupon or Yield</Attribute>
<Attribute default="Unknown" check="String" primary="False">Principal Amount</Attribute>
<Attribute default="Unknown" check="String" primary="False">Value of Collateral</Attribute>
<Attribute default="Unknown" check="String" primary="False">Ctgry Investments Rprsnts Collateral</Attribute>
</Attributes>
</Collateral_Issuer>
</Edgar>

93
Modules/EDGAR_&_EFD/FramesLookUp.py
View File

@@ -1,93 +0,0 @@
#!/usr/bin/env python3
class FramesLookUp:
# A string that is treated as the name of this resolution.
name = "Frames Look Up"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Frame Forms"
originTypes = {'Form Field'}
resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Phrase'}
parameters = {
'Quarter': {'description': 'Please Ensure that the selected Taxonomy matches the Form Field you typed',
'type': 'SingleChoice',
'value': {'January, February, and March (Q1)', 'April, May, and June (Q2)', 'July, August, and '
'September (Q3)',
'October, November, and December (Q4)'}},
'Year': {'description': 'Please enter the year to match.',
'type': 'String',
'default': '2021'},
'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
year = parameters['Year']
quarterChoice = parameters['Quarter']
quarter = quarterChoice[quarterChoice.find("(") + 1:quarterChoice.find(")")]
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
unit = entity['Unit']
taxonomy = entity['Taxonomy']
form_field = entity['Field Name'].split(' ')[1]
search_url = f'https://data.sec.gov/api/xbrl/frames/{taxonomy}/{form_field}/{unit}/CY{year}{quarter}I.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
data = r.json()
# print(data['data'])
if linkNumbers > len(data['data']):
linkNumbers = len(data['data'])
for i in range(linkNumbers):
# print(data['data'][i])
index_of_child = (len(returnResults))
returnResults.append([{'Company Name': data['data'][i]['entityName'],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Edgar Company',
'Notes': ''}}])
returnResults.append([{'CIK': str(data['data'][i]['cik']).zfill(10),
'Entity Type': 'Edgar ID'},
{index_of_child: {'Resolution': '',
'Notes': ''}}])
returnResults.append([{'Country Name': data['data'][i]['loc'],
'Entity Type': 'Country'},
{index_of_child: {'Resolution': '',
'Notes': ''}}])
if unit == 'USD':
returnResults.append([{'Amount': str(data['data'][i]['val']),
'Currency Type': 'USD',
'Entity Type': 'Currency'},
{index_of_child: {'Resolution': 'Form Filed Value',
'Notes': ''}}])
elif unit == 'shares':
returnResults.append([{'Phrase': 'Number of Shares: ' + str(data['data'][i]['val']),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Form Filed Value',
'Notes': ''}}])
return returnResults

96
Modules/EDGAR_&_EFD/Get10KForms.py
View File

@@ -1,96 +0,0 @@
#!/usr/bin/env python3
class Get10KForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 10-K Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 10-K Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form Field'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
data = r.json()
# print(data)
forms = list(data['facts'].keys())
for form in forms:
keys = list(data['facts'][form].keys())
for i in keys:
if 'Deprecated' not in data['facts'][form][i]['label']:
if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
if linkNumbers > len(data['facts'][form][i]['units']['USD']):
linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
for j in range(linkNumbers):
if '10-K' in data['facts'][form][i]['units']['USD'][j]['form']:
value = data['facts'][form][i]['units']['USD'][::-1][j]
returnResults.append([{'Field Name': cik + ' 10-K: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-K Field',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
if linkNumbers > len(data['facts'][form][i]['units']['shares']):
linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
for j in range(linkNumbers):
if '10-K' in data['facts'][form][i]['units']['shares'][j]['form']:
value = data['facts'][form][i]['units']['shares'][::-1][j]
returnResults.append([{'Field Name': cik + ' 10-K: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-K Field',
'Notes': ''}}])
return returnResults

94
Modules/EDGAR_&_EFD/Get10QForms.py
View File

@@ -1,94 +0,0 @@
#!/usr/bin/env python3
class Get10QForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 10-Q Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 10-Q Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form Field'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
data = r.json()
forms = list(data['facts'].keys())
for form in forms:
keys = list(data['facts'][form].keys())
for i in keys:
if 'Deprecated' not in data['facts'][form][i]['label']:
if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
if linkNumbers > len(data['facts'][form][i]['units']['USD']):
linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
for j in range(linkNumbers):
if '10-Q' in data['facts'][form][i]['units']['USD'][j]['form']:
value = data['facts'][form][i]['units']['USD'][::-1][j]
returnResults.append([{'Field Name': cik + ' 10-Q: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-Q Field',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
if linkNumbers > len(data['facts'][form][i]['units']['shares']):
linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
for j in range(linkNumbers):
if '10-Q' in data['facts'][form][i]['units']['shares'][j]['form']:
value = data['facts'][form][i]['units']['shares'][::-1][j]
returnResults.append([{'Field Name': cik + ' 10-Q: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-Q Field',
'Notes': ''}}])
return returnResults

106
Modules/EDGAR_&_EFD/Get13FForms.py
View File

@@ -1,106 +0,0 @@
#!/usr/bin/env python3
class Get13FForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 13F Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 13F Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form13F'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns 5 more recent by default',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
import xmltodict
import json
from bs4 import BeautifulSoup
from ast import literal_eval
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
name = ''
date = ''
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
archives_set = set()
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
f'={linkNumbers}&type=13F-HR'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor:
anchor = 'https://www.sec.gov' + anchor
archives_set.add(anchor)
for archive in archives_set:
time.sleep(1)
r = requests.get(archive, headers=headers)
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor and 'primary_doc.xml' in anchor \
and 'xslFormDX01' not in anchor and 'xslForm13F_X01' not in anchor:
time.sleep(1)
anchor = 'https://www.sec.gov' + anchor
r = requests.get(anchor, headers=headers)
data = literal_eval(json.dumps(xmltodict.parse(r.text)))
date = data['edgarSubmission']['headerData']['filerInfo']['periodOfReport']
name = data['edgarSubmission']['formData']['coverPage']['filingManager']['name']
elif '/Archives/edgar/data/' in anchor and 'infotable.xml' in anchor \
and 'xslFormDX01' not in anchor and 'xslForm13F_X01' not in anchor:
time.sleep(1)
anchor = 'https://www.sec.gov' + anchor
r = requests.get(anchor, headers=headers)
data = literal_eval(json.dumps(xmltodict.parse(r.text)))
for d in data['informationTable']['infoTable']:
returnResults.append([{'Name Of Issuer': '13F-HR: ' + name + ' ' + d['nameOfIssuer'] + ' '
+ date,
'Title Of Class': d['titleOfClass'],
'CUSIP': d['cusip'],
'Value': d['value'],
'Number Of Shares': d['shrsOrPrnAmt']['sshPrnamt'],
'Ssh Prnamt Type': d['shrsOrPrnAmt']['sshPrnamtType'],
'Investment Discretion': d['investmentDiscretion'],
'Notes': '',
'Entity Type': 'Form13F'},
{uid: {'Resolution': 'Form13F',
'Notes': ''}}])
return returnResults

95
Modules/EDGAR_&_EFD/Get20FForms.py
View File

@@ -1,95 +0,0 @@
#!/usr/bin/env python3
class Get20FForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 20-F Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 20-F Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form Field'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
# print(cik)
search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
data = r.json()
forms = list(data['facts'].keys())
for form in forms:
keys = list(data['facts'][form].keys())
for i in keys:
if 'Deprecated' not in data['facts'][form][i]['label']:
if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
if linkNumbers > len(data['facts'][form][i]['units']['USD']):
linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
for j in range(linkNumbers):
if '20-F' in data['facts'][form][i]['units']['USD'][j]['form']:
value = data['facts'][form][i]['units']['USD'][::-1][j]
returnResults.append([{'Field Name': cik + ' 20-F: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '20-F Field',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
if linkNumbers > len(data['facts'][form][i]['units']['shares']):
linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
for j in range(linkNumbers):
if '20-F' in data['facts'][form][i]['units']['shares'][j]['form']:
value = data['facts'][form][i]['units']['shares'][::-1][j]
returnResults.append([{'Field Name': cik + ' 20-F: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '20-F Field',
'Notes': ''}}])
return returnResults

108
Modules/EDGAR_&_EFD/Get3Forms.py
View File

@@ -1,108 +0,0 @@
#!/usr/bin/env python3
class Get3Forms:
# A string that is treated as the name of this resolution.
name = "Get Recent 3 Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 3 Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Person, Form3'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
import xmltodict
import json
from bs4 import BeautifulSoup
from ast import literal_eval
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
archives_set = set()
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
f'={linkNumbers}&type=3'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor:
anchor = 'https://www.sec.gov' + anchor
archives_set.add(anchor)
for archive in archives_set:
time.sleep(1)
r = requests.get(archive, headers=headers)
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor and 'ownership.xml' in anchor and 'xslF345X02' not in anchor:
time.sleep(1)
anchor = 'https://www.sec.gov' + anchor
r = requests.get(anchor, headers=headers)
data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
data = literal_eval(data)
# print(data)
name = data['ownershipDocument']['reportingOwner']['reportingOwnerId']['rptOwnerName']
remarks = \
data['ownershipDocument']['reportingOwner']['reportingOwnerRelationship']['officerTitle']
value = data['ownershipDocument']['nonDerivativeTable']['nonDerivativeHolding']
index_of_child = len(returnResults)
returnResults.append([{'Full Name': name,
'Notes': remarks,
'Entity Type': 'Person'},
{uid: {'Resolution': 'Reporting Owner',
'Notes': ''}}])
if value['ownershipNature']['directOrIndirectOwnership']['value'] == 'I':
nature = 'Indirect'
else:
nature = 'Direct'
returnResults.append([{'Security Title': name + ': ' + value['securityTitle']['value'] + ' ' +
data['ownershipDocument']['ownerSignature'][
'signatureDate'],
'Shares Owned Following Transaction':
value['postTransactionAmounts']['sharesOwnedFollowingTransaction'][
'value'],
'Direct Or Indirect Ownership': nature,
'Nature Of Ownership':
value['ownershipNature']['natureOfOwnership']['value'],
'Notes': '',
'Entity Type': 'Form3'},
{index_of_child: {'Resolution': 'Form 3',
'Notes': ''}}])
return returnResults

96
Modules/EDGAR_&_EFD/Get40FForms.py
View File

@@ -1,96 +0,0 @@
#!/usr/bin/env python3
class Get40FForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 40-F Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 40-F Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form Field'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
# print(r.content)
if r.status_code != 200:
return []
data = r.json()
forms = list(data['facts'].keys())
for form in forms:
keys = list(data['facts'][form].keys())
for i in keys:
if 'Deprecated' not in data['facts'][form][i]['label']:
if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
if linkNumbers > len(data['facts'][form][i]['units']['USD']):
linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
for j in range(linkNumbers):
if '40-F' in data['facts'][form][i]['units']['USD'][j]['form']:
value = data['facts'][form][i]['units']['USD'][::-1][j]
returnResults.append([{'Field Name': cik + ' 40-F: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '40-F Field',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
if linkNumbers > len(data['facts'][form][i]['units']['shares']):
linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
for j in range(linkNumbers):
if '40-F' in data['facts'][form][i]['units']['shares'][j]['form']:
value = data['facts'][form][i]['units']['shares'][::-1][j]
returnResults.append([{'Field Name': cik + ' 40-F: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '40-F Field',
'Notes': ''}}])
return returnResults

149
Modules/EDGAR_&_EFD/Get4Forms.py
View File

@@ -1,149 +0,0 @@
#!/usr/bin/env python3
class Get4Forms:
# A string that is treated as the name of this resolution.
name = "Get Recent 4 Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes D Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Person, Form4'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
import xmltodict
import json
from bs4 import BeautifulSoup
from ast import literal_eval
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
archives_set = set()
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
f'={linkNumbers}&type=4'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor:
anchor = 'https://www.sec.gov' + anchor
archives_set.add(anchor)
for archive in archives_set:
time.sleep(1)
r = requests.get(archive, headers=headers)
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor and '.xml' in anchor and 'xslF345X03' not in anchor:
time.sleep(1)
anchor = 'https://www.sec.gov' + anchor
r = requests.get(anchor, headers=headers)
data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
data = literal_eval(data)
# print(data)
index_of_child = len(returnResults)
try:
remarks = data['ownershipDocument']['remarks']
except KeyError:
remarks = ''
name = data['ownershipDocument']['reportingOwner']['reportingOwnerId']['rptOwnerName']
returnResults.append([{'Full Name': name,
'Notes': remarks,
'Entity Type': 'Person'},
{uid: {'Resolution': 'Reporting Owner',
'Notes': ''}}])
if type(data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']) == dict:
value = data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']
try:
footnote = value['transactionAmounts']['transactionShares']['footnoteId']['@id']
except KeyError:
footnote = value['transactionAmounts']['transactionShares']['value']
try:
footnotePerShare = \
value['transactionAmounts']['transactionPricePerShare']['footnoteId']['@id']
except KeyError:
footnotePerShare = value['transactionAmounts']['transactionPricePerShare']['value']
returnResults.append([{'Security Title': name + ': ' + value['securityTitle']['value'] + ' '
+ value['transactionCoding'][
'transactionCode'] + ' ' +
data['ownershipDocument']['ownerSignature'][
'signatureDate'],
'Deemed Execution Date': str(value['deemedExecutionDate']),
'Equity Swap Involved': value['transactionCoding'][
'equitySwapInvolved'],
'Transaction Timeliness': str(value['transactionTimeliness']),
'Transaction Shares': footnote,
'Transaction Price Per Share': footnotePerShare,
'Shares Owned Following Transaction':
value['postTransactionAmounts'][
'sharesOwnedFollowingTransaction'],
'Notes': (': '.join(
map(str, data['ownershipDocument']['footnotes']['footnote']))),
'Entity Type': 'Form4'},
{index_of_child: {'Resolution': 'Form 4',
'Notes': ''}}])
else:
for value in data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']:
try:
footnote = value['transactionAmounts']['transactionShares']['footnoteId']['@id']
except KeyError:
footnote = value['transactionAmounts']['transactionShares']['value']
try:
footnotePerShare = \
value['transactionAmounts']['transactionPricePerShare']['footnoteId']['@id']
except KeyError:
footnotePerShare = value['transactionAmounts']['transactionPricePerShare']['value']
returnResults.append(
[{'Security Title': name + ': ' + value['securityTitle']['value'] + ' '
+ value['transactionCoding']['transactionCode'] + ' ' +
data['ownershipDocument']['ownerSignature']['signatureDate'],
'Deemed Execution Date': str(value['deemedExecutionDate']),
'Equity Swap Involved': value['transactionCoding']['equitySwapInvolved'],
'Transaction Timeliness': str(value['transactionTimeliness']),
'Transaction Shares': footnote,
'Transaction Price Per Share': footnotePerShare,
'Shares Owned Following Transaction':
value['postTransactionAmounts']['sharesOwnedFollowingTransaction']['value'],
'Notes': (
': '.join(map(str, data['ownershipDocument']['footnotes']['footnote']))),
'Entity Type': 'Form4'},
{index_of_child: {'Resolution': 'Form 4',
'Notes': ''}}])
return returnResults

98
Modules/EDGAR_&_EFD/Get6KForms.py
View File

@@ -1,98 +0,0 @@
#!/usr/bin/env python3
class Get6KForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 6-K Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 6-K Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form Field'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
# print(cik)
search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
# print(r.content)
if r.status_code != 200:
return []
data = r.json()
# print(data)
forms = list(data['facts'].keys())
for form in forms:
keys = list(data['facts'][form].keys())
for i in keys:
if 'Deprecated' not in data['facts'][form][i]['label']:
if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
if linkNumbers > len(data['facts'][form][i]['units']['USD']):
linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
for j in range(linkNumbers):
if '6-K' in data['facts'][form][i]['units']['USD'][j]['form']:
value = data['facts'][form][i]['units']['USD'][::-1][j]
returnResults.append([{'Field Name': cik + ' 6-K: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '6-K Field',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
if linkNumbers > len(data['facts'][form][i]['units']['shares']):
linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
for j in range(linkNumbers):
if '6-K' in data['facts'][form][i]['units']['shares'][j]['form']:
value = data['facts'][form][i]['units']['shares'][::-1][j]
returnResults.append([{'Field Name': cik + ' 6-K: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '6-K Field',
'Notes': ''}}])
return returnResults

98
Modules/EDGAR_&_EFD/Get8KForms.py
View File

@@ -1,98 +0,0 @@
#!/usr/bin/env python3
class Get8KForms:
# A string that is treated as the name of this resolution.
name = "Get Recent 8-K Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes 8-K Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Form Field'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
# print(cik)
search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
time.sleep(1)
r = requests.get(search_url, headers=headers)
# print(r.content)
if r.status_code != 200:
return []
data = r.json()
# print(data)
forms = list(data['facts'].keys())
for form in forms:
keys = list(data['facts'][form].keys())
for i in keys:
if 'Deprecated' not in data['facts'][form][i]['label']:
if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
if linkNumbers > len(data['facts'][form][i]['units']['USD']):
linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
for j in range(linkNumbers):
if '8-K' in data['facts'][form][i]['units']['USD'][j]['form']:
value = data['facts'][form][i]['units']['USD'][::-1][j]
returnResults.append([{'Field Name': cik + ' 8-K: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '8-K Field',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
if linkNumbers > len(data['facts'][form][i]['units']['shares']):
linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
for j in range(linkNumbers):
if '8-K' in data['facts'][form][i]['units']['shares'][j]['form']:
value = data['facts'][form][i]['units']['shares'][::-1][j]
returnResults.append([{'Field Name': cik + ' 8-K: ' + i + ' ' + value['filed'],
'Account Number': value['accn'],
'Fiscal Year': value['fy'],
'Fiscal Period': value['fp'],
'Value': value['val'],
'Unit': list(data['facts'][form][i]['units'].keys())[0],
'Taxonomy': form,
'Notes': data['facts'][form][i]['label'],
'Entity Type': 'Form Field'},
{uid: {'Resolution': '8-K Field',
'Notes': ''}}])
return returnResults

149
Modules/EDGAR_&_EFD/GetDForms.py
View File

@@ -1,149 +0,0 @@
#!/usr/bin/env python3
class GetDForms:
# A string that is treated as the name of this resolution.
name = "Get Recent D Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes D Forms"
originTypes = {'Edgar ID'}
resultTypes = {'FormD, Person, Address, Phrase'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
import xmltodict
import json
from bs4 import BeautifulSoup
from ast import literal_eval
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
archives_set = set()
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
f'={linkNumbers}&type=D'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor:
anchor = 'https://www.sec.gov' + anchor
archives_set.add(anchor)
for archive in archives_set:
time.sleep(1)
r = requests.get(archive, headers=headers)
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor and 'primary_doc.xml' in anchor \
and 'xslFormDX01' not in anchor:
time.sleep(1)
anchor = 'https://www.sec.gov' + anchor
r = requests.get(anchor, headers=headers)
data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
data = literal_eval(data)
# print(data)
value = data['edgarSubmission']['offeringData']
index_of_child = len(returnResults)
returnResults.append([{'Company Name': 'D: ' + data['edgarSubmission']['primaryIssuer']
['entityName'] + ' ' + value['signatureBlock']['signature']['signatureDate'],
'Industry Group Type': value['industryGroup']['industryGroupType'],
'Investment Fund Type': value['industryGroup']['investmentFundInfo']
['investmentFundType'],
'Aggregate Net Asset Value Range': value['issuerSize']
['aggregateNetAssetValueRange'],
'Duration Of Offering': 'More Than one Year: ' +
value['durationOfOffering']['moreThanOneYear'],
'Types Of Securities Offered': 'Pooled Investment Fund Type: ' +
value['typesOfSecuritiesOffered'][
'isPooledInvestmentFundType'],
'Business Combination Transaction': 'Business Combination Transaction: '
+ value[
'businessCombinationTransaction'][
'isBusinessCombinationTransaction'],
'Minimum Investment Accepted': value['minimumInvestmentAccepted'],
'Total Offering Amount': value['offeringSalesAmounts']
['totalOfferingAmount'],
'Total Amount Sold': value['offeringSalesAmounts']['totalAmountSold'],
'Total Amount Remaining': value['offeringSalesAmounts'][
'totalRemaining'],
'Has Non Accredited Investors': 'Non Accredited Investors'
+ value['investors'][
'hasNonAccreditedInvestors'],
'Total Number Already Invested': value['investors']
['totalNumberAlreadyInvested'],
'Sales Commissions': value['salesCommissionsFindersFees']
['salesCommissions']['dollarAmount'],
'Finders Fees': value['salesCommissionsFindersFees']['findersFees']
['dollarAmount'],
'Gross Proceeds Used': value['useOfProceeds']['grossProceedsUsed']
['dollarAmount'],
'Notes': '',
'Entity Type': 'FormD'},
{uid: {'Resolution': 'D Form',
'Notes': ''}}])
people = data['edgarSubmission']['relatedPersonsList']['relatedPersonInfo']
for person in people:
child_of_child = len(returnResults)
returnResults.append([{'Full Name': person['relatedPersonName']['firstName'] + ' ' +
person['relatedPersonName']['lastName'],
'Entity Type': 'Person'},
{index_of_child: {'Resolution': 'Officer',
'Notes': ''}}])
returnResults.append(
[{'Street Address': person['relatedPersonAddress']['street1'],
'Locality': person['relatedPersonAddress']['city'],
'Postal Code': person['relatedPersonAddress']['zipCode'],
'Country': person['relatedPersonAddress']['stateOrCountryDescription'],
'Entity Type': 'Address'},
{child_of_child: {'Resolution': 'Location', 'Notes': ''}}])
returnResults.append(
[{'Phrase': person['relatedPersonRelationshipList']['relationship'],
'Entity Type': 'Phrase'},
{child_of_child: {'Resolution': 'Relationship', 'Notes': ''}}])
if person['relationshipClarification'] is not None:
returnResults.append(
[{'Phrase': person['relationshipClarification'],
'Entity Type': 'Phrase'},
{child_of_child: {'Resolution': 'Relationship', 'Notes': ''}}])
return returnResults

76
Modules/EDGAR_&_EFD/GetN8FForms.py
View File

@@ -1,76 +0,0 @@
#!/usr/bin/env python3
class GetN8FForms:
# A string that is treated as the name of this resolution.
name = "Get Recent N-8F Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes N-8F Forms Websites"
originTypes = {'Edgar ID'}
resultTypes = {'Website'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import time
from bs4 import BeautifulSoup
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
}
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Invalid integer provided in 'Max Results' parameter"
if linkNumbers <= 0:
return []
returnResults = []
for entity in entityJsonList:
archives_set = set()
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
f'={linkNumbers}&type=N-8F'
time.sleep(1)
r = requests.get(search_url, headers=headers)
if r.status_code != 200:
return []
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor:
anchor = 'https://www.sec.gov' + anchor
archives_set.add(anchor)
for archive in archives_set:
time.sleep(1)
r = requests.get(archive, headers=headers)
soup = BeautifulSoup(r.text, "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if '/Archives/edgar/data/' in anchor and '.htm' in anchor:
time.sleep(1)
anchor = 'https://www.sec.gov' + anchor
returnResults.append([{'URL': anchor,
'Entity Type': 'Website'},
{uid: {'Resolution': 'N-8F Form',
'Notes': ''}}])
return returnResults

275
Modules/EDGAR_&_EFD/GetNMFP2Forms.py
View File

@@ -1,275 +0,0 @@
#!/usr/bin/env python3
class GetNMFP2Forms:
# A string that is treated as the name of this resolution.
name = "Get Recent N-MFP2 Forms"
category = "EDGAR Info"
# A string that describes this resolution.
description = "Returns Nodes N-MFP2 Forms"
originTypes = {'Edgar ID'}
resultTypes = {'Collateral Issuer, Company, Phrase, FormNMFP2, CUSIP, LEIID, ISINID'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
'Returns the 5 most recent by default.',
'type': 'String',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import requests
import xmltodict
import json
from playwright.sync_api import sync_playwright, TimeoutError, Error
from bs4 import BeautifulSoup
from ast import literal_eval
headers = {
'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
}
try:
maxResults = int(parameters['Max Results'])
except ValueError:
return "Invalid integer value provided for 'Max Results' parameter."
if maxResults <= 0:
return []
returnResults = []
liquidAssets = ['totalValueDailyLiquidAssets', 'totalValueWeeklyLiquidAssets', 'percentageDailyLiquidAssets',
'percentageWeeklyLiquidAssets', 'netAssetValue']
seriesLevelInfoKeys = ['feederFundFlag', 'masterFundFlag', 'seriesFundInsuCmpnySepAccntFlag',
'fundExemptRetailFlag', 'averagePortfolioMaturity',
'averageLifeMaturity', 'cash',
'totalValuePortfolioSecurities', 'amortizedCostPortfolioSecurities',
'totalValueOtherAssets', 'totalValueLiabilities', 'netAssetOfSeries',
'numberOfSharesOutstanding', 'stablePricePerShare', 'sevenDayGrossYield']
classLevelInfoKeys = ['minInitialInvestment', 'netAssetsOfClass', 'numberOfSharesOutstanding',
'sevenDayNetYield', 'personPayForFundFlag']
securitiesInfoKeys = ['titleOfIssuer', 'investmentCategory', 'securityEligibilityFlag',
'investmentMaturityDateWAM', 'investmentMaturityDateWAL',
'finalLegalInvestmentMaturityDate', 'securityDemandFeatureFlag', 'securityGuaranteeFlag',
'securityEnhancementsFlag', 'yieldOfTheSecurityAsOfReportingDate',
'includingValueOfAnySponsorSupport', 'excludingValueOfAnySponsorSupport',
'percentageOfMoneyMarketFundNetAssets', 'securityCategorizedAtLevel3Flag',
'dailyLiquidAssetSecurityFlag', 'weeklyLiquidAssetSecurityFlag', 'illiquidSecurityFlag']
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
for entity in entityJsonList:
archives_set = set()
uid = entity['uid']
cik = entity['CIK']
if cik.lower().startswith('cik'):
cik = cik.split('cik')[1]
if len(cik) != 10:
cik = cik.zfill(10)
search_url = f'https://www.sec.gov/edgar/search/#/category=custom&entityName={cik}&forms=N-MFP2'
page.wait_for_timeout(1000)
pageResolved = False
for _ in range(3):
try:
page.goto(search_url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
continue
page.wait_for_timeout(1000)
soup = BeautifulSoup(page.content(), "lxml")
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['data-adsh'] if 'data-adsh' in link.attrs else ''
if anchor != '':
anchor = anchor.replace('-', '')
anchor = f'https://www.sec.gov/Archives/edgar/data/{cik}/{anchor}/primary_doc.xml'
archives_set.add(anchor)
for link in range(maxResults):
r = requests.get(list(archives_set)[link], headers=headers)
data = literal_eval(json.dumps(xmltodict.parse(r.text)).replace('null', 'None'))
fieldPath = data['edgarSubmission']['formData']['seriesLevelInfo']
seriesId = data['edgarSubmission']['formData']['generalInfo'][
'seriesId']
date = data['edgarSubmission']['formData']['generalInfo'][
'reportDate']
returnResults.append(
[{'Company Name': fieldPath['adviser']['adviserName'],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Adviser', 'Notes': ''}}])
returnResults.append(
[{'Company Name': fieldPath['indpPubAccountant']['name'],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Independent Pub Accountant', 'Notes': ''}}])
returnResults.append(
[{'Company Name': fieldPath['administrator']['administratorName'],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Administrator', 'Notes': ''}}])
returnResults.append(
[{'Company Name': fieldPath['transferAgent']['name'],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Transfer Agent', 'Notes': ''}}])
for value in seriesLevelInfoKeys:
returnResults.append(
[{'Phrase': f'N-MFP2:({value}) '
+ f'ID: {seriesId} Date: {date}',
'Notes': fieldPath[value],
'Entity Type': 'Phrase'},
{uid: {'Resolution': value, 'Notes': ''}}])
for value in fieldPath['moneyMarketFundCategory']:
returnResults.append(
[{'Phrase': value,
'Entity Type': 'Phrase'},
{uid: {'Resolution': value, 'Notes': ''}}])
for field in liquidAssets:
if 'Daily' in field:
returnResults.append([{'Field Name': cik + f' N-MFP2:({field})' + ' '
+ f'ID: {seriesId} Date: {date}',
'Friday 1': fieldPath[field][
'ns3:fridayDay1'],
'Friday 2': fieldPath[field][
'ns3:fridayDay2'],
'Friday 3': fieldPath[field][
'ns3:fridayDay3'],
'Friday 4': fieldPath[field][
'ns3:fridayDay4'],
'Friday 5': 'NO Value in Daily Measure',
'Entity Type': 'FormNMFP2'},
{uid: {'Resolution': field,
'Notes': ''}}])
else:
returnResults.append([{'Field Name': cik + f' N-MFP2:({field})' + ' '
+ f'ID: {seriesId} Date: {date}',
'Friday 1': fieldPath[field][
'ns3:fridayWeek1'],
'Friday 2': fieldPath[field][
'ns3:fridayWeek2'],
'Friday 3': fieldPath[field][
'ns3:fridayWeek3'],
'Friday 4': fieldPath[field][
'ns3:fridayWeek4'],
'Friday 5': fieldPath[field][
'ns3:fridayWeek5'],
'Entity Type': 'FormNMFP2'},
{uid: {'Resolution': field,
'Notes': ''}}])
classLevelInfo = data['edgarSubmission']['formData']['classLevelInfo']
for classInfo in classLevelInfo:
index_of_child = len(returnResults)
returnResults.append(
[{'Phrase': classInfo['classesId'],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Classes Id', 'Notes': ''}}])
returnResults.append([{'Field Name': cik + f' N-MFP2:(Net Asset Per Share)' + ' '
+ f'ID: {seriesId} Date: {date}',
'Friday 1': classInfo['netAssetPerShare'][
'ns3:fridayWeek1'],
'Friday 2': classInfo['netAssetPerShare'][
'ns3:fridayWeek2'],
'Friday 3': classInfo['netAssetPerShare'][
'ns3:fridayWeek3'],
'Friday 4': classInfo['netAssetPerShare'][
'ns3:fridayWeek4'],
'Friday 5': classInfo['netAssetPerShare'][
'ns3:fridayWeek5'],
'Entity Type': 'FormNMFP2'},
{index_of_child: {'Resolution': 'Net Asset Per Share',
'Notes': ''}}])
for weekCount in range(1, 6):
returnResults.append(
[{'Phrase': classInfo[f'fridayWeek{weekCount}']['weeklyGrossSubscriptions'],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': f'Friday Week {weekCount} Weekly Gross Subscriptions',
'Notes': ''}}])
returnResults.append(
[{'Phrase': classInfo[f'fridayWeek{weekCount}']['weeklyGrossRedemptions'],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': f'Friday Week {weekCount} Weekly Gross Redemptions',
'Notes': ''}}])
for value in classLevelInfoKeys:
returnResults.append(
[{'Phrase': classInfo[value],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': value,
'Notes': ''}}])
scheduleOfPortfolioSecurities = data['edgarSubmission']['formData'][
'scheduleOfPortfolioSecuritiesInfo']
instance = 0
for securitiesInfo in scheduleOfPortfolioSecurities:
index_of_child = len(returnResults)
returnResults.append(
[{'Company Name': securitiesInfo.get('nameOfIssuer') + ' ' + str(instance),
'Entity Type': 'Company'},
{uid: {'Resolution': 'Issuer', 'Notes': ''}}])
instance += 1
returnResults.append(
[{'CUSIP': securitiesInfo.get('CUSIPMember'),
'Entity Type': 'CUSIP'},
{index_of_child: {'Resolution': 'CUSIP', 'Notes': ''}}])
returnResults.append(
[{'LEIID': securitiesInfo.get('LEIID'),
'Entity Type': 'LEIID'},
{index_of_child: {'Resolution': 'LEIID', 'Notes': ''}}])
returnResults.append(
[{'ISINID': securitiesInfo.get('ISINId'),
'Entity Type': 'ISINID'},
{index_of_child: {'Resolution': 'ISINID', 'Notes': ''}}])
for value in securitiesInfoKeys:
returnResults.append(
[{'Phrase': securitiesInfo[value],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': value,
'Notes': ''}}])
for value in securitiesInfo['NRSRO']:
child_of_child = len(returnResults)
returnResults.append(
[{'Company Name': value.get('nameOfNRSRO'),
'Entity Type': 'Company'},
{index_of_child: {'Resolution': 'NRSRO',
'Notes': ''}}])
returnResults.append(
[{'Phrase': value.get('rating'),
'Entity Type': 'Phrase'},
{child_of_child: {'Resolution': 'Rating',
'Notes': ''}}])
try:
collateralIssuer = securitiesInfo['collateralIssuers']
for issuer in collateralIssuer:
returnResults.append([{'Name': issuer['nameOfCollateralIssuer'],
'Coupon or Yield': issuer['couponOrYield'],
'Principal Amount': issuer['principalAmountToTheNearestCent'],
'Value of Collateral': issuer[
'valueOfCollateralToTheNearestCent'],
'Ctgry Investments Rprsnts Collateral':
issuer['ctgryInvestmentsRprsntsCollateral'],
'Entity Type': 'Collateral Issuer'},
{index_of_child: {'Resolution': 'Collateral Issuer',
'Notes': ''}}])
except KeyError:
continue
page.close()
browser.close()
return returnResults

5
Modules/EDGAR_&_EFD/requirements.txt
View File

@@ -1,5 +0,0 @@
requests
bs4
xmltodict
playwright
datetime

10
Modules/ExampleModule/ExampleModuleEntities.xml
View File

@@ -1,10 +0,0 @@
<ExampleEntities>
<Example>
<Attributes>
<Attribute default="Default value for ExampleLabel attribute" check="String" primary="True">ExampleLabel</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Example>
</ExampleEntities>

69
Modules/ExampleModule/ExampleResolution.py
View File

@@ -1,69 +0,0 @@
#!/usr/bin/env python3
class ExampleResolution:
# A string that is treated as the name of this resolution.
name = "Example Resolution"
category = "Example"
# A string that describes this resolution.
description = "Resolves Nothing in particular"
# A set of entities that this resolution can be ran on.
originTypes = {'Person'}
# A set of entities that could be the result of this resolution.
resultTypes = {'Person'}
# A dictionary of properties for this resolution. The key is the property name,
# the value is the property attributes. The type of input expected from the user is determined by the
# variable type of the 'value' parameter.
parameters = {'String Example': {'description': 'Example String Description',
'type': 'String',
'value': ''},
'File Example': {'description': 'Example Choose File Description',
'type': 'File',
'value': ''},
'Choose One Example': {'description': 'Example Choose One Description',
'type': 'SingleChoice',
'value': {'one', 'two', 'three'}
},
'Choose Multiple Example': {'description': 'Example Choose Multiple Description',
'type': 'MultiChoice',
'value': {'one', 'two', 'three'}
}
}
def resolution(self, entityJsonList, parameters):
"""
eJsonList is a dictionary where the keys are the accepted origin
types, and the values are lists of json representations of
entities whose type matches the key.
parameters is a dictionary with the keys of the 'parameters' variable.
The value of each key is the user's input.
Example:
If the origin types of an entity are: {'Person', 'Alias'}
The input could be:
[Person1JSON, Person2JSON]
or:
[Person1JSON, Alias1JSON]
Returns a list of lists, where each inner list contains an entity produced as output, and a dict of dicts
where the keys of the outer dictionary are either UIDs of input nodes or indices of elements in the outer list.
The inner dict holds the 'Resolution' and 'Notes' characteristics of the link to create.
Example:
[[resultNodeJson1, {'inputNodeUID1': {'Resolution': 'LinkName1', 'Notes': 'LinkNotes1'},
resultNodeIndex1: {'Resolution': 'LinkName2', 'Notes': ''}}],
...]
"""
return []

1
Modules/ExampleModule/requirements.txt
View File

@@ -1 +0,0 @@
# Here all the python3 packages required for the module to function are listed.

70
Modules/GitHubRSAFinder/DorkingMethod.py
View File

@@ -1,70 +0,0 @@
#!/usr/bin/env python3
class DorkingMethod:
# A string that is treated as the name of this resolution.
name = "RSA Keys Startpage Dorking"
category = "Secrets & Leaks"
# A string that describes this resolution.
description = "Returns Nodes of github repos containing RSA keys"
originTypes = {'Phrase'}
resultTypes = {'Website'}
parameters = {}
def resolution(self, entityJsonList, parameters):
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright, TimeoutError, Error
urls = set()
returnResults = []
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
for entity in entityJsonList:
uid = entity['uid']
search_term = entity[list(entity)[1]]
search_url = f'https://www.startpage.com/do/dsearch?query={search_term}' \
f'+site:github.com' \
f'+-site:gist.github.com' \
f'+-inurl:issues' \
f'+-inurl:wiki' \
f'+-filetype:markdown' \
f'+-filetype:md' \
f'+"-----BEGIN+RSA+PRIVATE+KEY-----" '
pageResolved = False
for _ in range(3):
try:
page.goto(search_url, wait_until="networkidle", timeout=10000)
pageResolved = True
break
except TimeoutError:
pass
except Error:
break
if not pageResolved:
continue
soup = BeautifulSoup(page.content(), "lxml") # store the result from the search
for link in soup.find_all('a'):
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if 'github' in anchor and anchor.startswith('http') and anchor not in urls:
urls.add(anchor)
returnResults.append(
[{'URL': anchor,
'Entity Type': 'Website'},
{uid: {'Resolution': 'RSA Key',
'Notes': ''}}])
return returnResults

27
Modules/GitHubRSAFinder/GitHub.xml
View File

@@ -1,27 +0,0 @@
<GitHub>
<GitHub_Repository>
<Attributes>
<Attribute default="Github Repository Name" check="String" primary="True">Repository Name</Attribute>
</Attributes>
</GitHub_Repository>
<GitHub_Organisation>
<Attributes>
<Attribute default="Github Org Name" check="String" primary="True">Organisation Name</Attribute>
</Attributes>
</GitHub_Organisation>
<GitHub_FilePath>
<Attributes>
<Attribute default="Github FilePath" check="String" primary="True">Filepath</Attribute>
</Attributes>
</GitHub_FilePath>
<GitHub_Secret>
<Attributes>
<Attribute default="Github Secret" check="String" primary="True">Secret</Attribute>
</Attributes>
</GitHub_Secret>
<GitHub_Branch>
<Attributes>
<Attribute default="Github Repository Branch" check="String" primary="True">Branch</Attribute>
</Attributes>
</GitHub_Branch>
</GitHub>

128
Modules/GitHubRSAFinder/OrgSearch_GitAllSecrets.py
View File

@@ -1,128 +0,0 @@
#!/usr/bin/env python3
class OrgSearch_GitAllSecrets:
# A string that is treated as the name of this resolution.
name = "Git-All-Secrets OrgSearch"
category = "Secrets & Leaks"
# A string that describes this resolution.
description = "Searches Github Organization repositories for exposed secrets. Requires Docker to be installed."
originTypes = {'GitHub Organisation'}
resultTypes = {'GitHub Organisation', 'GitHub Secret', 'GitHub Repository', 'GitHub FilePath', 'Hash',
'GitHub Branch'}
parameters = {'Github Token': {'description': 'Github personal access token.\n'
'We need this because unauthenticated requests to the Github API '
'are likely to hit the rate limit.',
'type': 'String',
'value': '',
'global': True},
}
def resolution(self, entityJsonList, parameters):
import docker
import json
import tempfile
import re
from pathlib import Path
pattern = re.compile(r'\{(?:[^{}]|(?R))*\}')
ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
hogSecret = []
client = docker.from_env()
with tempfile.TemporaryDirectory() as tempDir:
tempPath = Path(tempDir).absolute()
client.containers.run('abhartiya/tools_gitallsecrets:latest',
f'-token={parameters["Token"]} '
f'-org={entity[list(entity)[1]]} -output=/home/out.txt',
volumes={str(tempPath): {'bind': '/home',
'mode': 'rw'}}, remove=True)
jsonFile = tempPath / 'out.txt'
if jsonFile.exists():
with open(jsonFile, 'r') as jsonFileHandler:
jsonContents = jsonFileHandler.read()
with open(jsonFile, 'r') as file:
hogSecret.extend(next(file) for line in file if "Commit" in line)
repoSupervisor = jsonContents.split('Tool: repo-supervisor')[1]
truffleHog = jsonContents.split('Tool: repo-supervisor')[0]
orgOrUser = [
line.split(' ')
for line in repoSupervisor.splitlines()
if line.startswith('OrgorUser')
]
data = pattern.findall(repoSupervisor)
for value in data:
data.append(json.loads(value))
for index, userOrg in enumerate(orgOrUser):
index_of_child = len(returnResults)
returnResults.append(
[{'Organisation Name': f'Org or User: {userOrg[1]}',
'Entity Type': 'GitHub Organisation'},
{uid: {'Resolution': 'Tool: repo-supervisor',
'Notes': ''}}
]
)
child_of_child = len(returnResults)
returnResults.append([{'Repository Name': userOrg[3],
'Entity Type': 'GitHub Repository'},
{index_of_child: {'Resolution': 'Repository of Organisation',
'Notes': ''}}])
childOfChild = 0
if userOrg[3] in truffleHog and userOrg[1] in truffleHog:
userContent = truffleHog.split(f'OrgorUser: {userOrg[1]} RepoName: {userOrg[3]}')[1]
hogIndex = 0
for line in userContent.splitlines():
line = line.strip()
if 'Reason' in line:
childOfChild = len(returnResults)
returnResults.append([{'Phrase': ansi_escape.sub('', line),
'Notes': ansi_escape.sub('', hogSecret[hogIndex]),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'GitHub Secret',
'Notes': ''}}])
hogIndex += 1
elif 'Hash' in line:
returnResults.append([{'Hash Value': ansi_escape.sub('', line),
'Entity Type': 'Hash'},
{childOfChild: {'Resolution': 'Tool: truffleHog',
'Notes': ''}}])
elif 'Filepath' in line:
returnResults.append([{'Filepath': ansi_escape.sub('', line),
'Entity Type': 'GitHub FilePath'},
{childOfChild: {'Resolution': 'GitHub FilePath',
'Notes': ''}}])
elif 'Branch' in line:
returnResults.append([{'Branch': ansi_escape.sub('', line),
'Entity Type': 'GitHub Branch'},
{childOfChild: {'Resolution': 'GitHub Branch',
'Notes': ''}}])
for result in data:
for secret in result['result']:
secrets = (secret.get('secrets'))
if userOrg[3] in secret.get('filepath'):
child_child = len(returnResults)
returnResults.append([{'Filepath': secret.get('filepath'),
'Entity Type': 'GitHub FilePath'},
{child_of_child: {'Resolution': 'GitHub FilePath',
'Notes': ''}}])
returnResults.extend(
[{'Secret': scrt,
'Entity Type': 'GitHub Secret'},
{child_child: {'Resolution': 'GitHub Secret',
'Notes': ''}}
]
for scrt in secrets
)
return returnResults

3
Modules/GitHubRSAFinder/requirements.txt
View File

@@ -1,3 +0,0 @@
docker
bs4
playwright

48
Modules/Hunter.io/HunterDomainEmailSearch.py
View File

@@ -1,48 +0,0 @@
#!/usr/bin/env python3
class HunterDomainEmailSearch:
name = "Hunter.io Domain Email Search"
category = "Online Identity"
description = "Find all the emails associated with a domain."
originTypes = {"Domain"}
resultTypes = {'Email Address'}
parameters = {'Hunter API Key': {'description': "Enter the api key under your profile after signing up at "
"https://hunter.io/",
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
return_result = []
futures = []
uidList = []
api_key = parameters['Hunter API Key']
url = "https://api.hunter.io/v2/"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]]
crafted_url = f"{url}domain-search?domain={primary_field}&api_key={api_key}"
futures.append(session.get(crafted_url))
response = {}
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
if future.result().status_code == 401:
return "The API Key provided is Invalid"
elif future.result().status_code == 200:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
for email in response['data']['emails']:
return_result.append([{'Email Address': email['value'],
'Entity Type': 'Email Address'},
{uid: {'Resolution': 'Hunter.io Domain Search', 'Notes': ''}}])
return return_result

47
Modules/Hunter.io/HunterEmailVerifier.py
View File

@@ -1,47 +0,0 @@
#!/usr/bin/env python3
class HunterEmailVerifier:
name = "Hunter.io Email Verifier"
category = "Online Identity"
description = "Verify an email using hunter.io."
originTypes = {"Email Address"}
resultTypes = {'Phrase'}
parameters = {'Hunter API Key': {'description': "Enter the api key under your profile after signing up at "
"https://hunter.io/",
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
return_result = []
futures = []
uidList = []
api_key = parameters['Hunter API Key']
url = "https://api.hunter.io/v2/"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]]
crafted_url = f"{url}email-verifier?email={primary_field}&api_key={api_key}"
futures.append(session.get(crafted_url))
response = {}
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
if future.result().status_code == 401:
return "The API Key provided is Invalid"
elif future.result().status_code == 200:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
return_result.append([{'Phrase': f"The email {primary_field} is {response['data']['status']}",
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Hunter.io Email Verifier', 'Notes': ''}}])
return return_result

55
Modules/Hunter.io/HunterPersonEmailFinder.py
View File

@@ -1,55 +0,0 @@
#!/usr/bin/env python3
class HunterPersonEmailFinder:
name = "Hunter.io Person Email Finder"
category = "Online Identity"
description = "Find the email of a person for a particular domain."
originTypes = {"Domain"}
resultTypes = {'Email Address'}
parameters = {'Hunter API Key': {'description': "Enter the api key under your profile after signing up at "
"https://hunter.io/",
'type': 'String',
'value': '',
'global': True},
'Full Name': {'description': "Enter the Full Name with space in between (John Doe)",
'type': 'String',
'value': ''}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
return_result = []
futures = []
uidList = []
api_key = parameters['Hunter API Key']
try:
first, last = parameters['Full Name'].strip().split(" ")
except ValueError:
return "Please enter only a first and last name. It has to be only two words with a space in between."
url = "https://api.hunter.io/v2/"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]]
crafted_url = \
f"{url}email-finder?domain={primary_field}&first_name={first}&last_name={last}&api_key={api_key}"
futures.append(session.get(crafted_url))
response = {}
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
if future.result().status_code == 401:
return "The API Key provided is Invalid"
elif future.result().status_code == 200:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
return_result.append([{'Email Address': response['data']['email'],
'Entity Type': 'Email Address'},
{uid: {'Resolution': 'Hunter.io Person Email Finder', 'Notes': ''}}])
return return_result

2
Modules/Hunter.io/requirements.txt
View File

@@ -1,2 +0,0 @@
requests
requests-futures

65
Modules/IPInfo/IPInfo.py
View File

@@ -1,65 +0,0 @@
#!/usr/bin/env python3
class IPInfo:
name = "Get IPInfo Results For IP"
category = "Geolocation"
description = "Find information about the location of a given IP Address"
originTypes = {'IP Address'}
resultTypes = {'Geocordinates', 'Organization', 'City'}
parameters = {'IPInfo Access Token': {'description': 'Enter your access token key under your profile after '
'signing up on https://ipinfo.io. Free usage of the API is '
'limited to 50,000 requests per month. '
'For any requests beyond that limit, no results will be '
'returned.',
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
from ipaddress import ip_address
return_result = []
futures = []
uidList = []
primaryFields = []
access_token = parameters['IPInfo Access Token'].strip()
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]].strip()
primaryFields.append(primary_field)
try:
ip_address(primary_field)
except ValueError:
return "The Entity Provided isn't a valid IP Address"
url = "https://ipinfo.io/" + str(primary_field) + "?token=" + str(access_token)
futures.append(session.get(url))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
if future.result().status_code == 429:
if len(return_result) != 0:
return return_result
return "The API Key provided is Invalid or you are sending requests above the rate limit"
else:
response = future.result().json()
latitude, longitude = response['loc'].split(",")
return_result.append([{'Label': str(primaryFields[futures.index(future)]) + " Location",
'Latitude': latitude,
'Longitude': longitude,
'Entity Type': 'GeoCoordinates'},
{uid: {'Resolution': 'IPInfo IP Geocordinates', 'Notes': ''}}])
return_result.append([{'Organization Name': response['org'],
'Entity Type': 'Organization'},
{uid: {'Resolution': 'IPInfo IP Organization', 'Notes': ''}}])
return_result.append([{'City Name': response['city'],
'Entity Type': 'City'},
{uid: {'Resolution': 'IPInfo IP City Name', 'Notes': ''}}])
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
return return_result

2
Modules/IPInfo/requirements.txt
View File

@@ -1,2 +0,0 @@
requests
requests-futures

78
Modules/IPQualityScore/IPQualityScore_Email.py
View File

@@ -1,78 +0,0 @@
#!/usr/bin/env python3
class IPQualityScore_Email:
name = "IP Quality Score Email"
category = "Threats & Malware"
description = "Find information about the location of a given IP Address or Validate an Email Address"
originTypes = {'Email Address'}
resultTypes = {'Phrase', 'Person', 'Email Address'}
parameters = {'IPQualityScore Private Key': {'description': 'Enter your private key under your profile after '
'signing up on https://ipqualityscore.com. The limit '
'per month for free accounts is 5000 lookups.',
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
return_result = []
uidList = []
primaryFields = []
futures = []
private_key = parameters['IPQualityScore Private Key']
url = "https://ipqualityscore.com/api/json/email/private_key/primary_field?timeout=7"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]].strip()
primaryFields.append(primary_field)
crafted_url = url.replace("primary_field", primary_field).replace("private_key", private_key)
futures.append(session.get(crafted_url))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
if response['success'] != "True" and response['message'] == "You have insufficient credits to make this " \
"query. Please contact IPQualityScore " \
" support if this error persists.":
return "Your account doesn't have sufficient credits to complete this operation."
valid = f"valid: {response['valid']}\n"
disposable = f"disposable: {response['disposable']}\n"
smtp_score = f"smtp_score: {response['smtp_score']}\n"
overall_score = f"overall_score: {response['overall_score']}\n"
generic = f"generic: {response['generic']}\n"
common = f"common: {response['common']}\n"
dns_valid = f"dns_valid: {response['dns_valid']}\n"
honeypot = f"honeypot: {response['honeypot']}\n"
deliverability = f"deliverability: {response['deliverability']}\n"
frequent_complainer = f"frequent_complainer: {response['frequent_complainer']}\n"
spam_trap_score = f"spam_trap_score: {response['spam_trap_score']}\n"
catch_all = f"catch_all: {response['catch_all']}\n"
suspect = f"suspect: {response['suspect']}\n"
recent_abuse = f"recent_abuse: {response['recent_abuse']}\n"
fraud_score = f"fraud_score: {response['fraud_score']}\n"
suggested_domain = f"suggested_domain: {response['suggested_domain']}\n"
leaked = f"leaked: {response['leaked']}\n"
return_result.append([{'Phrase': response['request_id'],
'Notes': valid + disposable + smtp_score + overall_score + generic + common +
dns_valid + honeypot + deliverability + frequent_complainer +
spam_trap_score + catch_all + suspect + recent_abuse + fraud_score +
suggested_domain + leaked,
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'IPQualityScore Scan ID', 'Notes': ''}}])
if response['first_name'] != "":
return_result.append([{'Full Name': response['first_name'],
'Entity Type': 'Person'},
{uid: {'Resolution': 'IPQualityScore First Name', 'Notes': ''}}])
if response['sanitized_email'] != primary_field:
return_result.append([{'Email Address': response['sanitized_email'],
'Entity Type': 'Email Address'},
{uid: {'Resolution': 'IPQualityScore Sanitized Email', 'Notes': ''}}])
return return_result

87
Modules/IPQualityScore/IPQualityScore_IP.py
View File

@@ -1,87 +0,0 @@
#!/usr/bin/env python3
class IPQualityScore_IP:
name = "IP Quality Score IP"
category = "Threats & Malware"
description = "Find information about the location of a given IP Address or Validate an Email Address"
originTypes = {'IP Address'}
resultTypes = {'Phrase', 'Autonomous System', 'Geocordinates', 'Organization', 'Country', 'City'}
parameters = {'IPQualityScore Private Key': {'description': 'Enter your private key under your profile after '
'signing up on https://ipqualityscore.com. The limit '
'per month for free accounts is 5000 lookups.',
'type': 'String',
'value': '',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed
import pycountry
return_result = []
uidList = []
primaryFields = []
futures = []
private_key = parameters['IPQualityScore Private Key']
url = "https://ipqualityscore.com/api/json/ip/private_key/primary_field?timeout=7"
with FuturesSession(max_workers=15) as session:
for entity in entityJsonList:
uidList.append(entity['uid'])
primary_field = entity[list(entity)[1]].strip()
primaryFields.append(primary_field)
crafted_url = url.replace("primary_field", primary_field).replace("private_key", private_key)
futures.append(session.get(crafted_url))
for future in as_completed(futures):
uid = uidList[futures.index(future)]
try:
response = future.result().json()
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
if response['success'] != "True" and response['message'] == "You have insufficient credits to make this " \
"query. Please contact IPQualityScore " \
" support if this error persists.":
return "Your account doesn't have sufficient credits to complete this operation."
Country_Code = response['country_code']
# Region = response['region']
City = response['city']
# ISP = response['ISP']
ASN = response['ASN']
Organization = response['organization']
latitude = response['latitude']
longitude = response['longitude']
fraud_score = f"fraud_score: {response['fraud_score']}\n"
proxy = f"proxy: {response['proxy']}\n"
vpn = f"vpn: {response['vpn']}\n"
tor = f"tor: {response['tor']}\n"
is_crawler = f"iscrawler {response['is_crawler']}\n"
active_vpn = f"active vpn: {response['active_vpn']}\n"
active_tor = f"active tor: {response['active_tor']}\n"
recent_abuse = f"recent abuse: {response['recent_abuse']}\n"
bot_status = f"bot status: {response['bot_status']}\n"
return_result.append([{'AS Number': f"AS{str(ASN)}",
'Entity Type': 'Autonomous System'},
{uid: {'Resolution': 'IPQualityScore AS Number', 'Notes': ''}}])
return_result.append([{'Organization Name': Organization,
'Entity Type': 'Organization'},
{uid: {'Resolution': 'IPQualityScore IP Organization', 'Notes': ''}}])
return_result.append([{'Country Name': pycountry.countries.get(alpha_2=Country_Code).name,
'Entity Type': 'Country'},
{uid: {'Resolution': 'IPQualityScore Scan', 'Notes': ''}}])
return_result.append([{'City Name': City,
'Entity Type': 'City'},
{uid: {'Resolution': 'IPQualityScore IP City Name', 'Notes': ''}}])
return_result.append([{'Label': f"{primary_field} Location",
'Latitude': latitude,
'Longitude': longitude,
'Entity Type': 'GeoCoordinates'},
{uid: {'Resolution': 'IPQualityScore IP Geolocation', 'Notes': ''}}])
return_result.append([{'Phrase': response['request_id'],
'Notes': fraud_score + is_crawler + proxy + vpn + tor + active_vpn + active_tor +
recent_abuse + bot_status,
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'IPQualityScore Scan ID', 'Notes': ''}}])
return return_result

3
Modules/IPQualityScore/requirements.txt
View File

@@ -1,3 +0,0 @@
pycountry
requests
requests-futures

71
Modules/InternetDB/InternetDB.py
View File

@@ -1,71 +0,0 @@
#!/usr/bin/env python3
class InternetDB:
name = "InternetDB IP lookup"
category = "Network Infrastructure"
description = "Convert the primary field of selected entities to a Phrase entity."
originTypes = {'IP Address'}
resultTypes = {'Domain', 'Phrase', 'Port'}
parameters = {'InternetDB Disclaimer': {'description': 'InternetDB access is free for non-commercial use. '
'If you are using this service for commercial purposes, '
'you need an enterprise license. You can get one at '
'https://enterprise.shodan.io/.\n'
'Type "Accept" (without quotes) to confirm your '
'understanding.',
'type': 'String',
'value': 'Type "Accept" (without quotes) to confirm your understanding.',
'global': True}}
def resolution(self, entityJsonList, parameters):
import requests
if parameters['InternetDB Disclaimer'].strip() != 'Accept':
return []
returnResults = []
for entity in entityJsonList:
primaryField = entity['IP Address']
entityUID = entity['uid']
requestResult = requests.get(f"https://internetdb.shodan.io/{primaryField}").json()
if "detail" in requestResult:
returnResults.append([{'Phrase': requestResult['detail'],
'Entity Type': 'Phrase'},
{entityUID: {'Resolution': 'InternetDB Lookup Result',
'Notes': ''}}])
elif "msg" in requestResult:
returnResults.append([{'Phrase': requestResult['msg'],
'Entity Type': 'Phrase'},
{entityUID: {'Resolution': 'InternetDB Lookup Result',
'Notes': ''}}])
else:
for cpe in requestResult['cpes']:
returnResults.append([{'Phrase': cpe,
'Entity Type': 'Phrase'},
{entityUID: {'Resolution': 'InternetDB IP CPE',
'Notes': ''}}])
for hostname in requestResult['hostnames']:
returnResults.append([{'Domain Name': hostname,
'Entity Type': 'Domain'},
{entityUID: {'Resolution': 'InternetDB IP Domain',
'Notes': ''}}])
for port in requestResult['ports']:
returnResults.append([{'Port': requestResult['ip'] + ":" + str(port),
'Entity Type': 'Port'},
{entityUID: {'Resolution': 'InternetDB IP Open Port',
'Notes': ''}}])
for tag in requestResult['tags']:
returnResults.append([{'Phrase': tag,
'Entity Type': 'Phrase'},
{entityUID: {'Resolution': 'InternetDB IP Tag',
'Notes': ''}}])
for vuln in requestResult['vulns']:
returnResults.append([{'Phrase': vuln,
'Entity Type': 'Phrase'},
{entityUID: {'Resolution': 'InternetDB IP Vuln',
'Notes': ''}}])
return returnResults

1
Modules/InternetDB/requirements.txt
View File

@@ -1 +0,0 @@
requests

26
Modules/Interpol/Interpol.xml
View File

@@ -1,26 +0,0 @@
<Interpol>
<Red_Notice>
<Attributes>
<Attribute default="0-0" check="String" primary="True">ID</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Red_Notice>
<Yellow_Notice>
<Attributes>
<Attribute default="0-0" check="String" primary="True">ID</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Yellow_Notice>
<UN_Notice>
<Attributes>
<Attribute default="0-0" check="String" primary="True">ID</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</UN_Notice>
</Interpol>

115
Modules/Interpol/InterpolRedNotices.py
View File

@@ -1,115 +0,0 @@
#!/usr/bin/env python3
class InterpolRedNotices:
name = "Interpol Red Notice Check"
category = "Crime"
description = "Find Interpol Red Notices about a person. Names from entities must be in the format " \
"Firstname Lastname."
originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
resultTypes = {'Red Notice'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
from PySide6.QtGui import QImage
firstRequestPart1 = "https://ws-public.interpol.int/notices/v1/red?forename="
firstRequestPart2 = "&ageMax=200&ageMin=0&page="
firstRequestPart3 = "&resultPerPage=160"
# Some notices may have missing info as 'null'.
null = None
returnResults = []
def handleRedNotice(redNoticeContents: dict) -> None:
for redNotice in redNoticeContents['_embedded']['notices']:
noticeLink = redNotice['_links']['self']['href']
noticeID = noticeLink.split('/red/')[1]
noticeContentsRaw = requests.get(noticeLink).json()
# Clean out values that are None as they cause issues with join()
noticeContents = {k: v for k, v in noticeContentsRaw.items() if v is not None}
noticeNotes = ""
for warrant in noticeContents['arrest_warrants']:
noticeNotes += f"CHARGE: {warrant.get('charge')}\nFROM COUNTRY: " \
f"{warrant.get('issuing_country_id')}\n\n"
try:
thumbnailPictureURL = noticeContents['_links']['thumbnail']['href']
thumbnailIconRequest = requests.get(thumbnailPictureURL)
thumbnailIconByteArray = QByteArray(thumbnailIconRequest.content)
thumbnailIconImageOriginal = QImage().fromData(thumbnailIconByteArray)
thumbnailIconImageScaled = thumbnailIconImageOriginal.scaled(QSize(40, 40))
thumbnailByteArrayFin = QByteArray()
thumbnailImageBuffer = QBuffer(thumbnailByteArrayFin)
thumbnailImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
thumbnailIconImageScaled.save(thumbnailImageBuffer, "PNG")
thumbnailImageBuffer.close()
except Exception:
thumbnailByteArrayFin = None
returnResults.append([{'ID': noticeID,
'Forename': noticeContents.get('forename', 'Unknown'),
'Name': noticeContents.get('name', 'Unknown'),
'Gender': noticeContents.get('sex_id', 'Unknown'),
'Date of Birth': noticeContents.get('date_of_birth', 'Unknown'),
'Country of Birth': noticeContents.get('country_of_birth_id', 'Unknown'),
'Place of Birth': noticeContents.get('place_of_birth', 'Unknown'),
'Weight': str(noticeContents.get('weight', 'Unknown')),
'Height': str(noticeContents.get('height', 'Unknown')),
'Distinguishing Features': noticeContents.get('distinguishing_marks',
'Unknown'),
'Languages Spoken': ', '.join(noticeContents.get('languages_spoken_ids',
['Unknown'])),
'Nationalities': ', '.join(noticeContents.get('nationalities',
['Unknown'])),
'Eye Colors': ', '.join(noticeContents.get('eyes_colors_id',
['Unknown'])),
'Hair Colors': ', '.join(noticeContents.get('hairs_id',
['Unknown'])),
'Entity Type': 'Red Notice',
'Date Created': noticeContents.get('date_of_birth', 'Unknown'),
'Icon': thumbnailByteArrayFin,
'Notes': noticeNotes},
{uid: {'Resolution': 'Red Notice',
'Notes': ''}}])
for entity in entityJsonList:
uid = entity['uid']
entityType = entity['Entity Type']
if entityType in ['Person', 'Politically Exposed Person']:
primaryField = entity['Full Name'].strip()
elif entityType == 'Phrase':
primaryField = entity['Phrase'].strip()
else:
continue
# We take the first part of the first name and the last part of the last name.
# This ensures that we will not miss any matches.
nameFragments = primaryField.split(' ')
firstName = nameFragments[0].upper()
lastName = None if len(nameFragments) == 1 else nameFragments[-1].upper()
firstRequestURL = firstRequestPart1 + firstName
if lastName is not None:
firstRequestURL += f"&name={lastName}"
firstRequestURL += firstRequestPart2
firstRequest = requests.get(f"{firstRequestURL}1{firstRequestPart3}")
pageContents = firstRequest.json()
lastPage = int(pageContents['_links']['last']['href'].split('&page=')[1].split('&')[0])
handleRedNotice(pageContents)
for pageIndex in range(2, lastPage + 1):
pageContents = requests.get(firstRequestURL + str(pageIndex) + firstRequestPart3)
handleRedNotice(pageContents.json())
return returnResults

115
Modules/Interpol/InterpolYellowNotices.py
View File

@@ -1,115 +0,0 @@
#!/usr/bin/env python3
class InterpolYellowNotices:
name = "Interpol Yellow Notice Check"
category = "Crime"
description = "Find Interpol Yellow Notices about a person. Names from entities must be in the format " \
"Firstname Lastname."
originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
resultTypes = {'Yellow Notice'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
from PySide6.QtGui import QImage
firstRequestPart1 = "https://ws-public.interpol.int/notices/v1/yellow?forename="
firstRequestPart2 = "&ageMax=200&ageMin=0&page="
firstRequestPart3 = "&resultPerPage=160"
# Some notices may have missing info as 'null'.
null = None
returnResults = []
def handleYellowNotice(yellowNoticeContents: dict) -> None:
for yellowNotice in yellowNoticeContents['_embedded']['notices']:
noticeLink = yellowNotice['_links']['self']['href']
noticeID = noticeLink.split('/yellow/')[1]
noticeContentsRaw = requests.get(noticeLink).json()
# Clean out values that are None as they cause issues with join()
noticeContents = {k: v for k, v in noticeContentsRaw.items() if v is not None}
try:
thumbnailPictureURL = noticeContents['_links']['thumbnail']['href']
thumbnailIconRequest = requests.get(thumbnailPictureURL)
thumbnailIconByteArray = QByteArray(thumbnailIconRequest.content)
thumbnailIconImageOriginal = QImage().fromData(thumbnailIconByteArray)
thumbnailIconImageScaled = thumbnailIconImageOriginal.scaled(QSize(40, 40))
thumbnailByteArrayFin = QByteArray()
thumbnailImageBuffer = QBuffer(thumbnailByteArrayFin)
thumbnailImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
thumbnailIconImageScaled.save(thumbnailImageBuffer, "PNG")
thumbnailImageBuffer.close()
except Exception:
thumbnailByteArrayFin = None
returnResults.append([{'ID': noticeID,
'Forename': noticeContents.get('forename', 'Unknown'),
'Name': noticeContents.get('name', 'Unknown'),
'Gender': noticeContents.get('sex_id', 'Unknown'),
'Date of Birth': noticeContents.get('date_of_birth', 'Unknown'),
'Country of Birth': noticeContents.get('country_of_birth_id', 'Unknown'),
'Place of Birth': noticeContents.get('place_of_birth', 'Unknown'),
'Mother Forename': noticeContents.get('mother_forename', 'Unknown'),
'Mother Name': noticeContents.get('mother_name', 'Unknown'),
'Father Forename': noticeContents.get('father_forename', 'Unknown'),
'Father Name': noticeContents.get('father_name', 'Unknown'),
'Weight': str(noticeContents.get('weight', 'Unknown')),
'Height': str(noticeContents.get('height', 'Unknown')),
'Distinguishing Features': noticeContents.get('distinguishing_marks',
'Unknown'),
'Languages Spoken': ', '.join(noticeContents.get('languages_spoken_ids',
['Unknown'])),
'Nationalities': ', '.join(noticeContents.get('nationalities',
['Unknown'])),
'Eye Colors': ', '.join(noticeContents.get('eyes_colors_id',
['Unknown'])),
'Hair Colors': ', '.join(noticeContents.get('hairs_id',
['Unknown'])),
'Place Of Event': noticeContents.get('place', 'Unknown'),
'Date Of Event': noticeContents.get('date_of_event', 'Unknown'),
'Entity Type': 'Yellow Notice',
'Date Created': noticeContents.get('date_of_birth', 'Unknown'),
'Icon': thumbnailByteArrayFin},
{uid: {'Resolution': 'Yellow Notice',
'Notes': ''}}])
for entity in entityJsonList:
uid = entity['uid']
entityType = entity['Entity Type']
if entityType in ['Person', 'Politically Exposed Person']:
primaryField = entity['Full Name'].strip()
elif entityType == 'Phrase':
primaryField = entity['Phrase'].strip()
else:
continue
# We take the first part of the first name and the last part of the last name.
# This ensures that we will not miss any matches.
nameFragments = primaryField.split(' ')
firstName = nameFragments[0].upper()
lastName = None if len(nameFragments) == 1 else nameFragments[-1].upper()
firstRequestURL = firstRequestPart1 + firstName
if lastName is not None:
firstRequestURL += f"&name={lastName}"
firstRequestURL += firstRequestPart2
firstRequest = requests.get(f"{firstRequestURL}1{firstRequestPart3}")
pageContents = firstRequest.json()
lastPage = int(pageContents['_links']['last']['href'].split('&page=')[1].split('&')[0])
handleYellowNotice(pageContents)
for pageIndex in range(2, lastPage + 1):
pageContents = requests.get(firstRequestURL + str(pageIndex) + firstRequestPart3)
handleYellowNotice(pageContents.json())
return returnResults

1
Modules/Interpol/requirements.txt
View File

@@ -1 +0,0 @@
requests

97
Modules/KYC/KYC.xml
View File

@@ -1,97 +0,0 @@
<KYC>
<Sanctioned_Person>
<Attributes>
<Attribute default="John Smith" check="String" primary="True">Full Name</Attribute>
<Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
<Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
<Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
<Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
</Attributes>
<Icon>
SanctionedPerson.svg
</Icon>
</Sanctioned_Person>
<Criminal>
<Attributes>
<Attribute default="John Smith" check="String" primary="True">Full Name</Attribute>
<Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
<Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
<Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
<Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
</Attributes>
<Icon>
SanctionedPerson.svg
</Icon>
</Criminal>
<Sanctioned_Organization>
<Attributes>
<Attribute default="Organization Name" check="String" primary="True">Organization Name</Attribute>
</Attributes>
<Icon>
SanctionedOrganization.svg
</Icon>
</Sanctioned_Organization>
<Politically_Exposed_Organization>
<Attributes>
<Attribute default="Organization Name" check="String" primary="True">Organization Name</Attribute>
</Attributes>
<Icon>
SanctionedOrganization.svg
</Icon>
</Politically_Exposed_Organization>
<Sanctioned_Company>
<Attributes>
<Attribute default="Company Name" check="String" primary="True">Company Name</Attribute>
</Attributes>
<Icon>
SanctionedCompany.svg
</Icon>
</Sanctioned_Company>
<Politically_Exposed_Company>
<Attributes>
<Attribute default="Company Name" check="String" primary="True">Company Name</Attribute>
</Attributes>
<Icon>
SanctionedCompany.svg
</Icon>
</Politically_Exposed_Company>
<KYC_Phrase>
<Attributes>
<Attribute default="Phrase" check="String" primary="True">KYC Phrase</Attribute>
</Attributes>
<Icon>
KYCPhrase.svg
</Icon>
</KYC_Phrase>
<Sanctioned_Crypto_Wallet>
<Attributes>
<Attribute default="Wallet Address" check="String" primary="True">Wallet Address</Attribute>
<Attribute default="Bitcoin" check="String" primary="False">Currency Name</Attribute>
</Attributes>
<Icon>
Sanctioned_CryptoWallet.svg
</Icon>
</Sanctioned_Crypto_Wallet>
<Sanctioned_Vessel>
<Attributes>
<Attribute default="IMO 0" check="String" primary="True">Vessel Registration</Attribute>
<Attribute default="Vessel" check="String" primary="False">Vessel Name</Attribute>
<Attribute default="000000" check="String" primary="False">Vessel CallSign</Attribute>
<Attribute default="XX" check="String" primary="False">Vessel Flag</Attribute>
</Attributes>
<Icon>
Sanctioned_Vessel.svg
</Icon>
</Sanctioned_Vessel>
<Sanctioned_Aircraft>
<Attributes>
<Attribute default="0000" check="String" primary="True">Aircraft Serial Number</Attribute>
<Attribute default="00-000" check="String" primary="False">Aircraft Registration</Attribute>
<Attribute default="Aircraft" check="String" primary="False">Aircraft Name</Attribute>
<Attribute default="Aircraft Model" check="String" primary="False">Aircraft Model</Attribute>
</Attributes>
<Icon>
Sanctioned_Aircraft.svg
</Icon>
</Sanctioned_Aircraft>
</KYC>

104
Modules/LittleSis_API/ExtractRelationship.py
View File

@@ -1,104 +0,0 @@
class ExtractRelationship:
# A string that is treated as the name of this resolution.
name = "LittleSis Relationship Extractor"
category = "LittleSis"
# A string that describes this resolution.
description = "Returns Nodes of Relationship Info"
originTypes = {'Little Sis ID'}
resultTypes = {'Currency', 'Politically Exposed Person', 'Little Sis ID'}
parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.',
'type': 'String',
'value': '',
'default': '5'}}
def resolution(self, entityJsonList, parameters):
import time
import requests
import json
returnResults = []
index_of_child = []
try:
linkNumbers = int(parameters['Max Results'])
except ValueError:
return "Non-integer specified for 'Max Results' parameter; cannot run resolution."
if linkNumbers <= 0:
return []
for entity in entityJsonList:
uid = entity['uid']
if not str(entity[list(entity)[1]]).startswith('LS:'):
search_term = 'LS:' + entity[list(entity)[1]]
else:
search_term = entity[list(entity)[1]]
term = (search_term.split(':')[1]).strip()
try:
apiRequest = requests.get(f'https://littlesis.org/api/entities/{term}/relationships')
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
if apiRequest.status_code != 200:
continue
data = apiRequest.json()
data = data['data'][:linkNumbers]
for relationship in data:
index_of_child.append(len(returnResults))
search_id = relationship['attributes']['entity1_id']
if relationship['attributes']['description2'] is not None:
returnResults.append([{'ID': 'LS: ' + str(search_id),
'Entity Type': 'Little Sis ID'},
{uid: {'Resolution': str(relationship['attributes']['description2']),
'Is Current': str(relationship['attributes']['is_current']),
'Notes': str(relationship['attributes']['description2'])}}])
elif relationship['attributes']['description1'] is not None:
returnResults.append([{'ID': 'LS: ' + str(search_id),
'Entity Type': 'Little Sis ID'},
{uid: {'Resolution': str(relationship['attributes']['description1']),
'Is Current': str(relationship['attributes']['is_current']),
'Notes': str(relationship['attributes']['description1'])}}])
else:
returnResults.append([{'ID': 'LS: ' + str(search_id),
'Entity Type': 'Little Sis ID'},
{uid: {'Resolution': str(relationship['attributes']['description']),
'Is Current': str(relationship['attributes']['is_current']),
'Notes': str(relationship['attributes']['description'])}}])
try:
res = requests.get(f'https://littlesis.org/api/entities/{search_id}')
time.sleep(0.25)
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
if apiRequest.status_code != 200:
return returnResults
try:
entity_data = res.json()
except json.decoder.JSONDecodeError:
continue
child_of_child = len(returnResults)
returnResults.append([{'Full Name': entity_data['data']['attributes']['name'],
'Occupation': ", ".join(entity_data['data']['attributes']['types']),
'Notes': entity_data['data']['attributes']['blurb'],
'Entity Type': 'Politically Exposed Person'},
{index_of_child[-1]: {'Resolution': 'Name in LittleSis DB',
'Notes': ''}}])
if relationship['attributes']['amount'] is not None:
returnResults.append([{'Amount': str(relationship['attributes']['amount']),
'Currency Type': relationship['attributes']['currency'],
'Entity Type': 'Currency'},
{child_of_child: {'Resolution': 'Contribution Amount',
'Notes': ''}}])
return returnResults

48
Modules/LittleSis_API/IDExtractorDB.py
View File

@@ -1,48 +0,0 @@
class IDExtractorDB:
# A string that is treated as the name of this resolution.
name = "LittleSis ID Extractor"
category = "LittleSis"
# A string that describes this resolution.
description = "Returns Nodes of ID Info"
originTypes = {'Person', 'Phrase', 'Politically Exposed Person'}
resultTypes = {'Politically Exposed Person', 'Little Sis ID'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
search_term = entity[list(entity)[1]]
try:
r = requests.get(
f'https://littlesis.org/api/entities/search?q={search_term}')
except requests.exceptions.ConnectionError:
return "Please check your internet connection"
if r.status_code != 200:
continue
data = r.json()
for data in data['data']:
index_of_child = len(returnResults)
returnResults.append([{'Full Name': data['attributes']['name'],
'Date of Birth': str(data['attributes']['start_date']),
'Occupation': ", ".join(data['attributes']['types']),
'Notes': str(data['attributes']['blurb']),
'Entity Type': 'Politically Exposed Person'},
{uid: {'Resolution': 'Name in LittleSis DB',
'Notes': ''}}])
returnResults.append([{'ID': 'LS:' + str(data['attributes']['id']),
'Entity Type': 'Little Sis ID'},
{index_of_child: {'Resolution': 'ID in LittleSis DB',
'Notes': ''}}])
return returnResults

10
Modules/LittleSis_API/LittleSis.xml
View File

@@ -1,10 +0,0 @@
<LittleSis>
<Little_Sis_ID>
<Attributes>
<Attribute default="LS:DefaultValue" check="String" primary="True">ID</Attribute>
</Attributes>
<Icon>
Default.svg
</Icon>
</Little_Sis_ID>
</LittleSis>

1
Modules/LittleSis_API/requirements.txt
View File

@@ -1 +0,0 @@
requests

1645
Modules/MailCat/Mailcat.py
View File

File diff suppressed because it is too large Load Diff

5
Modules/MailCat/requirements.txt
View File

@@ -1,5 +0,0 @@
aiohttp[speedups]
aiosmtplib
requests-html
aiohttp-socks
dnspython

39
Modules/MailDomainReputation/KickboxMailDisposable.py
View File

@@ -1,39 +0,0 @@
#!/usr/bin/env python3
class KickboxMailDisposable:
name = "Kickbox Disposable Email Check"
category = "Reputation Check"
description = "Check if an email address is from a disposable provider, or if a domain is known for providing " \
"disposable email addresses."
originTypes = {'Email Address', 'Domain'}
resultTypes = {'Phrase'}
parameters = {}
def resolution(self, entityJsonList, parameters):
import requests
returnResults = []
for entity in entityJsonList:
entityType = entity['Entity Type']
if entityType == 'Email Address':
emailDomain = entity['Email Address'][::-1].split('@', 1)[0][::-1]
elif entityType == 'Domain':
emailDomain = entity['Domain Name']
else:
continue
mailDisposable = requests.get('https://open.kickbox.com/v1/disposable/' + emailDomain).json()['disposable']
if mailDisposable:
returnResults.append([{'Phrase': 'Disposable: ' + emailDomain,
'Entity Type': 'Phrase'},
{entity['uid']: {'Resolution': 'Kickbox Disposable Email Check',
'Notes': ''}}])
else:
returnResults.append([{'Phrase': 'Not Disposable: ' + emailDomain,
'Entity Type': 'Phrase'},
{entity['uid']: {'Resolution': 'Kickbox Disposable Email Check',
'Notes': ''}}])
return returnResults

1
Modules/MailDomainReputation/requirements.txt
View File

@@ -1 +0,0 @@
requests

105
Modules/NameThatHash/NameThatHash.py
View File

@@ -1,105 +0,0 @@
#!/usr/bin/env python3
"""
Credit:
https://twitter.com/bee_sec_san
https://github.com/HashPals/Name-That-Hash
"""
class NameThatHash:
# A string that is treated as the name of this resolution.
name = "Identify Hash Type"
category = "String Operations"
# A string that describes this resolution.
description = "Identifies the algorithm that was used to generate the hash given as input."
# A set of entities that this resolution can be ran on.
originTypes = {'Hash', 'Phrase'}
# A set of entities that could be the result of this resolution.
resultTypes = {'Phrase'}
# A dictionary of properties for this resolution. The key is the property name,
# the value is the property attributes. The type of input expected from the user is determined by the
# variable type of the 'value' parameter.
parameters = {'Max Results Per Hash': {'description': 'Please enter the maximum results you want per hash. Results '
'are returned in order of likeliness, so the first few '
'results would be the hash types that would match the input '
'the best.',
'type': 'String',
'value': '',
'default': '5'},
'Hash Crack Options': {'description': 'Please select whether you want to generate entities that '
'specify the appropriate configuration option to use in hash '
'cracking software for each hash type.',
'type': 'MultiChoice',
'value': {'HashCat', 'John'},
'default': {'HashCat', 'John'}
}
}
def resolution(self, entityJsonList, parameters):
from name_that_hash import check_hashes, hash_namer, hashes
import logging
nth = hash_namer.Name_That_Hash(hashes.prototypes)
hashChecker = check_hashes.HashChecker({}, nth)
try:
maxResults = int(parameters['Max Results Per Hash'])
if maxResults < 1:
raise ValueError()
except ValueError:
return "Invalid integer provided for 'Max Results Per Hash' parameter: Input needs to be a positive " \
"integer that is bigger than 1."
returnResults = []
for entity in entityJsonList:
uid = entity['uid']
if entity['Entity Type'] == 'Hash':
hashChecker.single_hash(entity['Hash Value'])
elif entity['Entity Type'] == 'Phrase':
hashChecker.single_hash(entity['Phrase'])
else:
continue
try:
output = hashChecker.output[0]
resultsJson = output.get_prototypes()[:maxResults]
for count, result in enumerate(resultsJson, start=1):
childIndex = len(returnResults)
resultString = str(count) + ") " + result['name']
description = result.get('description') if result.get('description') is not None else ''
returnResults.append([{'Phrase': resultString,
'Entity Type': 'Phrase',
'Notes': description},
{uid: {'Resolution': 'Name That Hash'}}])
hashcat = str(result.get('hashcat'))
john = str(result.get('john'))
if hashcat is not None and 'HashCat' in parameters['Hash Crack Options']:
hashCatString = 'HashCat Option: ' + hashcat
returnResults.append([{'Phrase': hashCatString,
'Entity Type': 'Phrase'},
{childIndex: {'Resolution': 'HashCat Crack Option'}}])
if john is not None and 'John' in parameters['Hash Crack Options']:
johnString = 'John Option: ' + john
returnResults.append([{'Phrase': johnString,
'Entity Type': 'Phrase'},
{childIndex: {'Resolution': 'John Crack Option'}}])
except IndexError:
pass
# The module will enable logging, and we will remove it here.
rootLogger = logging.getLogger()
for handler in rootLogger.handlers:
rootLogger.removeHandler(handler)
return returnResults

1
Modules/NameThatHash/requirements.txt
View File

@@ -1 +0,0 @@
name-that-hash

10
Modules/Nessus/Nessus.xml
View File

@@ -1,10 +0,0 @@
<Nessus>
<Finding>
<Attributes>
<Attribute default="Issue Synopsis" check="String" primary="True">Issue Synopsis</Attribute>
<Attribute default="Unknown" check="String" primary="False">Solution</Attribute>
<Attribute default="Unknown" check="String" primary="False">CVSS2 Score</Attribute>
<Attribute default="Unknown" check="String" primary="False">CVSS2 Vector</Attribute>
</Attributes>
</Finding>
</Nessus>

192
Modules/Nessus/NessusExistingScan.py
View File

@@ -1,192 +0,0 @@
#!/usr/bin/python
class NessusExistingScan:
name = "Existing Nessus Template Scan"
category = "Nessus"
description = "Nessus vulnerability scanner"
originTypes = {'Phrase'}
resultTypes = {'Finding', 'Port', 'IP Address', 'IPv6 Address', 'Phrase', 'CVE'}
parameters = {
'Nessus Base URL': {'description': 'Enter the url where Nessus starts on your computer or company network '
'e.g: https://127.0.0.1:8834/',
'type': 'String',
'value': '',
'default': 'https://127.0.0.1:8834',
'global': True},
'Nessus Username': {'description': 'Enter the username used for Nessus.',
'type': 'String',
'value': '',
'global': True},
'Nessus Password': {'description': 'Enter the password used for Nessus.',
'type': 'String',
'value': '',
'global': True}
}
def resolution(self, entityJsonList, parameters):
from defusedxml.ElementTree import parse
from playwright.sync_api import sync_playwright, TimeoutError
nessusBaseURL = parameters['Nessus Base URL']
if not nessusBaseURL.endswith('/'):
nessusBaseURL = nessusBaseURL + "/"
nessusUsername = parameters['Nessus Username']
nessusPassword = parameters['Nessus Password']
return_results = []
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
ignore_https_errors=True,
accept_downloads=True
)
page = context.new_page()
pageResolved = False
for _ in range(3):
try:
page.goto(nessusBaseURL, wait_until="networkidle", timeout=10000)
page.fill("[placeholder=\"Username\"]", nessusUsername)
page.fill("[placeholder=\"Password\"]", nessusPassword)
pageResolved = True
break
except TimeoutError:
pass
if not pageResolved:
return "Could not access Nessus panel website."
with page.expect_navigation():
page.click("text=Sign In")
for entity in entityJsonList:
reportUID = entity['uid']
scanName = entity['Phrase']
page.click("text=All Scans")
page.fill("[placeholder=\"Search Scans\"]", scanName)
# Not required, strictly speaking, but doesn't hurt.
page.press("[placeholder=\"Search Scans\"]", "Enter")
try:
with page.expect_navigation():
page.click("text=" + scanName, timeout=10000)
except TimeoutError:
return "Scan name specified does not exist."
try:
page.click("li:has-text(\"Launch\")", timeout=3000)
except TimeoutError:
try:
page.click("text=Launch")
with page.expect_navigation():
page.click("text=Default")
except TimeoutError:
return "Scan specified is already running."
while True:
try:
page.click("text=Export")
break
except TimeoutError:
page.wait_for_timeout(5000)
with page.expect_download() as download_info:
page.click("li:has-text(\"Nessus\")")
download = download_info.value
root = parse(str(download.path()), forbid_dtd=True, forbid_entities=True,
forbid_external=True).getroot()
report = root.find('Report')
if report is None:
return "Report was not properly generated for the scan."
hostAddress = ""
for reportHost in report:
childIndex = len(return_results)
for tag in reportHost.find('HostProperties'):
if tag.attrib['name'] == "host-ip":
hostAddress = tag.text
if ":" in hostAddress:
return_results.append([{
'IPv6 Address': hostAddress,
'Entity Type': 'IPv6 Address'},
{reportUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
else:
return_results.append([{
'IP Address': hostAddress,
'Entity Type': 'IP Address'},
{reportUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
for reportItem in reportHost.findall('ReportItem'):
if reportItem.attrib['port'] == '0':
cvssScore = reportItem.find('cvss_base_score').text \
if reportItem.find('cvss_base_score') is not None else "0"
cvssVector = reportItem.find('cvss_vector').text \
if reportItem.find('cvss_vector') is not None else "N/A"
return_results.append([{
'Issue Synopsis': reportItem.find('synopsis').text,
'Solution': reportItem.find('solution').text,
'CVSS2 Score': cvssScore,
'CVSS2 Vector': cvssVector,
'Entity Type': 'Finding',
'Notes': reportItem.find('plugin_output').text
if reportItem.find('plugin_output') is not None else ""},
{childIndex: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
riskFactor = reportItem.find('risk_factor').text
return_results.append([{
'Phrase': "Risk Factor: " + riskFactor,
'Entity Type': 'Phrase'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cve = reportItem.find('cve').text if reportItem.find('cve') is not None else ""
if cve != "":
return_results.append([{
'CVE': cve,
'Entity Type': 'CVE'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
else:
return_results.append([{
'Port': hostAddress + ":" + reportItem.attrib['port'] + ":" + reportItem.attrib[
'protocol'],
'Entity Type': 'Port'},
{childIndex: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cvssScore = reportItem.find('cvss_base_score').text \
if reportItem.find('cvss_base_score') is not None else "0"
cvssVector = reportItem.find('cvss_vector').text \
if reportItem.find('cvss_vector') is not None else "N/A"
return_results.append([{
'Issue Synopsis': reportItem.find('synopsis').text,
'Solution': reportItem.find('solution').text,
'CVSS2 Score': cvssScore,
'CVSS2 Vector': cvssVector,
'Entity Type': 'Finding',
'Notes': reportItem.find('plugin_output').text
if reportItem.find('plugin_output') is not None else ""},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
riskFactor = reportItem.find('risk_factor').text
return_results.append([{
'Phrase': "Risk Factor: " + riskFactor,
'Entity Type': 'Phrase'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cve = reportItem.find('cve').text if reportItem.find('cve') is not None else ""
if cve != "":
return_results.append([{
'CVE': cve,
'Entity Type': 'CVE'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
download.delete()
page.close()
browser.close()
return return_results

207
Modules/Nessus/NessusNewTemplateScan.py
View File

@@ -1,207 +0,0 @@
#!/usr/bin/python
class NessusNewTemplateScan:
name = "New Nessus Template Scan"
category = "Nessus"
description = "Nessus vulnerability scanner"
originTypes = {'Website', 'IP Address', 'IPv6 Address', 'Domain'}
resultTypes = {'Finding', 'Port', 'IP Address', 'IPv6 Address', 'Phrase', 'CVE'}
parameters = {
'Nessus Base URL': {'description': 'Enter the url where Nessus starts on your computer or company network '
'e.g: https://127.0.0.1:8834/',
'type': 'String',
'value': '',
'default': 'https://127.0.0.1:8834',
'global': True},
'Nessus Username': {'description': 'Enter the username used for Nessus.',
'type': 'String',
'value': '',
'global': True},
'Nessus Password': {'description': 'Enter the password used for Nessus.',
'type': 'String',
'value': '',
'global': True},
'Scan Template': {'description': 'Enter the scan template to be used. Note that scans which are best performed '
'with credentials are not included here.',
'type': 'SingleChoice',
'value': {'Host Discovery', 'Web Application Tests', 'Ripple20 Remote Scan',
'Zerologon Remote Scan', 'Log4Shell Remote Checks'}}
}
def resolution(self, entityJsonList, parameters):
import tldextract
from uuid import uuid4
from defusedxml.ElementTree import parse
from playwright.sync_api import sync_playwright, TimeoutError
nessusBaseURL = parameters['Nessus Base URL']
if not nessusBaseURL.endswith('/'):
nessusBaseURL = nessusBaseURL + "/"
nessusUsername = parameters['Nessus Username']
nessusPassword = parameters['Nessus Password']
scanType = parameters['Scan Template']
return_results = []
entityPrimaryAndUIDFields = {}
# Primary fields should be, and are assumed to be, unique.
for entity in entityJsonList:
entityUID = entity['uid']
if entity['Entity Type'] == 'Website':
primaryField = tldextract.extract(entity['URL']).fqdn
elif entity['Entity Type'] == 'Domain':
primaryField = entity['Domain Name']
elif entity['Entity Type'] == 'IP Address':
primaryField = entity['IP Address']
elif entity['Entity Type'] == 'IPv6 Address':
primaryField = entity['IPv6 Address']
else:
continue
entityPrimaryAndUIDFields[primaryField] = entityUID
targets = ",".join([entityField for entityField in entityPrimaryAndUIDFields])
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
ignore_https_errors=True,
accept_downloads=True
)
page = context.new_page()
pageResolved = False
for _ in range(3):
try:
page.goto(nessusBaseURL, wait_until="networkidle", timeout=10000)
page.fill("[placeholder=\"Username\"]", nessusUsername)
page.fill("[placeholder=\"Password\"]", nessusPassword)
pageResolved = True
break
except TimeoutError:
pass
if not pageResolved:
return "Could not access Nessus panel website."
with page.expect_navigation():
page.click("text=Sign In")
page.click("text=New Scan")
with page.expect_navigation():
page.click("text=" + scanType)
scanName = "LinkScope Scan | " + scanType + " | " + str(uuid4())
page.fill("[aria-label=\"Name\"]", scanName)
page.fill("[aria-label=\"Targets\"]", targets)
page.click("text=Save")
page.fill("[placeholder=\"Search Scans\"]", scanName)
# Not required, strictly speaking, but doesn't hurt.
page.press("[placeholder=\"Search Scans\"]", "Enter")
with page.expect_navigation():
page.click("text=" + scanName)
with page.expect_navigation():
page.click("li:has-text(\"Launch\")")
while True:
try:
page.click("text=Export")
break
except TimeoutError:
page.wait_for_timeout(5000)
with page.expect_download() as download_info:
page.click("li:has-text(\"Nessus\")")
download = download_info.value
root = parse(str(download.path()), forbid_dtd=True, forbid_entities=True, forbid_external=True).getroot()
report = root.find('Report')
if report is None:
return "Report was not properly generated for the scan."
for reportHost in report:
hostName = reportHost.attrib['name']
hostUID = entityPrimaryAndUIDFields[hostName]
for tag in reportHost.find('HostProperties'):
if tag.attrib['name'] == "host-ip":
hostAddress = tag.text
if hostAddress != hostName:
if ":" in hostAddress:
return_results.append([{
'IPv6 Address': hostAddress,
'Entity Type': 'IPv6 Address'},
{hostUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
else:
return_results.append([{
'IP Address': hostAddress,
'Entity Type': 'IP Address'},
{hostUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
for reportItem in reportHost.findall('ReportItem'):
childIndex = len(return_results)
if reportItem.attrib['port'] == '0':
cvssScore = reportItem.find('cvss_base_score').text \
if reportItem.find('cvss_base_score') is not None else "0"
cvssVector = reportItem.find('cvss_vector').text \
if reportItem.find('cvss_vector') is not None else "N/A"
return_results.append([{
'Issue Synopsis': reportItem.find('synopsis').text,
'Solution': reportItem.find('solution').text,
'CVSS2 Score': cvssScore,
'CVSS2 Vector': cvssVector,
'Entity Type': 'Finding',
'Notes': reportItem.find('plugin_output').text
if reportItem.find('plugin_output') is not None else ""},
{hostUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
riskFactor = reportItem.find('risk_factor').text
return_results.append([{
'Phrase': "Risk Factor: " + riskFactor,
'Entity Type': 'Phrase'},
{childIndex: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cve = reportItem.find('cve').text if reportItem.find('cve') is not None else ""
if cve != "":
return_results.append([{
'CVE': cve,
'Entity Type': 'CVE'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
else:
return_results.append([{
'Port': hostAddress + ":" + reportItem.attrib['port'] + ":" + reportItem.attrib['protocol'],
'Entity Type': 'Port'},
{hostUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cvssScore = reportItem.find('cvss_base_score').text \
if reportItem.find('cvss_base_score') is not None else "0"
cvssVector = reportItem.find('cvss_vector').text \
if reportItem.find('cvss_vector') is not None else "N/A"
return_results.append([{
'Issue Synopsis': reportItem.find('synopsis').text,
'Solution': reportItem.find('solution').text,
'CVSS2 Score': cvssScore,
'CVSS2 Vector': cvssVector,
'Entity Type': 'Finding',
'Notes': reportItem.find('plugin_output').text
if reportItem.find('plugin_output') is not None else ""},
{childIndex: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
riskFactor = reportItem.find('risk_factor').text
return_results.append([{
'Phrase': "Risk Factor: " + riskFactor,
'Entity Type': 'Phrase'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cve = reportItem.find('cve').text if reportItem.find('cve') is not None else ""
if cve != "":
return_results.append([{
'CVE': cve,
'Entity Type': 'CVE'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
download.delete()
page.close()
browser.close()
return return_results

111
Modules/Nessus/Nessus_Import.py
View File

@@ -1,111 +0,0 @@
#!/usr/bin/python
class Nessus_Import:
name = "Nessus Import"
category = "Nessus"
description = "Import .nessus report findings"
originTypes = {'Document'}
resultTypes = {'Finding', 'Port', 'IP Address', 'IPv6 Address', 'Phrase', 'CVE'}
parameters = {}
def resolution(self, entityJsonList, parameters):
from pathlib import Path
from defusedxml.ElementTree import parse
return_results = []
for entity in entityJsonList:
reportUID = entity['uid']
filePath = Path(parameters['Project Files Directory']) / entity['File Path']
if not (filePath.exists() and filePath.is_file()):
continue
root = parse(str(filePath), forbid_dtd=True,
forbid_entities=True, forbid_external=True).getroot()
report = root.find('Report')
if report is None:
return "Report was not properly generated for the scan."
hostAddress = ""
for reportHost in report:
for tag in reportHost.find('HostProperties'):
if tag.attrib['name'] == "host-ip":
hostAddress = tag.text
if ":" in hostAddress:
return_results.append([{
'IP Address': hostAddress,
'Entity Type': 'IP Address'},
{reportUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
else:
return_results.append([{
'IPv6 Address': hostAddress,
'Entity Type': 'IPv6 Address'},
{reportUID: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
for reportItem in reportHost.findall('ReportItem'):
childIndex = len(return_results)
if reportItem.attrib['port'] == '0':
cvssScore = reportItem.find('cvss_base_score').text \
if reportItem.find('cvss_base_score') is not None else "0"
cvssVector = reportItem.find('cvss_vector').text \
if reportItem.find('cvss_vector') is not None else "N/A"
return_results.append([{
'Issue Synopsis': reportItem.find('synopsis').text,
'Solution': reportItem.find('solution').text,
'CVSS2 Score': cvssScore,
'CVSS2 Vector': cvssVector,
'Entity Type': 'Finding',
'Notes': reportItem.find('plugin_output').text
if reportItem.find('plugin_output') is not None else ""},
{childIndex: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
riskFactor = reportItem.find('risk_factor').text
return_results.append([{
'Phrase': "Risk Factor: " + riskFactor,
'Entity Type': 'Phrase'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cve = reportItem.find('cve').text if reportItem.find('cve') is not None else ""
if cve != "":
return_results.append([{
'CVE': cve,
'Entity Type': 'CVE'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
else:
return_results.append([{
'Port': hostAddress + ":" + reportItem.attrib['port'] + ":" + reportItem.attrib['protocol'],
'Entity Type': 'Port'},
{childIndex: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cvssScore = reportItem.find('cvss_base_score').text \
if reportItem.find('cvss_base_score') is not None else "0"
cvssVector = reportItem.find('cvss_vector').text \
if reportItem.find('cvss_vector') is not None else "N/A"
return_results.append([{
'Issue Synopsis': reportItem.find('synopsis').text,
'Solution': reportItem.find('solution').text,
'CVSS2 Score': cvssScore,
'CVSS2 Vector': cvssVector,
'Entity Type': 'Finding',
'Notes': reportItem.find('plugin_output').text
if reportItem.find('plugin_output') is not None else ""},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
riskFactor = reportItem.find('risk_factor').text
return_results.append([{
'Phrase': "Risk Factor: " + riskFactor,
'Entity Type': 'Phrase'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
cve = reportItem.find('cve').text if reportItem.find('cve') is not None else ""
if cve != "":
return_results.append([{
'CVE': cve,
'Entity Type': 'CVE'},
{len(return_results) - 1: {'Resolution': 'Nessus Scan', 'Notes': ''}}])
return return_results

3
Modules/Nessus/requirements.txt
View File

@@ -1,3 +0,0 @@
tldextract
defusedxml
playwright

Some files were not shown because too many files have changed in this diff Show More