maq/LinkScope_Client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Yet more updates and fixes to resolutions.

Browse Source 
File Extractor resolution moved from selenium to playwright.
This commit is contained in:
AccentuSoft
2021-12-20 01:37:48 -05:00
parent fb830de41a
commit 520cb612e9
8 changed files with 220 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Core/Resolutions/Core/AffiliateCodesExtractor.py
View File

@@ -43,6 +43,7 @@ class AffiliateCodesExtractor:
from playwright.sync_api import sync_playwright, TimeoutError
from bs4 import BeautifulSoup
import urllib
import tldextract
import re
returnResults = []
@@ -189,7 +190,7 @@ class AffiliateCodesExtractor:
continue
if not url.startswith('http://') and not url.startswith('https://'):
url = 'http://' + url
domain = ".".join(urllib.parse.urlparse(url).netloc.split('.')[-2:])
domain = tldextract.extract(url).fqdn
extractCodes(uid, url, maxDepth)
browser.close()
return returnResults

4
Core/Resolutions/Core/EmailExtractor.py
View File

@@ -46,7 +46,7 @@ class EmailExtractor:
def resolution(self, entityJsonList, parameters):
from playwright.sync_api import sync_playwright, TimeoutError
from bs4 import BeautifulSoup
import urllib
import tldextract
import re
from email_validator import validate_email, caching_resolver, EmailNotValidError
@@ -144,7 +144,7 @@ class EmailExtractor:
continue
if not url.startswith('http://') and not url.startswith('https://'):
url = 'http://' + url
domain = ".".join(urllib.parse.urlparse(url).netloc.split('.')[-2:])
domain = tldextract.extract(url).fqdn
extractEmails(uid, url, maxDepth)
browser.close()

325
Core/Resolutions/Core/FileExtractor.py
View File

@@ -1,169 +1,202 @@
#!/usr/bin/env python3
"""
It should be noted that the effectiveness of this resolution relies on the website in question to have the non-website
documents showing their extension in their links in the page. Since the extension is available the vast majority of the
time, the resolution should produce correct results just about every time.
The alternative is to analyze the contents of each page to determine if they are a non-web file, and then initiate a
download. This would require a lot of work for very little payoff, which would be error-prone in and of itself, so it
is not pursued.
A few sites may also present issues when downloads are attempted by a user agent without javascript.
Even if the download does not work however, the site containing the file will be represented as a node on the graph,
so the user can download the file themselves if there was any issue.
"""
class FileExtractor:
# A string that is treated as the name of this resolution.
name = "Get Files In Domain"
name = "Find Hosted File URLs"
# A string that describes this resolution.
description = "Returns Nodes of files in websites"
description = "Returns Nodes of files in websites and domains."
originTypes = {'Domain'}
originTypes = {'Domain', 'Website'}
resultTypes = {'Phrase'}
resultTypes = {'Website', 'Document', 'Image', 'Video', 'Archive'}
parameters = {'Max Webpages to Follow': {'description': 'Please enter the maximum number of webpages to follow. '
'Default number 20. The greater the number the longer the '
'resolution takes to complete. '
'Enter "0" (no quotes) to use the default value.',
'type': 'String',
'value': 'Max URLs to follow',
'default': '20'}}
parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
'explored to discover more entities. Each entity discovered after '
'exploring sites linked in the original website or domain is said to '
'have a "depth" value of 1. Entities found from exploring the links on '
'this page would have a "depth" of 2, and so on. A larger value could '
'result in EXPONENTIALLY more time taken to finish the resolution.\n'
'The default value is "0", which means only the provided website, or '
'the index page of the domain provided, is explored.',
'type': 'String',
'value': '0',
'default': '0'}}
def resolution(self, entityJsonList, parameters):
import requests.exceptions
import tldextract
from selenium import webdriver
import requests
from hashlib import md5
from binascii import hexlify
from pathlib import Path
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright, TimeoutError
try:
maxDepth = max(int(parameters['Max Depth']), 0)
except ValueError:
return "Invalid value provided for Max Webpages to follow."
fileTypes = (".sxw", ".odt", ".ods", ".odg", ".odp", ".docx", ".xlsx", ".pptx", ".ppsx", ".doc", ".xls",
".ppt", ".pps", ".pdf", ".wpd", ".raw", ".cr2", ".crw", ".indd", ".rdp", ".ica", ".ico", ".txt",
".text", ".bak", ".log", ".env", ".pub", ".docm", ".xlsm", ".old", ".csv", ".apk", ".sql", ".cfg",
".key", ".reg", ".yml", ".yaml", ".mail", ".eml", ".mbox", ".mbx", ".url", ".csr", ".config",
".mdb", ".user", ".adr", ".ini", ".plist", ".conf", ".dat", ".pcf", ".bok", ".properties", ".json",
".backup", ".sh", ".py", ".md", ".inc")
videoTypes = (".mp3", ".mp4")
imageTypes = (".jpg", ".jpeg", ".png", ".svg", ".svgz")
archiveTypes = (".zip", ".rar", ".7z", ".gz")
returnResults = []
try:
max_urls = int(parameters['Max Webpages to Follow'])
except ValueError:
max_urls = 20
fireFoxOptions = webdriver.FirefoxOptions()
fireFoxOptions.headless = True
driver = webdriver.Firefox(options=fireFoxOptions)
# Access requests via the `requests` attribute
for entity in entityJsonList:
uid = entity['uid']
primaryField = entity[list(entity)[1]]
if primaryField.startswith('http://') or primaryField.startswith('https://'):
url = primaryField
else:
url = 'https://' + primaryField
# a queue of urls to be crawled next
new_urls = {url} # deque([url])
# a set of urls that we have already processed
processed_urls = set()
# a set of domains inside the target website
local_urls = set()
# a set of urls containing pdf files
pdf_files = set()
# a set of urls containing doc files
doc_files = set()
# a set of domains outside the target website
foreign_urls = set()
# a set of broken urls
broken_urls = set()
# process urls one by one until we exhaust the queue
while len(new_urls):
# move url from the queue to processed url set
url = new_urls.pop()
# print the current url
poundlessUrl = url.split('#')[0]
if url in processed_urls:
continue
processed_urls.add(url)
# extract base url to resolve relative links
parts = tldextract.extract(poundlessUrl)
if parts.subdomain != '':
base = parts.subdomain + '.' + parts.domain + '.' + parts.suffix
else:
base = parts.domain + '.' + parts.suffix
strip_base = parts.domain + '.' + parts.suffix
base_url = 'https://' + base
if base_url != poundlessUrl and base_url in poundlessUrl:
paths = poundlessUrl.split(base_url, 1)[1]
path = poundlessUrl[:poundlessUrl.rfind('/') + 1] if '/' in paths else poundlessUrl
else:
path = poundlessUrl
def iterateOnDepth(currentURL, currentDepth: int):
urlsExplored.add(currentURL)
urlsToExplore = set()
for _ in range(3):
try:
response = requests.get(poundlessUrl)
if response.status_code == 404:
continue
elif base not in response.url:
foreign_urls.add(response.url)
continue
soup = BeautifulSoup(response.text, "lxml")
if response.status_code == 403:
driver.get(poundlessUrl)
pageSource = driver.page_source
soup = BeautifulSoup(pageSource, "lxml")
if base_url not in driver.current_url:
foreign_urls.add(driver.current_url)
continue
except(requests.exceptions.MissingSchema, requests.exceptions.ConnectionError,
requests.exceptions.InvalidURL,
requests.exceptions.InvalidSchema):
# add broken urls to it’s own set, then continue
broken_urls.add(poundlessUrl)
continue
for link in soup.find_all('a'):
# extract link url from the anchor
anchor = link.attrs['href'] if 'href' in link.attrs else ''
if anchor.startswith('/'):
local_link = base_url + anchor
local_link = local_link.split('#')[0]
local_urls.add(local_link)
if local_link not in processed_urls and base_url in local_link:
new_urls.add(local_link)
elif strip_base in anchor:
anchor = anchor.split('#')[0]
local_urls.add(anchor)
if anchor not in processed_urls and base_url in anchor:
new_urls.add(anchor)
elif not anchor.startswith('http'):
local_link = path + anchor
local_link = local_link.split('#')[0]
local_urls.add(local_link)
if local_link not in processed_urls and base_url in local_link:
new_urls.add(local_link)
else:
foreign_urls.add(anchor)
if len(processed_urls) > max_urls:
page.goto(currentURL, wait_until="networkidle", timeout=10000)
break
except TimeoutError:
pass
for link in local_urls:
if link.endswith('.pdf'):
pdf_files.add(link)
elif link.endswith('.doc') or link.endswith('.docx'):
doc_files.add(link)
for link in foreign_urls:
if link.endswith('.pdf'):
pdf_files.add(link)
elif link.endswith('.doc') or link.endswith('.docx'):
doc_files.add(link)
soupContents = BeautifulSoup(page.content(), 'lxml')
urlInPage = soupContents.find_all('a') + soupContents.find_all('link')
for tag in urlInPage:
link = tag.get('href', None)
if link is not None:
if not link.startswith('http'):
# We assume that we will be redirected to https if available.
link = 'http://' + domain + link
link = link.split('#')[0]
if link not in urlsExplored:
urlsExplored.add(link)
fileTypeIdentified = ''
# Material name is the part after the last slash of the URL, plus the sha512sum of the URL
if link.endswith(fileTypes):
fileTypeIdentified = 'Document'
elif link.endswith(videoTypes):
fileTypeIdentified = 'Video'
elif link.endswith(imageTypes):
fileTypeIdentified = 'Image'
elif link.endswith(archiveTypes):
fileTypeIdentified = 'Archive'
if fileTypeIdentified:
childIndex = len(returnResults)
returnResults.append([{'URL': link,
'Entity Type': 'Website'},
{uid: {'Resolution': 'File URL',
'Notes': ''}}])
docProperName = link.split('/')[-1]
docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName
docFullPath = Path(parameters['Project Files Directory']) / docFileName
try:
response = requests.get(link,
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
'x64; rv:94.0) Gecko/20100101 '
'Firefox/94.0'},
stream=True)
with open(docFullPath, 'wb') as fileToWrite:
for chunk in response.iter_content(4096):
fileToWrite.write(chunk)
returnResults.append([{fileTypeIdentified + ' Name': docProperName,
'File Path': docFileName,
'Entity Type': fileTypeIdentified},
{childIndex: {'Resolution': 'Downloaded File',
'Notes': ''}}])
except Exception:
pass
elif domain in link:
urlsToExplore.add(link)
linksInImgSrc = soupContents.find_all('img')
for tag in linksInImgSrc:
link = tag.get('src', None)
if link is not None:
if not link.startswith('http'):
# We assume that we will be redirected to https if available.
link = 'http://' + domain + link
link = link.split('#')[0]
if link not in urlsExplored:
urlsExplored.add(link)
childIndex = len(returnResults)
returnResults.append([{'URL': link,
'Entity Type': 'Website'},
{uid: {'Resolution': 'File URL',
'Notes': ''}}])
docProperName = link.split('/')[-1]
docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName
docFullPath = Path(parameters['Project Files Directory']) / docFileName
try:
response = requests.get(link,
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
'x64; rv:94.0) Gecko/20100101 '
'Firefox/94.0'},
stream=True)
with open(docFullPath, 'wb') as fileToWrite:
for chunk in response.iter_content(4096):
fileToWrite.write(chunk)
returnResults.append([{'Image Name': docProperName,
'File Path': docFileName,
'Entity Type': 'Image'},
{childIndex: {'Resolution': 'Downloaded File',
'Notes': ''}}])
except Exception:
pass
if currentDepth > 0:
newDepth = currentDepth - 1
for newURL in urlsToExplore:
iterateOnDepth(newURL, newDepth)
with sync_playwright() as p:
browser = p.firefox.launch()
context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
)
page = context.new_page()
for site in entityJsonList:
uid = site['uid']
url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
if url is None:
continue
if not url.startswith('http://') and not url.startswith('https://'):
url = 'http://' + url
domain = tldextract.extract(url).fqdn
# Because these do not persist across entities, it is possible to explore a URL multiple times.
# However, since different URLs may be encountered at different depths, this way should ensure
# that there are no false negatives, i.e. if something should be discovered, it will be.
urlsExplored = set()
iterateOnDepth(url, maxDepth)
for i in pdf_files:
returnResults.append([{'Document Name': i,
'Entity Type': 'Document'},
{uid: {'Resolution': 'PDF Document Found', 'Name': 'Document Found',
'Notes': ''}}])
for i in doc_files:
returnResults.append([{'Document Name': i,
'Entity Type': 'Document'},
{uid: {'Resolution': 'DOC Document Found', 'Name': 'Document Found',
'Notes': ''}}])
return returnResults

6
Core/Resolutions/Core/GetExternalURLs.py
View File

@@ -34,10 +34,11 @@ class GetExternalURLs:
}}
def resolution(self, entityJsonList, parameters):
import urllib
import tldextract
from playwright.sync_api import sync_playwright, TimeoutError
from bs4 import BeautifulSoup
import re
import urllib
returnResult = []
@@ -65,8 +66,7 @@ class GetExternalURLs:
continue
if not url.startswith('http://') and not url.startswith('https://'):
url = 'http://' + url
domain = ".".join(urllib.parse.urlparse(url).netloc.split('.')[-2:])
domain = tldextract.extract(url).fqdn
# Try to load the page a few times, in case of timeouts.
# I don't think making parts of this async actually helps in this case.

42
Core/Resolutions/Core/JSCodeExtractor.py
View File

@@ -18,26 +18,42 @@ class JSCodeExtractor:
from playwright.sync_api import sync_playwright
import re
returnResults = []
requestUrlsParsed = set()
uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
gtmRegex = re.compile(r'\bGTM-[A-Z0-9]{1,7}\b', re.IGNORECASE)
gRegex = re.compile(r'\bG-[A-Z0-9]{1,15}\b', re.IGNORECASE)
qualtricsRegex = re.compile(r'\bQ_ZID=[a-zA-Z_0-9]*\b', re.IGNORECASE)
# In the future, if tracking codes from more companies are supported, change this so that resolutions
# indicate which company the code is from.
def GetTrackingCodes(pageUid, requestUrl) -> None:
trackingCodes = set()
trackingCodes.update(set(uaRegex.findall(requestUrl)))
trackingCodes.update(set(pubRegex.findall(requestUrl)))
trackingCodes.update(set(gtmRegex.findall(requestUrl)))
trackingCodes.update(set(gRegex.findall(requestUrl)))
for trackingCode in trackingCodes:
returnResults.append([{'Phrase': trackingCode,
'Entity Type': 'Phrase'},
{pageUid: {'Resolution': 'Google Tracking Code',
'Notes': ''}}])
if requestUrl not in requestUrlsParsed:
requestUrlsParsed.add(requestUrl)
for uaCode in uaRegex.findall(requestUrl):
returnResults.append([{'Phrase': uaCode,
'Entity Type': 'Phrase'},
{pageUid: {'Resolution': 'Google UA Tracking Code',
'Notes': ''}}])
for pubCode in pubRegex.findall(requestUrl):
returnResults.append([{'Phrase': pubCode,
'Entity Type': 'Phrase'},
{pageUid: {'Resolution': 'Google AdSense ca-pub Tracking Code',
'Notes': ''}}])
for gtmCode in gtmRegex.findall(requestUrl):
returnResults.append([{'Phrase': gtmCode,
'Entity Type': 'Phrase'},
{pageUid: {'Resolution': 'Google GTM Tracking Code',
'Notes': ''}}])
for gCode in gRegex.findall(requestUrl):
returnResults.append([{'Phrase': gCode,
'Entity Type': 'Phrase'},
{pageUid: {'Resolution': 'Google G Tracking Code',
'Notes': ''}}])
for qCode in qualtricsRegex.findall(requestUrl):
returnResults.append([{'Phrase': qCode[6:],
'Entity Type': 'Phrase'},
{pageUid: {'Resolution': 'Qualtrics Tracking Code',
'Notes': ''}}])
with sync_playwright() as p:
browser = p.firefox.launch()

4
Core/Resolutions/Core/PhoneNumbersExtractor.py
View File

@@ -32,7 +32,7 @@ class PhoneNumbersExtractor:
def resolution(self, entityJsonList, parameters):
from playwright.sync_api import sync_playwright, TimeoutError
from bs4 import BeautifulSoup
import urllib
import tldextract
returnResults = []
@@ -99,7 +99,7 @@ class PhoneNumbersExtractor:
continue
if not url.startswith('http://') and not url.startswith('https://'):
url = 'http://' + url
domain = ".".join(urllib.parse.urlparse(url).netloc.split('.')[-2:])
domain = tldextract.extract(url).fqdn
extractTels(uid, url, maxDepth)
browser.close()

2
Modules/EDGAR_Api/EFDByToDate.py
View File

@@ -2,7 +2,7 @@
import bs4
class EFDByFromDate:
class EFDByToDate:
name = 'Get EFD Reports To Date'
description = 'Return Nodes Of Websites to Reports'
originTypes = {'Date'}

4
requirements.txt
View File

@@ -15,13 +15,13 @@ python-magic
pydot
networkx
tldextract
defusedxml
cryptography
msgpack
folium
pillow
lz4
urllib3
reportlab
pandas
svglib
@@ -41,7 +41,6 @@ requests
shodan
docx2python
PyPDF2
tldextract
beautifulsoup4
python-Wappalyzer
vtapi3
@@ -59,3 +58,4 @@ email-validator
docx2python
exif
xmltodict
urllib3